# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

import sys
sys.path.insert(0, '../')
sys.path.insert(0, '../pyrecodes/')

# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

project = 'pyrecodes'
copyright = '2023, Nikola Blagojević'
author = 'Nikola Blagojević'
release = 'v0.1.0'

# -- General configuration ---------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

extensions = [
    'sphinx.ext.autosummary',
    'sphinx.ext.autodoc',
    'sphinx_togglebutton',
    'sphinx_carousel.carousel',
    'sphinx.ext.githubpages'
]

templates_path = ['_templates']
exclude_patterns = []


# -- Options for HTML output -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

html_theme = 'furo'
html_static_path = ['_static']
html_logo = "./figures/pyrecodes_logo.png"
html_home = "./what_is_pyrecodes.html"

docs/source/conf.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2html.py 9115 2022-07-28 17:06:24Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing HTML.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description


description = ('Generates (X)HTML documents from standalone reStructuredText '
               'sources.  ' + default_description)

publish_cmdline(writer_name='html', description=description)

env_pyrecodes/bin/rst2html.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2html4.py 9115 2022-07-28 17:06:24Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing (X)HTML.

The output conforms to XHTML 1.0 transitional
and almost to HTML 4.01 transitional (except for closing empty tags).
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description


description = ('Generates (X)HTML documents from standalone reStructuredText '
               'sources.  ' + default_description)

publish_cmdline(writer_name='html4', description=description)

env_pyrecodes/bin/rst2html4.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3
# :Copyright: © 2015 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause
#
# Revision: $Revision: 9021 $
# Date: $Date: 2022-03-04 16:54:22 +0100 (Fr, 04. Mär 2022) $

"""
A minimal front end to the Docutils Publisher, producing HTML 5 documents.

The output is also valid XML.
"""

try:
    import locale  # module missing in Jython
    locale.setlocale(locale.LC_ALL, '')
except locale.Error:
    pass

from docutils.core import publish_cmdline, default_description

description = ('Generates HTML5 documents from standalone '
               'reStructuredText sources.\n'
               + default_description)

publish_cmdline(writer_name='html5', description=description)

env_pyrecodes/bin/rst2html5.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2latex.py 9115 2022-07-28 17:06:24Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing LaTeX.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline

description = ('Generates LaTeX documents from standalone reStructuredText '
               'sources. '
               'Reads from <source> (default is stdin) and writes to '
               '<destination> (default is stdout).  See '
               '<https://docutils.sourceforge.io/docs/user/latex.html> for '
               'the full reference.')

publish_cmdline(writer_name='latex', description=description)

env_pyrecodes/bin/rst2latex.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# Author:
# Contact: grubert@users.sf.net
# Copyright: This module has been placed in the public domain.

"""
man.py
======

This module provides a simple command line interface that uses the
man page writer to output from ReStructuredText source.
"""

import locale
try:
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description
from docutils.writers import manpage

description = ("Generates plain unix manual documents.  "
               + default_description)

publish_cmdline(writer=manpage.Writer(), description=description)

env_pyrecodes/bin/rst2man.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2odt.py 9115 2022-07-28 17:06:24Z milde $
# Author: Dave Kuhlman <dkuhlman@rexx.com>
# Copyright: This module has been placed in the public domain.

"""
A front end to the Docutils Publisher, producing OpenOffice documents.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline_to_binary, default_description
from docutils.writers.odf_odt import Writer, Reader


description = ('Generates OpenDocument/OpenOffice/ODF documents from '
               'standalone reStructuredText sources.  ' + default_description)


writer = Writer()
reader = Reader()
output = publish_cmdline_to_binary(reader=reader, writer=writer,
                                   description=description)

env_pyrecodes/bin/rst2odt.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# Copyright: This module has been placed in the public domain.

"""
Adapt a word-processor-generated styles.odt for odtwriter use:

Drop page size specifications from styles.xml in STYLE_FILE.odt.
See https://docutils.sourceforge.io/docs/user/odt.html#page-size

Provisional backwards compatibility stub (to be removed in Docutils >= 0.21).

The actual code moved to the "docutils" library package and can be started
with ``python -m docutils.writers.odf_odt.prepstyles``.
"""

from docutils.writers.odf_odt import prepstyles

if __name__ == '__main__':
    prepstyles.main()

env_pyrecodes/bin/rst2odt_prepstyles.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2pseudoxml.py 9115 2022-07-28 17:06:24Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing pseudo-XML.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description


description = ('Generates pseudo-XML from standalone reStructuredText '
               'sources (for testing purposes).  ' + default_description)

publish_cmdline(description=description)

env_pyrecodes/bin/rst2pseudoxml.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2s5.py 9115 2022-07-28 17:06:24Z milde $
# Author: Chris Liechti <cliechti@gmx.net>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing HTML slides using
the S5 template system.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description


description = ('Generates S5 (X)HTML slideshow documents from standalone '
               'reStructuredText sources.  ' + default_description)

publish_cmdline(writer_name='s5', description=description)

env_pyrecodes/bin/rst2s5.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2xetex.py 9115 2022-07-28 17:06:24Z milde $
# Author: Guenter Milde
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing Lua/XeLaTeX code.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline

description = ('Generates LaTeX documents from standalone reStructuredText '
               'sources for compilation with the Unicode-aware TeX variants '
               'XeLaTeX or LuaLaTeX. '
               'Reads from <source> (default is stdin) and writes to '
               '<destination> (default is stdout).  See '
               '<https://docutils.sourceforge.io/docs/user/latex.html> for '
               'the full reference.')

publish_cmdline(writer_name='xetex', description=description)

env_pyrecodes/bin/rst2xetex.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rst2xml.py 9115 2022-07-28 17:06:24Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing Docutils XML.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description


description = ('Generates Docutils-native XML from standalone '
               'reStructuredText sources.  ' + default_description)

publish_cmdline(writer_name='xml', description=description)

env_pyrecodes/bin/rst2xml.py
NikolaBlagojevic/pyrecodes
#!/Users/nikola/pyrecodes/env_pyrecodes/bin/python3

# $Id: rstpep2html.py 9115 2022-07-28 17:06:24Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A minimal front end to the Docutils Publisher, producing HTML from PEP
(Python Enhancement Proposal) documents.
"""

try:
    import locale
    locale.setlocale(locale.LC_ALL, '')
except Exception:
    pass

from docutils.core import publish_cmdline, default_description


description = ('Generates (X)HTML from reStructuredText-format PEP files.  '
               + default_description)

publish_cmdline(reader_name='pep', writer_name='pep_html',
                description=description)

env_pyrecodes/bin/rstpep2html.py
NikolaBlagojevic/pyrecodes
"""Affine transformation matrices

The Affine package is derived from Casey Duncan's Planar package. See the
copyright statement below.
"""

#############################################################################
# Copyright (c) 2010 by Casey Duncan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# * Neither the name(s) of the copyright holders nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AS IS AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#############################################################################

from collections import namedtuple
import math
import warnings


__all__ = ["Affine"]
__author__ = "Sean Gillies"
__version__ = "2.4.0"

EPSILON: float = 1e-5


class AffineError(Exception):
    pass


class TransformNotInvertibleError(AffineError):
    """The transform could not be inverted"""


class UndefinedRotationError(AffineError):
    """The rotation angle could not be computed for this transform"""


def cached_property(func):
    """Special property decorator that caches the computed
    property value in the object's instance dict the first
    time it is accessed.
    """
    name = func.__name__
    doc = func.__doc__

    def getter(self, name=name):
        try:
            return self.__dict__[name]
        except KeyError:
            self.__dict__[name] = value = func(self)
            return value

    getter.func_name = name
    return property(getter, doc=doc)


def cos_sin_deg(deg: float):
    """Return the cosine and sin for the given angle in degrees.

    With special-case handling of multiples of 90 for perfect right
    angles.
    """
    deg = deg % 360.0
    if deg == 90.0:
        return 0.0, 1.0
    elif deg == 180.0:
        return -1.0, 0
    elif deg == 270.0:
        return 0, -1.0
    rad = math.radians(deg)
    return math.cos(rad), math.sin(rad)


class Affine(namedtuple("Affine", ("a", "b", "c", "d", "e", "f", "g", "h", "i"))):
    """Two dimensional affine transform for 2D linear mapping.

    Parameters
    ----------
    a, b, c, d, e, f : float
        Coefficients of an augmented affine transformation matrix

        | x' |   | a  b  c | | x |
        | y' | = | d  e  f | | y |
        | 1  |   | 0  0  1 | | 1 |

        `a`, `b`, and `c` are the elements of the first row of the
        matrix. `d`, `e`, and `f` are the elements of the second row.

    Attributes
    ----------
    a, b, c, d, e, f, g, h, i : float
        The coefficients of the 3x3 augmented affine transformation
        matrix

        | x' |   | a  b  c | | x |
        | y' | = | d  e  f | | y |
        | 1  |   | g  h  i | | 1 |

        `g`, `h`, and `i` are always 0, 0, and 1.

    The Affine package is derived from Casey Duncan's Planar package.
    See the copyright statement below.  Parallel lines are preserved by
    these transforms. Affine transforms can perform any combination of
    translations, scales/flips, shears, and rotations.  Class methods
    are provided to conveniently compose transforms from these
    operations.

    Internally the transform is stored as a 3x3 transformation matrix.
    The transform may be constructed directly by specifying the first
    two rows of matrix values as 6 floats. Since the matrix is an affine
    transform, the last row is always ``(0, 0, 1)``.

    N.B.: multiplication of a transform and an (x, y) vector *always*
    returns the column vector that is the matrix multiplication product
    of the transform and (x, y) as a column vector, no matter which is
    on the left or right side. This is obviously not the case for
    matrices and vectors in general, but provides a convenience for
    users of this class.

    """

    precision = EPSILON

    def __new__(
        cls,
        a: float,
        b: float,
        c: float,
        d: float,
        e: float,
        f: float,
        g: float = 0.0,
        h: float = 0.0,
        i: float = 1.0,
    ):
        """Create a new object

        Parameters
        ----------
        a, b, c, d, e, f : float
            Elements of an augmented affine transformation matrix.
        """
        return tuple.__new__(
            cls,
            (
                a * 1.0,
                b * 1.0,
                c * 1.0,
                d * 1.0,
                e * 1.0,
                f * 1.0,
                g * 1.0,
                h * 1.0,
                i * 1.0,
            ),
        )

    @classmethod
    def from_gdal(cls, c: float, a: float, b: float, f: float, d: float, e: float):
        """Use same coefficient order as GDAL's GetGeoTransform().

        :param c, a, b, f, d, e: 6 floats ordered by GDAL.
        :rtype: Affine
        """
        return cls.__new__(cls, a, b, c, d, e, f)

    @classmethod
    def identity(cls):
        """Return the identity transform.

        :rtype: Affine
        """
        return identity

    @classmethod
    def translation(cls, xoff: float, yoff: float):
        """Create a translation transform from an offset vector.

        :param xoff: Translation x offset.
        :type xoff: float
        :param yoff: Translation y offset.
        :type yoff: float
        :rtype: Affine
        """
        return tuple.__new__(cls, (1.0, 0.0, xoff, 0.0, 1.0, yoff, 0.0, 0.0, 1.0))

    @classmethod
    def scale(cls, *scaling):
        """Create a scaling transform from a scalar or vector.

        :param scaling: The scaling factor. A scalar value will
            scale in both dimensions equally. A vector scaling
            value scales the dimensions independently.
        :type scaling: float or sequence
        :rtype: Affine
        """
        if len(scaling) == 1:
            sx = sy = float(scaling[0])
        else:
            sx, sy = scaling
        return tuple.__new__(cls, (sx, 0.0, 0.0, 0.0, sy, 0.0, 0.0, 0.0, 1.0))

    @classmethod
    def shear(cls, x_angle: float = 0, y_angle: float = 0):
        """Create a shear transform along one or both axes.

        :param x_angle: Shear angle in degrees parallel to the x-axis.
        :type x_angle: float
        :param y_angle: Shear angle in degrees parallel to the y-axis.
        :type y_angle: float
        :rtype: Affine
        """
        mx = math.tan(math.radians(x_angle))
        my = math.tan(math.radians(y_angle))
        return tuple.__new__(cls, (1.0, mx, 0.0, my, 1.0, 0.0, 0.0, 0.0, 1.0))

    @classmethod
    def rotation(cls, angle: float, pivot=None):
        """Create a rotation transform at the specified angle.

        A pivot point other than the coordinate system origin may be
        optionally specified.

        :param angle: Rotation angle in degrees, counter-clockwise
            about the pivot point.
        :type angle: float
        :param pivot: Point to rotate about, if omitted the rotation is
            about the origin.
        :type pivot: sequence
        :rtype: Affine
        """
        ca, sa = cos_sin_deg(angle)
        if pivot is None:
            return tuple.__new__(cls, (ca, -sa, 0.0, sa, ca, 0.0, 0.0, 0.0, 1.0))
        else:
            px, py = pivot
            return tuple.__new__(
                cls,
                (
                    ca,
                    -sa,
                    px - px * ca + py * sa,
                    sa,
                    ca,
                    py - px * sa - py * ca,
                    0.0,
                    0.0,
                    1.0,
                ),
            )

    @classmethod
    def permutation(cls, *scaling):
        """Create the permutation transform

        For 2x2 matrices, there is only one permutation matrix that is
        not the identity.

        :rtype: Affine
        """

        return tuple.__new__(cls, (0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0))

    def __str__(self) -> str:
        """Concise string representation."""
        return (
            "|% .2f,% .2f,% .2f|\n" "|% .2f,% .2f,% .2f|\n" "|% .2f,% .2f,% .2f|"
        ) % self

    def __repr__(self) -> str:
        """Precise string representation."""
        return ("Affine(%r, %r, %r,\n" "       %r, %r, %r)") % self[:6]

    def to_gdal(self):
        """Return same coefficient order as GDAL's SetGeoTransform().

        :rtype: tuple
        """
        return (self.c, self.a, self.b, self.f, self.d, self.e)

    def to_shapely(self):
        """Return an affine transformation matrix compatible with shapely

        Shapely's affinity module expects an affine transformation matrix
        in (a,b,d,e,xoff,yoff) order.

        :rtype: tuple
        """
        return (self.a, self.b, self.d, self.e, self.xoff, self.yoff)

    @property
    def xoff(self) -> float:
        """Alias for 'c'"""
        return self.c

    @property
    def yoff(self) -> float:
        """Alias for 'f'"""
        return self.f

    @cached_property
    def determinant(self) -> float:
        """The determinant of the transform matrix.

        This value is equal to the area scaling factor when the
        transform is applied to a shape.
        """
        a, b, c, d, e, f, g, h, i = self
        return a * e - b * d

    @property
    def _scaling(self):
        """The absolute scaling factors of the transformation.

        This tuple represents the absolute value of the scaling factors of the
        transformation, sorted from bigger to smaller.
        """
        a, b, _, d, e, _, _, _, _ = self

        # The singular values are the square root of the eigenvalues
        # of the matrix times its transpose, M M*
        # Computing trace and determinant of M M*
        trace = a ** 2 + b ** 2 + d ** 2 + e ** 2
        det = (a * e - b * d) ** 2

        delta = trace ** 2 / 4 - det
        if delta < 1e-12:
            delta = 0

        l1 = math.sqrt(trace / 2 + math.sqrt(delta))
        l2 = math.sqrt(trace / 2 - math.sqrt(delta))
        return l1, l2

    @property
    def eccentricity(self) -> float:
        """The eccentricity of the affine transformation.

        This value represents the eccentricity of an ellipse under
        this affine transformation.

        Raises NotImplementedError for improper transformations.
        """
        l1, l2 = self._scaling
        return math.sqrt(l1 ** 2 - l2 ** 2) / l1

    @property
    def rotation_angle(self) -> float:
        """The rotation angle in degrees of the affine transformation.

        This is the rotation angle in degrees of the affine transformation,
        assuming it is in the form M = R S, where R is a rotation and S is a
        scaling.

        Raises UndefinedRotationError for improper and degenerate
        transformations.
        """
        a, b, _, c, d, _, _, _, _ = self
        if self.is_proper or self.is_degenerate:
            l1, _ = self._scaling
            y, x = c / l1, a / l1
            return math.atan2(y, x) * 180 / math.pi
        else:
            raise UndefinedRotationError

    @property
    def is_identity(self) -> bool:
        """True if this transform equals the identity matrix,
        within rounding limits.
        """
        return self is identity or self.almost_equals(identity, self.precision)

    @property
    def is_rectilinear(self) -> bool:
        """True if the transform is rectilinear.

        i.e., whether a shape would remain axis-aligned, within rounding
        limits, after applying the transform.
        """
        a, b, c, d, e, f, g, h, i = self
        return (abs(a) < self.precision and abs(e) < self.precision) or (
            abs(d) < self.precision and abs(b) < self.precision
        )

    @property
    def is_conformal(self) -> bool:
        """True if the transform is conformal.

        i.e., if angles between points are preserved after applying the
        transform, within rounding limits.  This implies that the
        transform has no effective shear.
        """
        a, b, c, d, e, f, g, h, i = self
        return abs(a * b + d * e) < self.precision

    @property
    def is_orthonormal(self) -> bool:
        """True if the transform is orthonormal.

        Which means that the transform represents a rigid motion, which
        has no effective scaling or shear. Mathematically, this means
        that the axis vectors of the transform matrix are perpendicular
        and unit-length.  Applying an orthonormal transform to a shape
        always results in a congruent shape.
        """
        a, b, c, d, e, f, g, h, i = self
        return (
            self.is_conformal
            and abs(1.0 - (a * a + d * d)) < self.precision
            and abs(1.0 - (b * b + e * e)) < self.precision
        )

    @cached_property
    def is_degenerate(self) -> bool:
        """True if this transform is degenerate.

        Which means that it will collapse a shape to an effective area
        of zero. Degenerate transforms cannot be inverted.
        """
        return self.determinant == 0.0

    @cached_property
    def is_proper(self) -> bool:
        """True if this transform is proper.

        Which means that it does not include reflection.
        """
        return self.determinant > 0.0

    @property
    def column_vectors(self):
        """The values of the transform as three 2D column vectors"""
        a, b, c, d, e, f, _, _, _ = self
        return (a, d), (b, e), (c, f)

    def almost_equals(self, other, precision: float = EPSILON) -> bool:
        """Compare transforms for approximate equality.

        :param other: Transform being compared.
        :type other: Affine
        :return: True if absolute difference between each element
            of each respective transform matrix < ``self.precision``.
        """
        for i in (0, 1, 2, 3, 4, 5):
            if abs(self[i] - other[i]) >= precision:
                return False
        return True

    def __gt__(self, other) -> bool:
        return NotImplemented

    __ge__ = __lt__ = __le__ = __gt__

    # Override from base class. We do not support entrywise
    # addition, subtraction or scalar multiplication because
    # the result is not an affine transform

    def __add__(self, other):
        raise TypeError("Operation not supported")

    __iadd__ = __add__

    def __mul__(self, other):
        """Multiplication

        Apply the transform using matrix multiplication, creating
        a resulting object of the same type.  A transform may be applied
        to another transform, a vector, vector array, or shape.

        :param other: The object to transform.
        :type other: Affine, :class:`~planar.Vec2`,
            :class:`~planar.Vec2Array`, :class:`~planar.Shape`
        :rtype: Same as ``other``
        """
        sa, sb, sc, sd, se, sf, _, _, _ = self
        if isinstance(other, Affine):
            oa, ob, oc, od, oe, of, _, _, _ = other
            return tuple.__new__(
                self.__class__,
                (
                    sa * oa + sb * od,
                    sa * ob + sb * oe,
                    sa * oc + sb * of + sc,
                    sd * oa + se * od,
                    sd * ob + se * oe,
                    sd * oc + se * of + sf,
                    0.0,
                    0.0,
                    1.0,
                ),
            )
        else:
            try:
                vx, vy = other
                return (vx * sa + vy * sb + sc, vx * sd + vy * se + sf)
            except (ValueError, TypeError):
                return NotImplemented

    def __rmul__(self, other):
        """Right hand multiplication

        .. deprecated:: 2.3.0
            Right multiplication will be prohibited in version 3.0. This method
            will raise AffineError.

        Notes
        -----
        We should not be called if other is an affine instance This is
        just a guarantee, since we would potentially return the wrong
        answer in that case.
        """
        warnings.warn(
            "Right multiplication will be prohibited in version 3.0",
            DeprecationWarning,
            stacklevel=2,
        )
        assert not isinstance(other, Affine)
        return self.__mul__(other)

    def __imul__(self, other):
        if isinstance(other, Affine) or isinstance(other, tuple):
            return self.__mul__(other)
        else:
            return NotImplemented

    def itransform(self, seq) -> None:
        """Transform a sequence of points or vectors in place.

        :param seq: Mutable sequence of :class:`~planar.Vec2` to be
            transformed.
        :returns: None, the input sequence is mutated in place.
        """
        if self is not identity and self != identity:
            sa, sb, sc, sd, se, sf, _, _, _ = self
            for i, (x, y) in enumerate(seq):
                seq[i] = (x * sa + y * sb + sc, x * sd + y * se + sf)

    def __invert__(self):
        """Return the inverse transform.

        :raises: :except:`TransformNotInvertible` if the transform
            is degenerate.
        """
        if self.is_degenerate:
            raise TransformNotInvertibleError("Cannot invert degenerate transform")
        idet = 1.0 / self.determinant
        sa, sb, sc, sd, se, sf, _, _, _ = self
        ra = se * idet
        rb = -sb * idet
        rd = -sd * idet
        re = sa * idet
        return tuple.__new__(
            self.__class__,
            (ra, rb, -sc * ra - sf * rb, rd, re, -sc * rd - sf * re, 0.0, 0.0, 1.0),
        )

    __hash__ = tuple.__hash__  # hash is not inherited in Py 3

    def __getnewargs__(self):
        """Pickle protocol support

        Notes
        -----
        Normal unpickling creates a situation where __new__ receives all
        9 elements rather than the 6 that are required for the
        constructor.  This method ensures that only the 6 are provided.
        """
        return self.a, self.b, self.c, self.d, self.e, self.f


identity = Affine(1, 0, 0, 0, 1, 0)
"""The identity transform"""

# Miscellaneous utilities


def loadsw(s: str):
    """Returns Affine from the contents of a world file string.

    This method also translates the coefficients from center- to
    corner-based coordinates.

    :param s: str with 6 floats ordered in a world file.
    :rtype: Affine
    """
    if not hasattr(s, "split"):
        raise TypeError("Cannot split input string")
    coeffs = s.split()
    if len(coeffs) != 6:
        raise ValueError("Expected 6 coefficients, found %d" % len(coeffs))
    a, d, b, e, c, f = (float(x) for x in coeffs)
    center = tuple.__new__(Affine, [a, b, c, d, e, f, 0.0, 0.0, 1.0])
    return center * Affine.translation(-0.5, -0.5)


def dumpsw(obj) -> str:
    """Return string for a world file.

    This method also translates the coefficients from corner- to
    center-based coordinates.

    :rtype: str
    """
    center = obj * Affine.translation(0.5, 0.5)
    return "\n".join(repr(getattr(center, x)) for x in list("adbecf")) + "\n"

env_pyrecodes/lib/python3.9/site-packages/affine/__init__.py
NikolaBlagojevic/pyrecodes
"""Affine tests package"""

env_pyrecodes/lib/python3.9/site-packages/affine/tests/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
"""
Handy tools for working with URLs and APIs.
"""
#
#  Copyright (c) 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#

# this package
from apeye.url import URL, Domain, URLPath

__author__: str = "Dominic Davis-Foster"
__copyright__: str = "2020 Dominic Davis-Foster"
__license__: str = "GNU Lesser General Public License v3 or later (LGPLv3+)"
__version__: str = "1.4.1"
__email__: str = "dominic@davis-foster.co.uk"

__all__ = [
		"URL",
		"Domain",
		"URLPath",
		]

env_pyrecodes/lib/python3.9/site-packages/apeye/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  cache.py
"""
Caching functions for functions.

.. latex:vspace:: 10px

.. seealso::

	* The `cachier project <https://pypi.org/project/cachier/>`_
	* `DiskCache <http://www.grantjenks.com/docs/diskcache/index.html>`_

.. automodulesumm:: apeye.cache
	:autosummary-sections: ;;
"""
#
#  Copyright (c) 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#

# stdlib
import inspect
import json
import shutil
import warnings
from functools import wraps
from typing import Any, Callable, Dict, Iterable, Optional

# 3rd party
import platformdirs
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.utils import posargs2kwargs

__all__ = ["Cache"]


class Cache:
	"""
	Cache function arguments to and in-memory dictionary and a JSON file.

	:param app_name: The name of the app. This dictates the name of the cache directory.
	"""

	app_name: str  #: The name of the app. This dictates the name of the cache directory.
	cache_dir: PathPlus  #: The location of the cache directory on disk.
	caches: Dict[str, Dict[str, Any]]  #: Mapping of function names to their caches.

	def __init__(self, app_name: str):
		self.app_name: str = str(app_name)
		self.cache_dir = PathPlus(platformdirs.user_cache_dir(f"{self.app_name}_cache"))
		self.cache_dir.maybe_make(parents=True)

		# Mapping of function names to their caches
		self.caches: Dict[str, Dict[str, Any]] = {}

	def clear(self, func: Optional[Callable] = None) -> bool:
		"""
		Clear the cache.

		:param func: Optional function to clear the cache for.
			By default, the whole cache is cleared.
		:no-default func:

		:returns: True to indicate success. False otherwise.
		"""

		try:
			if func is None:
				shutil.rmtree(self.cache_dir)
				self.cache_dir.maybe_make()
				for function in self.caches:
					self.caches[function] = {}
			else:
				function_name = func.__name__
				cache_file = self.cache_dir / f"{function_name}.json"
				if cache_file.is_file():
					cache_file.unlink()

				if function_name in self.caches:
					del self.caches[function_name]
				self.caches[function_name] = {}

			return True

		except Exception as e:  # pragma: no cover
			warnings.warn(f"Could not remove cache. The error was: {e}")
			return False

	def load_cache(self, func: Callable) -> None:
		"""
		Loads the cache for the given function.

		:param func:
		"""

		cache_file: PathPlus = self.cache_dir / f"{func.__name__}.json"

		if cache_file.is_file():
			cache = json.loads(cache_file.read_text())
		else:
			cache = {}

		self.caches[func.__name__] = cache
		return cache

	def __call__(self, func: Callable):
		"""
		Decorator to cache the return values of a function based on its inputs.

		:param func:
		"""

		function_name = func.__name__
		posargs: Iterable[str] = inspect.getfullargspec(func).args
		cache_file: PathPlus = self.cache_dir / f"{function_name}.json"
		self.load_cache(func)

		@wraps(func)
		def wrapper(*args, **kwargs: Any):
			kwargs: Dict[str, Any] = posargs2kwargs(args, posargs, kwargs)  # type: ignore
			key: str = json.dumps(kwargs)
			response: Any

			cache = self.caches[function_name]
			if key in cache:
				# Return cached response
				response = json.loads(cache[key])
			else:
				response = func(**kwargs)

				if response is not None:
					# Don't cache None values.
					cache[key] = json.dumps(response)

			cache_file.write_text(json.dumps(cache))

			return response

		return wrapper

env_pyrecodes/lib/python3.9/site-packages/apeye/cache.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  url.py
"""
Email address validation functions.

.. versionadded:: 1.0.0

This module is a subset of https://pypi.org/project/email-validator

.. note::

	The classes in this module can instead be imported from the
	`apeye_core.email_validator <https://pypi.org/project/apeye-core/>`_ module instead.
"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
# Based on https://github.com/JoshData/python-email-validator
# Licensed under the CC0 License
#

# stdlib
import sys

# 3rd party
from apeye_core import email_validator

__all__ = [
		"EmailSyntaxError",
		"ValidatedEmail",
		"main",
		"validate_email",
		"validate_email_domain_part",
		"validate_email_local_part"
		]

EmailSyntaxError = email_validator.EmailSyntaxError
ValidatedEmail = email_validator.ValidatedEmail
ValidatedEmail.__module__ = "apeye.email_validator"

validate_email = email_validator.validate_email
validate_email_local_part = email_validator.validate_email_local_part
validate_email_domain_part = email_validator.validate_email_domain_part

main = email_validator.main

if __name__ == "__main__":
	sys.exit(email_validator.main())

env_pyrecodes/lib/python3.9/site-packages/apeye/email_validator.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  rate_limiter.py
"""
Rate limiters for making calls to external APIs in a polite manner.

.. extras-require:: limiter
	:pyproject:

.. latex:vspace:: -15px
"""
#
#  Copyright (c) 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on CacheControl
#  Copyright 2015 Eric Larson
#  https://github.com/ionrock/cachecontrol
#  Apache-2.0 Licensed
#

# stdlib
import datetime
import logging
import os
import shutil
import time
import warnings
import zlib
from functools import wraps
from typing import Any, Callable, Collection, Dict, Optional

# 3rd party
import platformdirs
import requests
from cachecontrol import CacheControl, CacheControlAdapter  # nodep
from cachecontrol.caches.file_cache import FileCache  # nodep
from cachecontrol.heuristics import ExpiresAfter  # nodep
from domdf_python_tools.paths import PathPlus

# import codetiming

__all__ = [
		"rate_limit",
		"RateLimitAdapter",
		"HTTPCache",
		]

# Fixes intersphinx links
requests.PreparedRequest.__module__ = "requests"
requests.Response.__module__ = "requests"
requests.Request.__module__ = "requests"
requests.Session.__module__ = "requests"


def rate_limit(min_time: float = 0.2, logger: Optional[logging.Logger] = None) -> Callable[[Callable], Any]:
	"""
	Decorator to force a function to run no less than ``min_time`` seconds after it last ran.
	Used for rate limiting.

	:param min_time: The minimum interval between subsequent runs of the decorated function.
	:default min_time: ``0.2``, which gives a maximum rate of 5 calls per second
	:param logger: Optional logger to log information about requests to. Defaults to the root logger.
	:no-default logger:
	"""

	if logger is None:
		# Log to root logger
		logger_ = logging.getLogger()
	else:  # pragma: no cover
		logger_ = logger

	def decorator(func: Callable) -> Callable:

		function_name = func.__name__
		last_ran_message = f"{function_name}: Last ran %.2f seconds ago."
		waiting_message = f"{function_name}: Waiting %.2f seconds."

		@wraps(func)
		def rate_limit_wrapper(*args, **kwargs):
			now = datetime.datetime.now()

			time_since_last_run = (now - rate_limit_wrapper.last_run_time).total_seconds()  # type: ignore
			logger_.debug(last_ran_message % time_since_last_run)

			if time_since_last_run < min_time:
				wait_time = min_time - time_since_last_run
				logger_.debug(waiting_message % wait_time)
				time.sleep(wait_time)

			rate_limit_wrapper.last_run_time = now  # type: ignore
			res = func(*args, **kwargs)
			return res

		rate_limit_wrapper.last_run_time = datetime.datetime.fromtimestamp(0)  # type: ignore

		return rate_limit_wrapper

	return decorator


class RateLimitAdapter(CacheControlAdapter):
	r"""
	Custom :class:`cachecontrol.adapter.CacheControlAdapter` to limit the rate of requests to 5 per second.

	:param cache:
	:param cache_etags:
	:param controller_class:
	:param serializer:
	:param heuristic:
	:param cacheable_methods:

	.. autosummary-widths:: 7/16
	.. latex:clearpage::
	"""

	def send(  # type: ignore[override]  # Latest cachecontrol has changed the signature to put cacheable_methods last
		self,
		request: requests.PreparedRequest,
		cacheable_methods: Optional[Collection[str]] = None,
		**kwargs,
		) -> requests.Response:
		r"""
		Send a request.

		Use the request information to see if it exists in the cache and cache the response if we need to and can.

		:param request: The :class:`requests.PreparedRequest` being sent.
		:param cacheable_methods:
		:param \*\*kwargs: Additional arguments taken by :meth:`requests.adapters.HTTPAdapter.send` (e.g. ``timeout``).
		"""

		cacheable = cacheable_methods or self.cacheable_methods
		if request.method in cacheable:
			try:
				cached_response = self.controller.cached_request(request)
			except zlib.error:  # pragma: no cover
				cached_response = None
			if cached_response:
				return self.build_response(request, cached_response, from_cache=True)

			# check for etags and add headers if appropriate
			request.headers.update(self.controller.conditional_headers(request))

		resp = self.rate_limited_send(request, **kwargs)

		return resp

	@rate_limit(0.2)
	def rate_limited_send(self, *args, **kwargs) -> requests.Response:
		"""
		Wrapper around :meth:`CacheControlAdapter.send <cachecontrol.adapter.CacheControlAdapter.send>`
		to limit the rate of requests.
		"""  # noqa: D400

		return super(CacheControlAdapter, self).send(*args, **kwargs)  # lgtm [py/super-not-enclosing-class]


class HTTPCache:
	"""
	Cache HTTP requests for up to 28 days and limit the rate of requests to no more than 5/second.

	:param app_name: The name of the app. This dictates the name of the cache directory.
	:param expires_after: The maximum time to cache responses for.
	"""

	app_name: str  #: The name of the app. This dictates the name of the cache directory.
	cache_dir: PathPlus  #: The location of the cache directory on disk.
	caches: Dict[str, Dict[str, Any]]  #: Mapping of function names to their caches.

	def __init__(self, app_name: str, expires_after: datetime.timedelta = datetime.timedelta(days=28)):
		self.app_name: str = str(app_name)
		self.cache_dir = PathPlus(platformdirs.user_cache_dir(self.app_name))
		self.cache_dir.maybe_make(parents=True)

		self.session: requests.Session = CacheControl(
				sess=requests.Session(),
				cache=FileCache(os.fspath(self.cache_dir)),
				heuristic=ExpiresAfter(
						days=expires_after.days,
						seconds=expires_after.seconds,
						microseconds=expires_after.microseconds,
						),
				adapter_class=RateLimitAdapter
				)

	def clear(self) -> bool:
		"""
		Clear the cache.

		:returns: True to indicate success. False otherwise.
		"""

		try:
			shutil.rmtree(self.cache_dir)
			return True

		except Exception as e:  # pragma: no cover
			warnings.warn(f"Could not remove cache. The error was: {e}")
			return False


#
# class ForceMinTime:
# 	"""
# 	Decorator to force a function to take an amount of time to run.
# 	Used for rate limiting to external APIs.
#
# 	:param min_time: The minimum run time in seconds
# 	:type min_time: float
# 	"""
#
# 	def __init__(self, min_time: float):
# 		"""
# 		If there are decorator arguments, the function
# 		to be decorated is not passed to the constructor!
# 		"""
#
# 		self.min_time = min_time
#
# 	def __call__(self, func):
# 		"""
# 		If there are decorator arguments, __call__() is only called
# 		once, as part of the decoration process! You can only give
# 		it a single argument, which is the function object.
# 		"""
#
# 		def wrapper(*args, **kwargs):
# 			with codetiming.Timer(logger=None) as t:
# 				r = func(*args, **kwargs)
#
# 			sleep_time = self.min_time - t.last
# 			print(t.last)
#
# 			if sleep_time > 0:
# 				time.sleep(sleep_time)
#
# 			return r
#
# 		return wrapper

env_pyrecodes/lib/python3.9/site-packages/apeye/rate_limiter.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  requests_url.py
"""
Extension of :class:`~apeye.url.URL` with support for interacting with the website using the :requests:`.` library.

.. versionadded:: 0.2.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Some docstrings from Requests <https://requests.readthedocs.io>
#  Copyright 2019 Kenneth Reitz
#  Licensed under the Apache License, Version 2.0

# stdlib
import sys
from typing import IO, Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, TypeVar, Union

# 3rd party
import requests

# this package
from apeye.url import URL

__all__ = ["RequestsURL", "TrailingRequestsURL", "_R"]

_ParamsMappingValueType = Union[str, bytes, int, float, Iterable[Union[str, bytes, int, float]]]
# _Data = Union[None, str, bytes, MutableMapping[str, Any], Iterable[Tuple[str, Optional[str]]], IO]
_Data = Union[
	None,
	str,
	bytes,
	MutableMapping[str, Any],
	List[Tuple[str, Optional[str]]],
	Tuple[Tuple[str, Optional[str]]],
	IO
	]
_ParamsType = Union[
	Mapping[Union[str, bytes, int, float], _ParamsMappingValueType],
	Union[str, bytes],
	Tuple[Union[str, bytes, int, float], _ParamsMappingValueType],
	None
	]

_R = TypeVar("_R", bound="RequestsURL")


# Ignore the LGTM warning as the "session" attribute should **not** affect equality.
class RequestsURL(URL):  # lgtm [py/missing-equals]
	"""
	Extension of :class:`~apeye.url.URL` with support for interacting with the website using the :requests:`.` library.

	The :class:`requests.Session` used for this object -- and all objects created using the
	``/`` or ``.parent`` operations -- can be accessed using the :attr:`~.session` attribute.
	If desired, this can be replaced with a different session object, such as one using caching.

	:param url: The url to construct the :class:`~apeye.url.URL` object from.

	.. latex:vspace:: 5px

	.. versionchanged:: 0.3.0  The ``url`` parameter can now be a string or a :class:`~.URL`.

	.. versionchanged:: 1.1.0

		When a :class:`~.RequestsURL` object is deleted or garbage collected,
		the underlying :class:`requests.Session` object it only closed if no objects hold references to the session.
		This prevents the session object of a global object from being inadvertently closed
		when one of its children is garbage collected.

	.. latex:vspace:: -4px
	"""

	#: The underlying requests session.
	session: requests.Session

	def __init__(self, url: Union[str, URL] = ''):
		super().__init__(url)
		self.session = requests.Session()

	def resolve(self: _R, timeout: Optional[int] = None) -> _R:
		"""
		Resolves the URL into its canonical form.

		This is done by making a ``HEAD`` request and following HTTP 302 redirects.

		.. versionadded:: 0.8.0

		.. versionchanged:: 1.1.0  Added the ``timeout`` argument.
		"""

		response: requests.Response = self.head(allow_redirects=True, timeout=timeout)

		if response.status_code != 200:
			raise requests.HTTPError(f"Could not resolve {self!r}: HTTP Status {response.status_code}")

		new_obj = self.__class__(response.url)
		new_obj.session = self.session
		return new_obj

	def get(self, params: _ParamsType = None, **kwargs) -> requests.Response:
		r"""
		Perform a GET request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/GET

		:param params: Dictionary, list of tuples or bytes to send
			in the query string for the :class:`requests.Request`.
		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.

		.. versionchanged:: 0.7.0

			If ``params`` is :py:obj:`None` but the URL has a query string,
			the query string will be parsed and used for ``params``.
		"""

		if params is None and self.query:
			params = self.query  # type: ignore

		return self.session.get(str(self.base_url), params=params, **kwargs)

	def options(self, **kwargs) -> requests.Response:
		r"""
		Send an OPTIONS request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS

		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.
		"""

		return self.session.options(str(self.base_url), **kwargs)

	def head(self, **kwargs) -> requests.Response:
		r"""
		Send a HEAD request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD

		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.
			If `allow_redirects` is not provided, it will be set to `False`
			(as opposed to the default :func:`requests.request` behavior).
		"""

		return self.session.head(str(self.base_url), **kwargs)

	def post(self, data: "_Data" = None, json=None, **kwargs) -> requests.Response:
		r"""
		Send a POST request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST

		:param data: Dictionary, list of tuples, bytes, or file-like
			object to send in the body of the :class:`requests.Request`.
		:param json: json data to send in the body of the :class:`requests.Request`.
		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.
		"""

		return self.session.post(str(self.base_url), data=data, json=json, **kwargs)

	def put(self, data: "_Data" = None, json=None, **kwargs) -> requests.Response:
		r"""
		Send a PUT request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/PUT

		:param data: Dictionary, list of tuples, bytes, or file-like
			object to send in the body of the :class:`requests.Request`.
		:param json: json data to send in the body of the :class:`requests.Request`.
		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.
		"""

		return self.session.put(str(self.base_url), data=data, json=json, **kwargs)

	def patch(self, data: "_Data" = None, json=None, **kwargs) -> requests.Response:
		r"""
		Send a PATCH request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/PATCH

		:param data: Dictionary, list of tuples, bytes, or file-like
			object to send in the body of the :class:`requests.Request`.
		:param json: json data to send in the body of the :class:`requests.Request`.
		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.
		"""

		return self.session.patch(str(self.base_url), data=data, json=json, **kwargs)

	def delete(self, **kwargs) -> requests.Response:
		r"""
		Send a DELETE request using :requests:`.`.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/DELETE

		:param \*\*kwargs: Optional arguments that :func:`requests.request` takes.
		"""

		return self.session.delete(str(self.base_url), **kwargs)

	def __del__(self):  # pragma: no cover
		"""
		Attempt to close session when garbage collected to avoid leaving connections open.
		"""

		try:
			if sys.getrefcount(self.session) <= 2:
				self.session.close()
		except Exception:  # nosec: B110  # pylint: disable=bare-except
			pass

	def __truediv__(self, other):
		"""
		Construct a new :class:`~apeye.url.URL` object for the given child of this :class:`~apeye.url.URL`.
		"""

		new_obj = super().__truediv__(other)

		if new_obj is not NotImplemented:
			new_obj.session = self.session

		return new_obj


class TrailingRequestsURL(RequestsURL):
	"""
	Extension of :class:`~apeye.requests_url.RequestsURL` which adds a trailing slash to the end of the URL.

	.. versionadded:: 0.5.0

	:param url: The url to construct the :class:`~apeye.url.URL` object from.

	.. autoclasssumm:: TrailingRequestsURL
		:autosummary-sections: ;;
	"""

	def __str__(self) -> str:
		"""
		Returns the :class:`~.TrailingRequestsURL` as a string.
		"""

		return super().__str__().rstrip('/') + '/'

env_pyrecodes/lib/python3.9/site-packages/apeye/requests_url.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  slumber_url.py
"""
Subclass of :class:`~apeye.url.URL` with support for interacting with
REST APIs with `Slumber <https://slumber.readthedocs.io>`__ and
`Requests <https://requests.readthedocs.io>`__.

.. versionadded:: 0.2.0
"""  # noqa: D400
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on the "pathlib" module from CPython.
#  Licensed under the Python Software Foundation License Version 2.
#  Copyright © 2001-2020 Python Software Foundation. All rights reserved.
#  Copyright © 2000 BeOpen.com. All rights reserved.
#  Copyright © 1995-2000 Corporation for National Research Initiatives. All rights reserved.
#  Copyright © 1991-1995 Stichting Mathematisch Centrum. All rights reserved.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#
#  Some docstrings from Requests <https://requests.readthedocs.io>
#  Copyright 2019 Kenneth Reitz
#  Licensed under the Apache License, Version 2.0
#

# stdlib
import copy
import sys
from typing import Callable, Dict, MutableMapping, Optional, Tuple, Union
from urllib.parse import unquote

# 3rd party
from requests import PreparedRequest, Session
from requests.auth import AuthBase
from requests.structures import CaseInsensitiveDict
from requests.utils import guess_json_utf

# this package
from apeye.requests_url import _Data
from apeye.slumber_url.exceptions import (
		HttpClientError,
		HttpNotFoundError,
		HttpServerError,
		SlumberBaseException,
		SlumberHttpBaseException
		)
from apeye.slumber_url.serializers import (
		JsonSerializer,
		Serializer,
		SerializerNotAvailable,
		SerializerRegistry,
		YamlSerializer
		)
from apeye.url import URL

__all__ = [
		"SlumberURL",
		"SerializerRegistry",
		"YamlSerializer",
		"Serializer",
		"JsonSerializer",
		"SlumberBaseException",
		"SlumberHttpBaseException",
		"HttpClientError",
		"HttpNotFoundError",
		"HttpServerError",
		"SerializerNotAvailable",
		]


# Ignore the LGTM warning as the "session" etc. attributes should **not** affect equality.
# Equality should only consider the URL, not its attributes.
class SlumberURL(URL):  # lgtm [py/missing-equals]
	"""
	Subclass of :class:`~apeye.url.URL` with support for interacting with
	REST APIs with `Slumber <https://slumber.readthedocs.io>`__ and
	`Requests <https://requests.readthedocs.io>`__.

	:param url: The url to construct the :class:`~apeye.slumber_url.SlumberURL` object from.
	:param auth:
	:param format:
	:param append_slash:
	:param session:
	:param serializer:
	:param timeout: How long to wait for the server to send data before giving up.
	:param allow_redirects: Whether to allow redirects.
	:param proxies: Dictionary mapping protocol or protocol and hostname to the URL of the proxy.
	:param verify: Either a boolean, in which case it controls whether we verify
		the server's TLS certificate, or a string, in which case it must be a path
		to a CA bundle to use.
	:param cert: Either the path to the SSL client cert file (``.pem``),
		or a tuple of ``('cert', 'key')``.

	``timeout``, ``allow_redirects``, ``proxies``, ``verify`` and ``cert`` are
	passed to Requests when making any HTTP requests, and are inherited by all children
	created from this URL.

	.. latex:vspace:: 10px
	.. versionchanged:: 0.3.0  The ``url`` parameter can now be a string or a :class:`~.URL`.

	.. versionchanged:: 1.1.0

		When a :class:`~.RequestsURL` object is deleted or garbage collected,
		the underlying :class:`requests.Session` object it only closed if no objects hold references to the session.
		This prevents the session object of a global object from being inadvertently closed
		when one of its children is garbage collected.
	"""  # noqa: D400

	serializer: SerializerRegistry
	"""
	The serializer used to (de)serialize the data when interacting with the API.

	.. versionadded:: 0.6.0
	"""

	session: Session
	"""
	The underlying requests session.

	.. versionadded:: 0.6.0
	"""

	#: How long to wait for the server to send data before giving up.
	timeout: Union[None, float, Tuple[float, float], Tuple[float, None]]

	#: Whether to allow redirects.
	allow_redirects: Optional[bool]

	#: Dictionary mapping protocol or protocol and hostname to the URL of the proxy.
	proxies: Optional[MutableMapping[str, str]]

	verify: Union[None, bool, str]
	"""
	Either a boolean, in which case it controls whether we verify
	the server's TLS certificate, or a string, in which case it must be a path
	to a CA bundle to use.
	"""

	#: The path to ssl client cert file or a tuple of ``('cert', 'key')``.
	cert: Union[str, Tuple[str, str], None]

	def __init__(
			self,
			url: Union[str, URL] = '',
			auth: Union[None, Tuple[str, str], AuthBase, Callable[[PreparedRequest], PreparedRequest]] = None,
			format: str = "json",  # noqa: A002  # pylint: disable=redefined-builtin
			append_slash: bool = True,
			session=None,
			serializer: Optional[SerializerRegistry] = None,
			*,
			timeout: Union[None, float, Tuple[float, float], Tuple[float, None]] = None,
			allow_redirects: Optional[bool] = True,
			proxies: Optional[MutableMapping[str, str]] = None,
			verify: Union[None, bool, str] = None,
			cert: Union[str, Tuple[str, str], None] = None,
			):
		super().__init__(url)

		if serializer is None:
			serializer = SerializerRegistry(default=format)

		if session is None:
			session = Session()

		self.serializer = serializer
		self.session = session

		if auth is not None:
			self.session.auth = auth

		self._store = {
				"format": format if format is not None else "json",
				"append_slash": append_slash,
				"session": self.session,
				"serializer": self.serializer,
				}

		self.timeout = timeout
		self.allow_redirects = allow_redirects
		self.proxies = proxies
		self.verify = verify
		self.cert = cert

	def url(self) -> str:
		"""
		Returns the URL as a string.
		"""

		url = str(self.base_url)

		if self._store["append_slash"] and not url.endswith('/'):
			url = url + '/'

		return url

	def _request(self, method, data=None, files=None, params=None):
		serializer = self.serializer
		url = self.url()

		headers = {"accept": serializer.get_content_type()}

		if not files:
			headers["content-type"] = serializer.get_content_type()
			if data is not None:
				data = serializer.dumps(data)

		resp = self.session.request(
				method,
				url,
				data=data,
				params=params,
				files=files,
				headers=headers,
				timeout=self.timeout,
				allow_redirects=self.allow_redirects,  # type: ignore[arg-type]
				proxies=self.proxies,
				verify=self.verify,
				cert=self.cert,
				)

		if 400 <= resp.status_code <= 499:
			exception_class = HttpNotFoundError if resp.status_code == 404 else HttpClientError
			raise exception_class(
					f"Client Error {resp.status_code}: {unquote(resp.url)}",
					response=resp,
					content=resp.content,
					)

		elif 500 <= resp.status_code <= 599:
			raise HttpServerError(
					f"Server Error {resp.status_code}: {unquote(resp.url)}",
					response=resp,
					content=resp.content,
					)

		self._ = resp

		return resp

	def _try_to_serialize_response(self, resp):
		s = self.serializer
		if resp.status_code in [204, 205]:
			return

		if resp.headers.get("content-type", None) and resp.content:
			content_type = resp.headers.get("content-type").split(';')[0].strip()

			try:
				stype = s.get_serializer(content_type=content_type)
			except SerializerNotAvailable:
				return resp.content

			if isinstance(resp.content, bytes):
				try:
					encoding = guess_json_utf(resp.content)
					return stype.loads(resp.content.decode(encoding))
				except Exception:
					return resp.content
			return stype.loads(resp.content)
		else:
			return resp.content

	def _process_response(self, resp):
		# TODO: something to expose headers and status

		if 200 <= resp.status_code <= 299:
			return self._try_to_serialize_response(resp)
		else:
			return  # @@@ We should probably do some sort of error here? (Is this even possible?)

	def get(self, **params) -> Dict:
		"""
		Perform a GET request using `Slumber <https://slumber.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/GET

		:param params: Parameters to send in the query string of the :class:`requests.Request`.
		"""

		resp = self._request("GET", params=params)
		return self._process_response(resp)

	def post(self, data: _Data = None, files=None, **params) -> Dict:
		"""
		Perform a POST request using `Slumber <https://slumber.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST

		:param data: Dictionary, list of tuples, bytes, or file-like
			object to send in the body of the :class:`requests.Request`.
		:param files: Dictionary of ``'name': file-like-objects``
			(or ``{'name': file-tuple}``) for multipart encoding upload.
			``file-tuple`` can be a 2-tuple ``('filename', fileobj)``,
			3-tuple ``('filename', fileobj, 'content_type')``
			or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``,
			where ``'content-type'`` is a string defining the content type of the
			given file and ``custom_headers`` a dict-like object containing additional
			headers to add for the file.
		:param params: Parameters to send in the query string of the :class:`requests.Request`.
		"""

		resp = self._request("POST", data=data, files=files, params=params)
		return self._process_response(resp)

	def patch(self, data=None, files=None, **params) -> Dict:
		"""
		Perform a PATCH request using `Slumber <https://slumber.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/PATCH

		:param data: Dictionary, list of tuples, bytes, or file-like
			object to send in the body of the :class:`requests.Request`.
		:param files: Dictionary of ``'name': file-like-objects``
			(or ``{'name': file-tuple}``) for multipart encoding upload.
			``file-tuple`` can be a 2-tuple ``('filename', fileobj)``,
			3-tuple ``('filename', fileobj, 'content_type')``
			or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``,
			where ``'content-type'`` is a string defining the content type of the
			given file and ``custom_headers`` a dict-like object containing additional
			headers to add for the file.
		:param params: Parameters to send in the query string of the :class:`requests.Request`.
		"""

		resp = self._request("PATCH", data=data, files=files, params=params)
		return self._process_response(resp)

	def put(self, data=None, files=None, **params) -> Dict:
		"""
		Perform a PUT request using `Slumber <https://slumber.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/PUT

		:param data: Dictionary, list of tuples, bytes, or file-like
			object to send in the body of the :class:`requests.Request`.
		:param files: Dictionary of ``'name': file-like-objects``
			(or ``{'name': file-tuple}``) for multipart encoding upload.
			``file-tuple`` can be a 2-tuple ``('filename', fileobj)``,
			3-tuple ``('filename', fileobj, 'content_type')``
			or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``,
			where ``'content-type'`` is a string defining the content type of the
			given file and ``custom_headers`` a dict-like object containing additional
			headers to add for the file.
		:param params: Parameters to send in the query string of the :class:`requests.Request`.
		"""

		resp = self._request("PUT", data=data, files=files, params=params)
		return self._process_response(resp)

	def delete(self, **params) -> bool:
		"""
		Perform a DELETE request using `Slumber <https://slumber.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/DELETE

		:param params: Parameters to send in the query string of the :class:`requests.Request`.

		:returns: :py:obj:`True` if the DELETE request succeeded. :py:obj:`False` otherwise.
		"""

		resp = self._request("DELETE", params=params)
		# if 200 <= resp.status_code <= 299:
		# 	if resp.status_code == 204:
		# 		return True
		# 	else:
		# 		return True
		# else:
		# 	return False
		return 200 <= resp.status_code <= 299

	def __del__(self):  # pragma: no cover
		"""
		Attempt to close session when garbage collected to avoid leaving connections open.
		"""

		try:
			if sys.getrefcount(self.session) <= 2:
				self.session.close()
		except Exception:  # nosec: B110  # pylint: disable=bare-except
			pass

	def options(self, **kwargs) -> str:
		"""
		Send an OPTIONS request using `Requests <https://requests.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS

		:param kwargs: Optional arguments that :func:`requests.request` takes.
		"""

		return self.session.options(str(self.base_url), **kwargs).headers.get("Allow", '')

	def head(self, **kwargs) -> CaseInsensitiveDict:
		"""
		Send a HEAD request using `Requests <https://requests.readthedocs.io>`__.

		https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD

		:param kwargs: Optional arguments that :func:`requests.request` takes.
			If `allow_redirects` is not provided, it will be set to :py:obj:`False`
			(as opposed to the default :func:`requests.request` behavior).
		"""

		return self.session.head(str(self.base_url), **kwargs).headers

	def __truediv__(self, other):
		"""
		Construct a new :class:`~apeye.url.URL` object for the given child of this :class:`~apeye.url.URL`.
		"""

		new_obj = super().__truediv__(other)

		if new_obj is not NotImplemented:
			new_obj._store = copy.copy(self._store)
			new_obj.serializer = self.serializer
			new_obj.session = self.session
			new_obj.timeout = self.timeout
			new_obj.allow_redirects = self.allow_redirects
			new_obj.proxies = self.proxies
			new_obj.verify = self.verify
			new_obj.cert = self.cert

		return new_obj

env_pyrecodes/lib/python3.9/site-packages/apeye/slumber_url/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  slumber_url.py
"""
Exceptions for :class:`~apeye.slumber_url.SlumberURL`.

.. versionadded:: 0.6.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#

__all__ = [
		"HttpClientError",
		"HttpNotFoundError",
		"HttpServerError",
		"SlumberBaseException",
		"SlumberHttpBaseException"
		]


class SlumberBaseException(Exception):
	"""
	All Slumber exceptions inherit from this exception.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.exceptions`
	"""


class SlumberHttpBaseException(SlumberBaseException):
	"""
	All Slumber HTTP Exceptions inherit from this exception.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.exceptions`
	"""

	def __init__(self, *args, **kwargs):
		for key, value in kwargs.items():
			setattr(self, key, value)
		super().__init__(*args)


class HttpClientError(SlumberHttpBaseException):
	"""
	Raised when the server tells us there was a client error (4xx).

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.exceptions`
	"""


class HttpNotFoundError(HttpClientError):
	"""
	Raised when the server sends a 404 error.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.exceptions`
	"""


class HttpServerError(SlumberHttpBaseException):
	"""
	Raised when the server tells us there was a server error (5xx).

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.exceptions`
	"""

env_pyrecodes/lib/python3.9/site-packages/apeye/slumber_url/exceptions.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  __init__.py
"""
JSON and YAML serializers for :class:`~apeye.slumber_url.SlumberURL`.

.. versionadded:: 0.6.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#

# stdlib
import json
from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Type

# this package
from apeye.slumber_url.exceptions import SlumberBaseException
from apeye.slumber_url.serializers._abc import Serializer

__all__ = ["JsonSerializer", "YamlSerializer", "SerializerRegistry", "Serializer", "SerializerNotAvailable"]


class JsonSerializer(Serializer):
	"""
	Serializer for JSON data.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`
	"""

	content_types = [
			"application/json",
			"application/x-javascript",
			"text/javascript",
			"text/x-javascript",
			"text/x-json",
			]
	key = "json"

	def loads(self, data: str) -> MutableMapping[str, Any]:
		"""
		Deserialize data using this :class:`~.Serializer`.

		:param data:
		"""

		return json.loads(data)

	def dumps(self, data: Mapping[str, Any]) -> str:
		"""
		Serialize data using this :class:`~.Serializer`.

		:param data:
		"""

		return json.dumps(data)


_SERIALIZERS: List[Type[Serializer]] = [JsonSerializer]

YamlSerializer: Type[Serializer]

try:
	# this package
	from apeye.slumber_url.serializers._pyyaml_serializer import YamlSerializer
	_SERIALIZERS.append(YamlSerializer)

except ImportError:
	try:
		# this package
		from apeye.slumber_url.serializers._ruamel_serializer import YamlSerializer
		_SERIALIZERS.append(YamlSerializer)

	except ImportError:

		class YamlSerializer(Serializer):  # type: ignore[no-redef]
			"""
			Serializer for YAML data.

			.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`

			.. attention::

				Either `PyYaml <https://pypi.org/project/PyYAML/>`_ or
				`ruamel.yaml <https://pypi.org/project/ruamel.yaml/>`_
				must be installed to use this serializer.
			"""

			content_types = ["text/yaml"]
			key = "yaml"

			def __init__(self):
				raise NotImplementedError("'PyYAML' or 'ruamel.yaml' package not available.")


class SerializerRegistry:
	"""
	Serializes and deserializes data for transfer to and from a REST API.

	:param default: The default serializer to use if none is specified.
		Corresponds to the :attr:`~.Serializer.key` of a :class:`~.Serializer`.
	:param serializers: List of :class:`~.Serializer` objects to use.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`
	"""

	def __init__(self, default: str = "json", serializers: Optional[List[Serializer]] = None):

		#: Mapping of formats to :class:`~.Serializer` objects.
		self.serializers: Dict[str, Serializer] = {}

		for serializer in serializers or [x() for x in _SERIALIZERS]:
			self.serializers[serializer.key] = serializer

		#: The default serializer to use if none is specified.
		self.default: str = default

	def get_serializer(self, name: Optional[str] = None, content_type: Optional[str] = None):
		"""
		Returns the first :class:`~.Serializer` that supports either the given
		format or the given content type.

		:param name:
		:param content_type:
		"""  # noqa: D400

		if content_type is None:
			if name is None:
				return self.serializers[self.default]
			else:
				if name not in self.serializers:
					raise SerializerNotAvailable(name)
				return self.serializers[name]
		else:
			for x in self.serializers.values():
				for ctype in x.content_types:
					if content_type == ctype:
						return x

			raise SerializerNotAvailable(content_type)

	def loads(
			self,
			data: str,
			format: Optional[str] = None,  # noqa: A002  # pylint: disable=redefined-builtin
			) -> MutableMapping[str, Any]:
		"""
		Deserialize data of the given format.

		:param data:
		:param format: The serialization format to use.
		"""

		s = self.get_serializer(format)
		return s.loads(data)

	def dumps(
			self,
			data: Mapping[str, Any],
			format: Optional[str] = None,  # noqa: A002  # pylint: disable=redefined-builtin
			) -> str:
		"""
		Serialize data of the given format.

		:param data:
		:param format: The serialization format to use.
		"""

		s = self.get_serializer(format)
		return s.dumps(data)

	def get_content_type(
			self,
			format: Optional[str] = None,  # noqa: A002  # pylint: disable=redefined-builtin
			):
		"""
		Returns the content type for the serializer that supports the given format.

		:param format: The desired serialization format.
		"""

		s = self.get_serializer(format)
		return s.get_content_type()


class SerializerNotAvailable(SlumberBaseException):
	"""
	The chosen :class:`Serializer` is not available.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`
	"""

	def __init__(self, content_type: str):
		super().__init__(f"No serializer available for {content_type!r}.")


Serializer.__module__ = JsonSerializer.__module__

env_pyrecodes/lib/python3.9/site-packages/apeye/slumber_url/serializers/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  _abc.py
"""
Abstract Base Class for JSON and YAML serializers.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#

# stdlib
from abc import ABC, abstractmethod
from typing import Any, List, Mapping, MutableMapping

__all__ = ["Serializer"]


class Serializer(ABC):
	"""
	Base class for serializers.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`
	"""

	@property
	@abstractmethod
	def content_types(self) -> List[str]:  # pragma: no cover
		"""
		List of supported content types.
		"""

		return NotImplemented

	@property
	@abstractmethod
	def key(self) -> str:  # pragma: no cover
		"""
		An identifier for the supported data type.

		For example, a YAML serializer would set this to ``'yaml'``.
		"""

		return NotImplemented

	def get_content_type(self) -> str:
		"""
		Returns the first value from :attr:`~.Serializer.content_types`.
		"""

		return self.content_types[0]

	@abstractmethod
	def loads(self, data: str) -> MutableMapping[str, Any]:
		"""
		Deserialize data using this :class:`~.Serializer`.

		:param data:
		"""

		raise NotImplementedError()

	@abstractmethod
	def dumps(self, data: Mapping[str, Any]) -> str:
		"""
		Serialize data using this :class:`~.Serializer`.

		:param data:
		"""

		raise NotImplementedError()

env_pyrecodes/lib/python3.9/site-packages/apeye/slumber_url/serializers/_abc.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  _pyyaml_serializer.py
"""
PyYAML-backed YAML serializer for :class:`~apeye.slumber_url.SlumberURL`.

.. versionadded:: 0.6.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#

# stdlib
from typing import Any, Mapping, MutableMapping

# 3rd party
import yaml  # nodep

# this package
from apeye.slumber_url.serializers._abc import Serializer

__all__ = ["YamlSerializer"]


class YamlSerializer(Serializer):
	"""
	Serializer for YAML data.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`

	.. attention::

		Either `PyYaml <https://pypi.org/project/PyYAML/>`_ or
		`ruamel.yaml <https://pypi.org/project/ruamel.yaml/>`_
		must be installed to use this serializer.
	"""

	content_types = ["text/yaml"]
	key = "yaml"

	def loads(self, data: str) -> MutableMapping[str, Any]:
		"""
		Deserialize data using this :class:`~.Serializer`.

		:param data:
		"""

		return yaml.safe_load(str(data))

	def dumps(self, data: Mapping[str, Any]) -> str:
		"""
		Serialize data using this :class:`~.Serializer`.

		:param data:
		"""

		return yaml.dump(data)

env_pyrecodes/lib/python3.9/site-packages/apeye/slumber_url/serializers/_pyyaml_serializer.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  _pyyaml_serializer.py
"""
ruamel.yaml-backed YAML serializer for :class:`~apeye.slumber_url.SlumberURL`.

.. versionadded:: 0.6.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#

# stdlib
from io import StringIO
from typing import Any, Mapping, MutableMapping

# 3rd party
import ruamel.yaml as yaml  # type: ignore[import]  # nodep

# this package
from apeye.slumber_url.serializers._abc import Serializer

__all__ = ["YamlSerializer"]


class YamlSerializer(Serializer):
	"""
	Serializer for YAML data.

	.. versionchanged:: 0.6.0  Moved to :mod:`apeye.slumber_url.serializers`

	.. attention::

		Either `PyYaml <https://pypi.org/project/PyYAML/>`_ or
		`ruamel.yaml <https://pypi.org/project/ruamel.yaml/>`_
		must be installed to use this serializer.
	"""

	content_types = ["text/yaml"]
	key = "yaml"

	def __init__(self):
		super().__init__()
		self._yaml = yaml.YAML()

	def loads(self, data: str) -> MutableMapping[str, Any]:
		"""
		Deserialize data using this :class:`~.Serializer`.

		:param data:
		"""

		y = yaml.YAML(typ="safe", pure=True)
		return y.load(str(data))

	def dumps(self, data: Mapping[str, Any]) -> str:
		"""
		Serialize data using this :class:`~.Serializer`.

		:param data:
		"""

		buf = StringIO()
		self._yaml.dump(data, buf)
		return buf.getvalue()

env_pyrecodes/lib/python3.9/site-packages/apeye/slumber_url/serializers/_ruamel_serializer.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  url.py
"""
:mod:`pathlib`-like approach to URLs.

.. versionchanged:: 0.2.0

	:class:`~apeye.slumber_url.SlumberURL` and :class:`~apeye.requests_url.RequestsURL`
	moved to :mod:`apeye.slumber_url` and :mod:`apeye.requests_url` respectively.

.. note::

	The classes in this module can instead be imported from the
	`apeye_core <https://pypi.org/project/apeye-core/>`_ module instead.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#

# stdlib
import os
import sys
from typing import TYPE_CHECKING, Dict, List, Optional

# 3rd party
import apeye_core
from apeye_core import URL as URL
from apeye_core import Domain as Domain
from apeye_core import URLPath as URLPath
from apeye_core import URLPathType as URLPathType
from apeye_core import URLType as URLType

__all__ = ["URL", "URLPath", "Domain", "URLType", "URLPathType"]

if TYPE_CHECKING:
	pass
elif "_APEYE_DOCS" in os.environ:

	class URL(apeye_core.URL):  # noqa: D101
		#: URL scheme specifier
		scheme: str

		#: Network location part of the URL
		netloc: str

		#: The hierarchical path of the URL
		path: URLPath

		query: Dict[str, List[str]]
		"""
		The query parameters of the URL, if present.

		.. versionadded:: 0.7.0
		"""

		fragment: Optional[str]
		"""
		The URL fragment, used to identify a part of the document. :py:obj:`None` if absent from the URL.

		.. versionadded:: 0.7.0
		"""


URLPath.__module__ = "apeye.url"
URL.__module__ = "apeye.url"
Domain.__module__ = "apeye.url"

if sys.version_info >= (3, 7):
	URLType.__module__ = "apeye.url"
	URLPathType.__module__ = "apeye.url"

env_pyrecodes/lib/python3.9/site-packages/apeye/url.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
"""
Core (offline) functionality for the apeye library.
"""
#
#  Copyright © 2020-2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Redistribution and use in source and binary forms, with or without modification,
#  are permitted provided that the following conditions are met:
#
#      * Redistributions of source code must retain the above copyright notice,
#        this list of conditions and the following disclaimer.
#      * Redistributions in binary form must reproduce the above copyright notice,
#        this list of conditions and the following disclaimer in the documentation
#        and/or other materials provided with the distribution.
#      * Neither the name of the copyright holder nor the names of its contributors
#        may be used to endorse or promote products derived from this software without
#        specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
#  OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#  Based on the "pathlib" module from CPython.
#  Licensed under the Python Software Foundation License Version 2.
#  Copyright © 2001-2020 Python Software Foundation. All rights reserved.
#  Copyright © 2000 BeOpen.com. All rights reserved.
#  Copyright © 1995-2000 Corporation for National Research Initiatives. All rights reserved.
#  Copyright © 1991-1995 Stichting Mathematisch Centrum. All rights reserved.
#
#  Based on Slumber <https://slumber.readthedocs.io>
#  Copyright (c) 2011 Donald Stufft
#  Licensed under the 2-clause BSD License
#

# stdlib
import ipaddress
import os
import pathlib
import re
import sys
from operator import attrgetter
from typing import (
		TYPE_CHECKING,
		Any,
		Dict,
		Iterable,
		List,
		Mapping,
		NamedTuple,
		Optional,
		Tuple,
		Type,
		TypeVar,
		Union,
		cast
		)
from urllib.parse import ParseResult, parse_qs, urlencode, urlparse, urlunparse

# 3rd party
from domdf_python_tools.doctools import prettify_docstrings
from domdf_python_tools.typing import PathLike

# this package
from apeye_core import _tld

if TYPE_CHECKING:
	# stdlib
	from typing import NoReturn

__author__: str = "Dominic Davis-Foster"
__copyright__: str = "2020-2022 Dominic Davis-Foster"
__license__: str = "BSD 3-Clause License"
__version__: str = "1.1.4"
__email__: str = "dominic@davis-foster.co.uk"

__all__ = ["URL", "URLPath", "Domain", "URLType", "URLPathType"]

# NOTE: Use the relevant version numbers for apeye itself.

URLType = TypeVar("URLType", bound="URL")

URLPathType = TypeVar("URLPathType", bound="URLPath")
"""
.. versionadded:: 1.1.0
"""


@prettify_docstrings
class URLPath(pathlib.PurePosixPath):
	"""
	Represents the path part of a URL.

	Subclass of :class:`pathlib.PurePosixPath` that provides a subset of its methods.

	.. versionchanged:: 1.1.0

		Implemented :meth:`~.apeye.url.URLPath.is_absolute`, :meth:`~.apeye.url.URLPath.joinpath`,
		:meth:`~.apeye.url.URLPath.relative_to`, :meth:`~.pathlib.PurePath.match`,
		``anchor``, ``drive``, and support for rich comparisons (``<``, ``<=``, ``>`` and ``>=``),
		which previously raised :exc:`NotImplementedError`.

	.. latex:clearpage::
	"""

	def __str__(self) -> str:
		"""
		Return the string representation of the path, suitable for passing to system calls.
		"""

		if not hasattr(self, "_root"):
			self._load_parts()  # type: ignore[attr-defined]

		try:
			return self._str  # type: ignore
		except AttributeError:
			if hasattr(self, "_parts"):
				parts = self._parts  # type: ignore[attr-defined]
			else:
				parts = self._tail  # type: ignore[attr-defined]

			self._str = self._format_parsed_parts(self._drv, self._root, parts) or ''  # type: ignore
			return self._str

	def __repr__(self) -> str:
		return super().__repr__()

	@classmethod
	def _format_parsed_parts(cls, drv, root, parts):

		if drv or root:
			if sys.version_info > (3, 12):
				return drv + root + '/'.join(parts)
			else:
				return drv + root + '/'.join(parts[1:])
		else:
			return '/'.join(parts)

	def is_absolute(self) -> bool:
		"""
		Returns whether the path is absolute (i.e. starts with ``/``).

		.. versionadded:: 1.1.0  previously raised :exc:`NotImplementedError`.
		"""

		return self.root == '/'

	def joinpath(self: URLPathType, *args) -> URLPathType:
		"""
		Combine this :class:`~.apeye.url.URLPath` with one or several arguments.

		.. versionadded:: 1.1.0  previously raised :exc:`NotImplementedError`.

		:returns: A new :class:`~.apeye.url.URLPath` representing either a subpath
			(if all arguments are relative paths) or a totally different path
			(if one of the arguments is absolute).
		"""

		return super().joinpath(*args)

	def relative_to(self: URLPathType, *other: PathLike) -> URLPathType:
		r"""
		Returns the relative path to another path identified by the passed arguments.

		The arguments are joined together to form a single path, and therefore the following behave identically:

		.. code-block:: pycon

			>>> URLPath("/news/sport").relative_to("/", "news")
			URLPath('sport')
			>>> URLPath("/news/sport").relative_to("/news")
			URLPath('sport')

		.. versionadded:: 1.1.0  previously raised :exc:`NotImplementedError`.

		:param \*other:

		:raises ValueError: if the operation is not possible (because this is not a subpath of the other path)

		.. latex:clearpage::

		.. seealso::

			:meth:`~.apeye.url.URL.relative_to`, which is recommended when constructing a relative path from a :class:`~URL`.
			This method cannot correctly handle some cases, such as:

			.. code-block:: pycon

				>>> URL("https://github.com/domdfcoding").path.relative_to(URL("https://github.com").path)
				Traceback (most recent call last):
				ValueError: '/domdfcoding' does not start with ''

			Since ``URL("https://github.com").path`` is ``URLPath('')``.

			Instead, use:

				>>> URL("https://github.com/domdfcoding").relative_to(URL("https://github.com"))
				URLPath('domdfcoding')
		"""

		return super().relative_to(*other)

	def as_uri(self, *args, **kwargs) -> "NoReturn":  # noqa: D102
		raise NotImplementedError


class URL(os.PathLike):
	r"""
	:mod:`pathlib`-like class for URLs.

	:param url: The URL to construct the :class:`~apeye.url.URL` object from.

	.. versionchanged:: 0.3.0  The ``url`` parameter can now be a string or a :class:`~.apeye.url.URL`.

	.. versionchanged:: 1.1.0

		Added support for sorting and rich comparisons (``<``, ``<=``, ``>`` and ``>=``).

	.. autoclasssumm:: URL
		:autosummary-sections: Methods
		:autosummary-exclude-members: __lt__,__le__,__gt__,__ge__,__init__,__hash__

	.. autosummary-widths:: 1/5

	.. autoclasssumm:: URL
		:autosummary-sections: Attributes
	"""

	#: URL scheme specifier
	scheme: str

	#: Network location part of the URL
	netloc: str

	#: The hierarchical path of the URL
	path: URLPath

	query: Dict[str, List[str]]
	"""
	The query parameters of the URL, if present.

	.. versionadded:: 0.7.0
	"""

	fragment: Optional[str]
	"""
	The URL fragment, used to identify a part of the document. :py:obj:`None` if absent from the URL.

	.. versionadded:: 0.7.0
	"""

	def __init__(self, url: Union[str, "URL"] = ''):
		if isinstance(url, URL):
			url = str(url)

		if not re.match("([A-Za-z-.]+:)?//", url):
			url = "//" + str(url)

		scheme, netloc, parts, params, query, fragment = urlparse(url)

		self.scheme: str = scheme
		self.netloc: str = netloc
		self.path = URLPath(parts)
		self.query = parse_qs(query or '')
		self.fragment = fragment or None

	@property
	def port(self) -> Optional[int]:
		"""
		The port of number of the URL as an integer, if present. Default :py:obj:`None`.

		.. versionadded:: 0.7.0
		"""

		if ':' not in self.netloc:
			return None
		else:
			return int(self.netloc.split(':')[-1])

	@classmethod
	def from_parts(
			cls: Type[URLType],
			scheme: str,
			netloc: str,
			path: PathLike,
			query: Optional[Mapping[Any, List]] = None,
			fragment: Optional[str] = None,
			) -> URLType:
		"""
		Construct a :class:`~apeye.url.URL` from a scheme, netloc and path.

		:param scheme: The scheme of the URL, e.g ``'http'``.
		:param netloc: The netloc of the URl, e.g. ``'bbc.co.uk:80'``.
		:param path: The path of the URL, e.g. ``'/news'``.
		:param query: The query parameters of the URL, if present.
		:param fragment: The URL fragment, used to identify a part of the document.
			:py:obj:`None` if absent from the URL.

		Put together, the resulting path would be ``'http://bbc.co.uk:80/news'``

		:rtype:

		.. versionchanged:: 0.7.0  Added the ``query`` and ``fragment`` arguments.
		"""

		obj = cls('')
		obj.scheme = scheme
		obj.netloc = netloc
		obj.query = dict(query or {})
		obj.fragment = fragment or None

		path = URLPath(path)

		if path.root == '/':
			obj.path = path
		else:
			obj.path = URLPath('/' + str(path))

		return obj

	def __str__(self) -> str:
		"""
		Returns the :class:`~apeye.url.URL` as a string.
		"""

		query = urlencode(self.query, doseq=True)
		url = urlunparse([self.scheme, self.netloc, str(self.path), None, query, self.fragment])

		if url.startswith("//"):
			return url[2:]
		else:
			return url

	def __repr__(self) -> str:
		"""
		Returns the string representation of the :class:`~apeye.url.URL`.
		"""

		return f"{self.__class__.__name__}({str(self)!r})"

	def __truediv__(self: URLType, key: Union[PathLike, int]) -> URLType:
		"""
		Construct a new :class:`~apeye.url.URL` object for the given child of this :class:`~apeye.url.URL`.

		:rtype:

		.. versionchanged:: 0.7.0

			* Added support for division by integers.

			* Now officially supports the new path having a URL fragment and/or query parameters.
			  Any URL fragment or query parameters from the parent URL are not inherited by its children.
		"""

		try:
			return self._make_child((key, ))
		except TypeError:
			return NotImplemented

	def _make_child(self: URLType, args: Iterable[Union[PathLike, int]]) -> URLType:
		"""
		Construct a new :class:`~apeye.url.URL` object by combining the given arguments with this instance's path part.

		.. versionadded:: 1.1.0  (private)

		Except for the final path element any queries and fragments are ignored.

		:returns: A new :class:`~.apeye.url.URL` representing either a subpath
			(if all arguments are relative paths) or a totally different path
			(if one of the arguments is absolute).
		"""

		parsed_args: List[ParseResult] = []

		for arg in args:

			raw_arg = arg

			if isinstance(arg, pathlib.PurePath):
				arg = arg.as_posix()
			elif isinstance(arg, os.PathLike):
				arg = os.fspath(arg)
			elif isinstance(arg, int):
				arg = str(arg)

			try:
				parse_result = urlparse(arg)
			except AttributeError as e:
				if str(e).endswith("'decode'"):
					msg = f"Cannot join {type(raw_arg).__name__!r} to a {type(self.path).__name__!r}"
					raise TypeError(msg) from None
				else:
					raise

			parsed_args.append(parse_result)

		try:
			new_path = self.from_parts(
					self.scheme,
					self.netloc,
					self.path.joinpath(*map(attrgetter("path"), parsed_args)),
					)
		except TypeError:
			return NotImplemented

		if parsed_args:
			new_path.query = parse_qs(parsed_args[-1].query)
			new_path.fragment = parsed_args[-1].fragment or None

		return new_path

	def joinurl(self: URLType, *args) -> URLType:
		"""
		Construct a new :class:`~apeye.url.URL` object by combining the given arguments with this instance's path part.

		.. versionadded:: 1.1.0

		Except for the final path element any queries and fragments are ignored.

		:returns: A new :class:`~.apeye.url.URL` representing either a subpath
			(if all arguments are relative paths) or a totally different path
			(if one of the arguments is absolute).
		"""

		return self._make_child(args)

	def __fspath__(self) -> str:
		"""
		Returns the file system path representation of the :class:`~.apeye.url.URL`.

		This is comprised of the ``netloc`` and ``path`` attributes.
		"""

		return f"{self.netloc}{self.path}"

	def __eq__(self, other) -> bool:
		"""
		Return ``self == other``.

		.. latex:vspace:: -10px

		.. attention::

			URL fragments and query parameters are not compared.

			.. seealso:: :meth:`.URL.strict_compare`, which *does* consider those attributes.

		.. latex:vspace:: -20px
		"""

		if isinstance(other, URL):
			return self.netloc == other.netloc and self.scheme == other.scheme and self.path == other.path
		else:
			return NotImplemented

	def __lt__(self, other):
		if isinstance(other, URL):
			return self._parts_port < other._parts_port
		else:
			return NotImplemented

	def __le__(self, other):
		if isinstance(other, URL):
			return self._parts_port <= other._parts_port
		else:
			return NotImplemented

	def __gt__(self, other):
		if isinstance(other, URL):
			return self._parts_port > other._parts_port
		else:
			return NotImplemented

	def __ge__(self, other):
		if isinstance(other, URL):
			return self._parts_port >= other._parts_port
		else:
			return NotImplemented

	def strict_compare(self, other) -> bool:
		"""
		Return ``self ≡ other``, comparing the scheme, netloc, path, fragment and query parameters.

		.. versionadded:: 0.7.0
		"""

		if isinstance(other, URL):
			return (
					self.netloc == other.netloc and self.scheme == other.scheme and self.path == other.path
					and self.query == other.query and self.fragment == other.fragment
					)
		else:
			return NotImplemented

	def __hash__(self) -> int:
		"""
		Returns the has of the :class:`~apeye.url.URL` .
		"""

		return hash((self.scheme, self.netloc, self.path))

	@property
	def name(self) -> str:
		"""
		The final path component, if any.
		"""

		return self.path.name

	@property
	def suffix(self) -> str:
		"""
		The final component's last suffix, if any.

		This includes the leading period. For example: ``'.txt'``.
		"""
		return self.path.suffix

	@property
	def suffixes(self) -> List[str]:
		"""
		A list of the final component's suffixes, if any.

		These include the leading periods. For example: ``['.tar', '.gz']``.
		"""
		return self.path.suffixes

	@property
	def stem(self) -> str:
		"""
		The final path component, minus its last suffix.
		"""

		return self.path.stem

	def with_name(self: URLType, name: str, inherit: bool = True) -> URLType:
		"""
		Return a new :class:`~apeye.url.URL` with the file name changed.

		:param name:
		:param inherit: Whether the new :class:`~apeye.url.URL` should inherit the query string
			and fragment from this :class:`~apeye.url.URL`.

		:rtype:

		.. versionchanged:: 0.7.0  Added the ``inherit`` parameter.
		"""

		if inherit:
			kwargs = {"query": self.query, "fragment": self.fragment}
		else:
			kwargs = {}

		return self.from_parts(
				self.scheme,
				self.netloc,
				self.path.with_name(name),
				**kwargs,  # type: ignore
				)

	def with_suffix(self: URLType, suffix: str, inherit: bool = True) -> URLType:
		"""
		Returns a new :class:`~apeye.url.URL` with the file suffix changed.

		If the :class:`~apeye.url.URL` has no suffix, add the given suffix.

		If the given suffix is an empty string, remove the suffix from the :class:`~apeye.url.URL`.

		:param suffix:
		:param inherit: Whether the new :class:`~apeye.url.URL` should inherit the query string
			and fragment from this :class:`~apeye.url.URL`.

		:rtype:

		.. versionchanged:: 0.7.0  Added the ``inherit`` parameter.
		"""

		if inherit:
			kwargs = {"query": self.query, "fragment": self.fragment}
		else:
			kwargs = {}

		return self.from_parts(
				self.scheme,
				self.netloc,
				self.path.with_suffix(suffix),
				**kwargs,  # type: ignore
				)

	@property
	def parts(self) -> Tuple[str, ...]:
		"""
		An object providing sequence-like access to the components in the URL.

		To retrieve only the parts of the path, use :meth:`URL.path.parts <URLPath.parts>`.
		"""

		return (
				self.scheme,
				self.domain.subdomain,
				self.domain.domain,
				self.domain.suffix,
				*('/' / self.path).parts[1:],
				)

	@property
	def _parts_port(self) -> Tuple:
		"""
		An object providing sequence-like access to the components in the URL.

		Unlike ``.parts`` this includes the port.

		To retrieve only the parts of the path, use :meth:`URL.path.parts <URLPath.parts>`.

		.. versionadded:: 1.1.0  (private)
		"""

		return (
				self.scheme,
				self.domain.subdomain,
				self.domain.domain,
				self.domain.suffix,
				self.port or 0,
				*('/' / self.path).parts[1:],
				)

	@property
	def parent(self: URLType) -> URLType:
		"""
		The logical parent of the :class:`~apeye.url.URL`.
		"""

		return self.from_parts(self.scheme, self.netloc, self.path.parent)

	@property
	def parents(self: URLType) -> Tuple[URLType, ...]:
		"""
		An immutable sequence providing access to the logical ancestors of the :class:`~apeye.url.URL`.
		"""

		return tuple(self.from_parts(self.scheme, self.netloc, path) for path in self.path.parents)

	@property
	def fqdn(self) -> str:
		"""
		Returns the Fully Qualified Domain Name of the :class:`~apeye.url.URL` .
		"""

		return self.domain.fqdn

	@property
	def domain(self) -> "Domain":
		"""
		Returns a :class:`apeye.url.Domain` object representing the domain part of the URL.
		"""

		return Domain._make(_tld.extract_tld(self.netloc))

	@property
	def base_url(self: URLType) -> URLType:
		"""
		Returns a :class:`apeye.url.URL` object representing the URL without query strings or URL fragments.

		.. versionadded:: 0.7.0
		"""

		return self.from_parts(
				self.scheme,
				self.netloc,
				self.path,
				)

	def relative_to(self, other: Union[str, "URL", URLPath]) -> URLPath:
		"""
		Returns a version of this URL's path relative to ``other``.

		.. versionadded:: 1.1.0

		:param other: Either a :class:`~.apeye.url.URL`, or a string or :class:`~.apeye.url.URLPath` representing an *absolute* path.
			If a :class:`~.apeye.url.URL`, the :attr:`~.apeye.url.URL.netloc` must match this URL's.

		:raises ValueError: if the operation is not possible
			(i.e. because this URL's path is not a subpath of the other path)
		"""

		if isinstance(other, URLPath):
			if not other.is_absolute():
				raise ValueError("'URL.relative_to' cannot be used with relative URLPath objects")
			else:
				other = URL('/') / other
		elif not isinstance(other, URL):
			# Parse other as a URL
			other = URL(other)

		# Compare netloc, if both have one
		if self.netloc and other.netloc and self.netloc.lower() != other.netloc.lower():
			raise ValueError(f"{self!r} does not start with {other!r}")

		# Make the paths absolute
		# If coming from a URL they must always be absolute
		our_path = '/' / self.path
		other_path = '/' / other.path

		relative_path = our_path.relative_to(other_path)

		return relative_path


class Domain(NamedTuple):
	"""
	:class:`typing.NamedTuple` of a URL's subdomain, domain, and suffix.
	"""

	subdomain: str
	domain: str
	suffix: str

	@property
	def registered_domain(self):
		"""
		Joins the domain and suffix fields with a dot, if they're both set.

		.. code-block:: python

			>>> URL('https://forums.bbc.co.uk').domain.registered_domain
			'bbc.co.uk'
			>>> URL('https://localhost:8080').domain.registered_domain
			''
		"""
		if self.domain and self.suffix:
			return self.domain + '.' + self.suffix
		return ''

	@property
	def fqdn(self):
		"""
		Returns a Fully Qualified Domain Name, if there is a proper domain/suffix.

		.. code-block:: python

			>>> URL('https://forums.bbc.co.uk/path/to/file').domain.fqdn
			'forums.bbc.co.uk'
			>>> URL('https://localhost:8080').domain.fqdn
			''
		"""
		if self.domain and self.suffix:
			# self is the namedtuple (subdomain domain suffix)
			return '.'.join(i for i in self if i)
		return ''

	@property
	def ipv4(self) -> Optional[ipaddress.IPv4Address]:
		"""
		Returns the ipv4 if that is what the presented domain/url is.

		.. code-block:: python

			>>> URL('https://127.0.0.1/path/to/file').domain.ipv4
			IPv4Address('127.0.0.1')
			>>> URL('https://127.0.0.1.1/path/to/file').domain.ipv4
			>>> URL('https://256.1.1.1').domain.ipv4
		"""

		if not (self.suffix or self.subdomain) and _tld.IP_RE.match(self.domain):
			return cast(ipaddress.IPv4Address, ipaddress.ip_address(self.domain))
		return None

	def __repr__(self) -> str:
		"""
		Return a string representation of the :class:`~.Domain`.
		"""

		# This is necessary to get the custom docstring

		repr_fmt = f"({', '.join(f'{name}=%r' for name in self._fields)})"
		return f"{self.__class__.__name__}{repr_fmt % self}"

env_pyrecodes/lib/python3.9/site-packages/apeye_core/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  _tld.py
"""
Functions for parsing top-level domains (TLDs).

A stripped back version of https://github.com/john-kurkowski/tldextract
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Adapted from https://github.com/john-kurkowski/tldextract
#  Licensed under the BSD 3-Clause License
#
#  Copyright (c) 2020, John Kurkowski
#  All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are met:
#
#  1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
#  2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import functools
import re
import socket
from typing import List, Tuple
from urllib.parse import scheme_chars

# 3rd party
import idna
from domdf_python_tools.compat import importlib_resources

__all__ = [
		"determine_suffix_index",
		"extract_tld",
		"extract_tlds_from_suffix_list",
		"load_suffix_list",
		"looks_like_ip"
		]

PUBLIC_SUFFIX_RE = re.compile(r"^(?P<suffix>[.*!]*\w[\S]*)", re.UNICODE | re.MULTILINE)
PUBLIC_PRIVATE_SUFFIX_SEPARATOR: str = "// ===BEGIN PRIVATE DOMAINS==="
IP_RE = re.compile(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$')  # pylint: disable=line-too-long
SCHEME_RE = re.compile(fr'^([{scheme_chars}]+:)?//')


def extract_tlds_from_suffix_list(suffix_list_text: str) -> Tuple[List[str], List[str]]:
	"""
	Parse the raw suffix list text for its different designations of suffixes.

	:param suffix_list_text:

	:returns: A two element tuple of ``(public TLDs, private TLDs)``.
	"""

	public_text, _, private_text = suffix_list_text.partition(PUBLIC_PRIVATE_SUFFIX_SEPARATOR)
	public_tlds = [m.group("suffix") for m in PUBLIC_SUFFIX_RE.finditer(public_text)]
	private_tlds = [m.group("suffix") for m in PUBLIC_SUFFIX_RE.finditer(private_text)]
	return public_tlds, private_tlds


@functools.lru_cache(1)
def load_suffix_list() -> Tuple[List[str], List[str]]:
	"""
	Load the public suffix list from disk.

	:returns: A two element tuple of ``(public TLDs, private TLDs)``.
	"""

	return extract_tlds_from_suffix_list(importlib_resources.read_text("apeye_core", "public_suffix_list.dat"))


def _decode_punycode(label: str) -> str:
	lowered = label.lower()
	looks_like_puny = lowered.startswith("xn--")

	if looks_like_puny:
		try:
			return idna.decode(label.encode("ascii")).lower()
		except (UnicodeError, IndexError):
			pass

	return lowered


def determine_suffix_index(tlds: List[str], elements: List[str]) -> int:
	"""
	Returns the index of the first suffix label, or ``len(elements)`` if no suffix is found.

	:param tlds: The list of public TLDs.
	:param elements: The elements of the domain, e.g. ``['forums', 'bbc', 'co', 'uk']``.
	"""

	length = len(elements)
	for i in range(length):
		maybe_tld = '.'.join(elements[i:])
		exception_tld = '!' + maybe_tld

		if exception_tld in tlds:
			return i + 1

		if maybe_tld in tlds:
			return i

		wildcard_tld = "*." + '.'.join(elements[i + 1:])
		if wildcard_tld in tlds:
			return i

	return length


def looks_like_ip(maybe_ip: str) -> bool:
	"""
	Returns whether the given string looks like an IP address.

	:param maybe_ip:
	"""

	if not maybe_ip[0].isdigit():
		return False

	try:
		socket.inet_aton(maybe_ip)
		return True

	except (AttributeError, UnicodeError):
		if IP_RE.match(maybe_ip):
			return True
		return False

	except OSError:
		return False


def extract_tld(url: str) -> Tuple[str, str, str]:
	"""
	Takes a string URL and splits it into its subdomain, domain, and suffix
	(effective TLD, gTLD, ccTLD, etc.) component.

	.. versionadded:: 1.0.0 (undocumented)

	.. code-block:: python

		>>> Domain._make(extract_tld('https://forums.news.cnn.com/'))
		Domain(subdomain='forums.news', domain='cnn', suffix='com')
		>>> Domain._make(extract_tld('https://forums.bbc.co.uk/')
		Domain(subdomain='forums', domain='bbc', suffix='co.uk')
	"""  # noqa: D400

	netloc_ = SCHEME_RE.sub('', url).partition('/')[0].partition('?')[0].partition('#')[0]
	netloc = netloc_.split('@')[-1].partition(':')[0].strip().rstrip('.')

	labels = netloc.split('.')

	translations = [_decode_punycode(label) for label in labels]
	tlds, _ = load_suffix_list()
	suffix_index = determine_suffix_index(tlds, translations)
	suffix = '.'.join(labels[suffix_index:])

	if not suffix and netloc and looks_like_ip(netloc):
		return '', netloc, ''

	subdomain = '.'.join(labels[:suffix_index - 1]) if suffix_index else ''
	domain = labels[suffix_index - 1] if suffix_index else ''
	return subdomain, domain, suffix

env_pyrecodes/lib/python3.9/site-packages/apeye_core/_tld.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  url.py
"""
Email address validation functions.

.. versionadded:: 1.0.0

This module is a subset of https://pypi.org/project/email-validator
"""
#
# Based on https://github.com/JoshData/python-email-validator
# Licensed under the CC0 License
#

# stdlib
import re
import sys
import unicodedata
from typing import Any, Dict, Optional, Union

# 3rd party
import idna  # implements IDNA 2008; Python's codec is only IDNA 2003
from domdf_python_tools.doctools import prettify_docstrings

__all__ = [
		"EmailSyntaxError",
		"ValidatedEmail",
		"main",
		"validate_email",
		"validate_email_domain_part",
		"validate_email_local_part"
		]

# Based on RFC 2822 section 3.2.4 / RFC 5322 section 3.2.3, these
# characters are permitted in email addresses (not taking into
# account internationalization):
ATEXT: str = r'a-zA-Z0-9_!#\$%&\'\*\+\-/=\?\^`\{\|\}~'

# A "dot atom text", per RFC 2822 3.2.4:
DOT_ATOM_TEXT: str = f'[{ATEXT}]+(?:\\.[{ATEXT}]+)*'

# RFC 6531 section 3.3 extends the allowed characters in internationalized
# addresses to also include three specific ranges of UTF8 defined in
# RFC3629 section 4, which appear to be the Unicode code points from
# U+0080 to U+10FFFF.
ATEXT_UTF8: str = ATEXT + "-􏿿"  # Don't convert to f-string
DOT_ATOM_TEXT_UTF8: str = f'[{ATEXT_UTF8}]+(?:\\.[{ATEXT_UTF8}]+)*'

# The domain part of the email address, after IDNA (ASCII) encoding,
# must also satisfy the requirements of RFC 952/RFC 1123 which restrict
# the allowed characters of hostnames further. The hyphen cannot be at
# the beginning or end of a *dot-atom component* of a hostname either.
ATEXT_HOSTNAME: str = r'(?:(?:[a-zA-Z0-9][a-zA-Z0-9\-]*)?[a-zA-Z0-9])'

# Length constants
# RFC 3696 + errata 1003 + errata 1690 (https://www.rfc-editor.org/errata_search.php?rfc=3696&eid=1690)
# explains the maximum length of an email address is 254 octets.
EMAIL_MAX_LENGTH: int = 254
LOCAL_PART_MAX_LENGTH: int = 64
DOMAIN_MAX_LENGTH: int = 255


class EmailSyntaxError(ValueError):
	"""
	Exception raised when an email address fails validation because of its form.
	"""


@prettify_docstrings
class ValidatedEmail:
	"""
	Represents the return type of the :func:`~apeye.email_validator.validate_email` function.

	This class holds the normalized form of the email address alongside other information.

	:param original_email: The original, unnormalized email address.
	:param email: The normalized email address, which should always be used in preference to the original address.
	:param local_part: The local part of the email address after Unicode normalization.
	:param domain: The domain part of the email address after Unicode normalization or conversion to Unicode from IDNA ascii.
	:param ascii_email: If not :py:obj:`None`, a form of the email address that uses 7-bit ASCII characters only.
	:param ascii_local_part: If not :py:obj:`None`, the local part of the email address using 7-bit ASCII characters only.
	:param ascii_domain: If not :py:obj:`None`, a form of the domain name that uses 7-bit ASCII characters only.
	:param smtputf8: Indicates whether SMTPUTF8 will be required to transmit messages to this address.

	.. autosummary-widths:: 1/4
	"""

	#: The email address that was passed to validate_email. (If passed as bytes, this will be a string.)
	original_email: str

	email: str
	"""
	The normalized email address, which should always be used in preference to the original address.

	The normalized address converts an IDNA ASCII domain name to Unicode, if possible,
	and performs Unicode normalization on the local part and on the domain (if originally Unicode).
	It is the concatenation of the local_part and domain attributes, separated by an @-sign.
	"""

	#: The local part of the email address after Unicode normalization.
	local_part: str

	#: The domain part of the email address after Unicode normalization or conversion to Unicode from IDNA ascii.
	domain: str

	#: If not :py:obj:`None`, a form of the email address that uses 7-bit ASCII characters only.
	ascii_email: Optional[str]

	#: If not :py:obj:`None`, the local part of the email address using 7-bit ASCII characters only.
	ascii_local_part: Optional[str]

	#: If not :py:obj:`None`, a form of the domain name that uses 7-bit ASCII characters only.
	ascii_domain: Optional[str]

	smtputf8: Optional[bool]
	"""
	If :py:obj:`True`, the SMTPUTF8 feature of your mail relay will be required to transmit messages
	to this address.

	This flag is :py:obj:`True` when :attr:`~.ascii_local_part` is missing.
	Otherwise it is :py:obj:`False`.
	"""

	def __init__(
			self,
			original_email: str,
			email: str,
			local_part: str,
			domain: str,
			*,
			ascii_email: Optional[str] = None,
			ascii_local_part: Optional[str] = None,
			ascii_domain: Optional[str] = None,
			smtputf8: Optional[bool] = None,
			):
		self.original_email = original_email
		self.email = email
		self.local_part = local_part
		self.domain = domain
		self.ascii_email = ascii_email
		self.ascii_local_part = ascii_local_part
		self.ascii_domain = ascii_domain
		self.smtputf8 = smtputf8

	def __str__(self) -> str:
		"""
		Return a string representation of the :class:`~apeye.email_validator.ValidatedEmail` object.
		"""

		return self.email

	def __repr__(self) -> str:
		"""
		Return a string representation of the :class:`~apeye.email_validator.ValidatedEmail` object.
		"""

		return f"<ValidatedEmail {self.email}>"

	def __eq__(self, other) -> bool:
		"""
		Return ``self == other``.
		"""

		return (
				self.email == other.email and self.local_part == other.local_part and self.domain == other.domain
				and self.ascii_email == other.ascii_email and self.ascii_local_part == other.ascii_local_part
				and self.ascii_domain == other.ascii_domain and self.smtputf8 == other.smtputf8
				)

	def as_dict(self) -> Dict[str, Any]:
		"""
		Convenience method for accessing the :class:`~apeye.email_validator.ValidatedEmail` as a dict.
		"""

		return {
				"original_email": self.original_email,
				"email": self.email,
				"local_part": self.local_part,
				"domain": self.domain,
				"ascii_email": self.ascii_email,
				"ascii_local_part": self.ascii_local_part,
				"ascii_domain": self.ascii_domain,
				"smtputf8": self.smtputf8,
				}


def __get_length_reason(addr: str, utf8: bool = False, limit: int = EMAIL_MAX_LENGTH):
	diff = len(addr) - limit
	prefix = "at least " if utf8 else ''
	suffix = 's' if diff > 1 else ''
	return f"({prefix}{diff} character{suffix} too many)"


def validate_email(
		email: Union[str, bytes],
		allow_smtputf8: bool = True,
		allow_empty_local: bool = False,
		) -> ValidatedEmail:
	"""
	Validates an email address.

	:param email: Either a string, or ASCII-encoded bytes.
	:param allow_smtputf8:
	:param allow_empty_local: Whether to allow the local part (the bit before the @-sign) to be empty.

	:raises EmailSyntaxError: if the address is not valid
	"""

	# Allow email to be a str or bytes instance. If bytes,
	# it must be ASCII because that's how the bytes work
	# on the wire with SMTP.
	if not isinstance(email, str):
		try:
			email = email.decode("ascii")
		except ValueError:
			raise EmailSyntaxError("The email address is not valid ASCII.")

	# At-sign.
	parts = email.split('@')
	if len(parts) != 2:
		raise EmailSyntaxError("The email address is not valid. It must have exactly one @-sign.")

	# Validate the email address's local part syntax and get a normalized form.
	local_part_info = validate_email_local_part(
			parts[0],
			allow_smtputf8=allow_smtputf8,
			allow_empty_local=allow_empty_local,
			)

	# Validate the email address's domain part syntax and get a normalized form.
	domain_part_info = validate_email_domain_part(parts[1])

	ret = ValidatedEmail(
			original_email=email,
			email=f"{local_part_info['local_part']}@{domain_part_info['domain']}",
			local_part=local_part_info["local_part"],
			domain=domain_part_info["domain"],
			ascii_local_part=local_part_info["ascii_local_part"],
			smtputf8=local_part_info["smtputf8"],
			ascii_domain=domain_part_info["ascii_domain"],
			)

	# If the email address has an ASCII form, add it.
	if not ret.smtputf8:
		ret.ascii_email = ret.ascii_local_part + '@' + ret.ascii_domain  # type: ignore[operator]

	# If the email address has an ASCII representation, then we assume it may be
	# transmitted in ASCII (we can't assume SMTPUTF8 will be used on all hops to
	# the destination) and the length limit applies to ASCII characters (which is
	# the same as octets). The number of characters in the internationalized form
	# may be many fewer (because IDNA ASCII is verbose) and could be less than 254
	# Unicode characters, and of course the number of octets over the limit may
	# not be the number of characters over the limit, so if the email address is
	# internationalized, we can't give any simple information about why the address
	# is too long.
	#
	# In addition, check that the UTF-8 encoding (i.e. not IDNA ASCII and not
	# Unicode characters) is at most 254 octets. If the addres is transmitted using
	# SMTPUTF8, then the length limit probably applies to the UTF-8 encoded octets.
	# If the email address has an ASCII form that differs from its internationalized
	# form, I don't think the internationalized form can be longer, and so the ASCII
	# form length check would be sufficient. If there is no ASCII form, then we have
	# to check the UTF-8 encoding. The UTF-8 encoding could be up to about four times
	# longer than the number of characters.
	#
	# See the length checks on the local part and the domain.

	if ret.ascii_email and len(ret.ascii_email) > EMAIL_MAX_LENGTH:
		if ret.ascii_email == ret.email:
			reason = __get_length_reason(ret.ascii_email)
		elif len(ret.email) > EMAIL_MAX_LENGTH:
			# If there are more than 254 characters, then the ASCII
			# form is definitely going to be too long.
			reason = __get_length_reason(ret.email, utf8=True)
		else:
			reason = "(when converted to IDNA ASCII)"
		raise EmailSyntaxError(f"The email address is too long {reason}.")

	if len(ret.email.encode("utf8")) > EMAIL_MAX_LENGTH:
		if len(ret.email) > EMAIL_MAX_LENGTH:
			# If there are more than 254 characters, then the UTF-8
			# encoding is definitely going to be too long.
			reason = __get_length_reason(ret.email, utf8=True)
		else:
			reason = "(when encoded in bytes)"
		raise EmailSyntaxError(f"The email address is too long {reason}.")

	return ret


def validate_email_local_part(
		local: str,
		allow_smtputf8: bool = True,
		allow_empty_local: bool = False,
		) -> Dict[str, Any]:
	"""
	Validates the local part of an email address  (the part before the @-sign).

	:param local:
	:param allow_smtputf8:
	:param allow_empty_local: Whether to allow the local part to be empty/
	"""

	if len(local) == 0:
		if not allow_empty_local:
			raise EmailSyntaxError("There must be something before the @-sign.")
		else:
			# The caller allows an empty local part. Useful for validating certain
			# Postfix aliases.
			return {
					"local_part": local,
					"ascii_local_part": local,
					"smtputf8": False,
					}

	# RFC 5321 4.5.3.1.1
	# We're checking the number of characters here. If the local part
	# is ASCII-only, then that's the same as bytes (octets). If it's
	# internationalized, then the UTF-8 encoding may be longer, but
	# that may not be relevant. We will check the total address length
	# instead.
	if len(local) > LOCAL_PART_MAX_LENGTH:
		reason = __get_length_reason(local, limit=LOCAL_PART_MAX_LENGTH)
		raise EmailSyntaxError(f"The email address is too long before the @-sign {reason}.")

	# Check the local part against the regular expression for the older ASCII requirements.
	m = re.match(DOT_ATOM_TEXT + "\\Z", local)
	if m:
		# Return the local part unchanged and flag that SMTPUTF8 is not needed.
		return {
				"local_part": local,
				"ascii_local_part": local,
				"smtputf8": False,
				}

	else:
		# The local part failed the ASCII check. Now try the extended internationalized requirements.
		m = re.match(DOT_ATOM_TEXT_UTF8 + "\\Z", local)
		if not m:
			# It's not a valid internationalized address either. Report which characters were not valid.
			bad_chars = ", ".join(
					sorted({
							c
							for c in local
							if not re.match('[' + (ATEXT if not allow_smtputf8 else ATEXT_UTF8) + ']', c)
							})
					)
			raise EmailSyntaxError(
					f"The email address contains invalid characters before the @-sign: {bad_chars}."
					)

		# It would be valid if internationalized characters were allowed by the caller.
		if not allow_smtputf8:
			raise EmailSyntaxError("Internationalized characters before the @-sign are not supported.")

		# It's valid.

		# RFC 6532 section 3.1 also says that Unicode NFC normalization should be applied,
		# so we'll return the normalized local part in the return value.
		local = unicodedata.normalize("NFC", local)

		# Flag that SMTPUTF8 will be required for deliverability.
		return {
				"local_part": local,
				"ascii_local_part": None,  # no ASCII form is possible
				"smtputf8": True,
				}


def validate_email_domain_part(domain: str) -> Dict[str, str]:
	"""
	Validate the domain part of an email address (the part after the @-sign).

	:param domain:
	"""

	# Empty?
	if len(domain) == 0:
		raise EmailSyntaxError("There must be something after the @-sign.")

	# Perform UTS-46 normalization, which includes casefolding, NFC normalization,
	# and converting all label separators (the period/full stop, fullwidth full stop,
	# ideographic full stop, and halfwidth ideographic full stop) to basic periods.
	# It will also raise an exception if there is an invalid character in the input,
	# such as "⒈" which is invalid because it would expand to include a period.
	try:
		domain = idna.uts46_remap(domain, std3_rules=False, transitional=False)
	except idna.IDNAError as e:
		raise EmailSyntaxError(f"The domain name {domain} contains invalid characters ({str(e)}).")

	# Now we can perform basic checks on the use of periods (since equivalent
	# symbols have been mapped to periods). These checks are needed because the
	# IDNA library doesn't handle well domains that have empty labels (i.e. initial
	# dot, trailing dot, or two dots in a row).
	if domain.endswith('.'):
		raise EmailSyntaxError("An email address cannot end with a period.")
	if domain.startswith('.'):
		raise EmailSyntaxError("An email address cannot have a period immediately after the @-sign.")
	if ".." in domain:
		raise EmailSyntaxError("An email address cannot have two periods in a row.")

	# Regardless of whether international characters are actually used,
	# first convert to IDNA ASCII. For ASCII-only domains, the transformation
	# does nothing. If internationalized characters are present, the MTA
	# must either support SMTPUTF8 or the mail client must convert the
	# domain name to IDNA before submission.
	#
	# Unfortunately this step incorrectly 'fixes' domain names with leading
	# periods by removing them, so we have to check for this above. It also gives
	# a funky error message ("No input") when there are two periods in a
	# row, also checked separately above.
	try:
		ascii_domain = idna.encode(domain, uts46=False).decode("ascii")
	except idna.IDNAError as e:
		if "Domain too long" in str(e):
			# We can't really be more specific because UTS-46 normalization means
			# the length check is applied to a string that is different from the
			# one the user supplied. Also I'm not sure if the length check applies
			# to the internationalized form, the IDNA ASCII form, or even both!
			raise EmailSyntaxError("The email address is too long after the @-sign.")
		raise EmailSyntaxError(f"The domain name {domain} contains invalid characters ({str(e)}).")

	# We may have been given an IDNA ASCII domain to begin with. Check
	# that the domain actually conforms to IDNA. It could look like IDNA
	# but not be actual IDNA. For ASCII-only domains, the conversion out
	# of IDNA just gives the same thing back.
	#
	# This gives us the canonical internationalized form of the domain,
	# which we should use in all error messages.
	try:
		domain_i18n = idna.decode(ascii_domain.encode("ascii"))
	except idna.IDNAError as e:
		raise EmailSyntaxError(f"The domain name {ascii_domain} is not valid IDNA ({str(e)}).")

	# RFC 5321 4.5.3.1.2
	# We're checking the number of bytes (octets) here, which can be much
	# higher than the number of characters in internationalized domains,
	# on the assumption that the domain may be transmitted without SMTPUTF8
	# as IDNA ASCII. This is also checked by idna.encode, so this exception
	# is never reached.
	if len(ascii_domain) > DOMAIN_MAX_LENGTH:
		raise EmailSyntaxError("The email address is too long after the @-sign.")

	# A "dot atom text", per RFC 2822 3.2.4, but using the restricted
	# characters allowed in a hostname (see ATEXT_HOSTNAME above).
	DOT_ATOM_TEXT = ATEXT_HOSTNAME + r'(?:\.' + ATEXT_HOSTNAME + r')*'

	# Check the regular expression. This is probably entirely redundant
	# with idna.decode, which also checks this format.
	m = re.match(DOT_ATOM_TEXT + "\\Z", ascii_domain)
	if not m:
		raise EmailSyntaxError("The email address contains invalid characters after the @-sign.")

	# All publicly deliverable addresses have domain named with at least
	# one period. We also know that all TLDs end with a letter.
	if '.' not in ascii_domain:
		raise EmailSyntaxError("The domain name %s is not valid. It should have a period." % domain_i18n)
	if not re.search(r"[A-Za-z]\Z", ascii_domain):
		raise EmailSyntaxError(
				"The domain name %s is not valid. It is not within a valid top-level domain." % domain_i18n
				)

	# Return the IDNA ASCII-encoded form of the domain, which is how it
	# would be transmitted on the wire (except when used with SMTPUTF8
	# possibly), as well as the canonical Unicode form of the domain,
	# which is better for display purposes. This should also take care
	# of RFC 6532 section 3.1's suggestion to apply Unicode NFC
	# normalization to addresses.
	return {
			"ascii_domain": ascii_domain,
			"domain": domain_i18n,
			}


def main():  # noqa: D103
	# stdlib
	import json

	if len(sys.argv) == 1:
		for line in sys.stdin:
			email = line.strip()
			try:
				validate_email(email)
			except EmailSyntaxError as e:
				print(f"{email} {e}")
	else:
		# Validate the email address passed on the command line.
		email = sys.argv[1]
		try:
			result = validate_email(email)
			print(json.dumps(result.as_dict(), indent=2, sort_keys=True, ensure_ascii=False))
		except EmailSyntaxError as e:
			print(e)


if __name__ == "__main__":
	sys.exit(main())

env_pyrecodes/lib/python3.9/site-packages/apeye_core/email_validator.py
NikolaBlagojevic/pyrecodes
"""Provides internationalization for arrow in over 60 languages and dialects."""

import sys
from math import trunc
from typing import (
    Any,
    ClassVar,
    Dict,
    List,
    Mapping,
    Optional,
    Sequence,
    Tuple,
    Type,
    Union,
    cast,
)

if sys.version_info < (3, 8):  # pragma: no cover
    from typing_extensions import Literal
else:
    from typing import Literal  # pragma: no cover

TimeFrameLiteral = Literal[
    "now",
    "second",
    "seconds",
    "minute",
    "minutes",
    "hour",
    "hours",
    "day",
    "days",
    "week",
    "weeks",
    "month",
    "months",
    "quarter",
    "quarters",
    "year",
    "years",
]

_TimeFrameElements = Union[
    str, Sequence[str], Mapping[str, str], Mapping[str, Sequence[str]]
]

_locale_map: Dict[str, Type["Locale"]] = {}


def get_locale(name: str) -> "Locale":
    """Returns an appropriate :class:`Locale <arrow.locales.Locale>`
    corresponding to an input locale name.

    :param name: the name of the locale.

    """

    normalized_locale_name = name.lower().replace("_", "-")
    locale_cls = _locale_map.get(normalized_locale_name)

    if locale_cls is None:
        raise ValueError(f"Unsupported locale {normalized_locale_name!r}.")

    return locale_cls()


def get_locale_by_class_name(name: str) -> "Locale":
    """Returns an appropriate :class:`Locale <arrow.locales.Locale>`
    corresponding to an locale class name.

    :param name: the name of the locale class.

    """
    locale_cls: Optional[Type[Locale]] = globals().get(name)

    if locale_cls is None:
        raise ValueError(f"Unsupported locale {name!r}.")

    return locale_cls()


class Locale:
    """Represents locale-specific data and functionality."""

    names: ClassVar[List[str]] = []

    timeframes: ClassVar[Mapping[TimeFrameLiteral, _TimeFrameElements]] = {
        "now": "",
        "second": "",
        "seconds": "",
        "minute": "",
        "minutes": "",
        "hour": "",
        "hours": "",
        "day": "",
        "days": "",
        "week": "",
        "weeks": "",
        "month": "",
        "months": "",
        "quarter": "",
        "quarters": "",
        "year": "",
        "years": "",
    }

    meridians: ClassVar[Dict[str, str]] = {"am": "", "pm": "", "AM": "", "PM": ""}

    past: ClassVar[str]
    future: ClassVar[str]
    and_word: ClassVar[Optional[str]] = None

    month_names: ClassVar[List[str]] = []
    month_abbreviations: ClassVar[List[str]] = []

    day_names: ClassVar[List[str]] = []
    day_abbreviations: ClassVar[List[str]] = []

    ordinal_day_re: ClassVar[str] = r"(\d+)"

    _month_name_to_ordinal: Optional[Dict[str, int]]

    def __init_subclass__(cls, **kwargs: Any) -> None:
        for locale_name in cls.names:
            if locale_name in _locale_map:
                raise LookupError(f"Duplicated locale name: {locale_name}")

            _locale_map[locale_name.lower().replace("_", "-")] = cls

    def __init__(self) -> None:

        self._month_name_to_ordinal = None

    def describe(
        self,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int] = 0,
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within a timeframe in plain language.

        :param timeframe: a string representing a timeframe.
        :param delta: a quantity representing a delta in a timeframe.
        :param only_distance: return only distance eg: "11 seconds" without "in" or "ago" keywords
        """

        humanized = self._format_timeframe(timeframe, trunc(delta))
        if not only_distance:
            humanized = self._format_relative(humanized, timeframe, delta)

        return humanized

    def describe_multi(
        self,
        timeframes: Sequence[Tuple[TimeFrameLiteral, Union[int, float]]],
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within multiple timeframes in plain language.

        :param timeframes: a list of string, quantity pairs each representing a timeframe and delta.
        :param only_distance: return only distance eg: "2 hours and 11 seconds" without "in" or "ago" keywords
        """

        parts = [
            self._format_timeframe(timeframe, trunc(delta))
            for timeframe, delta in timeframes
        ]
        if self.and_word:
            parts.insert(-1, self.and_word)
        humanized = " ".join(parts)

        if not only_distance:
            # Needed to determine the correct relative string to use
            timeframe_value = 0

            for _unit_name, unit_value in timeframes:
                if trunc(unit_value) != 0:
                    timeframe_value = trunc(unit_value)
                    break

            # Note it doesn't matter the timeframe unit we use on the call, only the value
            humanized = self._format_relative(humanized, "seconds", timeframe_value)

        return humanized

    def day_name(self, day: int) -> str:
        """Returns the day name for a specified day of the week.

        :param day: the ``int`` day of the week (1-7).

        """

        return self.day_names[day]

    def day_abbreviation(self, day: int) -> str:
        """Returns the day abbreviation for a specified day of the week.

        :param day: the ``int`` day of the week (1-7).

        """

        return self.day_abbreviations[day]

    def month_name(self, month: int) -> str:
        """Returns the month name for a specified month of the year.

        :param month: the ``int`` month of the year (1-12).

        """

        return self.month_names[month]

    def month_abbreviation(self, month: int) -> str:
        """Returns the month abbreviation for a specified month of the year.

        :param month: the ``int`` month of the year (1-12).

        """

        return self.month_abbreviations[month]

    def month_number(self, name: str) -> Optional[int]:
        """Returns the month number for a month specified by name or abbreviation.

        :param name: the month name or abbreviation.

        """

        if self._month_name_to_ordinal is None:
            self._month_name_to_ordinal = self._name_to_ordinal(self.month_names)
            self._month_name_to_ordinal.update(
                self._name_to_ordinal(self.month_abbreviations)
            )

        return self._month_name_to_ordinal.get(name)

    def year_full(self, year: int) -> str:
        """Returns the year for specific locale if available

        :param year: the ``int`` year (4-digit)
        """
        return f"{year:04d}"

    def year_abbreviation(self, year: int) -> str:
        """Returns the year for specific locale if available

        :param year: the ``int`` year (4-digit)
        """
        return f"{year:04d}"[2:]

    def meridian(self, hour: int, token: Any) -> Optional[str]:
        """Returns the meridian indicator for a specified hour and format token.

        :param hour: the ``int`` hour of the day.
        :param token: the format token.
        """

        if token == "a":
            return self.meridians["am"] if hour < 12 else self.meridians["pm"]
        if token == "A":
            return self.meridians["AM"] if hour < 12 else self.meridians["PM"]
        return None

    def ordinal_number(self, n: int) -> str:
        """Returns the ordinal format of a given integer

        :param n: an integer
        """
        return self._ordinal_number(n)

    def _ordinal_number(self, n: int) -> str:
        return f"{n}"

    def _name_to_ordinal(self, lst: Sequence[str]) -> Dict[str, int]:
        return {elem.lower(): i for i, elem in enumerate(lst[1:], 1)}

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        # TODO: remove cast
        return cast(str, self.timeframes[timeframe]).format(trunc(abs(delta)))

    def _format_relative(
        self,
        humanized: str,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int],
    ) -> str:

        if timeframe == "now":
            return humanized

        direction = self.past if delta < 0 else self.future

        return direction.format(humanized)


class EnglishLocale(Locale):
    names = [
        "en",
        "en-us",
        "en-gb",
        "en-au",
        "en-be",
        "en-jp",
        "en-za",
        "en-ca",
        "en-ph",
    ]

    past = "{0} ago"
    future = "in {0}"
    and_word = "and"

    timeframes = {
        "now": "just now",
        "second": "a second",
        "seconds": "{0} seconds",
        "minute": "a minute",
        "minutes": "{0} minutes",
        "hour": "an hour",
        "hours": "{0} hours",
        "day": "a day",
        "days": "{0} days",
        "week": "a week",
        "weeks": "{0} weeks",
        "month": "a month",
        "months": "{0} months",
        "quarter": "a quarter",
        "quarters": "{0} quarters",
        "year": "a year",
        "years": "{0} years",
    }

    meridians = {"am": "am", "pm": "pm", "AM": "AM", "PM": "PM"}

    month_names = [
        "",
        "January",
        "February",
        "March",
        "April",
        "May",
        "June",
        "July",
        "August",
        "September",
        "October",
        "November",
        "December",
    ]
    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mar",
        "Apr",
        "May",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Oct",
        "Nov",
        "Dec",
    ]

    day_names = [
        "",
        "Monday",
        "Tuesday",
        "Wednesday",
        "Thursday",
        "Friday",
        "Saturday",
        "Sunday",
    ]
    day_abbreviations = ["", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]

    ordinal_day_re = r"((?P<value>[2-3]?1(?=st)|[2-3]?2(?=nd)|[2-3]?3(?=rd)|[1-3]?[04-9](?=th)|1[1-3](?=th))(st|nd|rd|th))"

    def _ordinal_number(self, n: int) -> str:
        if n % 100 not in (11, 12, 13):
            remainder = abs(n) % 10
            if remainder == 1:
                return f"{n}st"
            elif remainder == 2:
                return f"{n}nd"
            elif remainder == 3:
                return f"{n}rd"
        return f"{n}th"

    def describe(
        self,
        timeframe: TimeFrameLiteral,
        delta: Union[int, float] = 0,
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within a timeframe in plain language.

        :param timeframe: a string representing a timeframe.
        :param delta: a quantity representing a delta in a timeframe.
        :param only_distance: return only distance eg: "11 seconds" without "in" or "ago" keywords
        """

        humanized = super().describe(timeframe, delta, only_distance)
        if only_distance and timeframe == "now":
            humanized = "instantly"

        return humanized


class ItalianLocale(Locale):
    names = ["it", "it-it"]
    past = "{0} fa"
    future = "tra {0}"
    and_word = "e"

    timeframes = {
        "now": "adesso",
        "second": "un secondo",
        "seconds": "{0} qualche secondo",
        "minute": "un minuto",
        "minutes": "{0} minuti",
        "hour": "un'ora",
        "hours": "{0} ore",
        "day": "un giorno",
        "days": "{0} giorni",
        "week": "una settimana,",
        "weeks": "{0} settimane",
        "month": "un mese",
        "months": "{0} mesi",
        "year": "un anno",
        "years": "{0} anni",
    }

    month_names = [
        "",
        "gennaio",
        "febbraio",
        "marzo",
        "aprile",
        "maggio",
        "giugno",
        "luglio",
        "agosto",
        "settembre",
        "ottobre",
        "novembre",
        "dicembre",
    ]
    month_abbreviations = [
        "",
        "gen",
        "feb",
        "mar",
        "apr",
        "mag",
        "giu",
        "lug",
        "ago",
        "set",
        "ott",
        "nov",
        "dic",
    ]

    day_names = [
        "",
        "lunedì",
        "martedì",
        "mercoledì",
        "giovedì",
        "venerdì",
        "sabato",
        "domenica",
    ]
    day_abbreviations = ["", "lun", "mar", "mer", "gio", "ven", "sab", "dom"]

    ordinal_day_re = r"((?P<value>[1-3]?[0-9](?=[ºª]))[ºª])"

    def _ordinal_number(self, n: int) -> str:
        return f"{n}º"


class SpanishLocale(Locale):
    names = ["es", "es-es"]
    past = "hace {0}"
    future = "en {0}"
    and_word = "y"

    timeframes = {
        "now": "ahora",
        "second": "un segundo",
        "seconds": "{0} segundos",
        "minute": "un minuto",
        "minutes": "{0} minutos",
        "hour": "una hora",
        "hours": "{0} horas",
        "day": "un día",
        "days": "{0} días",
        "week": "una semana",
        "weeks": "{0} semanas",
        "month": "un mes",
        "months": "{0} meses",
        "year": "un año",
        "years": "{0} años",
    }

    meridians = {"am": "am", "pm": "pm", "AM": "AM", "PM": "PM"}

    month_names = [
        "",
        "enero",
        "febrero",
        "marzo",
        "abril",
        "mayo",
        "junio",
        "julio",
        "agosto",
        "septiembre",
        "octubre",
        "noviembre",
        "diciembre",
    ]
    month_abbreviations = [
        "",
        "ene",
        "feb",
        "mar",
        "abr",
        "may",
        "jun",
        "jul",
        "ago",
        "sep",
        "oct",
        "nov",
        "dic",
    ]

    day_names = [
        "",
        "lunes",
        "martes",
        "miércoles",
        "jueves",
        "viernes",
        "sábado",
        "domingo",
    ]
    day_abbreviations = ["", "lun", "mar", "mie", "jue", "vie", "sab", "dom"]

    ordinal_day_re = r"((?P<value>[1-3]?[0-9](?=[ºª]))[ºª])"

    def _ordinal_number(self, n: int) -> str:
        return f"{n}º"


class FrenchBaseLocale(Locale):
    past = "il y a {0}"
    future = "dans {0}"
    and_word = "et"

    timeframes = {
        "now": "maintenant",
        "second": "une seconde",
        "seconds": "{0} secondes",
        "minute": "une minute",
        "minutes": "{0} minutes",
        "hour": "une heure",
        "hours": "{0} heures",
        "day": "un jour",
        "days": "{0} jours",
        "week": "une semaine",
        "weeks": "{0} semaines",
        "month": "un mois",
        "months": "{0} mois",
        "year": "un an",
        "years": "{0} ans",
    }

    month_names = [
        "",
        "janvier",
        "février",
        "mars",
        "avril",
        "mai",
        "juin",
        "juillet",
        "août",
        "septembre",
        "octobre",
        "novembre",
        "décembre",
    ]

    day_names = [
        "",
        "lundi",
        "mardi",
        "mercredi",
        "jeudi",
        "vendredi",
        "samedi",
        "dimanche",
    ]
    day_abbreviations = ["", "lun", "mar", "mer", "jeu", "ven", "sam", "dim"]

    ordinal_day_re = (
        r"((?P<value>\b1(?=er\b)|[1-3]?[02-9](?=e\b)|[1-3]1(?=e\b))(er|e)\b)"
    )

    def _ordinal_number(self, n: int) -> str:
        if abs(n) == 1:
            return f"{n}er"
        return f"{n}e"


class FrenchLocale(FrenchBaseLocale, Locale):
    names = ["fr", "fr-fr"]

    month_abbreviations = [
        "",
        "janv",
        "févr",
        "mars",
        "avr",
        "mai",
        "juin",
        "juil",
        "août",
        "sept",
        "oct",
        "nov",
        "déc",
    ]


class FrenchCanadianLocale(FrenchBaseLocale, Locale):
    names = ["fr-ca"]

    month_abbreviations = [
        "",
        "janv",
        "févr",
        "mars",
        "avr",
        "mai",
        "juin",
        "juill",
        "août",
        "sept",
        "oct",
        "nov",
        "déc",
    ]


class GreekLocale(Locale):
    names = ["el", "el-gr"]

    past = "{0} πριν"
    future = "σε {0}"
    and_word = "και"

    timeframes = {
        "now": "τώρα",
        "second": "ένα δεύτερο",
        "seconds": "{0} δευτερόλεπτα",
        "minute": "ένα λεπτό",
        "minutes": "{0} λεπτά",
        "hour": "μία ώρα",
        "hours": "{0} ώρες",
        "day": "μία μέρα",
        "days": "{0} μέρες",
        "week": "μία εβδομάδα",
        "weeks": "{0} εβδομάδες",
        "month": "ένα μήνα",
        "months": "{0} μήνες",
        "year": "ένα χρόνο",
        "years": "{0} χρόνια",
    }

    month_names = [
        "",
        "Ιανουαρίου",
        "Φεβρουαρίου",
        "Μαρτίου",
        "Απριλίου",
        "Μαΐου",
        "Ιουνίου",
        "Ιουλίου",
        "Αυγούστου",
        "Σεπτεμβρίου",
        "Οκτωβρίου",
        "Νοεμβρίου",
        "Δεκεμβρίου",
    ]
    month_abbreviations = [
        "",
        "Ιαν",
        "Φεβ",
        "Μαρ",
        "Απρ",
        "Μαϊ",
        "Ιον",
        "Ιολ",
        "Αυγ",
        "Σεπ",
        "Οκτ",
        "Νοε",
        "Δεκ",
    ]

    day_names = [
        "",
        "Δευτέρα",
        "Τρίτη",
        "Τετάρτη",
        "Πέμπτη",
        "Παρασκευή",
        "Σάββατο",
        "Κυριακή",
    ]
    day_abbreviations = ["", "Δευ", "Τρι", "Τετ", "Πεμ", "Παρ", "Σαβ", "Κυρ"]


class JapaneseLocale(Locale):
    names = ["ja", "ja-jp"]

    past = "{0}前"
    future = "{0}後"
    and_word = ""

    timeframes = {
        "now": "現在",
        "second": "1秒",
        "seconds": "{0}秒",
        "minute": "1分",
        "minutes": "{0}分",
        "hour": "1時間",
        "hours": "{0}時間",
        "day": "1日",
        "days": "{0}日",
        "week": "1週間",
        "weeks": "{0}週間",
        "month": "1ヶ月",
        "months": "{0}ヶ月",
        "year": "1年",
        "years": "{0}年",
    }

    month_names = [
        "",
        "1月",
        "2月",
        "3月",
        "4月",
        "5月",
        "6月",
        "7月",
        "8月",
        "9月",
        "10月",
        "11月",
        "12月",
    ]
    month_abbreviations = [
        "",
        " 1",
        " 2",
        " 3",
        " 4",
        " 5",
        " 6",
        " 7",
        " 8",
        " 9",
        "10",
        "11",
        "12",
    ]

    day_names = ["", "月曜日", "火曜日", "水曜日", "木曜日", "金曜日", "土曜日", "日曜日"]
    day_abbreviations = ["", "月", "火", "水", "木", "金", "土", "日"]


class SwedishLocale(Locale):
    names = ["sv", "sv-se"]

    past = "för {0} sen"
    future = "om {0}"
    and_word = "och"

    timeframes = {
        "now": "just nu",
        "second": "en sekund",
        "seconds": "{0} sekunder",
        "minute": "en minut",
        "minutes": "{0} minuter",
        "hour": "en timme",
        "hours": "{0} timmar",
        "day": "en dag",
        "days": "{0} dagar",
        "week": "en vecka",
        "weeks": "{0} veckor",
        "month": "en månad",
        "months": "{0} månader",
        "year": "ett år",
        "years": "{0} år",
    }

    month_names = [
        "",
        "januari",
        "februari",
        "mars",
        "april",
        "maj",
        "juni",
        "juli",
        "augusti",
        "september",
        "oktober",
        "november",
        "december",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "maj",
        "jun",
        "jul",
        "aug",
        "sep",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "måndag",
        "tisdag",
        "onsdag",
        "torsdag",
        "fredag",
        "lördag",
        "söndag",
    ]
    day_abbreviations = ["", "mån", "tis", "ons", "tor", "fre", "lör", "sön"]


class FinnishLocale(Locale):
    names = ["fi", "fi-fi"]

    # The finnish grammar is very complex, and its hard to convert
    # 1-to-1 to something like English.

    past = "{0} sitten"
    future = "{0} kuluttua"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "juuri nyt",
        "second": "sekunti",
        "seconds": {"past": "{0} muutama sekunti", "future": "{0} muutaman sekunnin"},
        "minute": {"past": "minuutti", "future": "minuutin"},
        "minutes": {"past": "{0} minuuttia", "future": "{0} minuutin"},
        "hour": {"past": "tunti", "future": "tunnin"},
        "hours": {"past": "{0} tuntia", "future": "{0} tunnin"},
        "day": "päivä",
        "days": {"past": "{0} päivää", "future": "{0} päivän"},
        "month": {"past": "kuukausi", "future": "kuukauden"},
        "months": {"past": "{0} kuukautta", "future": "{0} kuukauden"},
        "year": {"past": "vuosi", "future": "vuoden"},
        "years": {"past": "{0} vuotta", "future": "{0} vuoden"},
    }

    # Months and days are lowercase in Finnish
    month_names = [
        "",
        "tammikuu",
        "helmikuu",
        "maaliskuu",
        "huhtikuu",
        "toukokuu",
        "kesäkuu",
        "heinäkuu",
        "elokuu",
        "syyskuu",
        "lokakuu",
        "marraskuu",
        "joulukuu",
    ]

    month_abbreviations = [
        "",
        "tammi",
        "helmi",
        "maalis",
        "huhti",
        "touko",
        "kesä",
        "heinä",
        "elo",
        "syys",
        "loka",
        "marras",
        "joulu",
    ]

    day_names = [
        "",
        "maanantai",
        "tiistai",
        "keskiviikko",
        "torstai",
        "perjantai",
        "lauantai",
        "sunnuntai",
    ]

    day_abbreviations = ["", "ma", "ti", "ke", "to", "pe", "la", "su"]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]

        if isinstance(form, Mapping):
            if delta < 0:
                form = form["past"]
            else:
                form = form["future"]

        return form.format(abs(delta))

    def _ordinal_number(self, n: int) -> str:
        return f"{n}."


class ChineseCNLocale(Locale):
    names = ["zh", "zh-cn"]

    past = "{0}前"
    future = "{0}后"

    timeframes = {
        "now": "刚才",
        "second": "1秒",
        "seconds": "{0}秒",
        "minute": "1分钟",
        "minutes": "{0}分钟",
        "hour": "1小时",
        "hours": "{0}小时",
        "day": "1天",
        "days": "{0}天",
        "week": "1周",
        "weeks": "{0}周",
        "month": "1个月",
        "months": "{0}个月",
        "year": "1年",
        "years": "{0}年",
    }

    month_names = [
        "",
        "一月",
        "二月",
        "三月",
        "四月",
        "五月",
        "六月",
        "七月",
        "八月",
        "九月",
        "十月",
        "十一月",
        "十二月",
    ]
    month_abbreviations = [
        "",
        " 1",
        " 2",
        " 3",
        " 4",
        " 5",
        " 6",
        " 7",
        " 8",
        " 9",
        "10",
        "11",
        "12",
    ]

    day_names = ["", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"]
    day_abbreviations = ["", "一", "二", "三", "四", "五", "六", "日"]


class ChineseTWLocale(Locale):
    names = ["zh-tw"]

    past = "{0}前"
    future = "{0}後"
    and_word = "和"

    timeframes = {
        "now": "剛才",
        "second": "1秒",
        "seconds": "{0}秒",
        "minute": "1分鐘",
        "minutes": "{0}分鐘",
        "hour": "1小時",
        "hours": "{0}小時",
        "day": "1天",
        "days": "{0}天",
        "week": "1週",
        "weeks": "{0}週",
        "month": "1個月",
        "months": "{0}個月",
        "year": "1年",
        "years": "{0}年",
    }

    month_names = [
        "",
        "1月",
        "2月",
        "3月",
        "4月",
        "5月",
        "6月",
        "7月",
        "8月",
        "9月",
        "10月",
        "11月",
        "12月",
    ]
    month_abbreviations = [
        "",
        " 1",
        " 2",
        " 3",
        " 4",
        " 5",
        " 6",
        " 7",
        " 8",
        " 9",
        "10",
        "11",
        "12",
    ]

    day_names = ["", "週一", "週二", "週三", "週四", "週五", "週六", "週日"]
    day_abbreviations = ["", "一", "二", "三", "四", "五", "六", "日"]


class HongKongLocale(Locale):
    names = ["zh-hk"]

    past = "{0}前"
    future = "{0}後"

    timeframes = {
        "now": "剛才",
        "second": "1秒",
        "seconds": "{0}秒",
        "minute": "1分鐘",
        "minutes": "{0}分鐘",
        "hour": "1小時",
        "hours": "{0}小時",
        "day": "1天",
        "days": "{0}天",
        "week": "1星期",
        "weeks": "{0}星期",
        "month": "1個月",
        "months": "{0}個月",
        "year": "1年",
        "years": "{0}年",
    }

    month_names = [
        "",
        "1月",
        "2月",
        "3月",
        "4月",
        "5月",
        "6月",
        "7月",
        "8月",
        "9月",
        "10月",
        "11月",
        "12月",
    ]
    month_abbreviations = [
        "",
        " 1",
        " 2",
        " 3",
        " 4",
        " 5",
        " 6",
        " 7",
        " 8",
        " 9",
        "10",
        "11",
        "12",
    ]

    day_names = ["", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"]
    day_abbreviations = ["", "一", "二", "三", "四", "五", "六", "日"]


class KoreanLocale(Locale):
    names = ["ko", "ko-kr"]

    past = "{0} 전"
    future = "{0} 후"

    timeframes = {
        "now": "지금",
        "second": "1초",
        "seconds": "{0}초",
        "minute": "1분",
        "minutes": "{0}분",
        "hour": "한시간",
        "hours": "{0}시간",
        "day": "하루",
        "days": "{0}일",
        "week": "1주",
        "weeks": "{0}주",
        "month": "한달",
        "months": "{0}개월",
        "year": "1년",
        "years": "{0}년",
    }

    special_dayframes = {
        -3: "그끄제",
        -2: "그제",
        -1: "어제",
        1: "내일",
        2: "모레",
        3: "글피",
        4: "그글피",
    }

    special_yearframes = {-2: "제작년", -1: "작년", 1: "내년", 2: "내후년"}

    month_names = [
        "",
        "1월",
        "2월",
        "3월",
        "4월",
        "5월",
        "6월",
        "7월",
        "8월",
        "9월",
        "10월",
        "11월",
        "12월",
    ]
    month_abbreviations = [
        "",
        " 1",
        " 2",
        " 3",
        " 4",
        " 5",
        " 6",
        " 7",
        " 8",
        " 9",
        "10",
        "11",
        "12",
    ]

    day_names = ["", "월요일", "화요일", "수요일", "목요일", "금요일", "토요일", "일요일"]
    day_abbreviations = ["", "월", "화", "수", "목", "금", "토", "일"]

    def _ordinal_number(self, n: int) -> str:
        ordinals = ["0", "첫", "두", "세", "네", "다섯", "여섯", "일곱", "여덟", "아홉", "열"]
        if n < len(ordinals):
            return f"{ordinals[n]}번째"
        return f"{n}번째"

    def _format_relative(
        self,
        humanized: str,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int],
    ) -> str:
        if timeframe in ("day", "days"):
            special = self.special_dayframes.get(int(delta))
            if special:
                return special
        elif timeframe in ("year", "years"):
            special = self.special_yearframes.get(int(delta))
            if special:
                return special

        return super()._format_relative(humanized, timeframe, delta)


# derived locale types & implementations.
class DutchLocale(Locale):
    names = ["nl", "nl-nl"]

    past = "{0} geleden"
    future = "over {0}"

    timeframes = {
        "now": "nu",
        "second": "een seconde",
        "seconds": "{0} seconden",
        "minute": "een minuut",
        "minutes": "{0} minuten",
        "hour": "een uur",
        "hours": "{0} uur",
        "day": "een dag",
        "days": "{0} dagen",
        "week": "een week",
        "weeks": "{0} weken",
        "month": "een maand",
        "months": "{0} maanden",
        "year": "een jaar",
        "years": "{0} jaar",
    }

    # In Dutch names of months and days are not starting with a capital letter
    # like in the English language.
    month_names = [
        "",
        "januari",
        "februari",
        "maart",
        "april",
        "mei",
        "juni",
        "juli",
        "augustus",
        "september",
        "oktober",
        "november",
        "december",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mrt",
        "apr",
        "mei",
        "jun",
        "jul",
        "aug",
        "sep",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "maandag",
        "dinsdag",
        "woensdag",
        "donderdag",
        "vrijdag",
        "zaterdag",
        "zondag",
    ]
    day_abbreviations = ["", "ma", "di", "wo", "do", "vr", "za", "zo"]


class SlavicBaseLocale(Locale):
    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        delta = abs(delta)

        if isinstance(form, Mapping):
            if delta % 10 == 1 and delta % 100 != 11:
                form = form["singular"]
            elif 2 <= delta % 10 <= 4 and (delta % 100 < 10 or delta % 100 >= 20):
                form = form["dual"]
            else:
                form = form["plural"]

        return form.format(delta)


class BelarusianLocale(SlavicBaseLocale):
    names = ["be", "be-by"]

    past = "{0} таму"
    future = "праз {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "зараз",
        "second": "секунду",
        "seconds": "{0} некалькі секунд",
        "minute": "хвіліну",
        "minutes": {
            "singular": "{0} хвіліну",
            "dual": "{0} хвіліны",
            "plural": "{0} хвілін",
        },
        "hour": "гадзіну",
        "hours": {
            "singular": "{0} гадзіну",
            "dual": "{0} гадзіны",
            "plural": "{0} гадзін",
        },
        "day": "дзень",
        "days": {"singular": "{0} дзень", "dual": "{0} дні", "plural": "{0} дзён"},
        "month": "месяц",
        "months": {
            "singular": "{0} месяц",
            "dual": "{0} месяцы",
            "plural": "{0} месяцаў",
        },
        "year": "год",
        "years": {"singular": "{0} год", "dual": "{0} гады", "plural": "{0} гадоў"},
    }

    month_names = [
        "",
        "студзеня",
        "лютага",
        "сакавіка",
        "красавіка",
        "траўня",
        "чэрвеня",
        "ліпеня",
        "жніўня",
        "верасня",
        "кастрычніка",
        "лістапада",
        "снежня",
    ]
    month_abbreviations = [
        "",
        "студ",
        "лют",
        "сак",
        "крас",
        "трав",
        "чэрв",
        "ліп",
        "жнів",
        "вер",
        "каст",
        "ліст",
        "снеж",
    ]

    day_names = [
        "",
        "панядзелак",
        "аўторак",
        "серада",
        "чацвер",
        "пятніца",
        "субота",
        "нядзеля",
    ]
    day_abbreviations = ["", "пн", "ат", "ср", "чц", "пт", "сб", "нд"]


class PolishLocale(SlavicBaseLocale):
    names = ["pl", "pl-pl"]

    past = "{0} temu"
    future = "za {0}"

    # The nouns should be in genitive case (Polish: "dopełniacz")
    # in order to correctly form `past` & `future` expressions.
    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "teraz",
        "second": "sekundę",
        "seconds": {
            "singular": "{0} sekund",
            "dual": "{0} sekundy",
            "plural": "{0} sekund",
        },
        "minute": "minutę",
        "minutes": {
            "singular": "{0} minut",
            "dual": "{0} minuty",
            "plural": "{0} minut",
        },
        "hour": "godzinę",
        "hours": {
            "singular": "{0} godzin",
            "dual": "{0} godziny",
            "plural": "{0} godzin",
        },
        "day": "dzień",
        "days": "{0} dni",
        "week": "tydzień",
        "weeks": {
            "singular": "{0} tygodni",
            "dual": "{0} tygodnie",
            "plural": "{0} tygodni",
        },
        "month": "miesiąc",
        "months": {
            "singular": "{0} miesięcy",
            "dual": "{0} miesiące",
            "plural": "{0} miesięcy",
        },
        "year": "rok",
        "years": {"singular": "{0} lat", "dual": "{0} lata", "plural": "{0} lat"},
    }

    month_names = [
        "",
        "styczeń",
        "luty",
        "marzec",
        "kwiecień",
        "maj",
        "czerwiec",
        "lipiec",
        "sierpień",
        "wrzesień",
        "październik",
        "listopad",
        "grudzień",
    ]
    month_abbreviations = [
        "",
        "sty",
        "lut",
        "mar",
        "kwi",
        "maj",
        "cze",
        "lip",
        "sie",
        "wrz",
        "paź",
        "lis",
        "gru",
    ]

    day_names = [
        "",
        "poniedziałek",
        "wtorek",
        "środa",
        "czwartek",
        "piątek",
        "sobota",
        "niedziela",
    ]
    day_abbreviations = ["", "Pn", "Wt", "Śr", "Czw", "Pt", "So", "Nd"]


class RussianLocale(SlavicBaseLocale):
    names = ["ru", "ru-ru"]

    past = "{0} назад"
    future = "через {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "сейчас",
        "second": "секунда",
        "seconds": {
            "singular": "{0} секунду",
            "dual": "{0} секунды",
            "plural": "{0} секунд",
        },
        "minute": "минуту",
        "minutes": {
            "singular": "{0} минуту",
            "dual": "{0} минуты",
            "plural": "{0} минут",
        },
        "hour": "час",
        "hours": {"singular": "{0} час", "dual": "{0} часа", "plural": "{0} часов"},
        "day": "день",
        "days": {"singular": "{0} день", "dual": "{0} дня", "plural": "{0} дней"},
        "week": "неделю",
        "weeks": {
            "singular": "{0} неделю",
            "dual": "{0} недели",
            "plural": "{0} недель",
        },
        "month": "месяц",
        "months": {
            "singular": "{0} месяц",
            "dual": "{0} месяца",
            "plural": "{0} месяцев",
        },
        "quarter": "квартал",
        "quarters": {
            "singular": "{0} квартал",
            "dual": "{0} квартала",
            "plural": "{0} кварталов",
        },
        "year": "год",
        "years": {"singular": "{0} год", "dual": "{0} года", "plural": "{0} лет"},
    }

    month_names = [
        "",
        "января",
        "февраля",
        "марта",
        "апреля",
        "мая",
        "июня",
        "июля",
        "августа",
        "сентября",
        "октября",
        "ноября",
        "декабря",
    ]
    month_abbreviations = [
        "",
        "янв",
        "фев",
        "мар",
        "апр",
        "май",
        "июн",
        "июл",
        "авг",
        "сен",
        "окт",
        "ноя",
        "дек",
    ]

    day_names = [
        "",
        "понедельник",
        "вторник",
        "среда",
        "четверг",
        "пятница",
        "суббота",
        "воскресенье",
    ]
    day_abbreviations = ["", "пн", "вт", "ср", "чт", "пт", "сб", "вс"]


class AfrikaansLocale(Locale):
    names = ["af", "af-nl"]

    past = "{0} gelede"
    future = "in {0}"

    timeframes = {
        "now": "nou",
        "second": "n sekonde",
        "seconds": "{0} sekondes",
        "minute": "minuut",
        "minutes": "{0} minute",
        "hour": "uur",
        "hours": "{0} ure",
        "day": "een dag",
        "days": "{0} dae",
        "month": "een maand",
        "months": "{0} maande",
        "year": "een jaar",
        "years": "{0} jaar",
    }

    month_names = [
        "",
        "Januarie",
        "Februarie",
        "Maart",
        "April",
        "Mei",
        "Junie",
        "Julie",
        "Augustus",
        "September",
        "Oktober",
        "November",
        "Desember",
    ]
    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mrt",
        "Apr",
        "Mei",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Okt",
        "Nov",
        "Des",
    ]

    day_names = [
        "",
        "Maandag",
        "Dinsdag",
        "Woensdag",
        "Donderdag",
        "Vrydag",
        "Saterdag",
        "Sondag",
    ]
    day_abbreviations = ["", "Ma", "Di", "Wo", "Do", "Vr", "Za", "So"]


class BulgarianLocale(SlavicBaseLocale):
    names = ["bg", "bg-bg"]

    past = "{0} назад"
    future = "напред {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "сега",
        "second": "секунда",
        "seconds": "{0} няколко секунди",
        "minute": "минута",
        "minutes": {
            "singular": "{0} минута",
            "dual": "{0} минути",
            "plural": "{0} минути",
        },
        "hour": "час",
        "hours": {"singular": "{0} час", "dual": "{0} часа", "plural": "{0} часа"},
        "day": "ден",
        "days": {"singular": "{0} ден", "dual": "{0} дни", "plural": "{0} дни"},
        "month": "месец",
        "months": {
            "singular": "{0} месец",
            "dual": "{0} месеца",
            "plural": "{0} месеца",
        },
        "year": "година",
        "years": {
            "singular": "{0} година",
            "dual": "{0} години",
            "plural": "{0} години",
        },
    }

    month_names = [
        "",
        "януари",
        "февруари",
        "март",
        "април",
        "май",
        "юни",
        "юли",
        "август",
        "септември",
        "октомври",
        "ноември",
        "декември",
    ]
    month_abbreviations = [
        "",
        "ян",
        "февр",
        "март",
        "апр",
        "май",
        "юни",
        "юли",
        "авг",
        "септ",
        "окт",
        "ноем",
        "дек",
    ]

    day_names = [
        "",
        "понеделник",
        "вторник",
        "сряда",
        "четвъртък",
        "петък",
        "събота",
        "неделя",
    ]
    day_abbreviations = ["", "пон", "вт", "ср", "четв", "пет", "съб", "нед"]


class UkrainianLocale(SlavicBaseLocale):
    names = ["ua", "uk", "uk-ua"]

    past = "{0} тому"
    future = "за {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "зараз",
        "second": "секунда",
        "seconds": "{0} кілька секунд",
        "minute": "хвилину",
        "minutes": {
            "singular": "{0} хвилину",
            "dual": "{0} хвилини",
            "plural": "{0} хвилин",
        },
        "hour": "годину",
        "hours": {
            "singular": "{0} годину",
            "dual": "{0} години",
            "plural": "{0} годин",
        },
        "day": "день",
        "days": {"singular": "{0} день", "dual": "{0} дні", "plural": "{0} днів"},
        "month": "місяць",
        "months": {
            "singular": "{0} місяць",
            "dual": "{0} місяці",
            "plural": "{0} місяців",
        },
        "year": "рік",
        "years": {"singular": "{0} рік", "dual": "{0} роки", "plural": "{0} років"},
    }

    month_names = [
        "",
        "січня",
        "лютого",
        "березня",
        "квітня",
        "травня",
        "червня",
        "липня",
        "серпня",
        "вересня",
        "жовтня",
        "листопада",
        "грудня",
    ]
    month_abbreviations = [
        "",
        "січ",
        "лют",
        "бер",
        "квіт",
        "трав",
        "черв",
        "лип",
        "серп",
        "вер",
        "жовт",
        "лист",
        "груд",
    ]

    day_names = [
        "",
        "понеділок",
        "вівторок",
        "середа",
        "четвер",
        "п’ятниця",
        "субота",
        "неділя",
    ]
    day_abbreviations = ["", "пн", "вт", "ср", "чт", "пт", "сб", "нд"]


class MacedonianLocale(SlavicBaseLocale):
    names = ["mk", "mk-mk"]

    past = "пред {0}"
    future = "за {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "сега",
        "second": "една секунда",
        "seconds": {
            "singular": "{0} секунда",
            "dual": "{0} секунди",
            "plural": "{0} секунди",
        },
        "minute": "една минута",
        "minutes": {
            "singular": "{0} минута",
            "dual": "{0} минути",
            "plural": "{0} минути",
        },
        "hour": "еден саат",
        "hours": {"singular": "{0} саат", "dual": "{0} саати", "plural": "{0} саати"},
        "day": "еден ден",
        "days": {"singular": "{0} ден", "dual": "{0} дена", "plural": "{0} дена"},
        "week": "една недела",
        "weeks": {
            "singular": "{0} недела",
            "dual": "{0} недели",
            "plural": "{0} недели",
        },
        "month": "еден месец",
        "months": {
            "singular": "{0} месец",
            "dual": "{0} месеци",
            "plural": "{0} месеци",
        },
        "year": "една година",
        "years": {
            "singular": "{0} година",
            "dual": "{0} години",
            "plural": "{0} години",
        },
    }

    meridians = {"am": "дп", "pm": "пп", "AM": "претпладне", "PM": "попладне"}

    month_names = [
        "",
        "Јануари",
        "Февруари",
        "Март",
        "Април",
        "Мај",
        "Јуни",
        "Јули",
        "Август",
        "Септември",
        "Октомври",
        "Ноември",
        "Декември",
    ]
    month_abbreviations = [
        "",
        "Јан",
        "Фев",
        "Мар",
        "Апр",
        "Мај",
        "Јун",
        "Јул",
        "Авг",
        "Септ",
        "Окт",
        "Ноем",
        "Декем",
    ]

    day_names = [
        "",
        "Понеделник",
        "Вторник",
        "Среда",
        "Четврток",
        "Петок",
        "Сабота",
        "Недела",
    ]
    day_abbreviations = [
        "",
        "Пон",
        "Вт",
        "Сре",
        "Чет",
        "Пет",
        "Саб",
        "Нед",
    ]


class GermanBaseLocale(Locale):
    past = "vor {0}"
    future = "in {0}"
    and_word = "und"

    timeframes = {
        "now": "gerade eben",
        "second": "einer Sekunde",
        "seconds": "{0} Sekunden",
        "minute": "einer Minute",
        "minutes": "{0} Minuten",
        "hour": "einer Stunde",
        "hours": "{0} Stunden",
        "day": "einem Tag",
        "days": "{0} Tagen",
        "week": "einer Woche",
        "weeks": "{0} Wochen",
        "month": "einem Monat",
        "months": "{0} Monaten",
        "year": "einem Jahr",
        "years": "{0} Jahren",
    }

    timeframes_only_distance = timeframes.copy()
    timeframes_only_distance["second"] = "eine Sekunde"
    timeframes_only_distance["minute"] = "eine Minute"
    timeframes_only_distance["hour"] = "eine Stunde"
    timeframes_only_distance["day"] = "ein Tag"
    timeframes_only_distance["days"] = "{0} Tage"
    timeframes_only_distance["week"] = "eine Woche"
    timeframes_only_distance["month"] = "ein Monat"
    timeframes_only_distance["months"] = "{0} Monate"
    timeframes_only_distance["year"] = "ein Jahr"
    timeframes_only_distance["years"] = "{0} Jahre"

    month_names = [
        "",
        "Januar",
        "Februar",
        "März",
        "April",
        "Mai",
        "Juni",
        "Juli",
        "August",
        "September",
        "Oktober",
        "November",
        "Dezember",
    ]

    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mär",
        "Apr",
        "Mai",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Okt",
        "Nov",
        "Dez",
    ]

    day_names = [
        "",
        "Montag",
        "Dienstag",
        "Mittwoch",
        "Donnerstag",
        "Freitag",
        "Samstag",
        "Sonntag",
    ]

    day_abbreviations = ["", "Mo", "Di", "Mi", "Do", "Fr", "Sa", "So"]

    def _ordinal_number(self, n: int) -> str:
        return f"{n}."

    def describe(
        self,
        timeframe: TimeFrameLiteral,
        delta: Union[int, float] = 0,
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within a timeframe in plain language.

        :param timeframe: a string representing a timeframe.
        :param delta: a quantity representing a delta in a timeframe.
        :param only_distance: return only distance eg: "11 seconds" without "in" or "ago" keywords
        """

        if not only_distance:
            return super().describe(timeframe, delta, only_distance)

        # German uses a different case without 'in' or 'ago'
        humanized = self.timeframes_only_distance[timeframe].format(trunc(abs(delta)))

        return humanized


class GermanLocale(GermanBaseLocale, Locale):
    names = ["de", "de-de"]


class SwissLocale(GermanBaseLocale, Locale):
    names = ["de-ch"]


class AustrianLocale(GermanBaseLocale, Locale):
    names = ["de-at"]

    month_names = [
        "",
        "Jänner",
        "Februar",
        "März",
        "April",
        "Mai",
        "Juni",
        "Juli",
        "August",
        "September",
        "Oktober",
        "November",
        "Dezember",
    ]


class NorwegianLocale(Locale):
    names = ["nb", "nb-no"]

    past = "for {0} siden"
    future = "om {0}"

    timeframes = {
        "now": "nå nettopp",
        "second": "ett sekund",
        "seconds": "{0} sekunder",
        "minute": "ett minutt",
        "minutes": "{0} minutter",
        "hour": "en time",
        "hours": "{0} timer",
        "day": "en dag",
        "days": "{0} dager",
        "week": "en uke",
        "weeks": "{0} uker",
        "month": "en måned",
        "months": "{0} måneder",
        "year": "ett år",
        "years": "{0} år",
    }

    month_names = [
        "",
        "januar",
        "februar",
        "mars",
        "april",
        "mai",
        "juni",
        "juli",
        "august",
        "september",
        "oktober",
        "november",
        "desember",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "mai",
        "jun",
        "jul",
        "aug",
        "sep",
        "okt",
        "nov",
        "des",
    ]

    day_names = [
        "",
        "mandag",
        "tirsdag",
        "onsdag",
        "torsdag",
        "fredag",
        "lørdag",
        "søndag",
    ]
    day_abbreviations = ["", "ma", "ti", "on", "to", "fr", "lø", "sø"]

    def _ordinal_number(self, n: int) -> str:
        return f"{n}."


class NewNorwegianLocale(Locale):
    names = ["nn", "nn-no"]

    past = "for {0} sidan"
    future = "om {0}"

    timeframes = {
        "now": "no nettopp",
        "second": "eitt sekund",
        "seconds": "{0} sekund",
        "minute": "eitt minutt",
        "minutes": "{0} minutt",
        "hour": "ein time",
        "hours": "{0} timar",
        "day": "ein dag",
        "days": "{0} dagar",
        "week": "ei veke",
        "weeks": "{0} veker",
        "month": "ein månad",
        "months": "{0} månader",
        "year": "eitt år",
        "years": "{0} år",
    }

    month_names = [
        "",
        "januar",
        "februar",
        "mars",
        "april",
        "mai",
        "juni",
        "juli",
        "august",
        "september",
        "oktober",
        "november",
        "desember",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "mai",
        "jun",
        "jul",
        "aug",
        "sep",
        "okt",
        "nov",
        "des",
    ]

    day_names = [
        "",
        "måndag",
        "tysdag",
        "onsdag",
        "torsdag",
        "fredag",
        "laurdag",
        "sundag",
    ]
    day_abbreviations = ["", "må", "ty", "on", "to", "fr", "la", "su"]

    def _ordinal_number(self, n: int) -> str:
        return f"{n}."


class PortugueseLocale(Locale):
    names = ["pt", "pt-pt"]

    past = "há {0}"
    future = "em {0}"
    and_word = "e"

    timeframes = {
        "now": "agora",
        "second": "um segundo",
        "seconds": "{0} segundos",
        "minute": "um minuto",
        "minutes": "{0} minutos",
        "hour": "uma hora",
        "hours": "{0} horas",
        "day": "um dia",
        "days": "{0} dias",
        "week": "uma semana",
        "weeks": "{0} semanas",
        "month": "um mês",
        "months": "{0} meses",
        "year": "um ano",
        "years": "{0} anos",
    }

    month_names = [
        "",
        "Janeiro",
        "Fevereiro",
        "Março",
        "Abril",
        "Maio",
        "Junho",
        "Julho",
        "Agosto",
        "Setembro",
        "Outubro",
        "Novembro",
        "Dezembro",
    ]
    month_abbreviations = [
        "",
        "Jan",
        "Fev",
        "Mar",
        "Abr",
        "Mai",
        "Jun",
        "Jul",
        "Ago",
        "Set",
        "Out",
        "Nov",
        "Dez",
    ]

    day_names = [
        "",
        "Segunda-feira",
        "Terça-feira",
        "Quarta-feira",
        "Quinta-feira",
        "Sexta-feira",
        "Sábado",
        "Domingo",
    ]
    day_abbreviations = ["", "Seg", "Ter", "Qua", "Qui", "Sex", "Sab", "Dom"]


class BrazilianPortugueseLocale(PortugueseLocale):
    names = ["pt-br"]

    past = "faz {0}"


class TagalogLocale(Locale):
    names = ["tl", "tl-ph"]

    past = "nakaraang {0}"
    future = "{0} mula ngayon"

    timeframes = {
        "now": "ngayon lang",
        "second": "isang segundo",
        "seconds": "{0} segundo",
        "minute": "isang minuto",
        "minutes": "{0} minuto",
        "hour": "isang oras",
        "hours": "{0} oras",
        "day": "isang araw",
        "days": "{0} araw",
        "week": "isang linggo",
        "weeks": "{0} linggo",
        "month": "isang buwan",
        "months": "{0} buwan",
        "year": "isang taon",
        "years": "{0} taon",
    }

    month_names = [
        "",
        "Enero",
        "Pebrero",
        "Marso",
        "Abril",
        "Mayo",
        "Hunyo",
        "Hulyo",
        "Agosto",
        "Setyembre",
        "Oktubre",
        "Nobyembre",
        "Disyembre",
    ]
    month_abbreviations = [
        "",
        "Ene",
        "Peb",
        "Mar",
        "Abr",
        "May",
        "Hun",
        "Hul",
        "Ago",
        "Set",
        "Okt",
        "Nob",
        "Dis",
    ]

    day_names = [
        "",
        "Lunes",
        "Martes",
        "Miyerkules",
        "Huwebes",
        "Biyernes",
        "Sabado",
        "Linggo",
    ]
    day_abbreviations = ["", "Lun", "Mar", "Miy", "Huw", "Biy", "Sab", "Lin"]

    meridians = {"am": "nu", "pm": "nh", "AM": "ng umaga", "PM": "ng hapon"}

    def _ordinal_number(self, n: int) -> str:
        return f"ika-{n}"


class VietnameseLocale(Locale):
    names = ["vi", "vi-vn"]

    past = "{0} trước"
    future = "{0} nữa"

    timeframes = {
        "now": "hiện tại",
        "second": "một giây",
        "seconds": "{0} giây",
        "minute": "một phút",
        "minutes": "{0} phút",
        "hour": "một giờ",
        "hours": "{0} giờ",
        "day": "một ngày",
        "days": "{0} ngày",
        "week": "một tuần",
        "weeks": "{0} tuần",
        "month": "một tháng",
        "months": "{0} tháng",
        "year": "một năm",
        "years": "{0} năm",
    }

    month_names = [
        "",
        "Tháng Một",
        "Tháng Hai",
        "Tháng Ba",
        "Tháng Tư",
        "Tháng Năm",
        "Tháng Sáu",
        "Tháng Bảy",
        "Tháng Tám",
        "Tháng Chín",
        "Tháng Mười",
        "Tháng Mười Một",
        "Tháng Mười Hai",
    ]
    month_abbreviations = [
        "",
        "Tháng 1",
        "Tháng 2",
        "Tháng 3",
        "Tháng 4",
        "Tháng 5",
        "Tháng 6",
        "Tháng 7",
        "Tháng 8",
        "Tháng 9",
        "Tháng 10",
        "Tháng 11",
        "Tháng 12",
    ]

    day_names = [
        "",
        "Thứ Hai",
        "Thứ Ba",
        "Thứ Tư",
        "Thứ Năm",
        "Thứ Sáu",
        "Thứ Bảy",
        "Chủ Nhật",
    ]
    day_abbreviations = ["", "Thứ 2", "Thứ 3", "Thứ 4", "Thứ 5", "Thứ 6", "Thứ 7", "CN"]


class TurkishLocale(Locale):
    names = ["tr", "tr-tr"]

    past = "{0} önce"
    future = "{0} sonra"
    and_word = "ve"

    timeframes = {
        "now": "şimdi",
        "second": "bir saniye",
        "seconds": "{0} saniye",
        "minute": "bir dakika",
        "minutes": "{0} dakika",
        "hour": "bir saat",
        "hours": "{0} saat",
        "day": "bir gün",
        "days": "{0} gün",
        "week": "bir hafta",
        "weeks": "{0} hafta",
        "month": "bir ay",
        "months": "{0} ay",
        "year": "bir yıl",
        "years": "{0} yıl",
    }

    meridians = {"am": "öö", "pm": "ös", "AM": "ÖÖ", "PM": "ÖS"}

    month_names = [
        "",
        "Ocak",
        "Şubat",
        "Mart",
        "Nisan",
        "Mayıs",
        "Haziran",
        "Temmuz",
        "Ağustos",
        "Eylül",
        "Ekim",
        "Kasım",
        "Aralık",
    ]
    month_abbreviations = [
        "",
        "Oca",
        "Şub",
        "Mar",
        "Nis",
        "May",
        "Haz",
        "Tem",
        "Ağu",
        "Eyl",
        "Eki",
        "Kas",
        "Ara",
    ]

    day_names = [
        "",
        "Pazartesi",
        "Salı",
        "Çarşamba",
        "Perşembe",
        "Cuma",
        "Cumartesi",
        "Pazar",
    ]
    day_abbreviations = ["", "Pzt", "Sal", "Çar", "Per", "Cum", "Cmt", "Paz"]


class AzerbaijaniLocale(Locale):
    names = ["az", "az-az"]

    past = "{0} əvvəl"
    future = "{0} sonra"

    timeframes = {
        "now": "indi",
        "second": "bir saniyə",
        "seconds": "{0} saniyə",
        "minute": "bir dəqiqə",
        "minutes": "{0} dəqiqə",
        "hour": "bir saat",
        "hours": "{0} saat",
        "day": "bir gün",
        "days": "{0} gün",
        "week": "bir həftə",
        "weeks": "{0} həftə",
        "month": "bir ay",
        "months": "{0} ay",
        "year": "bir il",
        "years": "{0} il",
    }

    month_names = [
        "",
        "Yanvar",
        "Fevral",
        "Mart",
        "Aprel",
        "May",
        "İyun",
        "İyul",
        "Avqust",
        "Sentyabr",
        "Oktyabr",
        "Noyabr",
        "Dekabr",
    ]
    month_abbreviations = [
        "",
        "Yan",
        "Fev",
        "Mar",
        "Apr",
        "May",
        "İyn",
        "İyl",
        "Avq",
        "Sen",
        "Okt",
        "Noy",
        "Dek",
    ]

    day_names = [
        "",
        "Bazar ertəsi",
        "Çərşənbə axşamı",
        "Çərşənbə",
        "Cümə axşamı",
        "Cümə",
        "Şənbə",
        "Bazar",
    ]
    day_abbreviations = ["", "Ber", "Çax", "Çər", "Cax", "Cüm", "Şnb", "Bzr"]


class ArabicLocale(Locale):
    names = [
        "ar",
        "ar-ae",
        "ar-bh",
        "ar-dj",
        "ar-eg",
        "ar-eh",
        "ar-er",
        "ar-km",
        "ar-kw",
        "ar-ly",
        "ar-om",
        "ar-qa",
        "ar-sa",
        "ar-sd",
        "ar-so",
        "ar-ss",
        "ar-td",
        "ar-ye",
    ]

    past = "منذ {0}"
    future = "خلال {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "الآن",
        "second": "ثانية",
        "seconds": {"2": "ثانيتين", "ten": "{0} ثوان", "higher": "{0} ثانية"},
        "minute": "دقيقة",
        "minutes": {"2": "دقيقتين", "ten": "{0} دقائق", "higher": "{0} دقيقة"},
        "hour": "ساعة",
        "hours": {"2": "ساعتين", "ten": "{0} ساعات", "higher": "{0} ساعة"},
        "day": "يوم",
        "days": {"2": "يومين", "ten": "{0} أيام", "higher": "{0} يوم"},
        "month": "شهر",
        "months": {"2": "شهرين", "ten": "{0} أشهر", "higher": "{0} شهر"},
        "year": "سنة",
        "years": {"2": "سنتين", "ten": "{0} سنوات", "higher": "{0} سنة"},
    }

    month_names = [
        "",
        "يناير",
        "فبراير",
        "مارس",
        "أبريل",
        "مايو",
        "يونيو",
        "يوليو",
        "أغسطس",
        "سبتمبر",
        "أكتوبر",
        "نوفمبر",
        "ديسمبر",
    ]
    month_abbreviations = [
        "",
        "يناير",
        "فبراير",
        "مارس",
        "أبريل",
        "مايو",
        "يونيو",
        "يوليو",
        "أغسطس",
        "سبتمبر",
        "أكتوبر",
        "نوفمبر",
        "ديسمبر",
    ]

    day_names = [
        "",
        "الإثنين",
        "الثلاثاء",
        "الأربعاء",
        "الخميس",
        "الجمعة",
        "السبت",
        "الأحد",
    ]
    day_abbreviations = ["", "إثنين", "ثلاثاء", "أربعاء", "خميس", "جمعة", "سبت", "أحد"]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        delta = abs(delta)
        if isinstance(form, Mapping):
            if delta == 2:
                form = form["2"]
            elif 2 < delta <= 10:
                form = form["ten"]
            else:
                form = form["higher"]

        return form.format(delta)


class LevantArabicLocale(ArabicLocale):
    names = ["ar-iq", "ar-jo", "ar-lb", "ar-ps", "ar-sy"]
    month_names = [
        "",
        "كانون الثاني",
        "شباط",
        "آذار",
        "نيسان",
        "أيار",
        "حزيران",
        "تموز",
        "آب",
        "أيلول",
        "تشرين الأول",
        "تشرين الثاني",
        "كانون الأول",
    ]
    month_abbreviations = [
        "",
        "كانون الثاني",
        "شباط",
        "آذار",
        "نيسان",
        "أيار",
        "حزيران",
        "تموز",
        "آب",
        "أيلول",
        "تشرين الأول",
        "تشرين الثاني",
        "كانون الأول",
    ]


class AlgeriaTunisiaArabicLocale(ArabicLocale):
    names = ["ar-tn", "ar-dz"]
    month_names = [
        "",
        "جانفي",
        "فيفري",
        "مارس",
        "أفريل",
        "ماي",
        "جوان",
        "جويلية",
        "أوت",
        "سبتمبر",
        "أكتوبر",
        "نوفمبر",
        "ديسمبر",
    ]
    month_abbreviations = [
        "",
        "جانفي",
        "فيفري",
        "مارس",
        "أفريل",
        "ماي",
        "جوان",
        "جويلية",
        "أوت",
        "سبتمبر",
        "أكتوبر",
        "نوفمبر",
        "ديسمبر",
    ]


class MauritaniaArabicLocale(ArabicLocale):
    names = ["ar-mr"]
    month_names = [
        "",
        "يناير",
        "فبراير",
        "مارس",
        "إبريل",
        "مايو",
        "يونيو",
        "يوليو",
        "أغشت",
        "شتمبر",
        "أكتوبر",
        "نوفمبر",
        "دجمبر",
    ]
    month_abbreviations = [
        "",
        "يناير",
        "فبراير",
        "مارس",
        "إبريل",
        "مايو",
        "يونيو",
        "يوليو",
        "أغشت",
        "شتمبر",
        "أكتوبر",
        "نوفمبر",
        "دجمبر",
    ]


class MoroccoArabicLocale(ArabicLocale):
    names = ["ar-ma"]
    month_names = [
        "",
        "يناير",
        "فبراير",
        "مارس",
        "أبريل",
        "ماي",
        "يونيو",
        "يوليوز",
        "غشت",
        "شتنبر",
        "أكتوبر",
        "نونبر",
        "دجنبر",
    ]
    month_abbreviations = [
        "",
        "يناير",
        "فبراير",
        "مارس",
        "أبريل",
        "ماي",
        "يونيو",
        "يوليوز",
        "غشت",
        "شتنبر",
        "أكتوبر",
        "نونبر",
        "دجنبر",
    ]


class IcelandicLocale(Locale):
    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]

        if isinstance(form, Mapping):
            if delta < 0:
                form = form["past"]
            elif delta > 0:
                form = form["future"]
            else:
                raise ValueError(
                    "Icelandic Locale does not support units with a delta of zero. "
                    "Please consider making a contribution to fix this issue."
                )
                # FIXME: handle when delta is 0

        return form.format(abs(delta))

    names = ["is", "is-is"]

    past = "fyrir {0} síðan"
    future = "eftir {0}"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "rétt í þessu",
        "second": {"past": "sekúndu", "future": "sekúndu"},
        "seconds": {"past": "{0} nokkrum sekúndum", "future": "nokkrar sekúndur"},
        "minute": {"past": "einni mínútu", "future": "eina mínútu"},
        "minutes": {"past": "{0} mínútum", "future": "{0} mínútur"},
        "hour": {"past": "einum tíma", "future": "einn tíma"},
        "hours": {"past": "{0} tímum", "future": "{0} tíma"},
        "day": {"past": "einum degi", "future": "einn dag"},
        "days": {"past": "{0} dögum", "future": "{0} daga"},
        "month": {"past": "einum mánuði", "future": "einn mánuð"},
        "months": {"past": "{0} mánuðum", "future": "{0} mánuði"},
        "year": {"past": "einu ári", "future": "eitt ár"},
        "years": {"past": "{0} árum", "future": "{0} ár"},
    }

    meridians = {"am": "f.h.", "pm": "e.h.", "AM": "f.h.", "PM": "e.h."}

    month_names = [
        "",
        "janúar",
        "febrúar",
        "mars",
        "apríl",
        "maí",
        "júní",
        "júlí",
        "ágúst",
        "september",
        "október",
        "nóvember",
        "desember",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "maí",
        "jún",
        "júl",
        "ágú",
        "sep",
        "okt",
        "nóv",
        "des",
    ]

    day_names = [
        "",
        "mánudagur",
        "þriðjudagur",
        "miðvikudagur",
        "fimmtudagur",
        "föstudagur",
        "laugardagur",
        "sunnudagur",
    ]
    day_abbreviations = ["", "mán", "þri", "mið", "fim", "fös", "lau", "sun"]


class DanishLocale(Locale):
    names = ["da", "da-dk"]

    past = "for {0} siden"
    future = "om {0}"
    and_word = "og"

    timeframes = {
        "now": "lige nu",
        "second": "et sekund",
        "seconds": "{0} sekunder",
        "minute": "et minut",
        "minutes": "{0} minutter",
        "hour": "en time",
        "hours": "{0} timer",
        "day": "en dag",
        "days": "{0} dage",
        "week": "en uge",
        "weeks": "{0} uger",
        "month": "en måned",
        "months": "{0} måneder",
        "year": "et år",
        "years": "{0} år",
    }

    month_names = [
        "",
        "januar",
        "februar",
        "marts",
        "april",
        "maj",
        "juni",
        "juli",
        "august",
        "september",
        "oktober",
        "november",
        "december",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "maj",
        "jun",
        "jul",
        "aug",
        "sep",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "mandag",
        "tirsdag",
        "onsdag",
        "torsdag",
        "fredag",
        "lørdag",
        "søndag",
    ]
    day_abbreviations = ["", "man", "tir", "ons", "tor", "fre", "lør", "søn"]

    def _ordinal_number(self, n: int) -> str:
        return f"{n}."


class MalayalamLocale(Locale):
    names = ["ml"]

    past = "{0} മുമ്പ്"
    future = "{0} ശേഷം"

    timeframes = {
        "now": "ഇപ്പോൾ",
        "second": "ഒരു നിമിഷം",
        "seconds": "{0} സെക്കന്റ്‌",
        "minute": "ഒരു മിനിറ്റ്",
        "minutes": "{0} മിനിറ്റ്",
        "hour": "ഒരു മണിക്കൂർ",
        "hours": "{0} മണിക്കൂർ",
        "day": "ഒരു ദിവസം ",
        "days": "{0} ദിവസം ",
        "month": "ഒരു മാസം ",
        "months": "{0} മാസം ",
        "year": "ഒരു വർഷം ",
        "years": "{0} വർഷം ",
    }

    meridians = {
        "am": "രാവിലെ",
        "pm": "ഉച്ചക്ക് ശേഷം",
        "AM": "രാവിലെ",
        "PM": "ഉച്ചക്ക് ശേഷം",
    }

    month_names = [
        "",
        "ജനുവരി",
        "ഫെബ്രുവരി",
        "മാർച്ച്‌",
        "ഏപ്രിൽ ",
        "മെയ്‌ ",
        "ജൂണ്‍",
        "ജൂലൈ",
        "ഓഗസ്റ്റ്‌",
        "സെപ്റ്റംബർ",
        "ഒക്ടോബർ",
        "നവംബർ",
        "ഡിസംബർ",
    ]
    month_abbreviations = [
        "",
        "ജനു",
        "ഫെബ് ",
        "മാർ",
        "ഏപ്രിൽ",
        "മേയ്",
        "ജൂണ്‍",
        "ജൂലൈ",
        "ഓഗസ്റ",
        "സെപ്റ്റ",
        "ഒക്ടോ",
        "നവം",
        "ഡിസം",
    ]

    day_names = ["", "തിങ്കള്‍", "ചൊവ്വ", "ബുധന്‍", "വ്യാഴം", "വെള്ളി", "ശനി", "ഞായര്‍"]
    day_abbreviations = [
        "",
        "തിങ്കള്‍",
        "ചൊവ്വ",
        "ബുധന്‍",
        "വ്യാഴം",
        "വെള്ളി",
        "ശനി",
        "ഞായര്‍",
    ]


class HindiLocale(Locale):
    names = ["hi", "hi-in"]

    past = "{0} पहले"
    future = "{0} बाद"

    timeframes = {
        "now": "अभी",
        "second": "एक पल",
        "seconds": "{0} सेकंड्",
        "minute": "एक मिनट ",
        "minutes": "{0} मिनट ",
        "hour": "एक घंटा",
        "hours": "{0} घंटे",
        "day": "एक दिन",
        "days": "{0} दिन",
        "month": "एक माह ",
        "months": "{0} महीने ",
        "year": "एक वर्ष ",
        "years": "{0} साल ",
    }

    meridians = {"am": "सुबह", "pm": "शाम", "AM": "सुबह", "PM": "शाम"}

    month_names = [
        "",
        "जनवरी",
        "फरवरी",
        "मार्च",
        "अप्रैल ",
        "मई",
        "जून",
        "जुलाई",
        "अगस्त",
        "सितंबर",
        "अक्टूबर",
        "नवंबर",
        "दिसंबर",
    ]
    month_abbreviations = [
        "",
        "जन",
        "फ़र",
        "मार्च",
        "अप्रै",
        "मई",
        "जून",
        "जुलाई",
        "आग",
        "सित",
        "अकत",
        "नवे",
        "दिस",
    ]

    day_names = [
        "",
        "सोमवार",
        "मंगलवार",
        "बुधवार",
        "गुरुवार",
        "शुक्रवार",
        "शनिवार",
        "रविवार",
    ]
    day_abbreviations = ["", "सोम", "मंगल", "बुध", "गुरुवार", "शुक्र", "शनि", "रवि"]


class CzechLocale(Locale):
    names = ["cs", "cs-cz"]

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "Teď",
        "second": {"past": "vteřina", "future": "vteřina"},
        "seconds": {
            "zero": "vteřina",
            "past": "{0} sekundami",
            "future-singular": "{0} sekundy",
            "future-paucal": "{0} sekund",
        },
        "minute": {"past": "minutou", "future": "minutu"},
        "minutes": {
            "zero": "{0} minut",
            "past": "{0} minutami",
            "future-singular": "{0} minuty",
            "future-paucal": "{0} minut",
        },
        "hour": {"past": "hodinou", "future": "hodinu"},
        "hours": {
            "zero": "{0} hodin",
            "past": "{0} hodinami",
            "future-singular": "{0} hodiny",
            "future-paucal": "{0} hodin",
        },
        "day": {"past": "dnem", "future": "den"},
        "days": {
            "zero": "{0} dnů",
            "past": "{0} dny",
            "future-singular": "{0} dny",
            "future-paucal": "{0} dnů",
        },
        "week": {"past": "týdnem", "future": "týden"},
        "weeks": {
            "zero": "{0} týdnů",
            "past": "{0} týdny",
            "future-singular": "{0} týdny",
            "future-paucal": "{0} týdnů",
        },
        "month": {"past": "měsícem", "future": "měsíc"},
        "months": {
            "zero": "{0} měsíců",
            "past": "{0} měsíci",
            "future-singular": "{0} měsíce",
            "future-paucal": "{0} měsíců",
        },
        "year": {"past": "rokem", "future": "rok"},
        "years": {
            "zero": "{0} let",
            "past": "{0} lety",
            "future-singular": "{0} roky",
            "future-paucal": "{0} let",
        },
    }

    past = "Před {0}"
    future = "Za {0}"

    month_names = [
        "",
        "leden",
        "únor",
        "březen",
        "duben",
        "květen",
        "červen",
        "červenec",
        "srpen",
        "září",
        "říjen",
        "listopad",
        "prosinec",
    ]
    month_abbreviations = [
        "",
        "led",
        "úno",
        "bře",
        "dub",
        "kvě",
        "čvn",
        "čvc",
        "srp",
        "zář",
        "říj",
        "lis",
        "pro",
    ]

    day_names = [
        "",
        "pondělí",
        "úterý",
        "středa",
        "čtvrtek",
        "pátek",
        "sobota",
        "neděle",
    ]
    day_abbreviations = ["", "po", "út", "st", "čt", "pá", "so", "ne"]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        """Czech aware time frame format function, takes into account
        the differences between past and future forms."""
        abs_delta = abs(delta)
        form = self.timeframes[timeframe]

        if isinstance(form, str):
            return form.format(abs_delta)

        if delta == 0:
            key = "zero"  # And *never* use 0 in the singular!
        elif delta < 0:
            key = "past"
        else:
            # Needed since both regular future and future-singular and future-paucal cases
            if "future-singular" not in form:
                key = "future"
            elif 2 <= abs_delta % 10 <= 4 and (
                abs_delta % 100 < 10 or abs_delta % 100 >= 20
            ):
                key = "future-singular"
            else:
                key = "future-paucal"

        form: str = form[key]
        return form.format(abs_delta)


class SlovakLocale(Locale):
    names = ["sk", "sk-sk"]

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "Teraz",
        "second": {"past": "sekundou", "future": "sekundu"},
        "seconds": {
            "zero": "{0} sekúnd",
            "past": "{0} sekundami",
            "future-singular": "{0} sekundy",
            "future-paucal": "{0} sekúnd",
        },
        "minute": {"past": "minútou", "future": "minútu"},
        "minutes": {
            "zero": "{0} minút",
            "past": "{0} minútami",
            "future-singular": "{0} minúty",
            "future-paucal": "{0} minút",
        },
        "hour": {"past": "hodinou", "future": "hodinu"},
        "hours": {
            "zero": "{0} hodín",
            "past": "{0} hodinami",
            "future-singular": "{0} hodiny",
            "future-paucal": "{0} hodín",
        },
        "day": {"past": "dňom", "future": "deň"},
        "days": {
            "zero": "{0} dní",
            "past": "{0} dňami",
            "future-singular": "{0} dni",
            "future-paucal": "{0} dní",
        },
        "week": {"past": "týždňom", "future": "týždeň"},
        "weeks": {
            "zero": "{0} týždňov",
            "past": "{0} týždňami",
            "future-singular": "{0} týždne",
            "future-paucal": "{0} týždňov",
        },
        "month": {"past": "mesiacom", "future": "mesiac"},
        "months": {
            "zero": "{0} mesiacov",
            "past": "{0} mesiacmi",
            "future-singular": "{0} mesiace",
            "future-paucal": "{0} mesiacov",
        },
        "year": {"past": "rokom", "future": "rok"},
        "years": {
            "zero": "{0} rokov",
            "past": "{0} rokmi",
            "future-singular": "{0} roky",
            "future-paucal": "{0} rokov",
        },
    }

    past = "Pred {0}"
    future = "O {0}"
    and_word = "a"

    month_names = [
        "",
        "január",
        "február",
        "marec",
        "apríl",
        "máj",
        "jún",
        "júl",
        "august",
        "september",
        "október",
        "november",
        "december",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "máj",
        "jún",
        "júl",
        "aug",
        "sep",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "pondelok",
        "utorok",
        "streda",
        "štvrtok",
        "piatok",
        "sobota",
        "nedeľa",
    ]
    day_abbreviations = ["", "po", "ut", "st", "št", "pi", "so", "ne"]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        """Slovak aware time frame format function, takes into account
        the differences between past and future forms."""
        abs_delta = abs(delta)
        form = self.timeframes[timeframe]

        if isinstance(form, str):
            return form.format(abs_delta)

        if delta == 0:
            key = "zero"  # And *never* use 0 in the singular!
        elif delta < 0:
            key = "past"
        else:
            if "future-singular" not in form:
                key = "future"
            elif 2 <= abs_delta % 10 <= 4 and (
                abs_delta % 100 < 10 or abs_delta % 100 >= 20
            ):
                key = "future-singular"
            else:
                key = "future-paucal"

        form: str = form[key]
        return form.format(abs_delta)


class FarsiLocale(Locale):
    names = ["fa", "fa-ir"]

    past = "{0} قبل"
    future = "در {0}"

    timeframes = {
        "now": "اکنون",
        "second": "یک لحظه",
        "seconds": "{0} ثانیه",
        "minute": "یک دقیقه",
        "minutes": "{0} دقیقه",
        "hour": "یک ساعت",
        "hours": "{0} ساعت",
        "day": "یک روز",
        "days": "{0} روز",
        "month": "یک ماه",
        "months": "{0} ماه",
        "year": "یک سال",
        "years": "{0} سال",
    }

    meridians = {
        "am": "قبل از ظهر",
        "pm": "بعد از ظهر",
        "AM": "قبل از ظهر",
        "PM": "بعد از ظهر",
    }

    month_names = [
        "",
        "January",
        "February",
        "March",
        "April",
        "May",
        "June",
        "July",
        "August",
        "September",
        "October",
        "November",
        "December",
    ]
    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mar",
        "Apr",
        "May",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Oct",
        "Nov",
        "Dec",
    ]

    day_names = [
        "",
        "دو شنبه",
        "سه شنبه",
        "چهارشنبه",
        "پنجشنبه",
        "جمعه",
        "شنبه",
        "یکشنبه",
    ]
    day_abbreviations = ["", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]


class HebrewLocale(Locale):
    names = ["he", "he-il"]

    past = "לפני {0}"
    future = "בעוד {0}"
    and_word = "ו"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "הרגע",
        "second": "שנייה",
        "seconds": "{0} שניות",
        "minute": "דקה",
        "minutes": "{0} דקות",
        "hour": "שעה",
        "hours": {"2": "שעתיים", "ten": "{0} שעות", "higher": "{0} שעות"},
        "day": "יום",
        "days": {"2": "יומיים", "ten": "{0} ימים", "higher": "{0} יום"},
        "week": "שבוע",
        "weeks": {"2": "שבועיים", "ten": "{0} שבועות", "higher": "{0} שבועות"},
        "month": "חודש",
        "months": {"2": "חודשיים", "ten": "{0} חודשים", "higher": "{0} חודשים"},
        "year": "שנה",
        "years": {"2": "שנתיים", "ten": "{0} שנים", "higher": "{0} שנה"},
    }

    meridians = {
        "am": 'לפנ"צ',
        "pm": 'אחר"צ',
        "AM": "לפני הצהריים",
        "PM": "אחרי הצהריים",
    }

    month_names = [
        "",
        "ינואר",
        "פברואר",
        "מרץ",
        "אפריל",
        "מאי",
        "יוני",
        "יולי",
        "אוגוסט",
        "ספטמבר",
        "אוקטובר",
        "נובמבר",
        "דצמבר",
    ]
    month_abbreviations = [
        "",
        "ינו׳",
        "פבר׳",
        "מרץ",
        "אפר׳",
        "מאי",
        "יוני",
        "יולי",
        "אוג׳",
        "ספט׳",
        "אוק׳",
        "נוב׳",
        "דצמ׳",
    ]

    day_names = ["", "שני", "שלישי", "רביעי", "חמישי", "שישי", "שבת", "ראשון"]
    day_abbreviations = ["", "ב׳", "ג׳", "ד׳", "ה׳", "ו׳", "ש׳", "א׳"]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        delta = abs(delta)
        if isinstance(form, Mapping):
            if delta == 2:
                form = form["2"]
            elif delta == 0 or 2 < delta <= 10:
                form = form["ten"]
            else:
                form = form["higher"]

        return form.format(delta)

    def describe_multi(
        self,
        timeframes: Sequence[Tuple[TimeFrameLiteral, Union[int, float]]],
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within multiple timeframes in plain language.
        In Hebrew, the and word behaves a bit differently.

        :param timeframes: a list of string, quantity pairs each representing a timeframe and delta.
        :param only_distance: return only distance eg: "2 hours and 11 seconds" without "in" or "ago" keywords
        """

        humanized = ""
        for index, (timeframe, delta) in enumerate(timeframes):
            last_humanized = self._format_timeframe(timeframe, trunc(delta))
            if index == 0:
                humanized = last_humanized
            elif index == len(timeframes) - 1:  # Must have at least 2 items
                humanized += " " + self.and_word
                if last_humanized[0].isdecimal():
                    humanized += "־"
                humanized += last_humanized
            else:  # Don't add for the last one
                humanized += ", " + last_humanized

        if not only_distance:
            humanized = self._format_relative(humanized, timeframe, trunc(delta))

        return humanized


class MarathiLocale(Locale):
    names = ["mr"]

    past = "{0} आधी"
    future = "{0} नंतर"

    timeframes = {
        "now": "सद्य",
        "second": "एक सेकंद",
        "seconds": "{0} सेकंद",
        "minute": "एक मिनिट ",
        "minutes": "{0} मिनिट ",
        "hour": "एक तास",
        "hours": "{0} तास",
        "day": "एक दिवस",
        "days": "{0} दिवस",
        "month": "एक महिना ",
        "months": "{0} महिने ",
        "year": "एक वर्ष ",
        "years": "{0} वर्ष ",
    }

    meridians = {"am": "सकाळ", "pm": "संध्याकाळ", "AM": "सकाळ", "PM": "संध्याकाळ"}

    month_names = [
        "",
        "जानेवारी",
        "फेब्रुवारी",
        "मार्च",
        "एप्रिल",
        "मे",
        "जून",
        "जुलै",
        "अॉगस्ट",
        "सप्टेंबर",
        "अॉक्टोबर",
        "नोव्हेंबर",
        "डिसेंबर",
    ]
    month_abbreviations = [
        "",
        "जान",
        "फेब्रु",
        "मार्च",
        "एप्रि",
        "मे",
        "जून",
        "जुलै",
        "अॉग",
        "सप्टें",
        "अॉक्टो",
        "नोव्हें",
        "डिसें",
    ]

    day_names = [
        "",
        "सोमवार",
        "मंगळवार",
        "बुधवार",
        "गुरुवार",
        "शुक्रवार",
        "शनिवार",
        "रविवार",
    ]
    day_abbreviations = ["", "सोम", "मंगळ", "बुध", "गुरु", "शुक्र", "शनि", "रवि"]


class CatalanLocale(Locale):
    names = ["ca", "ca-es", "ca-ad", "ca-fr", "ca-it"]
    past = "Fa {0}"
    future = "En {0}"
    and_word = "i"

    timeframes = {
        "now": "Ara mateix",
        "second": "un segon",
        "seconds": "{0} segons",
        "minute": "un minut",
        "minutes": "{0} minuts",
        "hour": "una hora",
        "hours": "{0} hores",
        "day": "un dia",
        "days": "{0} dies",
        "month": "un mes",
        "months": "{0} mesos",
        "year": "un any",
        "years": "{0} anys",
    }

    month_names = [
        "",
        "gener",
        "febrer",
        "març",
        "abril",
        "maig",
        "juny",
        "juliol",
        "agost",
        "setembre",
        "octubre",
        "novembre",
        "desembre",
    ]
    month_abbreviations = [
        "",
        "gen.",
        "febr.",
        "març",
        "abr.",
        "maig",
        "juny",
        "jul.",
        "ag.",
        "set.",
        "oct.",
        "nov.",
        "des.",
    ]
    day_names = [
        "",
        "dilluns",
        "dimarts",
        "dimecres",
        "dijous",
        "divendres",
        "dissabte",
        "diumenge",
    ]
    day_abbreviations = [
        "",
        "dl.",
        "dt.",
        "dc.",
        "dj.",
        "dv.",
        "ds.",
        "dg.",
    ]


class BasqueLocale(Locale):
    names = ["eu", "eu-eu"]
    past = "duela {0}"
    future = "{0}"  # I don't know what's the right phrase in Basque for the future.

    timeframes = {
        "now": "Orain",
        "second": "segundo bat",
        "seconds": "{0} segundu",
        "minute": "minutu bat",
        "minutes": "{0} minutu",
        "hour": "ordu bat",
        "hours": "{0} ordu",
        "day": "egun bat",
        "days": "{0} egun",
        "month": "hilabete bat",
        "months": "{0} hilabet",
        "year": "urte bat",
        "years": "{0} urte",
    }

    month_names = [
        "",
        "urtarrilak",
        "otsailak",
        "martxoak",
        "apirilak",
        "maiatzak",
        "ekainak",
        "uztailak",
        "abuztuak",
        "irailak",
        "urriak",
        "azaroak",
        "abenduak",
    ]
    month_abbreviations = [
        "",
        "urt",
        "ots",
        "mar",
        "api",
        "mai",
        "eka",
        "uzt",
        "abu",
        "ira",
        "urr",
        "aza",
        "abe",
    ]
    day_names = [
        "",
        "astelehena",
        "asteartea",
        "asteazkena",
        "osteguna",
        "ostirala",
        "larunbata",
        "igandea",
    ]
    day_abbreviations = ["", "al", "ar", "az", "og", "ol", "lr", "ig"]


class HungarianLocale(Locale):
    names = ["hu", "hu-hu"]

    past = "{0} ezelőtt"
    future = "{0} múlva"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "éppen most",
        "second": {"past": "egy második", "future": "egy második"},
        "seconds": {"past": "{0} másodpercekkel", "future": "{0} pár másodperc"},
        "minute": {"past": "egy perccel", "future": "egy perc"},
        "minutes": {"past": "{0} perccel", "future": "{0} perc"},
        "hour": {"past": "egy órával", "future": "egy óra"},
        "hours": {"past": "{0} órával", "future": "{0} óra"},
        "day": {"past": "egy nappal", "future": "egy nap"},
        "days": {"past": "{0} nappal", "future": "{0} nap"},
        "month": {"past": "egy hónappal", "future": "egy hónap"},
        "months": {"past": "{0} hónappal", "future": "{0} hónap"},
        "year": {"past": "egy évvel", "future": "egy év"},
        "years": {"past": "{0} évvel", "future": "{0} év"},
    }

    month_names = [
        "",
        "január",
        "február",
        "március",
        "április",
        "május",
        "június",
        "július",
        "augusztus",
        "szeptember",
        "október",
        "november",
        "december",
    ]
    month_abbreviations = [
        "",
        "jan",
        "febr",
        "márc",
        "ápr",
        "máj",
        "jún",
        "júl",
        "aug",
        "szept",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "hétfő",
        "kedd",
        "szerda",
        "csütörtök",
        "péntek",
        "szombat",
        "vasárnap",
    ]
    day_abbreviations = ["", "hét", "kedd", "szer", "csüt", "pént", "szom", "vas"]

    meridians = {"am": "de", "pm": "du", "AM": "DE", "PM": "DU"}

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]

        if isinstance(form, Mapping):
            if delta > 0:
                form = form["future"]
            else:
                form = form["past"]

        return form.format(abs(delta))


class EsperantoLocale(Locale):
    names = ["eo", "eo-xx"]
    past = "antaŭ {0}"
    future = "post {0}"

    timeframes = {
        "now": "nun",
        "second": "sekundo",
        "seconds": "{0} kelkaj sekundoj",
        "minute": "unu minuto",
        "minutes": "{0} minutoj",
        "hour": "un horo",
        "hours": "{0} horoj",
        "day": "unu tago",
        "days": "{0} tagoj",
        "month": "unu monato",
        "months": "{0} monatoj",
        "year": "unu jaro",
        "years": "{0} jaroj",
    }

    month_names = [
        "",
        "januaro",
        "februaro",
        "marto",
        "aprilo",
        "majo",
        "junio",
        "julio",
        "aŭgusto",
        "septembro",
        "oktobro",
        "novembro",
        "decembro",
    ]
    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "maj",
        "jun",
        "jul",
        "aŭg",
        "sep",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "lundo",
        "mardo",
        "merkredo",
        "ĵaŭdo",
        "vendredo",
        "sabato",
        "dimanĉo",
    ]
    day_abbreviations = ["", "lun", "mar", "mer", "ĵaŭ", "ven", "sab", "dim"]

    meridians = {"am": "atm", "pm": "ptm", "AM": "ATM", "PM": "PTM"}

    ordinal_day_re = r"((?P<value>[1-3]?[0-9](?=a))a)"

    def _ordinal_number(self, n: int) -> str:
        return f"{n}a"


class ThaiLocale(Locale):
    names = ["th", "th-th"]

    past = "{0} ที่ผ่านมา"
    future = "ในอีก {0}"

    timeframes = {
        "now": "ขณะนี้",
        "second": "วินาที",
        "seconds": "{0} ไม่กี่วินาที",
        "minute": "1 นาที",
        "minutes": "{0} นาที",
        "hour": "1 ชั่วโมง",
        "hours": "{0} ชั่วโมง",
        "day": "1 วัน",
        "days": "{0} วัน",
        "month": "1 เดือน",
        "months": "{0} เดือน",
        "year": "1 ปี",
        "years": "{0} ปี",
    }

    month_names = [
        "",
        "มกราคม",
        "กุมภาพันธ์",
        "มีนาคม",
        "เมษายน",
        "พฤษภาคม",
        "มิถุนายน",
        "กรกฎาคม",
        "สิงหาคม",
        "กันยายน",
        "ตุลาคม",
        "พฤศจิกายน",
        "ธันวาคม",
    ]
    month_abbreviations = [
        "",
        "ม.ค.",
        "ก.พ.",
        "มี.ค.",
        "เม.ย.",
        "พ.ค.",
        "มิ.ย.",
        "ก.ค.",
        "ส.ค.",
        "ก.ย.",
        "ต.ค.",
        "พ.ย.",
        "ธ.ค.",
    ]

    day_names = ["", "จันทร์", "อังคาร", "พุธ", "พฤหัสบดี", "ศุกร์", "เสาร์", "อาทิตย์"]
    day_abbreviations = ["", "จ", "อ", "พ", "พฤ", "ศ", "ส", "อา"]

    meridians = {"am": "am", "pm": "pm", "AM": "AM", "PM": "PM"}

    BE_OFFSET = 543

    def year_full(self, year: int) -> str:
        """Thai always use Buddhist Era (BE) which is CE + 543"""
        year += self.BE_OFFSET
        return f"{year:04d}"

    def year_abbreviation(self, year: int) -> str:
        """Thai always use Buddhist Era (BE) which is CE + 543"""
        year += self.BE_OFFSET
        return f"{year:04d}"[2:]

    def _format_relative(
        self,
        humanized: str,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int],
    ) -> str:
        """Thai normally doesn't have any space between words"""
        if timeframe == "now":
            return humanized

        direction = self.past if delta < 0 else self.future
        relative_string = direction.format(humanized)

        if timeframe == "seconds":
            relative_string = relative_string.replace(" ", "")

        return relative_string


class LaotianLocale(Locale):

    names = ["lo", "lo-la"]

    past = "{0} ກ່ອນຫນ້ານີ້"
    future = "ໃນ {0}"

    timeframes = {
        "now": "ດຽວນີ້",
        "second": "ວິນາທີ",
        "seconds": "{0} ວິນາທີ",
        "minute": "ນາທີ",
        "minutes": "{0} ນາທີ",
        "hour": "ຊົ່ວໂມງ",
        "hours": "{0} ຊົ່ວໂມງ",
        "day": "ມື້",
        "days": "{0} ມື້",
        "week": "ອາທິດ",
        "weeks": "{0} ອາທິດ",
        "month": "ເດືອນ",
        "months": "{0} ເດືອນ",
        "year": "ປີ",
        "years": "{0} ປີ",
    }

    month_names = [
        "",
        "ມັງກອນ",  # mangkon
        "ກຸມພາ",  # kumpha
        "ມີນາ",  # mina
        "ເມສາ",  # mesa
        "ພຶດສະພາ",  # phudsapha
        "ມິຖຸນາ",  # mithuna
        "ກໍລະກົດ",  # kolakod
        "ສິງຫາ",  # singha
        "ກັນຍາ",  # knaia
        "ຕຸລາ",  # tula
        "ພະຈິກ",  # phachik
        "ທັນວາ",  # thanuaa
    ]
    month_abbreviations = [
        "",
        "ມັງກອນ",
        "ກຸມພາ",
        "ມີນາ",
        "ເມສາ",
        "ພຶດສະພາ",
        "ມິຖຸນາ",
        "ກໍລະກົດ",
        "ສິງຫາ",
        "ກັນຍາ",
        "ຕຸລາ",
        "ພະຈິກ",
        "ທັນວາ",
    ]

    day_names = [
        "",
        "ວັນຈັນ",  # vanchan
        "ວັນອັງຄານ",  # vnoangkhan
        "ວັນພຸດ",  # vanphud
        "ວັນພະຫັດ",  # vanphahad
        "ວັນ​ສຸກ",  # vansuk
        "ວັນເສົາ",  # vansao
        "ວັນອາທິດ",  # vnoathid
    ]
    day_abbreviations = [
        "",
        "ວັນຈັນ",
        "ວັນອັງຄານ",
        "ວັນພຸດ",
        "ວັນພະຫັດ",
        "ວັນ​ສຸກ",
        "ວັນເສົາ",
        "ວັນອາທິດ",
    ]

    BE_OFFSET = 543

    def year_full(self, year: int) -> str:
        """Lao always use Buddhist Era (BE) which is CE + 543"""
        year += self.BE_OFFSET
        return f"{year:04d}"

    def year_abbreviation(self, year: int) -> str:
        """Lao always use Buddhist Era (BE) which is CE + 543"""
        year += self.BE_OFFSET
        return f"{year:04d}"[2:]

    def _format_relative(
        self,
        humanized: str,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int],
    ) -> str:
        """Lao normally doesn't have any space between words"""
        if timeframe == "now":
            return humanized

        direction = self.past if delta < 0 else self.future
        relative_string = direction.format(humanized)

        if timeframe == "seconds":
            relative_string = relative_string.replace(" ", "")

        return relative_string


class BengaliLocale(Locale):
    names = ["bn", "bn-bd", "bn-in"]

    past = "{0} আগে"
    future = "{0} পরে"

    timeframes = {
        "now": "এখন",
        "second": "একটি দ্বিতীয়",
        "seconds": "{0} সেকেন্ড",
        "minute": "এক মিনিট",
        "minutes": "{0} মিনিট",
        "hour": "এক ঘণ্টা",
        "hours": "{0} ঘণ্টা",
        "day": "এক দিন",
        "days": "{0} দিন",
        "month": "এক মাস",
        "months": "{0} মাস ",
        "year": "এক বছর",
        "years": "{0} বছর",
    }

    meridians = {"am": "সকাল", "pm": "বিকাল", "AM": "সকাল", "PM": "বিকাল"}

    month_names = [
        "",
        "জানুয়ারি",
        "ফেব্রুয়ারি",
        "মার্চ",
        "এপ্রিল",
        "মে",
        "জুন",
        "জুলাই",
        "আগস্ট",
        "সেপ্টেম্বর",
        "অক্টোবর",
        "নভেম্বর",
        "ডিসেম্বর",
    ]
    month_abbreviations = [
        "",
        "জানু",
        "ফেব",
        "মার্চ",
        "এপ্রি",
        "মে",
        "জুন",
        "জুল",
        "অগা",
        "সেপ্ট",
        "অক্টো",
        "নভে",
        "ডিসে",
    ]

    day_names = [
        "",
        "সোমবার",
        "মঙ্গলবার",
        "বুধবার",
        "বৃহস্পতিবার",
        "শুক্রবার",
        "শনিবার",
        "রবিবার",
    ]
    day_abbreviations = ["", "সোম", "মঙ্গল", "বুধ", "বৃহঃ", "শুক্র", "শনি", "রবি"]

    def _ordinal_number(self, n: int) -> str:
        if n > 10 or n == 0:
            return f"{n}তম"
        if n in [1, 5, 7, 8, 9, 10]:
            return f"{n}ম"
        if n in [2, 3]:
            return f"{n}য়"
        if n == 4:
            return f"{n}র্থ"
        if n == 6:
            return f"{n}ষ্ঠ"


class RomanshLocale(Locale):
    names = ["rm", "rm-ch"]

    past = "avant {0}"
    future = "en {0}"

    timeframes = {
        "now": "en quest mument",
        "second": "in secunda",
        "seconds": "{0} secundas",
        "minute": "ina minuta",
        "minutes": "{0} minutas",
        "hour": "in'ura",
        "hours": "{0} ura",
        "day": "in di",
        "days": "{0} dis",
        "month": "in mais",
        "months": "{0} mais",
        "year": "in onn",
        "years": "{0} onns",
    }

    month_names = [
        "",
        "schaner",
        "favrer",
        "mars",
        "avrigl",
        "matg",
        "zercladur",
        "fanadur",
        "avust",
        "settember",
        "october",
        "november",
        "december",
    ]

    month_abbreviations = [
        "",
        "schan",
        "fav",
        "mars",
        "avr",
        "matg",
        "zer",
        "fan",
        "avu",
        "set",
        "oct",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "glindesdi",
        "mardi",
        "mesemna",
        "gievgia",
        "venderdi",
        "sonda",
        "dumengia",
    ]

    day_abbreviations = ["", "gli", "ma", "me", "gie", "ve", "so", "du"]


class RomanianLocale(Locale):
    names = ["ro", "ro-ro"]

    past = "{0} în urmă"
    future = "peste {0}"
    and_word = "și"

    timeframes = {
        "now": "acum",
        "second": "o secunda",
        "seconds": "{0} câteva secunde",
        "minute": "un minut",
        "minutes": "{0} minute",
        "hour": "o oră",
        "hours": "{0} ore",
        "day": "o zi",
        "days": "{0} zile",
        "month": "o lună",
        "months": "{0} luni",
        "year": "un an",
        "years": "{0} ani",
    }

    month_names = [
        "",
        "ianuarie",
        "februarie",
        "martie",
        "aprilie",
        "mai",
        "iunie",
        "iulie",
        "august",
        "septembrie",
        "octombrie",
        "noiembrie",
        "decembrie",
    ]
    month_abbreviations = [
        "",
        "ian",
        "febr",
        "mart",
        "apr",
        "mai",
        "iun",
        "iul",
        "aug",
        "sept",
        "oct",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "luni",
        "marți",
        "miercuri",
        "joi",
        "vineri",
        "sâmbătă",
        "duminică",
    ]
    day_abbreviations = ["", "Lun", "Mar", "Mie", "Joi", "Vin", "Sâm", "Dum"]


class SlovenianLocale(Locale):
    names = ["sl", "sl-si"]

    past = "pred {0}"
    future = "čez {0}"
    and_word = "in"

    timeframes = {
        "now": "zdaj",
        "second": "sekundo",
        "seconds": "{0} sekund",
        "minute": "minuta",
        "minutes": "{0} minutami",
        "hour": "uro",
        "hours": "{0} ur",
        "day": "dan",
        "days": "{0} dni",
        "month": "mesec",
        "months": "{0} mesecev",
        "year": "leto",
        "years": "{0} let",
    }

    meridians = {"am": "", "pm": "", "AM": "", "PM": ""}

    month_names = [
        "",
        "Januar",
        "Februar",
        "Marec",
        "April",
        "Maj",
        "Junij",
        "Julij",
        "Avgust",
        "September",
        "Oktober",
        "November",
        "December",
    ]

    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mar",
        "Apr",
        "Maj",
        "Jun",
        "Jul",
        "Avg",
        "Sep",
        "Okt",
        "Nov",
        "Dec",
    ]

    day_names = [
        "",
        "Ponedeljek",
        "Torek",
        "Sreda",
        "Četrtek",
        "Petek",
        "Sobota",
        "Nedelja",
    ]

    day_abbreviations = ["", "Pon", "Tor", "Sre", "Čet", "Pet", "Sob", "Ned"]


class IndonesianLocale(Locale):
    names = ["id", "id-id"]

    past = "{0} yang lalu"
    future = "dalam {0}"
    and_word = "dan"

    timeframes = {
        "now": "baru saja",
        "second": "1 sebentar",
        "seconds": "{0} detik",
        "minute": "1 menit",
        "minutes": "{0} menit",
        "hour": "1 jam",
        "hours": "{0} jam",
        "day": "1 hari",
        "days": "{0} hari",
        "week": "1 minggu",
        "weeks": "{0} minggu",
        "month": "1 bulan",
        "months": "{0} bulan",
        "quarter": "1 kuartal",
        "quarters": "{0} kuartal",
        "year": "1 tahun",
        "years": "{0} tahun",
    }

    meridians = {"am": "", "pm": "", "AM": "", "PM": ""}

    month_names = [
        "",
        "Januari",
        "Februari",
        "Maret",
        "April",
        "Mei",
        "Juni",
        "Juli",
        "Agustus",
        "September",
        "Oktober",
        "November",
        "Desember",
    ]

    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mar",
        "Apr",
        "Mei",
        "Jun",
        "Jul",
        "Ags",
        "Sept",
        "Okt",
        "Nov",
        "Des",
    ]

    day_names = ["", "Senin", "Selasa", "Rabu", "Kamis", "Jumat", "Sabtu", "Minggu"]

    day_abbreviations = [
        "",
        "Senin",
        "Selasa",
        "Rabu",
        "Kamis",
        "Jumat",
        "Sabtu",
        "Minggu",
    ]


class NepaliLocale(Locale):
    names = ["ne", "ne-np"]

    past = "{0} पहिले"
    future = "{0} पछी"

    timeframes = {
        "now": "अहिले",
        "second": "एक सेकेन्ड",
        "seconds": "{0} सेकण्ड",
        "minute": "मिनेट",
        "minutes": "{0} मिनेट",
        "hour": "एक घण्टा",
        "hours": "{0} घण्टा",
        "day": "एक दिन",
        "days": "{0} दिन",
        "month": "एक महिना",
        "months": "{0} महिना",
        "year": "एक बर्ष",
        "years": "{0} बर्ष",
    }

    meridians = {"am": "पूर्वाह्न", "pm": "अपरान्ह", "AM": "पूर्वाह्न", "PM": "अपरान्ह"}

    month_names = [
        "",
        "जनवरी",
        "फेब्रुअरी",
        "मार्च",
        "एप्रील",
        "मे",
        "जुन",
        "जुलाई",
        "अगष्ट",
        "सेप्टेम्बर",
        "अक्टोबर",
        "नोवेम्बर",
        "डिसेम्बर",
    ]
    month_abbreviations = [
        "",
        "जन",
        "फेब",
        "मार्च",
        "एप्रील",
        "मे",
        "जुन",
        "जुलाई",
        "अग",
        "सेप",
        "अक्ट",
        "नोव",
        "डिस",
    ]

    day_names = [
        "",
        "सोमवार",
        "मंगलवार",
        "बुधवार",
        "बिहिवार",
        "शुक्रवार",
        "शनिवार",
        "आइतवार",
    ]

    day_abbreviations = ["", "सोम", "मंगल", "बुध", "बिहि", "शुक्र", "शनि", "आइत"]


class EstonianLocale(Locale):
    names = ["ee", "et"]

    past = "{0} tagasi"
    future = "{0} pärast"
    and_word = "ja"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Mapping[str, str]]] = {
        "now": {"past": "just nüüd", "future": "just nüüd"},
        "second": {"past": "üks sekund", "future": "ühe sekundi"},
        "seconds": {"past": "{0} sekundit", "future": "{0} sekundi"},
        "minute": {"past": "üks minut", "future": "ühe minuti"},
        "minutes": {"past": "{0} minutit", "future": "{0} minuti"},
        "hour": {"past": "tund aega", "future": "tunni aja"},
        "hours": {"past": "{0} tundi", "future": "{0} tunni"},
        "day": {"past": "üks päev", "future": "ühe päeva"},
        "days": {"past": "{0} päeva", "future": "{0} päeva"},
        "month": {"past": "üks kuu", "future": "ühe kuu"},
        "months": {"past": "{0} kuud", "future": "{0} kuu"},
        "year": {"past": "üks aasta", "future": "ühe aasta"},
        "years": {"past": "{0} aastat", "future": "{0} aasta"},
    }

    month_names = [
        "",
        "Jaanuar",
        "Veebruar",
        "Märts",
        "Aprill",
        "Mai",
        "Juuni",
        "Juuli",
        "August",
        "September",
        "Oktoober",
        "November",
        "Detsember",
    ]
    month_abbreviations = [
        "",
        "Jan",
        "Veb",
        "Mär",
        "Apr",
        "Mai",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Okt",
        "Nov",
        "Dets",
    ]

    day_names = [
        "",
        "Esmaspäev",
        "Teisipäev",
        "Kolmapäev",
        "Neljapäev",
        "Reede",
        "Laupäev",
        "Pühapäev",
    ]
    day_abbreviations = ["", "Esm", "Teis", "Kolm", "Nelj", "Re", "Lau", "Püh"]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        if delta > 0:
            _form = form["future"]
        else:
            _form = form["past"]
        return _form.format(abs(delta))


class LatvianLocale(Locale):
    names = ["lv", "lv-lv"]

    past = "pirms {0}"
    future = "pēc {0}"
    and_word = "un"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "tagad",
        "second": "sekundes",
        "seconds": "{0} sekundēm",
        "minute": "minūtes",
        "minutes": "{0} minūtēm",
        "hour": "stundas",
        "hours": "{0} stundām",
        "day": "dienas",
        "days": "{0} dienām",
        "week": "nedēļas",
        "weeks": "{0} nedēļām",
        "month": "mēneša",
        "months": "{0} mēnešiem",
        "year": "gada",
        "years": "{0} gadiem",
    }

    month_names = [
        "",
        "janvāris",
        "februāris",
        "marts",
        "aprīlis",
        "maijs",
        "jūnijs",
        "jūlijs",
        "augusts",
        "septembris",
        "oktobris",
        "novembris",
        "decembris",
    ]

    month_abbreviations = [
        "",
        "jan",
        "feb",
        "marts",
        "apr",
        "maijs",
        "jūnijs",
        "jūlijs",
        "aug",
        "sept",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "pirmdiena",
        "otrdiena",
        "trešdiena",
        "ceturtdiena",
        "piektdiena",
        "sestdiena",
        "svētdiena",
    ]

    day_abbreviations = [
        "",
        "pi",
        "ot",
        "tr",
        "ce",
        "pi",
        "se",
        "sv",
    ]


class SwahiliLocale(Locale):
    names = [
        "sw",
        "sw-ke",
        "sw-tz",
    ]

    past = "{0} iliyopita"
    future = "muda wa {0}"
    and_word = "na"

    timeframes = {
        "now": "sasa hivi",
        "second": "sekunde",
        "seconds": "sekunde {0}",
        "minute": "dakika moja",
        "minutes": "dakika {0}",
        "hour": "saa moja",
        "hours": "saa {0}",
        "day": "siku moja",
        "days": "siku {0}",
        "week": "wiki moja",
        "weeks": "wiki {0}",
        "month": "mwezi moja",
        "months": "miezi {0}",
        "year": "mwaka moja",
        "years": "miaka {0}",
    }

    meridians = {"am": "asu", "pm": "mch", "AM": "ASU", "PM": "MCH"}

    month_names = [
        "",
        "Januari",
        "Februari",
        "Machi",
        "Aprili",
        "Mei",
        "Juni",
        "Julai",
        "Agosti",
        "Septemba",
        "Oktoba",
        "Novemba",
        "Desemba",
    ]
    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mac",
        "Apr",
        "Mei",
        "Jun",
        "Jul",
        "Ago",
        "Sep",
        "Okt",
        "Nov",
        "Des",
    ]

    day_names = [
        "",
        "Jumatatu",
        "Jumanne",
        "Jumatano",
        "Alhamisi",
        "Ijumaa",
        "Jumamosi",
        "Jumapili",
    ]
    day_abbreviations = [
        "",
        "Jumatatu",
        "Jumanne",
        "Jumatano",
        "Alhamisi",
        "Ijumaa",
        "Jumamosi",
        "Jumapili",
    ]


class CroatianLocale(Locale):
    names = ["hr", "hr-hr"]

    past = "prije {0}"
    future = "za {0}"
    and_word = "i"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "upravo sad",
        "second": "sekundu",
        "seconds": {"double": "{0} sekunde", "higher": "{0} sekundi"},
        "minute": "minutu",
        "minutes": {"double": "{0} minute", "higher": "{0} minuta"},
        "hour": "sat",
        "hours": {"double": "{0} sata", "higher": "{0} sati"},
        "day": "jedan dan",
        "days": {"double": "{0} dana", "higher": "{0} dana"},
        "week": "tjedan",
        "weeks": {"double": "{0} tjedna", "higher": "{0} tjedana"},
        "month": "mjesec",
        "months": {"double": "{0} mjeseca", "higher": "{0} mjeseci"},
        "year": "godinu",
        "years": {"double": "{0} godine", "higher": "{0} godina"},
    }

    month_names = [
        "",
        "siječanj",
        "veljača",
        "ožujak",
        "travanj",
        "svibanj",
        "lipanj",
        "srpanj",
        "kolovoz",
        "rujan",
        "listopad",
        "studeni",
        "prosinac",
    ]

    month_abbreviations = [
        "",
        "siječ",
        "velj",
        "ožuj",
        "trav",
        "svib",
        "lip",
        "srp",
        "kol",
        "ruj",
        "list",
        "stud",
        "pros",
    ]

    day_names = [
        "",
        "ponedjeljak",
        "utorak",
        "srijeda",
        "četvrtak",
        "petak",
        "subota",
        "nedjelja",
    ]

    day_abbreviations = [
        "",
        "po",
        "ut",
        "sr",
        "če",
        "pe",
        "su",
        "ne",
    ]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        delta = abs(delta)
        if isinstance(form, Mapping):
            if 1 < delta <= 4:
                form = form["double"]
            else:
                form = form["higher"]

        return form.format(delta)


class LatinLocale(Locale):
    names = ["la", "la-va"]

    past = "ante {0}"
    future = "in {0}"
    and_word = "et"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "nunc",
        "second": "secundum",
        "seconds": "{0} secundis",
        "minute": "minutam",
        "minutes": "{0} minutis",
        "hour": "horam",
        "hours": "{0} horas",
        "day": "diem",
        "days": "{0} dies",
        "week": "hebdomadem",
        "weeks": "{0} hebdomades",
        "month": "mensem",
        "months": "{0} mensis",
        "year": "annum",
        "years": "{0} annos",
    }

    month_names = [
        "",
        "Ianuarius",
        "Februarius",
        "Martius",
        "Aprilis",
        "Maius",
        "Iunius",
        "Iulius",
        "Augustus",
        "September",
        "October",
        "November",
        "December",
    ]

    month_abbreviations = [
        "",
        "Ian",
        "Febr",
        "Mart",
        "Apr",
        "Mai",
        "Iun",
        "Iul",
        "Aug",
        "Sept",
        "Oct",
        "Nov",
        "Dec",
    ]

    day_names = [
        "",
        "dies Lunae",
        "dies Martis",
        "dies Mercurii",
        "dies Iovis",
        "dies Veneris",
        "dies Saturni",
        "dies Solis",
    ]

    day_abbreviations = [
        "",
        "dies Lunae",
        "dies Martis",
        "dies Mercurii",
        "dies Iovis",
        "dies Veneris",
        "dies Saturni",
        "dies Solis",
    ]


class LithuanianLocale(Locale):
    names = ["lt", "lt-lt"]

    past = "prieš {0}"
    future = "po {0}"
    and_word = "ir"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "dabar",
        "second": "sekundės",
        "seconds": "{0} sekundžių",
        "minute": "minutės",
        "minutes": "{0} minučių",
        "hour": "valandos",
        "hours": "{0} valandų",
        "day": "dieną",
        "days": "{0} dienų",
        "week": "savaitės",
        "weeks": "{0} savaičių",
        "month": "mėnesio",
        "months": "{0} mėnesių",
        "year": "metų",
        "years": "{0} metų",
    }

    month_names = [
        "",
        "sausis",
        "vasaris",
        "kovas",
        "balandis",
        "gegužė",
        "birželis",
        "liepa",
        "rugpjūtis",
        "rugsėjis",
        "spalis",
        "lapkritis",
        "gruodis",
    ]

    month_abbreviations = [
        "",
        "saus",
        "vas",
        "kovas",
        "bal",
        "geg",
        "birž",
        "liepa",
        "rugp",
        "rugs",
        "spalis",
        "lapkr",
        "gr",
    ]

    day_names = [
        "",
        "pirmadienis",
        "antradienis",
        "trečiadienis",
        "ketvirtadienis",
        "penktadienis",
        "šeštadienis",
        "sekmadienis",
    ]

    day_abbreviations = [
        "",
        "pi",
        "an",
        "tr",
        "ke",
        "pe",
        "še",
        "se",
    ]


class MalayLocale(Locale):
    names = ["ms", "ms-my", "ms-bn"]

    past = "{0} yang lalu"
    future = "dalam {0}"
    and_word = "dan"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "sekarang",
        "second": "saat",
        "seconds": "{0} saat",
        "minute": "minit",
        "minutes": "{0} minit",
        "hour": "jam",
        "hours": "{0} jam",
        "day": "hari",
        "days": "{0} hari",
        "week": "minggu",
        "weeks": "{0} minggu",
        "month": "bulan",
        "months": "{0} bulan",
        "year": "tahun",
        "years": "{0} tahun",
    }

    month_names = [
        "",
        "Januari",
        "Februari",
        "Mac",
        "April",
        "Mei",
        "Jun",
        "Julai",
        "Ogos",
        "September",
        "Oktober",
        "November",
        "Disember",
    ]

    month_abbreviations = [
        "",
        "Jan.",
        "Feb.",
        "Mac",
        "Apr.",
        "Mei",
        "Jun",
        "Julai",
        "Og.",
        "Sept.",
        "Okt.",
        "Nov.",
        "Dis.",
    ]

    day_names = [
        "",
        "Isnin",
        "Selasa",
        "Rabu",
        "Khamis",
        "Jumaat",
        "Sabtu",
        "Ahad",
    ]

    day_abbreviations = [
        "",
        "Isnin",
        "Selasa",
        "Rabu",
        "Khamis",
        "Jumaat",
        "Sabtu",
        "Ahad",
    ]


class MalteseLocale(Locale):
    names = ["mt", "mt-mt"]

    past = "{0} ilu"
    future = "fi {0}"
    and_word = "u"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "issa",
        "second": "sekonda",
        "seconds": "{0} sekondi",
        "minute": "minuta",
        "minutes": "{0} minuti",
        "hour": "siegħa",
        "hours": {"dual": "{0} sagħtejn", "plural": "{0} sigħat"},
        "day": "jum",
        "days": {"dual": "{0} jumejn", "plural": "{0} ijiem"},
        "week": "ġimgħa",
        "weeks": {"dual": "{0} ġimagħtejn", "plural": "{0} ġimgħat"},
        "month": "xahar",
        "months": {"dual": "{0} xahrejn", "plural": "{0} xhur"},
        "year": "sena",
        "years": {"dual": "{0} sentejn", "plural": "{0} snin"},
    }

    month_names = [
        "",
        "Jannar",
        "Frar",
        "Marzu",
        "April",
        "Mejju",
        "Ġunju",
        "Lulju",
        "Awwissu",
        "Settembru",
        "Ottubru",
        "Novembru",
        "Diċembru",
    ]

    month_abbreviations = [
        "",
        "Jan",
        "Fr",
        "Mar",
        "Apr",
        "Mejju",
        "Ġun",
        "Lul",
        "Aw",
        "Sett",
        "Ott",
        "Nov",
        "Diċ",
    ]

    day_names = [
        "",
        "It-Tnejn",
        "It-Tlieta",
        "L-Erbgħa",
        "Il-Ħamis",
        "Il-Ġimgħa",
        "Is-Sibt",
        "Il-Ħadd",
    ]

    day_abbreviations = [
        "",
        "T",
        "TL",
        "E",
        "Ħ",
        "Ġ",
        "S",
        "Ħ",
    ]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        delta = abs(delta)
        if isinstance(form, Mapping):
            if delta == 2:
                form = form["dual"]
            else:
                form = form["plural"]

        return form.format(delta)


class SamiLocale(Locale):
    names = ["se", "se-fi", "se-no", "se-se"]

    past = "{0} dassái"
    future = "{0} "  # NOTE: couldn't find preposition for Sami here, none needed?

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "dál",
        "second": "sekunda",
        "seconds": "{0} sekundda",
        "minute": "minuhta",
        "minutes": "{0} minuhta",
        "hour": "diimmu",
        "hours": "{0} diimmu",
        "day": "beaivvi",
        "days": "{0} beaivvi",
        "week": "vahku",
        "weeks": "{0} vahku",
        "month": "mánu",
        "months": "{0} mánu",
        "year": "jagi",
        "years": "{0} jagi",
    }

    month_names = [
        "",
        "Ođđajagimánnu",
        "Guovvamánnu",
        "Njukčamánnu",
        "Cuoŋománnu",
        "Miessemánnu",
        "Geassemánnu",
        "Suoidnemánnu",
        "Borgemánnu",
        "Čakčamánnu",
        "Golggotmánnu",
        "Skábmamánnu",
        "Juovlamánnu",
    ]

    month_abbreviations = [
        "",
        "Ođđajagimánnu",
        "Guovvamánnu",
        "Njukčamánnu",
        "Cuoŋománnu",
        "Miessemánnu",
        "Geassemánnu",
        "Suoidnemánnu",
        "Borgemánnu",
        "Čakčamánnu",
        "Golggotmánnu",
        "Skábmamánnu",
        "Juovlamánnu",
    ]

    day_names = [
        "",
        "Mánnodat",
        "Disdat",
        "Gaskavahkku",
        "Duorastat",
        "Bearjadat",
        "Lávvordat",
        "Sotnabeaivi",
    ]

    day_abbreviations = [
        "",
        "Mánnodat",
        "Disdat",
        "Gaskavahkku",
        "Duorastat",
        "Bearjadat",
        "Lávvordat",
        "Sotnabeaivi",
    ]


class OdiaLocale(Locale):
    names = ["or", "or-in"]

    past = "{0} ପୂର୍ବେ"
    future = "{0} ପରେ"

    timeframes = {
        "now": "ବର୍ତ୍ତମାନ",
        "second": "ଏକ ସେକେଣ୍ଡ",
        "seconds": "{0} ସେକେଣ୍ଡ",
        "minute": "ଏକ ମିନଟ",
        "minutes": "{0} ମିନଟ",
        "hour": "ଏକ ଘଣ୍ଟା",
        "hours": "{0} ଘଣ୍ଟା",
        "day": "ଏକ ଦିନ",
        "days": "{0} ଦିନ",
        "month": "ଏକ ମାସ",
        "months": "{0} ମାସ ",
        "year": "ଏକ ବର୍ଷ",
        "years": "{0} ବର୍ଷ",
    }

    meridians = {"am": "ପୂର୍ବାହ୍ନ", "pm": "ଅପରାହ୍ନ", "AM": "ପୂର୍ବାହ୍ନ", "PM": "ଅପରାହ୍ନ"}

    month_names = [
        "",
        "ଜାନୁଆରୀ",
        "ଫେବୃଆରୀ",
        "ମାର୍ଚ୍ଚ୍",
        "ଅପ୍ରେଲ",
        "ମଇ",
        "ଜୁନ୍",
        "ଜୁଲାଇ",
        "ଅଗଷ୍ଟ",
        "ସେପ୍ଟେମ୍ବର",
        "ଅକ୍ଟୋବର୍",
        "ନଭେମ୍ବର୍",
        "ଡିସେମ୍ବର୍",
    ]
    month_abbreviations = [
        "",
        "ଜାନୁ",
        "ଫେବୃ",
        "ମାର୍ଚ୍ଚ୍",
        "ଅପ୍ରେ",
        "ମଇ",
        "ଜୁନ୍",
        "ଜୁଲା",
        "ଅଗ",
        "ସେପ୍ଟେ",
        "ଅକ୍ଟୋ",
        "ନଭେ",
        "ଡିସେ",
    ]

    day_names = [
        "",
        "ସୋମବାର",
        "ମଙ୍ଗଳବାର",
        "ବୁଧବାର",
        "ଗୁରୁବାର",
        "ଶୁକ୍ରବାର",
        "ଶନିବାର",
        "ରବିବାର",
    ]
    day_abbreviations = [
        "",
        "ସୋମ",
        "ମଙ୍ଗଳ",
        "ବୁଧ",
        "ଗୁରୁ",
        "ଶୁକ୍ର",
        "ଶନି",
        "ରବି",
    ]

    def _ordinal_number(self, n: int) -> str:
        if n > 10 or n == 0:
            return f"{n}ତମ"
        if n in [1, 5, 7, 8, 9, 10]:
            return f"{n}ମ"
        if n in [2, 3]:
            return f"{n}ୟ"
        if n == 4:
            return f"{n}ର୍ଥ"
        if n == 6:
            return f"{n}ଷ୍ଠ"
        return ""


class SerbianLocale(Locale):
    names = ["sr", "sr-rs", "sr-sp"]

    past = "pre {0}"
    future = "za {0}"
    and_word = "i"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[str, Mapping[str, str]]]] = {
        "now": "sada",
        "second": "sekundu",
        "seconds": {"double": "{0} sekunde", "higher": "{0} sekundi"},
        "minute": "minutu",
        "minutes": {"double": "{0} minute", "higher": "{0} minuta"},
        "hour": "sat",
        "hours": {"double": "{0} sata", "higher": "{0} sati"},
        "day": "dan",
        "days": {"double": "{0} dana", "higher": "{0} dana"},
        "week": "nedelju",
        "weeks": {"double": "{0} nedelje", "higher": "{0} nedelja"},
        "month": "mesec",
        "months": {"double": "{0} meseca", "higher": "{0} meseci"},
        "year": "godinu",
        "years": {"double": "{0} godine", "higher": "{0} godina"},
    }

    month_names = [
        "",
        "januar",  # јануар
        "februar",  # фебруар
        "mart",  # март
        "april",  # април
        "maj",  # мај
        "jun",  # јун
        "jul",  # јул
        "avgust",  # август
        "septembar",  # септембар
        "oktobar",  # октобар
        "novembar",  # новембар
        "decembar",  # децембар
    ]

    month_abbreviations = [
        "",
        "jan",
        "feb",
        "mar",
        "apr",
        "maj",
        "jun",
        "jul",
        "avg",
        "sep",
        "okt",
        "nov",
        "dec",
    ]

    day_names = [
        "",
        "ponedeljak",  # понедељак
        "utorak",  # уторак
        "sreda",  # среда
        "četvrtak",  # четвртак
        "petak",  # петак
        "subota",  # субота
        "nedelja",  # недеља
    ]

    day_abbreviations = [
        "",
        "po",  # по
        "ut",  # ут
        "sr",  # ср
        "če",  # че
        "pe",  # пе
        "su",  # су
        "ne",  # не
    ]

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        form = self.timeframes[timeframe]
        delta = abs(delta)
        if isinstance(form, Mapping):
            if 1 < delta <= 4:
                form = form["double"]
            else:
                form = form["higher"]

        return form.format(delta)


class LuxembourgishLocale(Locale):
    names = ["lb", "lb-lu"]

    past = "virun {0}"
    future = "an {0}"
    and_word = "an"

    timeframes = {
        "now": "just elo",
        "second": "enger Sekonn",
        "seconds": "{0} Sekonnen",
        "minute": "enger Minutt",
        "minutes": "{0} Minutten",
        "hour": "enger Stonn",
        "hours": "{0} Stonnen",
        "day": "engem Dag",
        "days": "{0} Deeg",
        "week": "enger Woch",
        "weeks": "{0} Wochen",
        "month": "engem Mount",
        "months": "{0} Méint",
        "year": "engem Joer",
        "years": "{0} Jahren",
    }

    timeframes_only_distance = timeframes.copy()
    timeframes_only_distance["second"] = "eng Sekonn"
    timeframes_only_distance["minute"] = "eng Minutt"
    timeframes_only_distance["hour"] = "eng Stonn"
    timeframes_only_distance["day"] = "een Dag"
    timeframes_only_distance["days"] = "{0} Deeg"
    timeframes_only_distance["week"] = "eng Woch"
    timeframes_only_distance["month"] = "ee Mount"
    timeframes_only_distance["months"] = "{0} Méint"
    timeframes_only_distance["year"] = "ee Joer"
    timeframes_only_distance["years"] = "{0} Joer"

    month_names = [
        "",
        "Januar",
        "Februar",
        "Mäerz",
        "Abrëll",
        "Mee",
        "Juni",
        "Juli",
        "August",
        "September",
        "Oktouber",
        "November",
        "Dezember",
    ]

    month_abbreviations = [
        "",
        "Jan",
        "Feb",
        "Mäe",
        "Abr",
        "Mee",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Okt",
        "Nov",
        "Dez",
    ]

    day_names = [
        "",
        "Méindeg",
        "Dënschdeg",
        "Mëttwoch",
        "Donneschdeg",
        "Freideg",
        "Samschdeg",
        "Sonndeg",
    ]

    day_abbreviations = ["", "Méi", "Dën", "Mët", "Don", "Fre", "Sam", "Son"]

    def _ordinal_number(self, n: int) -> str:
        return f"{n}."

    def describe(
        self,
        timeframe: TimeFrameLiteral,
        delta: Union[int, float] = 0,
        only_distance: bool = False,
    ) -> str:
        if not only_distance:
            return super().describe(timeframe, delta, only_distance)

        # Luxembourgish uses a different case without 'in' or 'ago'
        humanized = self.timeframes_only_distance[timeframe].format(trunc(abs(delta)))

        return humanized


class ZuluLocale(Locale):
    names = ["zu", "zu-za"]

    past = "{0} edlule"
    future = "{0} "
    and_word = "futhi"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[Mapping[str, str], str]]] = {
        "now": "manje",
        "second": {"past": "umzuzwana", "future": "ngomzuzwana"},
        "seconds": {"past": "{0} imizuzwana", "future": "{0} ngemizuzwana"},
        "minute": {"past": "umzuzu", "future": "ngomzuzu"},
        "minutes": {"past": "{0} imizuzu", "future": "{0} ngemizuzu"},
        "hour": {"past": "ihora", "future": "ngehora"},
        "hours": {"past": "{0} amahora", "future": "{0} emahoreni"},
        "day": {"past": "usuku", "future": "ngosuku"},
        "days": {"past": "{0} izinsuku", "future": "{0} ezinsukwini"},
        "week": {"past": "isonto", "future": "ngesonto"},
        "weeks": {"past": "{0} amasonto", "future": "{0} emasontweni"},
        "month": {"past": "inyanga", "future": "ngenyanga"},
        "months": {"past": "{0} izinyanga", "future": "{0} ezinyangeni"},
        "year": {"past": "unyaka", "future": "ngonyak"},
        "years": {"past": "{0} iminyaka", "future": "{0} eminyakeni"},
    }

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        """Zulu aware time frame format function, takes into account
        the differences between past and future forms."""
        abs_delta = abs(delta)
        form = self.timeframes[timeframe]

        if isinstance(form, str):
            return form.format(abs_delta)

        if delta > 0:
            key = "future"
        else:
            key = "past"
        form = form[key]

        return form.format(abs_delta)

    month_names = [
        "",
        "uMasingane",
        "uNhlolanja",
        "uNdasa",
        "UMbasa",
        "UNhlaba",
        "UNhlangulana",
        "uNtulikazi",
        "UNcwaba",
        "uMandulo",
        "uMfumfu",
        "uLwezi",
        "uZibandlela",
    ]

    month_abbreviations = [
        "",
        "uMasingane",
        "uNhlolanja",
        "uNdasa",
        "UMbasa",
        "UNhlaba",
        "UNhlangulana",
        "uNtulikazi",
        "UNcwaba",
        "uMandulo",
        "uMfumfu",
        "uLwezi",
        "uZibandlela",
    ]

    day_names = [
        "",
        "uMsombuluko",
        "uLwesibili",
        "uLwesithathu",
        "uLwesine",
        "uLwesihlanu",
        "uMgqibelo",
        "iSonto",
    ]

    day_abbreviations = [
        "",
        "uMsombuluko",
        "uLwesibili",
        "uLwesithathu",
        "uLwesine",
        "uLwesihlanu",
        "uMgqibelo",
        "iSonto",
    ]


class TamilLocale(Locale):
    names = ["ta", "ta-in", "ta-lk"]

    past = "{0} நேரத்திற்கு முன்பு"
    future = "இல் {0}"

    timeframes = {
        "now": "இப்போது",
        "second": "ஒரு இரண்டாவது",
        "seconds": "{0} விநாடிகள்",
        "minute": "ஒரு நிமிடம்",
        "minutes": "{0} நிமிடங்கள்",
        "hour": "ஒரு மணி",
        "hours": "{0} மணிநேரம்",
        "day": "ஒரு நாள்",
        "days": "{0} நாட்கள்",
        "week": "ஒரு வாரம்",
        "weeks": "{0} வாரங்கள்",
        "month": "ஒரு மாதம்",
        "months": "{0} மாதங்கள்",
        "year": "ஒரு ஆண்டு",
        "years": "{0} ஆண்டுகள்",
    }

    month_names = [
        "",
        "சித்திரை",
        "வைகாசி",
        "ஆனி",
        "ஆடி",
        "ஆவணி",
        "புரட்டாசி",
        "ஐப்பசி",
        "கார்த்திகை",
        "மார்கழி",
        "தை",
        "மாசி",
        "பங்குனி",
    ]

    month_abbreviations = [
        "",
        "ஜன",
        "பிப்",
        "மார்",
        "ஏப்",
        "மே",
        "ஜூன்",
        "ஜூலை",
        "ஆக",
        "செப்",
        "அக்",
        "நவ",
        "டிச",
    ]

    day_names = [
        "",
        "திங்கட்கிழமை",
        "செவ்வாய்க்கிழமை",
        "புதன்கிழமை",
        "வியாழக்கிழமை",
        "வெள்ளிக்கிழமை",
        "சனிக்கிழமை",
        "ஞாயிற்றுக்கிழமை",
    ]

    day_abbreviations = [
        "",
        "திங்கட்",
        "செவ்வாய்",
        "புதன்",
        "வியாழன்",
        "வெள்ளி",
        "சனி",
        "ஞாயிறு",
    ]

    def _ordinal_number(self, n: int) -> str:
        if n == 1:
            return f"{n}வது"
        elif n >= 0:
            return f"{n}ஆம்"
        else:
            return ""


class AlbanianLocale(Locale):
    names = ["sq", "sq-al"]

    past = "{0} më parë"
    future = "në {0}"
    and_word = "dhe"

    timeframes = {
        "now": "tani",
        "second": "sekondë",
        "seconds": "{0} sekonda",
        "minute": "minutë",
        "minutes": "{0} minuta",
        "hour": "orë",
        "hours": "{0} orë",
        "day": "ditë",
        "days": "{0} ditë",
        "week": "javë",
        "weeks": "{0} javë",
        "month": "muaj",
        "months": "{0} muaj",
        "year": "vit",
        "years": "{0} vjet",
    }

    month_names = [
        "",
        "janar",
        "shkurt",
        "mars",
        "prill",
        "maj",
        "qershor",
        "korrik",
        "gusht",
        "shtator",
        "tetor",
        "nëntor",
        "dhjetor",
    ]

    month_abbreviations = [
        "",
        "jan",
        "shk",
        "mar",
        "pri",
        "maj",
        "qer",
        "korr",
        "gush",
        "sht",
        "tet",
        "nën",
        "dhj",
    ]

    day_names = [
        "",
        "e hënë",
        "e martë",
        "e mërkurë",
        "e enjte",
        "e premte",
        "e shtunë",
        "e diel",
    ]

    day_abbreviations = [
        "",
        "hën",
        "mar",
        "mër",
        "enj",
        "pre",
        "sht",
        "die",
    ]


class GeorgianLocale(Locale):
    names = ["ka", "ka-ge"]

    past = "{0} წინ"  # ts’in
    future = "{0} შემდეგ"  # shemdeg
    and_word = "და"  # da

    timeframes = {
        "now": "ახლა",  # akhla
        # When a cardinal qualifies a noun, it stands in the singular
        "second": "წამის",  # ts’amis
        "seconds": "{0} წამის",
        "minute": "წუთის",  # ts’utis
        "minutes": "{0} წუთის",
        "hour": "საათის",  # saatis
        "hours": "{0} საათის",
        "day": "დღის",  # dghis
        "days": "{0} დღის",
        "week": "კვირის",  # k’viris
        "weeks": "{0} კვირის",
        "month": "თვის",  # tvis
        "months": "{0} თვის",
        "year": "წლის",  # ts’lis
        "years": "{0} წლის",
    }

    month_names = [
        # modern month names
        "",
        "იანვარი",  # Ianvari
        "თებერვალი",  # Tebervali
        "მარტი",  # Mart'i
        "აპრილი",  # Ap'rili
        "მაისი",  # Maisi
        "ივნისი",  # Ivnisi
        "ივლისი",  # Ivlisi
        "აგვისტო",  # Agvist'o
        "სექტემბერი",  # Sekt'emberi
        "ოქტომბერი",  # Okt'omberi
        "ნოემბერი",  # Noemberi
        "დეკემბერი",  # Dek'emberi
    ]

    month_abbreviations = [
        # no abbr. found yet
        "",
        "იანვარი",  # Ianvari
        "თებერვალი",  # Tebervali
        "მარტი",  # Mart'i
        "აპრილი",  # Ap'rili
        "მაისი",  # Maisi
        "ივნისი",  # Ivnisi
        "ივლისი",  # Ivlisi
        "აგვისტო",  # Agvist'o
        "სექტემბერი",  # Sekt'emberi
        "ოქტომბერი",  # Okt'omberi
        "ნოემბერი",  # Noemberi
        "დეკემბერი",  # Dek'emberi
    ]

    day_names = [
        "",
        "ორშაბათი",  # orshabati
        "სამშაბათი",  # samshabati
        "ოთხშაბათი",  # otkhshabati
        "ხუთშაბათი",  # khutshabati
        "პარასკევი",  # p’arask’evi
        "შაბათი",  # shabati
        # "k’vira" also serves as week; to avoid confusion "k’vira-dge" can be used for Sunday
        "კვირა",  # k’vira
    ]

    day_abbreviations = [
        "",
        "ორშაბათი",  # orshabati
        "სამშაბათი",  # samshabati
        "ოთხშაბათი",  # otkhshabati
        "ხუთშაბათი",  # khutshabati
        "პარასკევი",  # p’arask’evi
        "შაბათი",  # shabati
        "კვირა",  # k’vira
    ]


class SinhalaLocale(Locale):
    names = ["si", "si-lk"]

    past = "{0}ට පෙර"
    future = "{0}"
    and_word = "සහ"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[Mapping[str, str], str]]] = {
        "now": "දැන්",
        "second": {
            "past": "තත්පරයක",
            "future": "තත්පරයකින්",
        },  # ක් is the article
        "seconds": {
            "past": "තත්පර {0} ක",
            "future": "තත්පර {0} කින්",
        },
        "minute": {
            "past": "විනාඩියක",
            "future": "විනාඩියකින්",
        },
        "minutes": {
            "past": "විනාඩි {0} ක",
            "future": "මිනිත්තු {0} කින්",
        },
        "hour": {"past": "පැයක", "future": "පැයකින්"},
        "hours": {
            "past": "පැය {0} ක",
            "future": "පැය {0} කින්",
        },
        "day": {"past": "දිනක", "future": "දිනකට"},
        "days": {
            "past": "දින {0} ක",
            "future": "දින {0} කින්",
        },
        "week": {"past": "සතියක", "future": "සතියකින්"},
        "weeks": {
            "past": "සති {0} ක",
            "future": "සති {0} කින්",
        },
        "month": {"past": "මාසයක", "future": "එය මාසය තුළ"},
        "months": {
            "past": "මාස {0} ක",
            "future": "මාස {0} කින්",
        },
        "year": {"past": "වසරක", "future": "වසරක් තුළ"},
        "years": {
            "past": "අවුරුදු {0} ක",
            "future": "අවුරුදු {0} තුළ",
        },
    }
    # Sinhala: the general format to describe timeframe is different from past and future,
    # so we do not copy the original timeframes dictionary
    timeframes_only_distance = {}
    timeframes_only_distance["second"] = "තත්පරයක්"
    timeframes_only_distance["seconds"] = "තත්පර {0}"
    timeframes_only_distance["minute"] = "මිනිත්තුවක්"
    timeframes_only_distance["minutes"] = "විනාඩි {0}"
    timeframes_only_distance["hour"] = "පැයක්"
    timeframes_only_distance["hours"] = "පැය {0}"
    timeframes_only_distance["day"] = "දවසක්"
    timeframes_only_distance["days"] = "දවස් {0}"
    timeframes_only_distance["week"] = "සතියක්"
    timeframes_only_distance["weeks"] = "සති {0}"
    timeframes_only_distance["month"] = "මාසයක්"
    timeframes_only_distance["months"] = "මාස {0}"
    timeframes_only_distance["year"] = "අවුරුද්දක්"
    timeframes_only_distance["years"] = "අවුරුදු {0}"

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        """
        Sinhala awares time frame format function, takes into account
        the differences between general, past, and future forms (three different suffixes).
        """
        abs_delta = abs(delta)
        form = self.timeframes[timeframe]

        if isinstance(form, str):
            return form.format(abs_delta)

        if delta > 0:
            key = "future"
        else:
            key = "past"
        form = form[key]

        return form.format(abs_delta)

    def describe(
        self,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int] = 1,  # key is always future when only_distance=False
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within a timeframe in plain language.

        :param timeframe: a string representing a timeframe.
        :param delta: a quantity representing a delta in a timeframe.
        :param only_distance: return only distance eg: "11 seconds" without "in" or "ago" keywords
        """

        if not only_distance:
            return super().describe(timeframe, delta, only_distance)
        # Sinhala uses a different case without 'in' or 'ago'
        humanized = self.timeframes_only_distance[timeframe].format(trunc(abs(delta)))

        return humanized

    month_names = [
        "",
        "ජනවාරි",
        "පෙබරවාරි",
        "මාර්තු",
        "අප්‍රේල්",
        "මැයි",
        "ජූනි",
        "ජූලි",
        "අගෝස්තු",
        "සැප්තැම්බර්",
        "ඔක්තෝබර්",
        "නොවැම්බර්",
        "දෙසැම්බර්",
    ]

    month_abbreviations = [
        "",
        "ජන",
        "පෙබ",
        "මාර්",
        "අප්‍රේ",
        "මැයි",
        "ජුනි",
        "ජූලි",
        "අගෝ",
        "සැප්",
        "ඔක්",
        "නොවැ",
        "දෙසැ",
    ]

    day_names = [
        "",
        "සදුදා",
        "අඟහරැවදා",
        "බදාදා",
        "බ්‍රහස්‍පතින්‍දා",
        "සිකුරාදා",
        "සෙනසුරාදා",
        "ඉරිදා",
    ]

    day_abbreviations = [
        "",
        "සදුද",
        "බදා",
        "බදා",
        "සිකු",
        "සෙන",
        "අ",
        "ඉරිදා",
    ]


class UrduLocale(Locale):
    names = ["ur", "ur-pk"]

    past = "پہلے {0}"
    future = "میں {0}"
    and_word = "اور"

    timeframes = {
        "now": "ابھی",
        "second": "ایک سیکنڈ",
        "seconds": "{0} سیکنڈ",
        "minute": "ایک منٹ",
        "minutes": "{0} منٹ",
        "hour": "ایک گھنٹے",
        "hours": "{0} گھنٹے",
        "day": "ایک دن",
        "days": "{0} دن",
        "week": "ایک ہفتے",
        "weeks": "{0} ہفتے",
        "month": "ایک مہینہ",
        "months": "{0} ماہ",
        "year": "ایک سال",
        "years": "{0} سال",
    }

    month_names = [
        "",
        "جنوری",
        "فروری",
        "مارچ",
        "اپریل",
        "مئی",
        "جون",
        "جولائی",
        "اگست",
        "ستمبر",
        "اکتوبر",
        "نومبر",
        "دسمبر",
    ]

    month_abbreviations = [
        "",
        "جنوری",
        "فروری",
        "مارچ",
        "اپریل",
        "مئی",
        "جون",
        "جولائی",
        "اگست",
        "ستمبر",
        "اکتوبر",
        "نومبر",
        "دسمبر",
    ]

    day_names = [
        "",
        "سوموار",
        "منگل",
        "بدھ",
        "جمعرات",
        "جمعہ",
        "ہفتہ",
        "اتوار",
    ]

    day_abbreviations = [
        "",
        "سوموار",
        "منگل",
        "بدھ",
        "جمعرات",
        "جمعہ",
        "ہفتہ",
        "اتوار",
    ]


class KazakhLocale(Locale):
    names = ["kk", "kk-kz"]

    past = "{0} бұрын"
    future = "{0} кейін"
    timeframes = {
        "now": "қазір",
        "second": "бір секунд",
        "seconds": "{0} секунд",
        "minute": "бір минут",
        "minutes": "{0} минут",
        "hour": "бір сағат",
        "hours": "{0} сағат",
        "day": "бір күн",
        "days": "{0} күн",
        "week": "бір апта",
        "weeks": "{0} апта",
        "month": "бір ай",
        "months": "{0} ай",
        "year": "бір жыл",
        "years": "{0} жыл",
    }

    month_names = [
        "",
        "Қаңтар",
        "Ақпан",
        "Наурыз",
        "Сәуір",
        "Мамыр",
        "Маусым",
        "Шілде",
        "Тамыз",
        "Қыркүйек",
        "Қазан",
        "Қараша",
        "Желтоқсан",
    ]
    month_abbreviations = [
        "",
        "Қан",
        "Ақп",
        "Нау",
        "Сәу",
        "Мам",
        "Мау",
        "Шіл",
        "Там",
        "Қыр",
        "Қаз",
        "Қар",
        "Жел",
    ]

    day_names = [
        "",
        "Дүйсембі",
        "Сейсенбі",
        "Сәрсенбі",
        "Бейсенбі",
        "Жұма",
        "Сенбі",
        "Жексенбі",
    ]
    day_abbreviations = ["", "Дс", "Сс", "Ср", "Бс", "Жм", "Сб", "Жс"]


class AmharicLocale(Locale):
    names = ["am", "am-et"]

    past = "{0} በፊት"
    future = "{0} ውስጥ"
    and_word = "እና"

    timeframes: ClassVar[Mapping[TimeFrameLiteral, Union[Mapping[str, str], str]]] = {
        "now": "አሁን",
        "second": {
            "past": "ከአንድ ሰከንድ",
            "future": "በአንድ ሰከንድ",
        },
        "seconds": {
            "past": "ከ {0} ሰከንድ",
            "future": "በ {0} ሰከንድ",
        },
        "minute": {
            "past": "ከአንድ ደቂቃ",
            "future": "በአንድ ደቂቃ",
        },
        "minutes": {
            "past": "ከ {0} ደቂቃዎች",
            "future": "በ {0} ደቂቃዎች",
        },
        "hour": {
            "past": "ከአንድ ሰዓት",
            "future": "በአንድ ሰዓት",
        },
        "hours": {
            "past": "ከ {0} ሰዓታት",
            "future": "በ {0} ሰከንድ",
        },
        "day": {
            "past": "ከአንድ ቀን",
            "future": "በአንድ ቀን",
        },
        "days": {
            "past": "ከ {0} ቀናት",
            "future": "በ {0} ቀናት",
        },
        "week": {
            "past": "ከአንድ ሳምንት",
            "future": "በአንድ ሳምንት",
        },
        "weeks": {
            "past": "ከ {0} ሳምንታት",
            "future": "በ {0} ሳምንታት",
        },
        "month": {
            "past": "ከአንድ ወር",
            "future": "በአንድ ወር",
        },
        "months": {
            "past": "ከ {0} ወር",
            "future": "በ {0} ወራት",
        },
        "year": {
            "past": "ከአንድ አመት",
            "future": "በአንድ አመት",
        },
        "years": {
            "past": "ከ {0} ዓመታት",
            "future": "በ {0} ዓመታት",
        },
    }
    # Amharic: the general format to describe timeframe is different from past and future,
    # so we do not copy the original timeframes dictionary
    timeframes_only_distance = {
        "second": "አንድ ሰከንድ",
        "seconds": "{0} ሰከንድ",
        "minute": "አንድ ደቂቃ",
        "minutes": "{0} ደቂቃዎች",
        "hour": "አንድ ሰዓት",
        "hours": "{0} ሰዓት",
        "day": "አንድ ቀን",
        "days": "{0} ቀናት",
        "week": "አንድ ሳምንት",
        "weeks": "{0} ሳምንት",
        "month": "አንድ ወር",
        "months": "{0} ወራት",
        "year": "አንድ አመት",
        "years": "{0} ዓመታት",
    }

    month_names = [
        "",
        "ጃንዩወሪ",
        "ፌብሩወሪ",
        "ማርች",
        "ኤፕሪል",
        "ሜይ",
        "ጁን",
        "ጁላይ",
        "ኦገስት",
        "ሴፕቴምበር",
        "ኦክቶበር",
        "ኖቬምበር",
        "ዲሴምበር",
    ]

    month_abbreviations = [
        "",
        "ጃንዩ",
        "ፌብሩ",
        "ማርች",
        "ኤፕሪ",
        "ሜይ",
        "ጁን",
        "ጁላይ",
        "ኦገስ",
        "ሴፕቴ",
        "ኦክቶ",
        "ኖቬም",
        "ዲሴም",
    ]

    day_names = [
        "",
        "ሰኞ",
        "ማክሰኞ",
        "ረቡዕ",
        "ሐሙስ",
        "ዓርብ",
        "ቅዳሜ",
        "እሑድ",
    ]
    day_abbreviations = ["", "እ", "ሰ", "ማ", "ረ", "ሐ", "ዓ", "ቅ"]

    def _ordinal_number(self, n: int) -> str:
        return f"{n}ኛ"

    def _format_timeframe(self, timeframe: TimeFrameLiteral, delta: int) -> str:
        """
        Amharic awares time frame format function, takes into account
        the differences between general, past, and future forms (three different suffixes).
        """
        abs_delta = abs(delta)
        form = self.timeframes[timeframe]

        if isinstance(form, str):
            return form.format(abs_delta)

        if delta > 0:
            key = "future"
        else:
            key = "past"
        form = form[key]

        return form.format(abs_delta)

    def describe(
        self,
        timeframe: TimeFrameLiteral,
        delta: Union[float, int] = 1,  # key is always future when only_distance=False
        only_distance: bool = False,
    ) -> str:
        """Describes a delta within a timeframe in plain language.

        :param timeframe: a string representing a timeframe.
        :param delta: a quantity representing a delta in a timeframe.
        :param only_distance: return only distance eg: "11 seconds" without "in" or "ago" keywords
        """

        if not only_distance:
            return super().describe(timeframe, delta, only_distance)
        humanized = self.timeframes_only_distance[timeframe].format(trunc(abs(delta)))

        return humanized


class ArmenianLocale(Locale):
    names = ["hy", "hy-am"]
    past = "{0} առաջ"
    future = "{0}ից"
    and_word = "Եվ"  # Yev

    timeframes = {
        "now": "հիմա",
        "second": "վայրկյան",
        "seconds": "{0} վայրկյան",
        "minute": "րոպե",
        "minutes": "{0} րոպե",
        "hour": "ժամ",
        "hours": "{0} ժամ",
        "day": "օր",
        "days": "{0} օր",
        "month": "ամիս",
        "months": "{0} ամիս",
        "year": "տարին",
        "years": "{0} տարին",
        "week": "շաբաթ",
        "weeks": "{0} շաբաթ",
    }

    meridians = {
        "am": "Ամ",
        "pm": "պ.մ.",
        "AM": "Ամ",
        "PM": "պ.մ.",
    }

    month_names = [
        "",
        "հունվար",
        "փետրվար",
        "մարտ",
        "ապրիլ",
        "մայիս",
        "հունիս",
        "հուլիս",
        "օգոստոս",
        "սեպտեմբեր",
        "հոկտեմբեր",
        "նոյեմբեր",
        "դեկտեմբեր",
    ]

    month_abbreviations = [
        "",
        "հունվար",
        "փետրվար",
        "մարտ",
        "ապրիլ",
        "մայիս",
        "հունիս",
        "հուլիս",
        "օգոստոս",
        "սեպտեմբեր",
        "հոկտեմբեր",
        "նոյեմբեր",
        "դեկտեմբեր",
    ]

    day_names = [
        "",
        "երկուշաբթի",
        "երեքշաբթի",
        "չորեքշաբթի",
        "հինգշաբթի",
        "ուրբաթ",
        "շաբաթ",
        "կիրակի",
    ]

    day_abbreviations = [
        "",
        "երկ.",
        "երեք.",
        "չորեք.",
        "հինգ.",
        "ուրբ.",
        "շաբ.",
        "կիր.",
    ]


class UzbekLocale(Locale):
    names = ["uz", "uz-uz"]
    past = "{0}dan avval"
    future = "{0}dan keyin"
    timeframes = {
        "now": "hozir",
        "second": "bir soniya",
        "seconds": "{0} soniya",
        "minute": "bir daqiqa",
        "minutes": "{0} daqiqa",
        "hour": "bir soat",
        "hours": "{0} soat",
        "day": "bir kun",
        "days": "{0} kun",
        "week": "bir hafta",
        "weeks": "{0} hafta",
        "month": "bir oy",
        "months": "{0} oy",
        "year": "bir yil",
        "years": "{0} yil",
    }

    month_names = [
        "",
        "Yanvar",
        "Fevral",
        "Mart",
        "Aprel",
        "May",
        "Iyun",
        "Iyul",
        "Avgust",
        "Sentyabr",
        "Oktyabr",
        "Noyabr",
        "Dekabr",
    ]

    month_abbreviations = [
        "",
        "Yan",
        "Fev",
        "Mar",
        "Apr",
        "May",
        "Iyn",
        "Iyl",
        "Avg",
        "Sen",
        "Okt",
        "Noy",
        "Dek",
    ]

    day_names = [
        "",
        "Dushanba",
        "Seshanba",
        "Chorshanba",
        "Payshanba",
        "Juma",
        "Shanba",
        "Yakshanba",
    ]

    day_abbreviations = ["", "Dush", "Sesh", "Chor", "Pay", "Jum", "Shan", "Yak"]

env_pyrecodes/lib/python3.9/site-packages/arrow/locales.py
NikolaBlagojevic/pyrecodes

# This file was generated by 'versioneer.py' (0.28) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.

import json

version_json = '''
{
 "date": "2023-05-02T14:34:07+0200",
 "dirty": false,
 "error": null,
 "full-revisionid": "7009e5ec0de58ac2675269829ca602447113d6e2",
 "version": "0.2.11"
}
'''  # END VERSION_JSON


def get_versions():
    return json.loads(version_json)

env_pyrecodes/lib/python3.9/site-packages/autodocsumm/_version.py
NikolaBlagojevic/pyrecodes
"""Beautiful Soup Elixir and Tonic - "The Screen-Scraper's Friend".

http://www.crummy.com/software/BeautifulSoup/

Beautiful Soup uses a pluggable XML or HTML parser to parse a
(possibly invalid) document into a tree representation. Beautiful Soup
provides methods and Pythonic idioms that make it easy to navigate,
search, and modify the parse tree.

Beautiful Soup works with Python 3.6 and up. It works better if lxml
and/or html5lib is installed.

For more than you ever wanted to know about Beautiful Soup, see the
documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/
"""

__author__ = "Leonard Richardson (leonardr@segfault.org)"
__version__ = "4.12.2"
__copyright__ = "Copyright (c) 2004-2023 Leonard Richardson"
# Use of this source code is governed by the MIT license.
__license__ = "MIT"

__all__ = ['BeautifulSoup']

from collections import Counter
import os
import re
import sys
import traceback
import warnings

# The very first thing we do is give a useful error if someone is
# running this code under Python 2.
if sys.version_info.major < 3:
    raise ImportError('You are trying to use a Python 3-specific version of Beautiful Soup under Python 2. This will not work. The final version of Beautiful Soup to support Python 2 was 4.9.3.')

from .builder import (
    builder_registry,
    ParserRejectedMarkup,
    XMLParsedAsHTMLWarning,
    HTMLParserTreeBuilder
)
from .dammit import UnicodeDammit
from .element import (
    CData,
    Comment,
    CSS,
    DEFAULT_OUTPUT_ENCODING,
    Declaration,
    Doctype,
    NavigableString,
    PageElement,
    ProcessingInstruction,
    PYTHON_SPECIFIC_ENCODINGS,
    ResultSet,
    Script,
    Stylesheet,
    SoupStrainer,
    Tag,
    TemplateString,
    )

# Define some custom warnings.
class GuessedAtParserWarning(UserWarning):
    """The warning issued when BeautifulSoup has to guess what parser to
    use -- probably because no parser was specified in the constructor.
    """

class MarkupResemblesLocatorWarning(UserWarning):
    """The warning issued when BeautifulSoup is given 'markup' that
    actually looks like a resource locator -- a URL or a path to a file
    on disk.
    """

   
class BeautifulSoup(Tag):
    """A data structure representing a parsed HTML or XML document.

    Most of the methods you'll call on a BeautifulSoup object are inherited from
    PageElement or Tag.

    Internally, this class defines the basic interface called by the
    tree builders when converting an HTML/XML document into a data
    structure. The interface abstracts away the differences between
    parsers. To write a new tree builder, you'll need to understand
    these methods as a whole.

    These methods will be called by the BeautifulSoup constructor:
      * reset()
      * feed(markup)

    The tree builder may call these methods from its feed() implementation:
      * handle_starttag(name, attrs) # See note about return value
      * handle_endtag(name)
      * handle_data(data) # Appends to the current data node
      * endData(containerClass) # Ends the current data node

    No matter how complicated the underlying parser is, you should be
    able to build a tree using 'start tag' events, 'end tag' events,
    'data' events, and "done with data" events.

    If you encounter an empty-element tag (aka a self-closing tag,
    like HTML's <br> tag), call handle_starttag and then
    handle_endtag.
    """

    # Since BeautifulSoup subclasses Tag, it's possible to treat it as
    # a Tag with a .name. This name makes it clear the BeautifulSoup
    # object isn't a real markup tag.
    ROOT_TAG_NAME = '[document]'

    # If the end-user gives no indication which tree builder they
    # want, look for one with these features.
    DEFAULT_BUILDER_FEATURES = ['html', 'fast']

    # A string containing all ASCII whitespace characters, used in
    # endData() to detect data chunks that seem 'empty'.
    ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'

    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
   
    def __init__(self, markup="", features=None, builder=None,
                 parse_only=None, from_encoding=None, exclude_encodings=None,
                 element_classes=None, **kwargs):
        """Constructor.

        :param markup: A string or a file-like object representing
         markup to be parsed.

        :param features: Desirable features of the parser to be
         used. This may be the name of a specific parser ("lxml",
         "lxml-xml", "html.parser", or "html5lib") or it may be the
         type of markup to be used ("html", "html5", "xml"). It's
         recommended that you name a specific parser, so that
         Beautiful Soup gives you the same results across platforms
         and virtual environments.

        :param builder: A TreeBuilder subclass to instantiate (or
         instance to use) instead of looking one up based on
         `features`. You only need to use this if you've implemented a
         custom TreeBuilder.

        :param parse_only: A SoupStrainer. Only parts of the document
         matching the SoupStrainer will be considered. This is useful
         when parsing part of a document that would otherwise be too
         large to fit into memory.

        :param from_encoding: A string indicating the encoding of the
         document to be parsed. Pass this in if Beautiful Soup is
         guessing wrongly about the document's encoding.

        :param exclude_encodings: A list of strings indicating
         encodings known to be wrong. Pass this in if you don't know
         the document's encoding but you know Beautiful Soup's guess is
         wrong.

        :param element_classes: A dictionary mapping BeautifulSoup
         classes like Tag and NavigableString, to other classes you'd
         like to be instantiated instead as the parse tree is
         built. This is useful for subclassing Tag or NavigableString
         to modify default behavior.

        :param kwargs: For backwards compatibility purposes, the
         constructor accepts certain keyword arguments used in
         Beautiful Soup 3. None of these arguments do anything in
         Beautiful Soup 4; they will result in a warning and then be
         ignored.
         
         Apart from this, any keyword arguments passed into the
         BeautifulSoup constructor are propagated to the TreeBuilder
         constructor. This makes it possible to configure a
         TreeBuilder by passing in arguments, not just by saying which
         one to use.
        """
        if 'convertEntities' in kwargs:
            del kwargs['convertEntities']
            warnings.warn(
                "BS4 does not respect the convertEntities argument to the "
                "BeautifulSoup constructor. Entities are always converted "
                "to Unicode characters.")

        if 'markupMassage' in kwargs:
            del kwargs['markupMassage']
            warnings.warn(
                "BS4 does not respect the markupMassage argument to the "
                "BeautifulSoup constructor. The tree builder is responsible "
                "for any necessary markup massage.")

        if 'smartQuotesTo' in kwargs:
            del kwargs['smartQuotesTo']
            warnings.warn(
                "BS4 does not respect the smartQuotesTo argument to the "
                "BeautifulSoup constructor. Smart quotes are always converted "
                "to Unicode characters.")

        if 'selfClosingTags' in kwargs:
            del kwargs['selfClosingTags']
            warnings.warn(
                "BS4 does not respect the selfClosingTags argument to the "
                "BeautifulSoup constructor. The tree builder is responsible "
                "for understanding self-closing tags.")

        if 'isHTML' in kwargs:
            del kwargs['isHTML']
            warnings.warn(
                "BS4 does not respect the isHTML argument to the "
                "BeautifulSoup constructor. Suggest you use "
                "features='lxml' for HTML and features='lxml-xml' for "
                "XML.")

        def deprecated_argument(old_name, new_name):
            if old_name in kwargs:
                warnings.warn(
                    'The "%s" argument to the BeautifulSoup constructor '
                    'has been renamed to "%s."' % (old_name, new_name),
                    DeprecationWarning, stacklevel=3
                )
                return kwargs.pop(old_name)
            return None

        parse_only = parse_only or deprecated_argument(
            "parseOnlyThese", "parse_only")

        from_encoding = from_encoding or deprecated_argument(
            "fromEncoding", "from_encoding")

        if from_encoding and isinstance(markup, str):
            warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
            from_encoding = None

        self.element_classes = element_classes or dict()

        # We need this information to track whether or not the builder
        # was specified well enough that we can omit the 'you need to
        # specify a parser' warning.
        original_builder = builder
        original_features = features
            
        if isinstance(builder, type):
            # A builder class was passed in; it needs to be instantiated.
            builder_class = builder
            builder = None
        elif builder is None:
            if isinstance(features, str):
                features = [features]
            if features is None or len(features) == 0:
                features = self.DEFAULT_BUILDER_FEATURES
            builder_class = builder_registry.lookup(*features)
            if builder_class is None:
                raise FeatureNotFound(
                    "Couldn't find a tree builder with the features you "
                    "requested: %s. Do you need to install a parser library?"
                    % ",".join(features))

        # At this point either we have a TreeBuilder instance in
        # builder, or we have a builder_class that we can instantiate
        # with the remaining **kwargs.
        if builder is None:
            builder = builder_class(**kwargs)
            if not original_builder and not (
                    original_features == builder.NAME or
                    original_features in builder.ALTERNATE_NAMES
            ) and markup:
                # The user did not tell us which TreeBuilder to use,
                # and we had to guess. Issue a warning.
                if builder.is_xml:
                    markup_type = "XML"
                else:
                    markup_type = "HTML"

                # This code adapted from warnings.py so that we get the same line
                # of code as our warnings.warn() call gets, even if the answer is wrong
                # (as it may be in a multithreading situation).
                caller = None
                try:
                    caller = sys._getframe(1)
                except ValueError:
                    pass
                if caller:
                    globals = caller.f_globals
                    line_number = caller.f_lineno
                else:
                    globals = sys.__dict__
                    line_number= 1                    
                filename = globals.get('__file__')
                if filename:
                    fnl = filename.lower()
                    if fnl.endswith((".pyc", ".pyo")):
                        filename = filename[:-1]
                if filename:
                    # If there is no filename at all, the user is most likely in a REPL,
                    # and the warning is not necessary.
                    values = dict(
                        filename=filename,
                        line_number=line_number,
                        parser=builder.NAME,
                        markup_type=markup_type
                    )
                    warnings.warn(
                        self.NO_PARSER_SPECIFIED_WARNING % values,
                        GuessedAtParserWarning, stacklevel=2
                    )
        else:
            if kwargs:
                warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
                    
        self.builder = builder
        self.is_xml = builder.is_xml
        self.known_xml = self.is_xml
        self._namespaces = dict()
        self.parse_only = parse_only

        if hasattr(markup, 'read'):        # It's a file-type object.
            markup = markup.read()
        elif len(markup) <= 256 and (
                (isinstance(markup, bytes) and not b'<' in markup)
                or (isinstance(markup, str) and not '<' in markup)
        ):
            # Issue warnings for a couple beginner problems
            # involving passing non-markup to Beautiful Soup.
            # Beautiful Soup will still parse the input as markup,
            # since that is sometimes the intended behavior.
            if not self._markup_is_url(markup):
                self._markup_resembles_filename(markup)                

        rejections = []
        success = False
        for (self.markup, self.original_encoding, self.declared_html_encoding,
         self.contains_replacement_characters) in (
             self.builder.prepare_markup(
                 markup, from_encoding, exclude_encodings=exclude_encodings)):
            self.reset()
            self.builder.initialize_soup(self)
            try:
                self._feed()
                success = True
                break
            except ParserRejectedMarkup as e:
                rejections.append(e)
                pass

        if not success:
            other_exceptions = [str(e) for e in rejections]
            raise ParserRejectedMarkup(
                "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
            )

        # Clear out the markup and remove the builder's circular
        # reference to this object.
        self.markup = None
        self.builder.soup = None

    def _clone(self):
        """Create a new BeautifulSoup object with the same TreeBuilder,
        but not associated with any markup.

        This is the first step of the deepcopy process.
        """
        clone = type(self)("", None, self.builder)

        # Keep track of the encoding of the original document,
        # since we won't be parsing it again.
        clone.original_encoding = self.original_encoding
        return clone
        
    def __getstate__(self):
        # Frequently a tree builder can't be pickled.
        d = dict(self.__dict__)
        if 'builder' in d and d['builder'] is not None and not self.builder.picklable:
            d['builder'] = type(self.builder)
        # Store the contents as a Unicode string.
        d['contents'] = []
        d['markup'] = self.decode()

        # If _most_recent_element is present, it's a Tag object left
        # over from initial parse. It might not be picklable and we
        # don't need it.
        if '_most_recent_element' in d:
            del d['_most_recent_element']
        return d

    def __setstate__(self, state):
        # If necessary, restore the TreeBuilder by looking it up.
        self.__dict__ = state
        if isinstance(self.builder, type):
            self.builder = self.builder()
        elif not self.builder:
            # We don't know which builder was used to build this
            # parse tree, so use a default we know is always available.
            self.builder = HTMLParserTreeBuilder()
        self.builder.soup = self
        self.reset()
        self._feed()
        return state

    
    @classmethod
    def _decode_markup(cls, markup):
        """Ensure `markup` is bytes so it's safe to send into warnings.warn.

        TODO: warnings.warn had this problem back in 2010 but it might not
        anymore.
        """
        if isinstance(markup, bytes):
            decoded = markup.decode('utf-8', 'replace')
        else:
            decoded = markup
        return decoded

    @classmethod
    def _markup_is_url(cls, markup):
        """Error-handling method to raise a warning if incoming markup looks
        like a URL.

        :param markup: A string.
        :return: Whether or not the markup resembles a URL
            closely enough to justify a warning.
        """
        if isinstance(markup, bytes):
            space = b' '
            cant_start_with = (b"http:", b"https:")
        elif isinstance(markup, str):
            space = ' '
            cant_start_with = ("http:", "https:")
        else:
            return False

        if any(markup.startswith(prefix) for prefix in cant_start_with):
            if not space in markup:
                warnings.warn(
                    'The input looks more like a URL than markup. You may want to use'
                    ' an HTTP client like requests to get the document behind'
                    ' the URL, and feed that document to Beautiful Soup.',
                    MarkupResemblesLocatorWarning,
                    stacklevel=3
                )
                return True
        return False

    @classmethod
    def _markup_resembles_filename(cls, markup):
        """Error-handling method to raise a warning if incoming markup
        resembles a filename.

        :param markup: A bytestring or string.
        :return: Whether or not the markup resembles a filename
            closely enough to justify a warning.
        """
        path_characters = '/\\'
        extensions = ['.html', '.htm', '.xml', '.xhtml', '.txt']
        if isinstance(markup, bytes):
            path_characters = path_characters.encode("utf8")
            extensions = [x.encode('utf8') for x in extensions]
        filelike = False
        if any(x in markup for x in path_characters):
            filelike = True
        else:
            lower = markup.lower()
            if any(lower.endswith(ext) for ext in extensions):
                filelike = True
        if filelike:
            warnings.warn(
                'The input looks more like a filename than markup. You may'
                ' want to open this file and pass the filehandle into'
                ' Beautiful Soup.',
                MarkupResemblesLocatorWarning, stacklevel=3
            )
            return True
        return False
    
    def _feed(self):
        """Internal method that parses previously set markup, creating a large
        number of Tag and NavigableString objects.
        """
        # Convert the document to Unicode.
        self.builder.reset()

        self.builder.feed(self.markup)
        # Close out any unfinished strings and close all the open tags.
        self.endData()
        while self.currentTag.name != self.ROOT_TAG_NAME:
            self.popTag()

    def reset(self):
        """Reset this object to a state as though it had never parsed any
        markup.
        """
        Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
        self.hidden = 1
        self.builder.reset()
        self.current_data = []
        self.currentTag = None
        self.tagStack = []
        self.open_tag_counter = Counter()
        self.preserve_whitespace_tag_stack = []
        self.string_container_stack = []
        self._most_recent_element = None
        self.pushTag(self)

    def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
                sourceline=None, sourcepos=None, **kwattrs):
        """Create a new Tag associated with this BeautifulSoup object.

        :param name: The name of the new Tag.
        :param namespace: The URI of the new Tag's XML namespace, if any.
        :param prefix: The prefix for the new Tag's XML namespace, if any.
        :param attrs: A dictionary of this Tag's attribute values; can
            be used instead of `kwattrs` for attributes like 'class'
            that are reserved words in Python.
        :param sourceline: The line number where this tag was
            (purportedly) found in its source document.
        :param sourcepos: The character position within `sourceline` where this
            tag was (purportedly) found.
        :param kwattrs: Keyword arguments for the new Tag's attribute values.

        """
        kwattrs.update(attrs)
        return self.element_classes.get(Tag, Tag)(
            None, self.builder, name, namespace, nsprefix, kwattrs,
            sourceline=sourceline, sourcepos=sourcepos
        )

    def string_container(self, base_class=None):
        container = base_class or NavigableString
        
        # There may be a general override of NavigableString.
        container = self.element_classes.get(
            container, container
        )

        # On top of that, we may be inside a tag that needs a special
        # container class.
        if self.string_container_stack and container is NavigableString:
            container = self.builder.string_containers.get(
                self.string_container_stack[-1].name, container
            )
        return container
        
    def new_string(self, s, subclass=None):
        """Create a new NavigableString associated with this BeautifulSoup
        object.
        """
        container = self.string_container(subclass)
        return container(s)

    def insert_before(self, *args):
        """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
        it because there is nothing before or after it in the parse tree.
        """
        raise NotImplementedError("BeautifulSoup objects don't support insert_before().")

    def insert_after(self, *args):
        """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
        it because there is nothing before or after it in the parse tree.
        """
        raise NotImplementedError("BeautifulSoup objects don't support insert_after().")

    def popTag(self):
        """Internal method called by _popToTag when a tag is closed."""
        tag = self.tagStack.pop()
        if tag.name in self.open_tag_counter:
            self.open_tag_counter[tag.name] -= 1
        if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
            self.preserve_whitespace_tag_stack.pop()
        if self.string_container_stack and tag == self.string_container_stack[-1]:
            self.string_container_stack.pop()
        #print("Pop", tag.name)
        if self.tagStack:
            self.currentTag = self.tagStack[-1]
        return self.currentTag

    def pushTag(self, tag):
        """Internal method called by handle_starttag when a tag is opened."""
        #print("Push", tag.name)
        if self.currentTag is not None:
            self.currentTag.contents.append(tag)
        self.tagStack.append(tag)
        self.currentTag = self.tagStack[-1]
        if tag.name != self.ROOT_TAG_NAME:
            self.open_tag_counter[tag.name] += 1
        if tag.name in self.builder.preserve_whitespace_tags:
            self.preserve_whitespace_tag_stack.append(tag)
        if tag.name in self.builder.string_containers:
            self.string_container_stack.append(tag)

    def endData(self, containerClass=None):
        """Method called by the TreeBuilder when the end of a data segment
        occurs.
        """       
        if self.current_data:
            current_data = ''.join(self.current_data)
            # If whitespace is not preserved, and this string contains
            # nothing but ASCII spaces, replace it with a single space
            # or newline.
            if not self.preserve_whitespace_tag_stack:
                strippable = True
                for i in current_data:
                    if i not in self.ASCII_SPACES:
                        strippable = False
                        break
                if strippable:
                    if '\n' in current_data:
                        current_data = '\n'
                    else:
                        current_data = ' '

            # Reset the data collector.
            self.current_data = []

            # Should we add this string to the tree at all?
            if self.parse_only and len(self.tagStack) <= 1 and \
                   (not self.parse_only.text or \
                    not self.parse_only.search(current_data)):
                return

            containerClass = self.string_container(containerClass)
            o = containerClass(current_data)
            self.object_was_parsed(o)

    def object_was_parsed(self, o, parent=None, most_recent_element=None):
        """Method called by the TreeBuilder to integrate an object into the parse tree."""
        if parent is None:
            parent = self.currentTag
        if most_recent_element is not None:
            previous_element = most_recent_element
        else:
            previous_element = self._most_recent_element

        next_element = previous_sibling = next_sibling = None
        if isinstance(o, Tag):
            next_element = o.next_element
            next_sibling = o.next_sibling
            previous_sibling = o.previous_sibling
            if previous_element is None:
                previous_element = o.previous_element

        fix = parent.next_element is not None

        o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)

        self._most_recent_element = o
        parent.contents.append(o)

        # Check if we are inserting into an already parsed node.
        if fix:
            self._linkage_fixer(parent)

    def _linkage_fixer(self, el):
        """Make sure linkage of this fragment is sound."""

        first = el.contents[0]
        child = el.contents[-1]
        descendant = child

        if child is first and el.parent is not None:
            # Parent should be linked to first child
            el.next_element = child
            # We are no longer linked to whatever this element is
            prev_el = child.previous_element
            if prev_el is not None and prev_el is not el:
                prev_el.next_element = None
            # First child should be linked to the parent, and no previous siblings.
            child.previous_element = el
            child.previous_sibling = None

        # We have no sibling as we've been appended as the last.
        child.next_sibling = None

        # This index is a tag, dig deeper for a "last descendant"
        if isinstance(child, Tag) and child.contents:
            descendant = child._last_descendant(False)

        # As the final step, link last descendant. It should be linked
        # to the parent's next sibling (if found), else walk up the chain
        # and find a parent with a sibling. It should have no next sibling.
        descendant.next_element = None
        descendant.next_sibling = None
        target = el
        while True:
            if target is None:
                break
            elif target.next_sibling is not None:
                descendant.next_element = target.next_sibling
                target.next_sibling.previous_element = child
                break
            target = target.parent

    def _popToTag(self, name, nsprefix=None, inclusivePop=True):
        """Pops the tag stack up to and including the most recent
        instance of the given tag.

        If there are no open tags with the given name, nothing will be
        popped.

        :param name: Pop up to the most recent tag with this name.
        :param nsprefix: The namespace prefix that goes with `name`.
        :param inclusivePop: It this is false, pops the tag stack up
          to but *not* including the most recent instqance of the
          given tag.

        """
        #print("Popping to %s" % name)
        if name == self.ROOT_TAG_NAME:
            # The BeautifulSoup object itself can never be popped.
            return

        most_recently_popped = None

        stack_size = len(self.tagStack)
        for i in range(stack_size - 1, 0, -1):
            if not self.open_tag_counter.get(name):
                break
            t = self.tagStack[i]
            if (name == t.name and nsprefix == t.prefix):
                if inclusivePop:
                    most_recently_popped = self.popTag()
                break
            most_recently_popped = self.popTag()

        return most_recently_popped

    def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
                        sourcepos=None, namespaces=None):
        """Called by the tree builder when a new tag is encountered.

        :param name: Name of the tag.
        :param nsprefix: Namespace prefix for the tag.
        :param attrs: A dictionary of attribute values.
        :param sourceline: The line number where this tag was found in its
            source document.
        :param sourcepos: The character position within `sourceline` where this
            tag was found.
        :param namespaces: A dictionary of all namespace prefix mappings 
            currently in scope in the document.

        If this method returns None, the tag was rejected by an active
        SoupStrainer. You should proceed as if the tag had not occurred
        in the document. For instance, if this was a self-closing tag,
        don't call handle_endtag.
        """
        # print("Start tag %s: %s" % (name, attrs))
        self.endData()

        if (self.parse_only and len(self.tagStack) <= 1
            and (self.parse_only.text
                 or not self.parse_only.search_tag(name, attrs))):
            return None

        tag = self.element_classes.get(Tag, Tag)(
            self, self.builder, name, namespace, nsprefix, attrs,
            self.currentTag, self._most_recent_element,
            sourceline=sourceline, sourcepos=sourcepos,
            namespaces=namespaces
        )
        if tag is None:
            return tag
        if self._most_recent_element is not None:
            self._most_recent_element.next_element = tag
        self._most_recent_element = tag
        self.pushTag(tag)
        return tag

    def handle_endtag(self, name, nsprefix=None):
        """Called by the tree builder when an ending tag is encountered.

        :param name: Name of the tag.
        :param nsprefix: Namespace prefix for the tag.
        """
        #print("End tag: " + name)
        self.endData()
        self._popToTag(name, nsprefix)
        
    def handle_data(self, data):
        """Called by the tree builder when a chunk of textual data is encountered."""
        self.current_data.append(data)
       
    def decode(self, pretty_print=False,
               eventual_encoding=DEFAULT_OUTPUT_ENCODING,
               formatter="minimal", iterator=None):
        """Returns a string or Unicode representation of the parse tree
            as an HTML or XML document.

        :param pretty_print: If this is True, indentation will be used to
            make the document more readable.
        :param eventual_encoding: The encoding of the final document.
            If this is None, the document will be a Unicode string.
        """
        if self.is_xml:
            # Print the XML declaration
            encoding_part = ''
            if eventual_encoding in PYTHON_SPECIFIC_ENCODINGS:
                # This is a special Python encoding; it can't actually
                # go into an XML document because it means nothing
                # outside of Python.
                eventual_encoding = None
            if eventual_encoding != None:
                encoding_part = ' encoding="%s"' % eventual_encoding
            prefix = '<?xml version="1.0"%s?>\n' % encoding_part
        else:
            prefix = ''
        if not pretty_print:
            indent_level = None
        else:
            indent_level = 0
        return prefix + super(BeautifulSoup, self).decode(
            indent_level, eventual_encoding, formatter, iterator)

# Aliases to make it easier to get started quickly, e.g. 'from bs4 import _soup'
_s = BeautifulSoup
_soup = BeautifulSoup

class BeautifulStoneSoup(BeautifulSoup):
    """Deprecated interface to an XML parser."""

    def __init__(self, *args, **kwargs):
        kwargs['features'] = 'xml'
        warnings.warn(
            'The BeautifulStoneSoup class is deprecated. Instead of using '
            'it, pass features="xml" into the BeautifulSoup constructor.',
            DeprecationWarning, stacklevel=2
        )
        super(BeautifulStoneSoup, self).__init__(*args, **kwargs)


class StopParsing(Exception):
    """Exception raised by a TreeBuilder if it's unable to continue parsing."""
    pass

class FeatureNotFound(ValueError):
    """Exception raised by the BeautifulSoup constructor if no parser with the
    requested features is found.
    """
    pass


#If this file is run as a script, act as an HTML pretty-printer.
if __name__ == '__main__':
    import sys
    soup = BeautifulSoup(sys.stdin)
    print((soup.prettify()))

env_pyrecodes/lib/python3.9/site-packages/bs4/__init__.py
NikolaBlagojevic/pyrecodes
# Use of this source code is governed by the MIT license.
__license__ = "MIT"

from collections import defaultdict
import itertools
import re
import warnings
import sys
from bs4.element import (
    CharsetMetaAttributeValue,
    ContentMetaAttributeValue,
    RubyParenthesisString,
    RubyTextString,
    Stylesheet,
    Script,
    TemplateString,
    nonwhitespace_re
)

__all__ = [
    'HTMLTreeBuilder',
    'SAXTreeBuilder',
    'TreeBuilder',
    'TreeBuilderRegistry',
    ]

# Some useful features for a TreeBuilder to have.
FAST = 'fast'
PERMISSIVE = 'permissive'
STRICT = 'strict'
XML = 'xml'
HTML = 'html'
HTML_5 = 'html5'

class XMLParsedAsHTMLWarning(UserWarning):
    """The warning issued when an HTML parser is used to parse
    XML that is not XHTML.
    """
    MESSAGE = """It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor."""


class TreeBuilderRegistry(object):
    """A way of looking up TreeBuilder subclasses by their name or by desired
    features.
    """
    
    def __init__(self):
        self.builders_for_feature = defaultdict(list)
        self.builders = []

    def register(self, treebuilder_class):
        """Register a treebuilder based on its advertised features.

        :param treebuilder_class: A subclass of Treebuilder. its .features
           attribute should list its features.
        """
        for feature in treebuilder_class.features:
            self.builders_for_feature[feature].insert(0, treebuilder_class)
        self.builders.insert(0, treebuilder_class)

    def lookup(self, *features):
        """Look up a TreeBuilder subclass with the desired features.

        :param features: A list of features to look for. If none are
            provided, the most recently registered TreeBuilder subclass
            will be used.
        :return: A TreeBuilder subclass, or None if there's no
            registered subclass with all the requested features.
        """
        if len(self.builders) == 0:
            # There are no builders at all.
            return None

        if len(features) == 0:
            # They didn't ask for any features. Give them the most
            # recently registered builder.
            return self.builders[0]

        # Go down the list of features in order, and eliminate any builders
        # that don't match every feature.
        features = list(features)
        features.reverse()
        candidates = None
        candidate_set = None
        while len(features) > 0:
            feature = features.pop()
            we_have_the_feature = self.builders_for_feature.get(feature, [])
            if len(we_have_the_feature) > 0:
                if candidates is None:
                    candidates = we_have_the_feature
                    candidate_set = set(candidates)
                else:
                    # Eliminate any candidates that don't have this feature.
                    candidate_set = candidate_set.intersection(
                        set(we_have_the_feature))

        # The only valid candidates are the ones in candidate_set.
        # Go through the original list of candidates and pick the first one
        # that's in candidate_set.
        if candidate_set is None:
            return None
        for candidate in candidates:
            if candidate in candidate_set:
                return candidate
        return None

# The BeautifulSoup class will take feature lists from developers and use them
# to look up builders in this registry.
builder_registry = TreeBuilderRegistry()

class TreeBuilder(object):
    """Turn a textual document into a Beautiful Soup object tree."""

    NAME = "[Unknown tree builder]"
    ALTERNATE_NAMES = []
    features = []

    is_xml = False
    picklable = False
    empty_element_tags = None # A tag will be considered an empty-element
                              # tag when and only when it has no contents.
    
    # A value for these tag/attribute combinations is a space- or
    # comma-separated list of CDATA, rather than a single CDATA.
    DEFAULT_CDATA_LIST_ATTRIBUTES = defaultdict(list)

    # Whitespace should be preserved inside these tags.
    DEFAULT_PRESERVE_WHITESPACE_TAGS = set()

    # The textual contents of tags with these names should be
    # instantiated with some class other than NavigableString.
    DEFAULT_STRING_CONTAINERS = {}
    
    USE_DEFAULT = object()

    # Most parsers don't keep track of line numbers.
    TRACKS_LINE_NUMBERS = False
    
    def __init__(self, multi_valued_attributes=USE_DEFAULT,
                 preserve_whitespace_tags=USE_DEFAULT,
                 store_line_numbers=USE_DEFAULT,
                 string_containers=USE_DEFAULT,
    ):
        """Constructor.

        :param multi_valued_attributes: If this is set to None, the
         TreeBuilder will not turn any values for attributes like
         'class' into lists. Setting this to a dictionary will
         customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
         for an example.

         Internally, these are called "CDATA list attributes", but that
         probably doesn't make sense to an end-user, so the argument name
         is `multi_valued_attributes`.

        :param preserve_whitespace_tags: A list of tags to treat
         the way <pre> tags are treated in HTML. Tags in this list
         are immune from pretty-printing; their contents will always be
         output as-is.

        :param string_containers: A dictionary mapping tag names to
        the classes that should be instantiated to contain the textual
        contents of those tags. The default is to use NavigableString
        for every tag, no matter what the name. You can override the
        default by changing DEFAULT_STRING_CONTAINERS.

        :param store_line_numbers: If the parser keeps track of the
         line numbers and positions of the original markup, that
         information will, by default, be stored in each corresponding
         `Tag` object. You can turn this off by passing
         store_line_numbers=False. If the parser you're using doesn't 
         keep track of this information, then setting store_line_numbers=True
         will do nothing.
        """
        self.soup = None
        if multi_valued_attributes is self.USE_DEFAULT:
            multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
        self.cdata_list_attributes = multi_valued_attributes
        if preserve_whitespace_tags is self.USE_DEFAULT:
            preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
        self.preserve_whitespace_tags = preserve_whitespace_tags
        if store_line_numbers == self.USE_DEFAULT:
            store_line_numbers = self.TRACKS_LINE_NUMBERS
        self.store_line_numbers = store_line_numbers 
        if string_containers == self.USE_DEFAULT:
            string_containers = self.DEFAULT_STRING_CONTAINERS
        self.string_containers = string_containers
        
    def initialize_soup(self, soup):
        """The BeautifulSoup object has been initialized and is now
        being associated with the TreeBuilder.

        :param soup: A BeautifulSoup object.
        """
        self.soup = soup
        
    def reset(self):
        """Do any work necessary to reset the underlying parser
        for a new document.

        By default, this does nothing.
        """
        pass

    def can_be_empty_element(self, tag_name):
        """Might a tag with this name be an empty-element tag?

        The final markup may or may not actually present this tag as
        self-closing.

        For instance: an HTMLBuilder does not consider a <p> tag to be
        an empty-element tag (it's not in
        HTMLBuilder.empty_element_tags). This means an empty <p> tag
        will be presented as "<p></p>", not "<p/>" or "<p>".

        The default implementation has no opinion about which tags are
        empty-element tags, so a tag will be presented as an
        empty-element tag if and only if it has no children.
        "<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will
        be left alone.

        :param tag_name: The name of a markup tag.
        """
        if self.empty_element_tags is None:
            return True
        return tag_name in self.empty_element_tags
    
    def feed(self, markup):
        """Run some incoming markup through some parsing process,
        populating the `BeautifulSoup` object in self.soup.

        This method is not implemented in TreeBuilder; it must be
        implemented in subclasses.

        :return: None.
        """
        raise NotImplementedError()

    def prepare_markup(self, markup, user_specified_encoding=None,
                       document_declared_encoding=None, exclude_encodings=None):
        """Run any preliminary steps necessary to make incoming markup
        acceptable to the parser.

        :param markup: Some markup -- probably a bytestring.
        :param user_specified_encoding: The user asked to try this encoding.
        :param document_declared_encoding: The markup itself claims to be
            in this encoding. NOTE: This argument is not used by the
            calling code and can probably be removed.
        :param exclude_encodings: The user asked _not_ to try any of
            these encodings.

        :yield: A series of 4-tuples:
         (markup, encoding, declared encoding,
          has undergone character replacement)

         Each 4-tuple represents a strategy for converting the
         document to Unicode and parsing it. Each strategy will be tried 
         in turn.

         By default, the only strategy is to parse the markup
         as-is. See `LXMLTreeBuilderForXML` and
         `HTMLParserTreeBuilder` for implementations that take into
         account the quirks of particular parsers.
        """
        yield markup, None, None, False

    def test_fragment_to_document(self, fragment):
        """Wrap an HTML fragment to make it look like a document.

        Different parsers do this differently. For instance, lxml
        introduces an empty <head> tag, and html5lib
        doesn't. Abstracting this away lets us write simple tests
        which run HTML fragments through the parser and compare the
        results against other HTML fragments.

        This method should not be used outside of tests.

        :param fragment: A string -- fragment of HTML.
        :return: A string -- a full HTML document.
        """
        return fragment

    def set_up_substitutions(self, tag):
        """Set up any substitutions that will need to be performed on 
        a `Tag` when it's output as a string.

        By default, this does nothing. See `HTMLTreeBuilder` for a
        case where this is used.

        :param tag: A `Tag`
        :return: Whether or not a substitution was performed.
        """
        return False

    def _replace_cdata_list_attribute_values(self, tag_name, attrs):
        """When an attribute value is associated with a tag that can
        have multiple values for that attribute, convert the string
        value to a list of strings.

        Basically, replaces class="foo bar" with class=["foo", "bar"]

        NOTE: This method modifies its input in place.

        :param tag_name: The name of a tag.
        :param attrs: A dictionary containing the tag's attributes.
           Any appropriate attribute values will be modified in place.
        """
        if not attrs:
            return attrs
        if self.cdata_list_attributes:
            universal = self.cdata_list_attributes.get('*', [])
            tag_specific = self.cdata_list_attributes.get(
                tag_name.lower(), None)
            for attr in list(attrs.keys()):
                if attr in universal or (tag_specific and attr in tag_specific):
                    # We have a "class"-type attribute whose string
                    # value is a whitespace-separated list of
                    # values. Split it into a list.
                    value = attrs[attr]
                    if isinstance(value, str):
                        values = nonwhitespace_re.findall(value)
                    else:
                        # html5lib sometimes calls setAttributes twice
                        # for the same tag when rearranging the parse
                        # tree. On the second call the attribute value
                        # here is already a list.  If this happens,
                        # leave the value alone rather than trying to
                        # split it again.
                        values = value
                    attrs[attr] = values
        return attrs
    
class SAXTreeBuilder(TreeBuilder):
    """A Beautiful Soup treebuilder that listens for SAX events.

    This is not currently used for anything, but it demonstrates
    how a simple TreeBuilder would work.
    """

    def feed(self, markup):
        raise NotImplementedError()

    def close(self):
        pass

    def startElement(self, name, attrs):
        attrs = dict((key[1], value) for key, value in list(attrs.items()))
        #print("Start %s, %r" % (name, attrs))
        self.soup.handle_starttag(name, attrs)

    def endElement(self, name):
        #print("End %s" % name)
        self.soup.handle_endtag(name)

    def startElementNS(self, nsTuple, nodeName, attrs):
        # Throw away (ns, nodeName) for now.
        self.startElement(nodeName, attrs)

    def endElementNS(self, nsTuple, nodeName):
        # Throw away (ns, nodeName) for now.
        self.endElement(nodeName)
        #handler.endElementNS((ns, node.nodeName), node.nodeName)

    def startPrefixMapping(self, prefix, nodeValue):
        # Ignore the prefix for now.
        pass

    def endPrefixMapping(self, prefix):
        # Ignore the prefix for now.
        # handler.endPrefixMapping(prefix)
        pass

    def characters(self, content):
        self.soup.handle_data(content)

    def startDocument(self):
        pass

    def endDocument(self):
        pass


class HTMLTreeBuilder(TreeBuilder):
    """This TreeBuilder knows facts about HTML.

    Such as which tags are empty-element tags.
    """

    empty_element_tags = set([
        # These are from HTML5.
        'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
        
        # These are from earlier versions of HTML and are removed in HTML5.
        'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
    ])

    # The HTML standard defines these as block-level elements. Beautiful
    # Soup does not treat these elements differently from other elements,
    # but it may do so eventually, and this information is available if
    # you need to use it.
    block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])

    # These HTML tags need special treatment so they can be
    # represented by a string class other than NavigableString.
    #
    # For some of these tags, it's because the HTML standard defines
    # an unusual content model for them. I made this list by going
    # through the HTML spec
    # (https://html.spec.whatwg.org/#metadata-content) and looking for
    # "metadata content" elements that can contain strings.
    #
    # The Ruby tags (<rt> and <rp>) are here despite being normal
    # "phrasing content" tags, because the content they contain is
    # qualitatively different from other text in the document, and it
    # can be useful to be able to distinguish it.
    #
    # TODO: Arguably <noscript> could go here but it seems
    # qualitatively different from the other tags.
    DEFAULT_STRING_CONTAINERS = {
        'rt' : RubyTextString,
        'rp' : RubyParenthesisString,
        'style': Stylesheet,
        'script': Script,
        'template': TemplateString,
    }    
    
    # The HTML standard defines these attributes as containing a
    # space-separated list of values, not a single value. That is,
    # class="foo bar" means that the 'class' attribute has two values,
    # 'foo' and 'bar', not the single value 'foo bar'.  When we
    # encounter one of these attributes, we will parse its value into
    # a list of values if possible. Upon output, the list will be
    # converted back into a string.
    DEFAULT_CDATA_LIST_ATTRIBUTES = {
        "*" : ['class', 'accesskey', 'dropzone'],
        "a" : ['rel', 'rev'],
        "link" :  ['rel', 'rev'],
        "td" : ["headers"],
        "th" : ["headers"],
        "td" : ["headers"],
        "form" : ["accept-charset"],
        "object" : ["archive"],

        # These are HTML5 specific, as are *.accesskey and *.dropzone above.
        "area" : ["rel"],
        "icon" : ["sizes"],
        "iframe" : ["sandbox"],
        "output" : ["for"],
        }

    DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])

    def set_up_substitutions(self, tag):
        """Replace the declared encoding in a <meta> tag with a placeholder,
        to be substituted when the tag is output to a string.

        An HTML document may come in to Beautiful Soup as one
        encoding, but exit in a different encoding, and the <meta> tag
        needs to be changed to reflect this.

        :param tag: A `Tag`
        :return: Whether or not a substitution was performed.
        """
        # We are only interested in <meta> tags
        if tag.name != 'meta':
            return False

        http_equiv = tag.get('http-equiv')
        content = tag.get('content')
        charset = tag.get('charset')

        # We are interested in <meta> tags that say what encoding the
        # document was originally in. This means HTML 5-style <meta>
        # tags that provide the "charset" attribute. It also means
        # HTML 4-style <meta> tags that provide the "content"
        # attribute and have "http-equiv" set to "content-type".
        #
        # In both cases we will replace the value of the appropriate
        # attribute with a standin object that can take on any
        # encoding.
        meta_encoding = None
        if charset is not None:
            # HTML 5 style:
            # <meta charset="utf8">
            meta_encoding = charset
            tag['charset'] = CharsetMetaAttributeValue(charset)

        elif (content is not None and http_equiv is not None
              and http_equiv.lower() == 'content-type'):
            # HTML 4 style:
            # <meta http-equiv="content-type" content="text/html; charset=utf8">
            tag['content'] = ContentMetaAttributeValue(content)

        return (meta_encoding is not None)

class DetectsXMLParsedAsHTML(object):
    """A mixin class for any class (a TreeBuilder, or some class used by a
    TreeBuilder) that's in a position to detect whether an XML
    document is being incorrectly parsed as HTML, and issue an
    appropriate warning.

    This requires being able to observe an incoming processing
    instruction that might be an XML declaration, and also able to
    observe tags as they're opened. If you can't do that for a given
    TreeBuilder, there's a less reliable implementation based on
    examining the raw markup.
    """

    # Regular expression for seeing if markup has an <html> tag.
    LOOKS_LIKE_HTML = re.compile("<[^ +]html", re.I)
    LOOKS_LIKE_HTML_B = re.compile(b"<[^ +]html", re.I)

    XML_PREFIX = '<?xml'
    XML_PREFIX_B = b'<?xml'
    
    @classmethod
    def warn_if_markup_looks_like_xml(cls, markup):
        """Perform a check on some markup to see if it looks like XML
        that's not XHTML. If so, issue a warning.

        This is much less reliable than doing the check while parsing,
        but some of the tree builders can't do that.

        :return: True if the markup looks like non-XHTML XML, False
        otherwise.
        """
        if isinstance(markup, bytes):
            prefix = cls.XML_PREFIX_B
            looks_like_html = cls.LOOKS_LIKE_HTML_B
        else:
            prefix = cls.XML_PREFIX
            looks_like_html = cls.LOOKS_LIKE_HTML
        
        if (markup is not None
            and markup.startswith(prefix)
            and not looks_like_html.search(markup[:500])
        ):
            cls._warn()
            return True
        return False

    @classmethod
    def _warn(cls):
        """Issue a warning about XML being parsed as HTML."""
        warnings.warn(
            XMLParsedAsHTMLWarning.MESSAGE, XMLParsedAsHTMLWarning
        )
        
    def _initialize_xml_detector(self):
        """Call this method before parsing a document."""
        self._first_processing_instruction = None
        self._root_tag = None
       
    def _document_might_be_xml(self, processing_instruction):
        """Call this method when encountering an XML declaration, or a
        "processing instruction" that might be an XML declaration.
        """
        if (self._first_processing_instruction is not None
            or self._root_tag is not None):
            # The document has already started. Don't bother checking
            # anymore.
            return

        self._first_processing_instruction = processing_instruction

        # We won't know until we encounter the first tag whether or
        # not this is actually a problem.
        
    def _root_tag_encountered(self, name):
        """Call this when you encounter the document's root tag.

        This is where we actually check whether an XML document is
        being incorrectly parsed as HTML, and issue the warning.
        """
        if self._root_tag is not None:
            # This method was incorrectly called multiple times. Do
            # nothing.
            return

        self._root_tag = name
        if (name != 'html' and self._first_processing_instruction is not None
            and self._first_processing_instruction.lower().startswith('xml ')):
            # We encountered an XML declaration and then a tag other
            # than 'html'. This is a reliable indicator that a
            # non-XHTML document is being parsed as XML.
            self._warn()

    
def register_treebuilders_from(module):
    """Copy TreeBuilders from the given module into this module."""
    this_module = sys.modules[__name__]
    for name in module.__all__:
        obj = getattr(module, name)

        if issubclass(obj, TreeBuilder):
            setattr(this_module, name, obj)
            this_module.__all__.append(name)
            # Register the builder while we're at it.
            this_module.builder_registry.register(obj)

class ParserRejectedMarkup(Exception):
    """An Exception to be raised when the underlying parser simply
    refuses to parse the given markup.
    """
    def __init__(self, message_or_exception):
        """Explain why the parser rejected the given markup, either
        with a textual explanation or another exception.
        """
        if isinstance(message_or_exception, Exception):
            e = message_or_exception
            message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
        super(ParserRejectedMarkup, self).__init__(message_or_exception)
            
# Builders are registered in reverse order of priority, so that custom
# builder registrations will take precedence. In general, we want lxml
# to take precedence over html5lib, because it's faster. And we only
# want to use HTMLParser as a last resort.
from . import _htmlparser
register_treebuilders_from(_htmlparser)
try:
    from . import _html5lib
    register_treebuilders_from(_html5lib)
except ImportError:
    # They don't have html5lib installed.
    pass
try:
    from . import _lxml
    register_treebuilders_from(_lxml)
except ImportError:
    # They don't have lxml installed.
    pass

env_pyrecodes/lib/python3.9/site-packages/bs4/builder/__init__.py
NikolaBlagojevic/pyrecodes
# Use of this source code is governed by the MIT license.
__license__ = "MIT"

__all__ = [
    'HTML5TreeBuilder',
    ]

import warnings
import re
from bs4.builder import (
    DetectsXMLParsedAsHTML,
    PERMISSIVE,
    HTML,
    HTML_5,
    HTMLTreeBuilder,
    )
from bs4.element import (
    NamespacedAttribute,
    nonwhitespace_re,
)
import html5lib
from html5lib.constants import (
    namespaces,
    prefixes,
    )
from bs4.element import (
    Comment,
    Doctype,
    NavigableString,
    Tag,
    )

try:
    # Pre-0.99999999
    from html5lib.treebuilders import _base as treebuilder_base
    new_html5lib = False
except ImportError as e:
    # 0.99999999 and up
    from html5lib.treebuilders import base as treebuilder_base
    new_html5lib = True

class HTML5TreeBuilder(HTMLTreeBuilder):
    """Use html5lib to build a tree.

    Note that this TreeBuilder does not support some features common
    to HTML TreeBuilders. Some of these features could theoretically
    be implemented, but at the very least it's quite difficult,
    because html5lib moves the parse tree around as it's being built.

    * This TreeBuilder doesn't use different subclasses of NavigableString
      based on the name of the tag in which the string was found.

    * You can't use a SoupStrainer to parse only part of a document.
    """

    NAME = "html5lib"

    features = [NAME, PERMISSIVE, HTML_5, HTML]

    # html5lib can tell us which line number and position in the
    # original file is the source of an element.
    TRACKS_LINE_NUMBERS = True
    
    def prepare_markup(self, markup, user_specified_encoding,
                       document_declared_encoding=None, exclude_encodings=None):
        # Store the user-specified encoding for use later on.
        self.user_specified_encoding = user_specified_encoding

        # document_declared_encoding and exclude_encodings aren't used
        # ATM because the html5lib TreeBuilder doesn't use
        # UnicodeDammit.
        if exclude_encodings:
            warnings.warn(
                "You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.",
                stacklevel=3
            )

        # html5lib only parses HTML, so if it's given XML that's worth
        # noting.
        DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)

        yield (markup, None, None, False)

    # These methods are defined by Beautiful Soup.
    def feed(self, markup):
        if self.soup.parse_only is not None:
            warnings.warn(
                "You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.",
                stacklevel=4
            )
        parser = html5lib.HTMLParser(tree=self.create_treebuilder)
        self.underlying_builder.parser = parser
        extra_kwargs = dict()
        if not isinstance(markup, str):
            if new_html5lib:
                extra_kwargs['override_encoding'] = self.user_specified_encoding
            else:
                extra_kwargs['encoding'] = self.user_specified_encoding
        doc = parser.parse(markup, **extra_kwargs)
        
        # Set the character encoding detected by the tokenizer.
        if isinstance(markup, str):
            # We need to special-case this because html5lib sets
            # charEncoding to UTF-8 if it gets Unicode input.
            doc.original_encoding = None
        else:
            original_encoding = parser.tokenizer.stream.charEncoding[0]
            if not isinstance(original_encoding, str):
                # In 0.99999999 and up, the encoding is an html5lib
                # Encoding object. We want to use a string for compatibility
                # with other tree builders.
                original_encoding = original_encoding.name
            doc.original_encoding = original_encoding
        self.underlying_builder.parser = None
            
    def create_treebuilder(self, namespaceHTMLElements):
        self.underlying_builder = TreeBuilderForHtml5lib(
            namespaceHTMLElements, self.soup,
            store_line_numbers=self.store_line_numbers
        )
        return self.underlying_builder

    def test_fragment_to_document(self, fragment):
        """See `TreeBuilder`."""
        return '<html><head></head><body>%s</body></html>' % fragment


class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
    
    def __init__(self, namespaceHTMLElements, soup=None,
                 store_line_numbers=True, **kwargs):
        if soup:
            self.soup = soup
        else:
            from bs4 import BeautifulSoup
            # TODO: Why is the parser 'html.parser' here? To avoid an
            # infinite loop?
            self.soup = BeautifulSoup(
                "", "html.parser", store_line_numbers=store_line_numbers,
                **kwargs
            )
        # TODO: What are **kwargs exactly? Should they be passed in
        # here in addition to/instead of being passed to the BeautifulSoup
        # constructor?
        super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)

        # This will be set later to an html5lib.html5parser.HTMLParser
        # object, which we can use to track the current line number.
        self.parser = None
        self.store_line_numbers = store_line_numbers
        
    def documentClass(self):
        self.soup.reset()
        return Element(self.soup, self.soup, None)

    def insertDoctype(self, token):
        name = token["name"]
        publicId = token["publicId"]
        systemId = token["systemId"]

        doctype = Doctype.for_name_and_ids(name, publicId, systemId)
        self.soup.object_was_parsed(doctype)

    def elementClass(self, name, namespace):
        kwargs = {}
        if self.parser and self.store_line_numbers:
            # This represents the point immediately after the end of the
            # tag. We don't know when the tag started, but we do know
            # where it ended -- the character just before this one.
            sourceline, sourcepos = self.parser.tokenizer.stream.position()
            kwargs['sourceline'] = sourceline
            kwargs['sourcepos'] = sourcepos-1
        tag = self.soup.new_tag(name, namespace, **kwargs)

        return Element(tag, self.soup, namespace)

    def commentClass(self, data):
        return TextNode(Comment(data), self.soup)

    def fragmentClass(self):
        from bs4 import BeautifulSoup
        # TODO: Why is the parser 'html.parser' here? To avoid an
        # infinite loop?
        self.soup = BeautifulSoup("", "html.parser")
        self.soup.name = "[document_fragment]"
        return Element(self.soup, self.soup, None)

    def appendChild(self, node):
        # XXX This code is not covered by the BS4 tests.
        self.soup.append(node.element)

    def getDocument(self):
        return self.soup

    def getFragment(self):
        return treebuilder_base.TreeBuilder.getFragment(self).element

    def testSerializer(self, element):
        from bs4 import BeautifulSoup
        rv = []
        doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')

        def serializeElement(element, indent=0):
            if isinstance(element, BeautifulSoup):
                pass
            if isinstance(element, Doctype):
                m = doctype_re.match(element)
                if m:
                    name = m.group(1)
                    if m.lastindex > 1:
                        publicId = m.group(2) or ""
                        systemId = m.group(3) or m.group(4) or ""
                        rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
                                  (' ' * indent, name, publicId, systemId))
                    else:
                        rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
                else:
                    rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
            elif isinstance(element, Comment):
                rv.append("|%s<!-- %s -->" % (' ' * indent, element))
            elif isinstance(element, NavigableString):
                rv.append("|%s\"%s\"" % (' ' * indent, element))
            else:
                if element.namespace:
                    name = "%s %s" % (prefixes[element.namespace],
                                      element.name)
                else:
                    name = element.name
                rv.append("|%s<%s>" % (' ' * indent, name))
                if element.attrs:
                    attributes = []
                    for name, value in list(element.attrs.items()):
                        if isinstance(name, NamespacedAttribute):
                            name = "%s %s" % (prefixes[name.namespace], name.name)
                        if isinstance(value, list):
                            value = " ".join(value)
                        attributes.append((name, value))

                    for name, value in sorted(attributes):
                        rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
                indent += 2
                for child in element.children:
                    serializeElement(child, indent)
        serializeElement(element, 0)

        return "\n".join(rv)

class AttrList(object):
    def __init__(self, element):
        self.element = element
        self.attrs = dict(self.element.attrs)
    def __iter__(self):
        return list(self.attrs.items()).__iter__()
    def __setitem__(self, name, value):
        # If this attribute is a multi-valued attribute for this element,
        # turn its value into a list.
        list_attr = self.element.cdata_list_attributes or {}
        if (name in list_attr.get('*', [])
            or (self.element.name in list_attr
                and name in list_attr.get(self.element.name, []))):
            # A node that is being cloned may have already undergone
            # this procedure.
            if not isinstance(value, list):
                value = nonwhitespace_re.findall(value)
        self.element[name] = value
    def items(self):
        return list(self.attrs.items())
    def keys(self):
        return list(self.attrs.keys())
    def __len__(self):
        return len(self.attrs)
    def __getitem__(self, name):
        return self.attrs[name]
    def __contains__(self, name):
        return name in list(self.attrs.keys())


class Element(treebuilder_base.Node):
    def __init__(self, element, soup, namespace):
        treebuilder_base.Node.__init__(self, element.name)
        self.element = element
        self.soup = soup
        self.namespace = namespace

    def appendChild(self, node):
        string_child = child = None
        if isinstance(node, str):
            # Some other piece of code decided to pass in a string
            # instead of creating a TextElement object to contain the
            # string.
            string_child = child = node
        elif isinstance(node, Tag):
            # Some other piece of code decided to pass in a Tag
            # instead of creating an Element object to contain the
            # Tag.
            child = node
        elif node.element.__class__ == NavigableString:
            string_child = child = node.element
            node.parent = self
        else:
            child = node.element
            node.parent = self

        if not isinstance(child, str) and child.parent is not None:
            node.element.extract()

        if (string_child is not None and self.element.contents
            and self.element.contents[-1].__class__ == NavigableString):
            # We are appending a string onto another string.
            # TODO This has O(n^2) performance, for input like
            # "a</a>a</a>a</a>..."
            old_element = self.element.contents[-1]
            new_element = self.soup.new_string(old_element + string_child)
            old_element.replace_with(new_element)
            self.soup._most_recent_element = new_element
        else:
            if isinstance(node, str):
                # Create a brand new NavigableString from this string.
                child = self.soup.new_string(node)

            # Tell Beautiful Soup to act as if it parsed this element
            # immediately after the parent's last descendant. (Or
            # immediately after the parent, if it has no children.)
            if self.element.contents:
                most_recent_element = self.element._last_descendant(False)
            elif self.element.next_element is not None:
                # Something from further ahead in the parse tree is
                # being inserted into this earlier element. This is
                # very annoying because it means an expensive search
                # for the last element in the tree.
                most_recent_element = self.soup._last_descendant()
            else:
                most_recent_element = self.element

            self.soup.object_was_parsed(
                child, parent=self.element,
                most_recent_element=most_recent_element)

    def getAttributes(self):
        if isinstance(self.element, Comment):
            return {}
        return AttrList(self.element)

    def setAttributes(self, attributes):
        if attributes is not None and len(attributes) > 0:
            converted_attributes = []
            for name, value in list(attributes.items()):
                if isinstance(name, tuple):
                    new_name = NamespacedAttribute(*name)
                    del attributes[name]
                    attributes[new_name] = value

            self.soup.builder._replace_cdata_list_attribute_values(
                self.name, attributes)
            for name, value in list(attributes.items()):
                self.element[name] = value

            # The attributes may contain variables that need substitution.
            # Call set_up_substitutions manually.
            #
            # The Tag constructor called this method when the Tag was created,
            # but we just set/changed the attributes, so call it again.
            self.soup.builder.set_up_substitutions(self.element)
    attributes = property(getAttributes, setAttributes)

    def insertText(self, data, insertBefore=None):
        text = TextNode(self.soup.new_string(data), self.soup)
        if insertBefore:
            self.insertBefore(text, insertBefore)
        else:
            self.appendChild(text)

    def insertBefore(self, node, refNode):
        index = self.element.index(refNode.element)
        if (node.element.__class__ == NavigableString and self.element.contents
            and self.element.contents[index-1].__class__ == NavigableString):
            # (See comments in appendChild)
            old_node = self.element.contents[index-1]
            new_str = self.soup.new_string(old_node + node.element)
            old_node.replace_with(new_str)
        else:
            self.element.insert(index, node.element)
            node.parent = self

    def removeChild(self, node):
        node.element.extract()

    def reparentChildren(self, new_parent):
        """Move all of this tag's children into another tag."""
        # print("MOVE", self.element.contents)
        # print("FROM", self.element)
        # print("TO", new_parent.element)

        element = self.element
        new_parent_element = new_parent.element
        # Determine what this tag's next_element will be once all the children
        # are removed.
        final_next_element = element.next_sibling

        new_parents_last_descendant = new_parent_element._last_descendant(False, False)
        if len(new_parent_element.contents) > 0:
            # The new parent already contains children. We will be
            # appending this tag's children to the end.
            new_parents_last_child = new_parent_element.contents[-1]
            new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
        else:
            # The new parent contains no children.
            new_parents_last_child = None
            new_parents_last_descendant_next_element = new_parent_element.next_element

        to_append = element.contents
        if len(to_append) > 0:
            # Set the first child's previous_element and previous_sibling
            # to elements within the new parent
            first_child = to_append[0]
            if new_parents_last_descendant is not None:
                first_child.previous_element = new_parents_last_descendant
            else:
                first_child.previous_element = new_parent_element
            first_child.previous_sibling = new_parents_last_child
            if new_parents_last_descendant is not None:
                new_parents_last_descendant.next_element = first_child
            else:
                new_parent_element.next_element = first_child
            if new_parents_last_child is not None:
                new_parents_last_child.next_sibling = first_child

            # Find the very last element being moved. It is now the
            # parent's last descendant. It has no .next_sibling and
            # its .next_element is whatever the previous last
            # descendant had.
            last_childs_last_descendant = to_append[-1]._last_descendant(False, True)

            last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
            if new_parents_last_descendant_next_element is not None:
                # TODO: This code has no test coverage and I'm not sure
                # how to get html5lib to go through this path, but it's
                # just the other side of the previous line.
                new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
            last_childs_last_descendant.next_sibling = None

        for child in to_append:
            child.parent = new_parent_element
            new_parent_element.contents.append(child)

        # Now that this element has no children, change its .next_element.
        element.contents = []
        element.next_element = final_next_element

        # print("DONE WITH MOVE")
        # print("FROM", self.element)
        # print("TO", new_parent_element)

    def cloneNode(self):
        tag = self.soup.new_tag(self.element.name, self.namespace)
        node = Element(tag, self.soup, self.namespace)
        for key,value in self.attributes:
            node.attributes[key] = value
        return node

    def hasContent(self):
        return self.element.contents

    def getNameTuple(self):
        if self.namespace == None:
            return namespaces["html"], self.name
        else:
            return self.namespace, self.name

    nameTuple = property(getNameTuple)

class TextNode(Element):
    def __init__(self, element, soup):
        treebuilder_base.Node.__init__(self, None)
        self.element = element
        self.soup = soup

    def cloneNode(self):
        raise NotImplementedError

env_pyrecodes/lib/python3.9/site-packages/bs4/builder/_html5lib.py
NikolaBlagojevic/pyrecodes
# encoding: utf-8
"""Use the HTMLParser library to parse HTML files that aren't too bad."""

# Use of this source code is governed by the MIT license.
__license__ = "MIT"

__all__ = [
    'HTMLParserTreeBuilder',
    ]

from html.parser import HTMLParser

import sys
import warnings

from bs4.element import (
    CData,
    Comment,
    Declaration,
    Doctype,
    ProcessingInstruction,
    )
from bs4.dammit import EntitySubstitution, UnicodeDammit

from bs4.builder import (
    DetectsXMLParsedAsHTML,
    ParserRejectedMarkup,
    HTML,
    HTMLTreeBuilder,
    STRICT,
    )


HTMLPARSER = 'html.parser'

class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
    """A subclass of the Python standard library's HTMLParser class, which
    listens for HTMLParser events and translates them into calls
    to Beautiful Soup's tree construction API.
    """

    # Strategies for handling duplicate attributes
    IGNORE = 'ignore'
    REPLACE = 'replace'
    
    def __init__(self, *args, **kwargs):
        """Constructor.

        :param on_duplicate_attribute: A strategy for what to do if a
            tag includes the same attribute more than once. Accepted
            values are: REPLACE (replace earlier values with later
            ones, the default), IGNORE (keep the earliest value
            encountered), or a callable. A callable must take three
            arguments: the dictionary of attributes already processed,
            the name of the duplicate attribute, and the most recent value
            encountered.           
        """
        self.on_duplicate_attribute = kwargs.pop(
            'on_duplicate_attribute', self.REPLACE
        )
        HTMLParser.__init__(self, *args, **kwargs)

        # Keep a list of empty-element tags that were encountered
        # without an explicit closing tag. If we encounter a closing tag
        # of this type, we'll associate it with one of those entries.
        #
        # This isn't a stack because we don't care about the
        # order. It's a list of closing tags we've already handled and
        # will ignore, assuming they ever show up.
        self.already_closed_empty_element = []

        self._initialize_xml_detector()

    def error(self, message):
        # NOTE: This method is required so long as Python 3.9 is
        # supported. The corresponding code is removed from HTMLParser
        # in 3.5, but not removed from ParserBase until 3.10.
        # https://github.com/python/cpython/issues/76025
        #
        # The original implementation turned the error into a warning,
        # but in every case I discovered, this made HTMLParser
        # immediately crash with an error message that was less
        # helpful than the warning. The new implementation makes it
        # more clear that html.parser just can't parse this
        # markup. The 3.10 implementation does the same, though it
        # raises AssertionError rather than calling a method. (We
        # catch this error and wrap it in a ParserRejectedMarkup.)
        raise ParserRejectedMarkup(message)

    def handle_startendtag(self, name, attrs):
        """Handle an incoming empty-element tag.

        This is only called when the markup looks like <tag/>.

        :param name: Name of the tag.
        :param attrs: Dictionary of the tag's attributes.
        """
        # is_startend() tells handle_starttag not to close the tag
        # just because its name matches a known empty-element tag. We
        # know that this is an empty-element tag and we want to call
        # handle_endtag ourselves.
        tag = self.handle_starttag(name, attrs, handle_empty_element=False)
        self.handle_endtag(name)
        
    def handle_starttag(self, name, attrs, handle_empty_element=True):
        """Handle an opening tag, e.g. '<tag>'

        :param name: Name of the tag.
        :param attrs: Dictionary of the tag's attributes.
        :param handle_empty_element: True if this tag is known to be
            an empty-element tag (i.e. there is not expected to be any
            closing tag).
        """
        # XXX namespace
        attr_dict = {}
        for key, value in attrs:
            # Change None attribute values to the empty string
            # for consistency with the other tree builders.
            if value is None:
                value = ''
            if key in attr_dict:
                # A single attribute shows up multiple times in this
                # tag. How to handle it depends on the
                # on_duplicate_attribute setting.
                on_dupe = self.on_duplicate_attribute
                if on_dupe == self.IGNORE:
                    pass
                elif on_dupe in (None, self.REPLACE):
                    attr_dict[key] = value
                else:
                    on_dupe(attr_dict, key, value)
            else:
                attr_dict[key] = value
            attrvalue = '""'
        #print("START", name)
        sourceline, sourcepos = self.getpos()
        tag = self.soup.handle_starttag(
            name, None, None, attr_dict, sourceline=sourceline,
            sourcepos=sourcepos
        )
        if tag and tag.is_empty_element and handle_empty_element:
            # Unlike other parsers, html.parser doesn't send separate end tag
            # events for empty-element tags. (It's handled in
            # handle_startendtag, but only if the original markup looked like
            # <tag/>.)
            #
            # So we need to call handle_endtag() ourselves. Since we
            # know the start event is identical to the end event, we
            # don't want handle_endtag() to cross off any previous end
            # events for tags of this name.
            self.handle_endtag(name, check_already_closed=False)

            # But we might encounter an explicit closing tag for this tag
            # later on. If so, we want to ignore it.
            self.already_closed_empty_element.append(name)

        if self._root_tag is None:
            self._root_tag_encountered(name)
            
    def handle_endtag(self, name, check_already_closed=True):
        """Handle a closing tag, e.g. '</tag>'
        
        :param name: A tag name.
        :param check_already_closed: True if this tag is expected to
           be the closing portion of an empty-element tag,
           e.g. '<tag></tag>'.
        """
        #print("END", name)
        if check_already_closed and name in self.already_closed_empty_element:
            # This is a redundant end tag for an empty-element tag.
            # We've already called handle_endtag() for it, so just
            # check it off the list.
            #print("ALREADY CLOSED", name)
            self.already_closed_empty_element.remove(name)
        else:
            self.soup.handle_endtag(name)
            
    def handle_data(self, data):
        """Handle some textual data that shows up between tags."""
        self.soup.handle_data(data)

    def handle_charref(self, name):
        """Handle a numeric character reference by converting it to the
        corresponding Unicode character and treating it as textual
        data.

        :param name: Character number, possibly in hexadecimal.
        """
        # TODO: This was originally a workaround for a bug in
        # HTMLParser. (http://bugs.python.org/issue13633) The bug has
        # been fixed, but removing this code still makes some
        # Beautiful Soup tests fail. This needs investigation.
        if name.startswith('x'):
            real_name = int(name.lstrip('x'), 16)
        elif name.startswith('X'):
            real_name = int(name.lstrip('X'), 16)
        else:
            real_name = int(name)

        data = None
        if real_name < 256:
            # HTML numeric entities are supposed to reference Unicode
            # code points, but sometimes they reference code points in
            # some other encoding (ahem, Windows-1252). E.g. &#147;
            # instead of &#201; for LEFT DOUBLE QUOTATION MARK. This
            # code tries to detect this situation and compensate.
            for encoding in (self.soup.original_encoding, 'windows-1252'):
                if not encoding:
                    continue
                try:
                    data = bytearray([real_name]).decode(encoding)
                except UnicodeDecodeError as e:
                    pass
        if not data:
            try:
                data = chr(real_name)
            except (ValueError, OverflowError) as e:
                pass
        data = data or "\N{REPLACEMENT CHARACTER}"
        self.handle_data(data)

    def handle_entityref(self, name):
        """Handle a named entity reference by converting it to the
        corresponding Unicode character(s) and treating it as textual
        data.

        :param name: Name of the entity reference.
        """
        character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
        if character is not None:
            data = character
        else:
            # If this were XML, it would be ambiguous whether "&foo"
            # was an character entity reference with a missing
            # semicolon or the literal string "&foo". Since this is
            # HTML, we have a complete list of all character entity references,
            # and this one wasn't found, so assume it's the literal string "&foo".
            data = "&%s" % name
        self.handle_data(data)

    def handle_comment(self, data):
        """Handle an HTML comment.

        :param data: The text of the comment.
        """
        self.soup.endData()
        self.soup.handle_data(data)
        self.soup.endData(Comment)

    def handle_decl(self, data):
        """Handle a DOCTYPE declaration.

        :param data: The text of the declaration.
        """
        self.soup.endData()
        data = data[len("DOCTYPE "):]
        self.soup.handle_data(data)
        self.soup.endData(Doctype)

    def unknown_decl(self, data):
        """Handle a declaration of unknown type -- probably a CDATA block.

        :param data: The text of the declaration.
        """
        if data.upper().startswith('CDATA['):
            cls = CData
            data = data[len('CDATA['):]
        else:
            cls = Declaration
        self.soup.endData()
        self.soup.handle_data(data)
        self.soup.endData(cls)

    def handle_pi(self, data):
        """Handle a processing instruction.

        :param data: The text of the instruction.
        """
        self.soup.endData()
        self.soup.handle_data(data)
        self._document_might_be_xml(data)
        self.soup.endData(ProcessingInstruction)


class HTMLParserTreeBuilder(HTMLTreeBuilder):
    """A Beautiful soup `TreeBuilder` that uses the `HTMLParser` parser,
    found in the Python standard library.
    """
    is_xml = False
    picklable = True
    NAME = HTMLPARSER
    features = [NAME, HTML, STRICT]

    # The html.parser knows which line number and position in the
    # original file is the source of an element.
    TRACKS_LINE_NUMBERS = True

    def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
        """Constructor.

        :param parser_args: Positional arguments to pass into 
            the BeautifulSoupHTMLParser constructor, once it's
            invoked.
        :param parser_kwargs: Keyword arguments to pass into 
            the BeautifulSoupHTMLParser constructor, once it's
            invoked.
        :param kwargs: Keyword arguments for the superclass constructor.
        """
        # Some keyword arguments will be pulled out of kwargs and placed
        # into parser_kwargs.
        extra_parser_kwargs = dict()
        for arg in ('on_duplicate_attribute',):
            if arg in kwargs:
                value = kwargs.pop(arg)
                extra_parser_kwargs[arg] = value
        super(HTMLParserTreeBuilder, self).__init__(**kwargs)
        parser_args = parser_args or []
        parser_kwargs = parser_kwargs or {}
        parser_kwargs.update(extra_parser_kwargs)
        parser_kwargs['convert_charrefs'] = False
        self.parser_args = (parser_args, parser_kwargs)
        
    def prepare_markup(self, markup, user_specified_encoding=None,
                       document_declared_encoding=None, exclude_encodings=None):

        """Run any preliminary steps necessary to make incoming markup
        acceptable to the parser.

        :param markup: Some markup -- probably a bytestring.
        :param user_specified_encoding: The user asked to try this encoding.
        :param document_declared_encoding: The markup itself claims to be
            in this encoding.
        :param exclude_encodings: The user asked _not_ to try any of
            these encodings.

        :yield: A series of 4-tuples:
         (markup, encoding, declared encoding,
          has undergone character replacement)

         Each 4-tuple represents a strategy for converting the
         document to Unicode and parsing it. Each strategy will be tried 
         in turn.
        """
        if isinstance(markup, str):
            # Parse Unicode as-is.
            yield (markup, None, None, False)
            return

        # Ask UnicodeDammit to sniff the most likely encoding.

        # This was provided by the end-user; treat it as a known
        # definite encoding per the algorithm laid out in the HTML5
        # spec.  (See the EncodingDetector class for details.)
        known_definite_encodings = [user_specified_encoding]

        # This was found in the document; treat it as a slightly lower-priority
        # user encoding.
        user_encodings = [document_declared_encoding]

        try_encodings = [user_specified_encoding, document_declared_encoding]
        dammit = UnicodeDammit(
            markup,
            known_definite_encodings=known_definite_encodings,
            user_encodings=user_encodings,
            is_html=True,
            exclude_encodings=exclude_encodings
        )
        yield (dammit.markup, dammit.original_encoding,
               dammit.declared_html_encoding,
               dammit.contains_replacement_characters)

    def feed(self, markup):
        """Run some incoming markup through some parsing process,
        populating the `BeautifulSoup` object in self.soup.
        """
        args, kwargs = self.parser_args
        parser = BeautifulSoupHTMLParser(*args, **kwargs)
        parser.soup = self.soup
        try:
            parser.feed(markup)
        except AssertionError as e:
            # html.parser raises AssertionError in rare cases to
            # indicate a fatal problem with the markup, especially
            # when there's an error in the doctype declaration.
            raise ParserRejectedMarkup(e)
        parser.close()
        parser.already_closed_empty_element = []

env_pyrecodes/lib/python3.9/site-packages/bs4/builder/_htmlparser.py
NikolaBlagojevic/pyrecodes
# Use of this source code is governed by the MIT license.
__license__ = "MIT"

__all__ = [
    'LXMLTreeBuilderForXML',
    'LXMLTreeBuilder',
    ]

try:
    from collections.abc import Callable # Python 3.6
except ImportError as e:
    from collections import Callable

from io import BytesIO
from io import StringIO
from lxml import etree
from bs4.element import (
    Comment,
    Doctype,
    NamespacedAttribute,
    ProcessingInstruction,
    XMLProcessingInstruction,
)
from bs4.builder import (
    DetectsXMLParsedAsHTML,
    FAST,
    HTML,
    HTMLTreeBuilder,
    PERMISSIVE,
    ParserRejectedMarkup,
    TreeBuilder,
    XML)
from bs4.dammit import EncodingDetector

LXML = 'lxml'

def _invert(d):
    "Invert a dictionary."
    return dict((v,k) for k, v in list(d.items()))

class LXMLTreeBuilderForXML(TreeBuilder):
    DEFAULT_PARSER_CLASS = etree.XMLParser

    is_xml = True
    processing_instruction_class = XMLProcessingInstruction

    NAME = "lxml-xml"
    ALTERNATE_NAMES = ["xml"]

    # Well, it's permissive by XML parser standards.
    features = [NAME, LXML, XML, FAST, PERMISSIVE]

    CHUNK_SIZE = 512

    # This namespace mapping is specified in the XML Namespace
    # standard.
    DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')

    DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)

    # NOTE: If we parsed Element objects and looked at .sourceline,
    # we'd be able to see the line numbers from the original document.
    # But instead we build an XMLParser or HTMLParser object to serve
    # as the target of parse messages, and those messages don't include
    # line numbers.
    # See: https://bugs.launchpad.net/lxml/+bug/1846906
    
    def initialize_soup(self, soup):
        """Let the BeautifulSoup object know about the standard namespace
        mapping.

        :param soup: A `BeautifulSoup`.
        """
        super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
        self._register_namespaces(self.DEFAULT_NSMAPS)

    def _register_namespaces(self, mapping):
        """Let the BeautifulSoup object know about namespaces encountered
        while parsing the document.

        This might be useful later on when creating CSS selectors.

        This will track (almost) all namespaces, even ones that were
        only in scope for part of the document. If two namespaces have
        the same prefix, only the first one encountered will be
        tracked. Un-prefixed namespaces are not tracked.

        :param mapping: A dictionary mapping namespace prefixes to URIs.
        """
        for key, value in list(mapping.items()):
            # This is 'if key' and not 'if key is not None' because we
            # don't track un-prefixed namespaces. Soupselect will
            # treat an un-prefixed namespace as the default, which
            # causes confusion in some cases.
            if key and key not in self.soup._namespaces:
                # Let the BeautifulSoup object know about a new namespace.
                # If there are multiple namespaces defined with the same
                # prefix, the first one in the document takes precedence.
                self.soup._namespaces[key] = value
                
    def default_parser(self, encoding):
        """Find the default parser for the given encoding.

        :param encoding: A string.
        :return: Either a parser object or a class, which
          will be instantiated with default arguments.
        """
        if self._default_parser is not None:
            return self._default_parser
        return etree.XMLParser(
            target=self, strip_cdata=False, recover=True, encoding=encoding)

    def parser_for(self, encoding):
        """Instantiate an appropriate parser for the given encoding.

        :param encoding: A string.
        :return: A parser object such as an `etree.XMLParser`.
        """
        # Use the default parser.
        parser = self.default_parser(encoding)

        if isinstance(parser, Callable):
            # Instantiate the parser with default arguments
            parser = parser(
                target=self, strip_cdata=False, recover=True, encoding=encoding
            )
        return parser

    def __init__(self, parser=None, empty_element_tags=None, **kwargs):
        # TODO: Issue a warning if parser is present but not a
        # callable, since that means there's no way to create new
        # parsers for different encodings.
        self._default_parser = parser
        if empty_element_tags is not None:
            self.empty_element_tags = set(empty_element_tags)
        self.soup = None
        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
        self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)]
        super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
        
    def _getNsTag(self, tag):
        # Split the namespace URL out of a fully-qualified lxml tag
        # name. Copied from lxml's src/lxml/sax.py.
        if tag[0] == '{':
            return tuple(tag[1:].split('}', 1))
        else:
            return (None, tag)

    def prepare_markup(self, markup, user_specified_encoding=None,
                       exclude_encodings=None,
                       document_declared_encoding=None):
        """Run any preliminary steps necessary to make incoming markup
        acceptable to the parser.

        lxml really wants to get a bytestring and convert it to
        Unicode itself. So instead of using UnicodeDammit to convert
        the bytestring to Unicode using different encodings, this
        implementation uses EncodingDetector to iterate over the
        encodings, and tell lxml to try to parse the document as each
        one in turn.

        :param markup: Some markup -- hopefully a bytestring.
        :param user_specified_encoding: The user asked to try this encoding.
        :param document_declared_encoding: The markup itself claims to be
            in this encoding.
        :param exclude_encodings: The user asked _not_ to try any of
            these encodings.

        :yield: A series of 4-tuples:
         (markup, encoding, declared encoding,
          has undergone character replacement)

         Each 4-tuple represents a strategy for converting the
         document to Unicode and parsing it. Each strategy will be tried 
         in turn.
        """
        is_html = not self.is_xml
        if is_html:
            self.processing_instruction_class = ProcessingInstruction
            # We're in HTML mode, so if we're given XML, that's worth
            # noting.
            DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
        else:
            self.processing_instruction_class = XMLProcessingInstruction

        if isinstance(markup, str):
            # We were given Unicode. Maybe lxml can parse Unicode on
            # this system?

            # TODO: This is a workaround for
            # https://bugs.launchpad.net/lxml/+bug/1948551.
            # We can remove it once the upstream issue is fixed.
            if len(markup) > 0 and markup[0] == u'\N{BYTE ORDER MARK}':
                markup = markup[1:]
            yield markup, None, document_declared_encoding, False

        if isinstance(markup, str):
            # No, apparently not. Convert the Unicode to UTF-8 and
            # tell lxml to parse it as UTF-8.
            yield (markup.encode("utf8"), "utf8",
                   document_declared_encoding, False)

        # This was provided by the end-user; treat it as a known
        # definite encoding per the algorithm laid out in the HTML5
        # spec.  (See the EncodingDetector class for details.)
        known_definite_encodings = [user_specified_encoding]

        # This was found in the document; treat it as a slightly lower-priority
        # user encoding.
        user_encodings = [document_declared_encoding]
        detector = EncodingDetector(
            markup, known_definite_encodings=known_definite_encodings,
            user_encodings=user_encodings, is_html=is_html,
            exclude_encodings=exclude_encodings
        )
        for encoding in detector.encodings:
            yield (detector.markup, encoding, document_declared_encoding, False)

    def feed(self, markup):
        if isinstance(markup, bytes):
            markup = BytesIO(markup)
        elif isinstance(markup, str):
            markup = StringIO(markup)

        # Call feed() at least once, even if the markup is empty,
        # or the parser won't be initialized.
        data = markup.read(self.CHUNK_SIZE)
        try:
            self.parser = self.parser_for(self.soup.original_encoding)
            self.parser.feed(data)
            while len(data) != 0:
                # Now call feed() on the rest of the data, chunk by chunk.
                data = markup.read(self.CHUNK_SIZE)
                if len(data) != 0:
                    self.parser.feed(data)
            self.parser.close()
        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
            raise ParserRejectedMarkup(e)

    def close(self):
        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]

    def start(self, name, attrs, nsmap={}):
        # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
        attrs = dict(attrs)
        nsprefix = None
        # Invert each namespace map as it comes in.
        if len(nsmap) == 0 and len(self.nsmaps) > 1:
                # There are no new namespaces for this tag, but
                # non-default namespaces are in play, so we need a
                # separate tag stack to know when they end.
                self.nsmaps.append(None)
        elif len(nsmap) > 0:
            # A new namespace mapping has come into play.

            # First, Let the BeautifulSoup object know about it.
            self._register_namespaces(nsmap)

            # Then, add it to our running list of inverted namespace
            # mappings.
            self.nsmaps.append(_invert(nsmap))

            # The currently active namespace prefixes have
            # changed. Calculate the new mapping so it can be stored
            # with all Tag objects created while these prefixes are in
            # scope.
            current_mapping = dict(self.active_namespace_prefixes[-1])
            current_mapping.update(nsmap)

            # We should not track un-prefixed namespaces as we can only hold one
            # and it will be recognized as the default namespace by soupsieve,
            # which may be confusing in some situations.
            if '' in current_mapping:
                del current_mapping['']
            self.active_namespace_prefixes.append(current_mapping)
            
            # Also treat the namespace mapping as a set of attributes on the
            # tag, so we can recreate it later.
            attrs = attrs.copy()
            for prefix, namespace in list(nsmap.items()):
                attribute = NamespacedAttribute(
                    "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
                attrs[attribute] = namespace

        # Namespaces are in play. Find any attributes that came in
        # from lxml with namespaces attached to their names, and
        # turn then into NamespacedAttribute objects.
        new_attrs = {}
        for attr, value in list(attrs.items()):
            namespace, attr = self._getNsTag(attr)
            if namespace is None:
                new_attrs[attr] = value
            else:
                nsprefix = self._prefix_for_namespace(namespace)
                attr = NamespacedAttribute(nsprefix, attr, namespace)
                new_attrs[attr] = value
        attrs = new_attrs

        namespace, name = self._getNsTag(name)
        nsprefix = self._prefix_for_namespace(namespace)
        self.soup.handle_starttag(
            name, namespace, nsprefix, attrs,
            namespaces=self.active_namespace_prefixes[-1]
        )
        
    def _prefix_for_namespace(self, namespace):
        """Find the currently active prefix for the given namespace."""
        if namespace is None:
            return None
        for inverted_nsmap in reversed(self.nsmaps):
            if inverted_nsmap is not None and namespace in inverted_nsmap:
                return inverted_nsmap[namespace]
        return None

    def end(self, name):
        self.soup.endData()
        completed_tag = self.soup.tagStack[-1]
        namespace, name = self._getNsTag(name)
        nsprefix = None
        if namespace is not None:
            for inverted_nsmap in reversed(self.nsmaps):
                if inverted_nsmap is not None and namespace in inverted_nsmap:
                    nsprefix = inverted_nsmap[namespace]
                    break
        self.soup.handle_endtag(name, nsprefix)
        if len(self.nsmaps) > 1:
            # This tag, or one of its parents, introduced a namespace
            # mapping, so pop it off the stack.
            out_of_scope_nsmap = self.nsmaps.pop()

            if out_of_scope_nsmap is not None:
                # This tag introduced a namespace mapping which is no
                # longer in scope. Recalculate the currently active
                # namespace prefixes.
                self.active_namespace_prefixes.pop()
            
    def pi(self, target, data):
        self.soup.endData()
        data = target + ' ' + data
        self.soup.handle_data(data)
        self.soup.endData(self.processing_instruction_class)
        
    def data(self, content):
        self.soup.handle_data(content)

    def doctype(self, name, pubid, system):
        self.soup.endData()
        doctype = Doctype.for_name_and_ids(name, pubid, system)
        self.soup.object_was_parsed(doctype)

    def comment(self, content):
        "Handle comments as Comment objects."
        self.soup.endData()
        self.soup.handle_data(content)
        self.soup.endData(Comment)

    def test_fragment_to_document(self, fragment):
        """See `TreeBuilder`."""
        return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment


class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):

    NAME = LXML
    ALTERNATE_NAMES = ["lxml-html"]

    features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
    is_xml = False
    processing_instruction_class = ProcessingInstruction

    def default_parser(self, encoding):
        return etree.HTMLParser

    def feed(self, markup):
        encoding = self.soup.original_encoding
        try:
            self.parser = self.parser_for(encoding)
            self.parser.feed(markup)
            self.parser.close()
        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
            raise ParserRejectedMarkup(e)


    def test_fragment_to_document(self, fragment):
        """See `TreeBuilder`."""
        return '<html><body>%s</body></html>' % fragment

env_pyrecodes/lib/python3.9/site-packages/bs4/builder/_lxml.py
NikolaBlagojevic/pyrecodes
"""Integration code for CSS selectors using Soup Sieve (pypi: soupsieve)."""

import warnings
try:
    import soupsieve
except ImportError as e:
    soupsieve = None
    warnings.warn(
        'The soupsieve package is not installed. CSS selectors cannot be used.'
    )


class CSS(object):
    """A proxy object against the soupsieve library, to simplify its
    CSS selector API.

    Acquire this object through the .css attribute on the
    BeautifulSoup object, or on the Tag you want to use as the
    starting point for a CSS selector.

    The main advantage of doing this is that the tag to be selected
    against doesn't need to be explicitly specified in the function
    calls, since it's already scoped to a tag.
    """

    def __init__(self, tag, api=soupsieve):
        """Constructor.

        You don't need to instantiate this class yourself; instead,
        access the .css attribute on the BeautifulSoup object, or on
        the Tag you want to use as the starting point for your CSS
        selector.

        :param tag: All CSS selectors will use this as their starting
        point.

        :param api: A plug-in replacement for the soupsieve module,
        designed mainly for use in tests.
        """
        if api is None:
            raise NotImplementedError(
                "Cannot execute CSS selectors because the soupsieve package is not installed."
            )
        self.api = api
        self.tag = tag

    def escape(self, ident):
        """Escape a CSS identifier.

        This is a simple wrapper around soupselect.escape(). See the
        documentation for that function for more information.
        """
        if soupsieve is None:
            raise NotImplementedError(
                "Cannot escape CSS identifiers because the soupsieve package is not installed."
            )
        return self.api.escape(ident)

    def _ns(self, ns, select):
        """Normalize a dictionary of namespaces."""
        if not isinstance(select, self.api.SoupSieve) and ns is None:
            # If the selector is a precompiled pattern, it already has
            # a namespace context compiled in, which cannot be
            # replaced.
            ns = self.tag._namespaces
        return ns

    def _rs(self, results):
        """Normalize a list of results to a Resultset.

        A ResultSet is more consistent with the rest of Beautiful
        Soup's API, and ResultSet.__getattr__ has a helpful error
        message if you try to treat a list of results as a single
        result (a common mistake).
        """
        # Import here to avoid circular import
        from bs4.element import ResultSet
        return ResultSet(None, results)

    def compile(self, select, namespaces=None, flags=0, **kwargs):
        """Pre-compile a selector and return the compiled object.

        :param selector: A CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
           used in the CSS selector to namespace URIs. By default,
           Beautiful Soup will use the prefixes it encountered while
           parsing the document.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.compile() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
           soupsieve.compile() method.

        :return: A precompiled selector object.
        :rtype: soupsieve.SoupSieve
        """
        return self.api.compile(
            select, self._ns(namespaces, select), flags, **kwargs
        )

    def select_one(self, select, namespaces=None, flags=0, **kwargs):
        """Perform a CSS selection operation on the current Tag and return the
        first result.

        This uses the Soup Sieve library. For more information, see
        that library's documentation for the soupsieve.select_one()
        method.

        :param selector: A CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
           used in the CSS selector to namespace URIs. By default,
           Beautiful Soup will use the prefixes it encountered while
           parsing the document.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.select_one() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
           soupsieve.select_one() method.

        :return: A Tag, or None if the selector has no match.
        :rtype: bs4.element.Tag

        """
        return self.api.select_one(
            select, self.tag, self._ns(namespaces, select), flags, **kwargs
        )

    def select(self, select, namespaces=None, limit=0, flags=0, **kwargs):
        """Perform a CSS selection operation on the current Tag.

        This uses the Soup Sieve library. For more information, see
        that library's documentation for the soupsieve.select()
        method.

        :param selector: A string containing a CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
            used in the CSS selector to namespace URIs. By default,
            Beautiful Soup will pass in the prefixes it encountered while
            parsing the document.

        :param limit: After finding this number of results, stop looking.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.select() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
            soupsieve.select() method.

        :return: A ResultSet of Tag objects.
        :rtype: bs4.element.ResultSet

        """
        if limit is None:
            limit = 0

        return self._rs(
            self.api.select(
                select, self.tag, self._ns(namespaces, select), limit, flags,
                **kwargs
            )
        )

    def iselect(self, select, namespaces=None, limit=0, flags=0, **kwargs):
        """Perform a CSS selection operation on the current Tag.

        This uses the Soup Sieve library. For more information, see
        that library's documentation for the soupsieve.iselect()
        method. It is the same as select(), but it returns a generator
        instead of a list.

        :param selector: A string containing a CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
            used in the CSS selector to namespace URIs. By default,
            Beautiful Soup will pass in the prefixes it encountered while
            parsing the document.

        :param limit: After finding this number of results, stop looking.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.iselect() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
            soupsieve.iselect() method.

        :return: A generator
        :rtype: types.GeneratorType
        """
        return self.api.iselect(
            select, self.tag, self._ns(namespaces, select), limit, flags, **kwargs
        )

    def closest(self, select, namespaces=None, flags=0, **kwargs):
        """Find the Tag closest to this one that matches the given selector.

        This uses the Soup Sieve library. For more information, see
        that library's documentation for the soupsieve.closest()
        method.

        :param selector: A string containing a CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
            used in the CSS selector to namespace URIs. By default,
            Beautiful Soup will pass in the prefixes it encountered while
            parsing the document.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.closest() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
            soupsieve.closest() method.

        :return: A Tag, or None if there is no match.
        :rtype: bs4.Tag

        """
        return self.api.closest(
            select, self.tag, self._ns(namespaces, select), flags, **kwargs
        )

    def match(self, select, namespaces=None, flags=0, **kwargs):
        """Check whether this Tag matches the given CSS selector.

        This uses the Soup Sieve library. For more information, see
        that library's documentation for the soupsieve.match()
        method.

        :param: a CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
            used in the CSS selector to namespace URIs. By default,
            Beautiful Soup will pass in the prefixes it encountered while
            parsing the document.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.match() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
            soupsieve.match() method.

        :return: True if this Tag matches the selector; False otherwise.
        :rtype: bool
        """
        return self.api.match(
            select, self.tag, self._ns(namespaces, select), flags, **kwargs
        )

    def filter(self, select, namespaces=None, flags=0, **kwargs):
        """Filter this Tag's direct children based on the given CSS selector.

        This uses the Soup Sieve library. It works the same way as
        passing this Tag into that library's soupsieve.filter()
        method. More information, for more information see the
        documentation for soupsieve.filter().

        :param namespaces: A dictionary mapping namespace prefixes
            used in the CSS selector to namespace URIs. By default,
            Beautiful Soup will pass in the prefixes it encountered while
            parsing the document.

        :param flags: Flags to be passed into Soup Sieve's
            soupsieve.filter() method.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
            soupsieve.filter() method.

        :return: A ResultSet of Tag objects.
        :rtype: bs4.element.ResultSet

        """
        return self._rs(
            self.api.filter(
                select, self.tag, self._ns(namespaces, select), flags, **kwargs
            )
        )

env_pyrecodes/lib/python3.9/site-packages/bs4/css.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
"""Beautiful Soup bonus library: Unicode, Dammit

This library converts a bytestream to Unicode through any means
necessary. It is heavily based on code from Mark Pilgrim's Universal
Feed Parser. It works best on XML and HTML, but it does not rewrite the
XML or HTML to reflect a new encoding; that's the tree builder's job.
"""
# Use of this source code is governed by the MIT license.
__license__ = "MIT"

from html.entities import codepoint2name
from collections import defaultdict
import codecs
import re
import logging
import string

# Import a library to autodetect character encodings. We'll support
# any of a number of libraries that all support the same API:
#
# * cchardet
# * chardet
# * charset-normalizer
chardet_module = None
try:
    #  PyPI package: cchardet
    import cchardet as chardet_module
except ImportError:
    try:
        #  Debian package: python-chardet
        #  PyPI package: chardet
        import chardet as chardet_module
    except ImportError:
        try:
            # PyPI package: charset-normalizer
            import charset_normalizer as chardet_module
        except ImportError:
            # No chardet available.
            chardet_module = None

if chardet_module:
    def chardet_dammit(s):
        if isinstance(s, str):
            return None
        return chardet_module.detect(s)['encoding']
else:
    def chardet_dammit(s):
        return None

# Build bytestring and Unicode versions of regular expressions for finding
# a declared encoding inside an XML or HTML document.
xml_encoding = '^\\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
encoding_res = dict()
encoding_res[bytes] = {
    'html' : re.compile(html_meta.encode("ascii"), re.I),
    'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
}
encoding_res[str] = {
    'html' : re.compile(html_meta, re.I),
    'xml' : re.compile(xml_encoding, re.I)
}

from html.entities import html5

class EntitySubstitution(object):
    """The ability to substitute XML or HTML entities for certain characters."""

    def _populate_class_variables():
        """Initialize variables used by this class to manage the plethora of
        HTML5 named entities.

        This function returns a 3-tuple containing two dictionaries
        and a regular expression:

        unicode_to_name - A mapping of Unicode strings like "⦨" to
        entity names like "angmsdaa". When a single Unicode string has
        multiple entity names, we try to choose the most commonly-used
        name.

        name_to_unicode: A mapping of entity names like "angmsdaa" to 
        Unicode strings like "⦨".

        named_entity_re: A regular expression matching (almost) any
        Unicode string that corresponds to an HTML5 named entity.
        """
        unicode_to_name = {}
        name_to_unicode = {}

        short_entities = set()
        long_entities_by_first_character = defaultdict(set)
        
        for name_with_semicolon, character in sorted(html5.items()):
            # "It is intentional, for legacy compatibility, that many
            # code points have multiple character reference names. For
            # example, some appear both with and without the trailing
            # semicolon, or with different capitalizations."
            # - https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references
            #
            # The parsers are in charge of handling (or not) character
            # references with no trailing semicolon, so we remove the
            # semicolon whenever it appears.
            if name_with_semicolon.endswith(';'):
                name = name_with_semicolon[:-1]
            else:
                name = name_with_semicolon

            # When parsing HTML, we want to recognize any known named
            # entity and convert it to a sequence of Unicode
            # characters.
            if name not in name_to_unicode:
                name_to_unicode[name] = character

            # When _generating_ HTML, we want to recognize special
            # character sequences that _could_ be converted to named
            # entities.
            unicode_to_name[character] = name

            # We also need to build a regular expression that lets us
            # _find_ those characters in output strings so we can
            # replace them.
            #
            # This is tricky, for two reasons.

            if (len(character) == 1 and ord(character) < 128
                and character not in '<>&'):
                # First, it would be annoying to turn single ASCII
                # characters like | into named entities like
                # &verbar;. The exceptions are <>&, which we _must_
                # turn into named entities to produce valid HTML.
                continue

            if len(character) > 1 and all(ord(x) < 128 for x in character):
                # We also do not want to turn _combinations_ of ASCII
                # characters like 'fj' into named entities like '&fjlig;',
                # though that's more debateable.
                continue

            # Second, some named entities have a Unicode value that's
            # a subset of the Unicode value for some _other_ named
            # entity.  As an example, \u2267' is &GreaterFullEqual;,
            # but '\u2267\u0338' is &NotGreaterFullEqual;. Our regular
            # expression needs to match the first two characters of
            # "\u2267\u0338foo", but only the first character of
            # "\u2267foo".
            #
            # In this step, we build two sets of characters that
            # _eventually_ need to go into the regular expression. But
            # we won't know exactly what the regular expression needs
            # to look like until we've gone through the entire list of
            # named entities.
            if len(character) == 1:
                short_entities.add(character)
            else:
                long_entities_by_first_character[character[0]].add(character)

        # Now that we've been through the entire list of entities, we
        # can create a regular expression that matches any of them.
        particles = set()
        for short in short_entities:
            long_versions = long_entities_by_first_character[short]
            if not long_versions:
                particles.add(short)
            else:
                ignore = "".join([x[1] for x in long_versions])
                # This finds, e.g. \u2267 but only if it is _not_
                # followed by \u0338.
                particles.add("%s(?![%s])" % (short, ignore))
        
        for long_entities in list(long_entities_by_first_character.values()):
            for long_entity in long_entities:
                particles.add(long_entity)

        re_definition = "(%s)" % "|".join(particles)
                
        # If an entity shows up in both html5 and codepoint2name, it's
        # likely that HTML5 gives it several different names, such as
        # 'rsquo' and 'rsquor'. When converting Unicode characters to
        # named entities, the codepoint2name name should take
        # precedence where possible, since that's the more easily
        # recognizable one.
        for codepoint, name in list(codepoint2name.items()):
            character = chr(codepoint)
            unicode_to_name[character] = name

        return unicode_to_name, name_to_unicode, re.compile(re_definition)
    (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
     CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()

    CHARACTER_TO_XML_ENTITY = {
        "'": "apos",
        '"': "quot",
        "&": "amp",
        "<": "lt",
        ">": "gt",
        }

    BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
                                           "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
                                           ")")

    AMPERSAND_OR_BRACKET = re.compile("([<>&])")

    @classmethod
    def _substitute_html_entity(cls, matchobj):
        """Used with a regular expression to substitute the
        appropriate HTML entity for a special character string."""
        entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
        return "&%s;" % entity

    @classmethod
    def _substitute_xml_entity(cls, matchobj):
        """Used with a regular expression to substitute the
        appropriate XML entity for a special character string."""
        entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
        return "&%s;" % entity

    @classmethod
    def quoted_attribute_value(self, value):
        """Make a value into a quoted XML attribute, possibly escaping it.

         Most strings will be quoted using double quotes.

          Bob's Bar -> "Bob's Bar"

         If a string contains double quotes, it will be quoted using
         single quotes.

          Welcome to "my bar" -> 'Welcome to "my bar"'

         If a string contains both single and double quotes, the
         double quotes will be escaped, and the string will be quoted
         using double quotes.

          Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
        """
        quote_with = '"'
        if '"' in value:
            if "'" in value:
                # The string contains both single and double
                # quotes.  Turn the double quotes into
                # entities. We quote the double quotes rather than
                # the single quotes because the entity name is
                # "&quot;" whether this is HTML or XML.  If we
                # quoted the single quotes, we'd have to decide
                # between &apos; and &squot;.
                replace_with = "&quot;"
                value = value.replace('"', replace_with)
            else:
                # There are double quotes but no single quotes.
                # We can use single quotes to quote the attribute.
                quote_with = "'"
        return quote_with + value + quote_with

    @classmethod
    def substitute_xml(cls, value, make_quoted_attribute=False):
        """Substitute XML entities for special XML characters.

        :param value: A string to be substituted. The less-than sign
          will become &lt;, the greater-than sign will become &gt;,
          and any ampersands will become &amp;. If you want ampersands
          that appear to be part of an entity definition to be left
          alone, use substitute_xml_containing_entities() instead.

        :param make_quoted_attribute: If True, then the string will be
         quoted, as befits an attribute value.
        """
        # Escape angle brackets and ampersands.
        value = cls.AMPERSAND_OR_BRACKET.sub(
            cls._substitute_xml_entity, value)

        if make_quoted_attribute:
            value = cls.quoted_attribute_value(value)
        return value

    @classmethod
    def substitute_xml_containing_entities(
        cls, value, make_quoted_attribute=False):
        """Substitute XML entities for special XML characters.

        :param value: A string to be substituted. The less-than sign will
          become &lt;, the greater-than sign will become &gt;, and any
          ampersands that are not part of an entity defition will
          become &amp;.

        :param make_quoted_attribute: If True, then the string will be
         quoted, as befits an attribute value.
        """
        # Escape angle brackets, and ampersands that aren't part of
        # entities.
        value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
            cls._substitute_xml_entity, value)

        if make_quoted_attribute:
            value = cls.quoted_attribute_value(value)
        return value

    @classmethod
    def substitute_html(cls, s):
        """Replace certain Unicode characters with named HTML entities.

        This differs from data.encode(encoding, 'xmlcharrefreplace')
        in that the goal is to make the result more readable (to those
        with ASCII displays) rather than to recover from
        errors. There's absolutely nothing wrong with a UTF-8 string
        containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
        character with "&eacute;" will make it more readable to some
        people.

        :param s: A Unicode string.
        """
        return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
            cls._substitute_html_entity, s)


class EncodingDetector:
    """Suggests a number of possible encodings for a bytestring.

    Order of precedence:

    1. Encodings you specifically tell EncodingDetector to try first
    (the known_definite_encodings argument to the constructor).

    2. An encoding determined by sniffing the document's byte-order mark.

    3. Encodings you specifically tell EncodingDetector to try if
    byte-order mark sniffing fails (the user_encodings argument to the
    constructor).

    4. An encoding declared within the bytestring itself, either in an
    XML declaration (if the bytestring is to be interpreted as an XML
    document), or in a <meta> tag (if the bytestring is to be
    interpreted as an HTML document.)

    5. An encoding detected through textual analysis by chardet,
    cchardet, or a similar external library.

    4. UTF-8.

    5. Windows-1252.

    """
    def __init__(self, markup, known_definite_encodings=None,
                 is_html=False, exclude_encodings=None,
                 user_encodings=None, override_encodings=None):
        """Constructor.

        :param markup: Some markup in an unknown encoding.

        :param known_definite_encodings: When determining the encoding
            of `markup`, these encodings will be tried first, in
            order. In HTML terms, this corresponds to the "known
            definite encoding" step defined here:
            https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding

        :param user_encodings: These encodings will be tried after the
            `known_definite_encodings` have been tried and failed, and
            after an attempt to sniff the encoding by looking at a
            byte order mark has failed. In HTML terms, this
            corresponds to the step "user has explicitly instructed
            the user agent to override the document's character
            encoding", defined here:
            https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding

        :param override_encodings: A deprecated alias for
            known_definite_encodings. Any encodings here will be tried
            immediately after the encodings in
            known_definite_encodings.

        :param is_html: If True, this markup is considered to be
            HTML. Otherwise it's assumed to be XML.

        :param exclude_encodings: These encodings will not be tried,
            even if they otherwise would be.

        """
        self.known_definite_encodings = list(known_definite_encodings or [])
        if override_encodings:
            self.known_definite_encodings += override_encodings
        self.user_encodings = user_encodings or []
        exclude_encodings = exclude_encodings or []
        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
        self.chardet_encoding = None
        self.is_html = is_html
        self.declared_encoding = None

        # First order of business: strip a byte-order mark.
        self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)

    def _usable(self, encoding, tried):
        """Should we even bother to try this encoding?

        :param encoding: Name of an encoding.
        :param tried: Encodings that have already been tried. This will be modified
            as a side effect.
        """
        if encoding is not None:
            encoding = encoding.lower()
            if encoding in self.exclude_encodings:
                return False
            if encoding not in tried:
                tried.add(encoding)
                return True
        return False

    @property
    def encodings(self):
        """Yield a number of encodings that might work for this markup.

        :yield: A sequence of strings.
        """
        tried = set()

        # First, try the known definite encodings
        for e in self.known_definite_encodings:
            if self._usable(e, tried):
                yield e

        # Did the document originally start with a byte-order mark
        # that indicated its encoding?
        if self._usable(self.sniffed_encoding, tried):
            yield self.sniffed_encoding

        # Sniffing the byte-order mark did nothing; try the user
        # encodings.
        for e in self.user_encodings:
            if self._usable(e, tried):
                yield e
            
        # Look within the document for an XML or HTML encoding
        # declaration.
        if self.declared_encoding is None:
            self.declared_encoding = self.find_declared_encoding(
                self.markup, self.is_html)
        if self._usable(self.declared_encoding, tried):
            yield self.declared_encoding

        # Use third-party character set detection to guess at the
        # encoding.
        if self.chardet_encoding is None:
            self.chardet_encoding = chardet_dammit(self.markup)
        if self._usable(self.chardet_encoding, tried):
            yield self.chardet_encoding

        # As a last-ditch effort, try utf-8 and windows-1252.
        for e in ('utf-8', 'windows-1252'):
            if self._usable(e, tried):
                yield e

    @classmethod
    def strip_byte_order_mark(cls, data):
        """If a byte-order mark is present, strip it and return the encoding it implies.

        :param data: Some markup.
        :return: A 2-tuple (modified data, implied encoding)
        """
        encoding = None
        if isinstance(data, str):
            # Unicode data cannot have a byte-order mark.
            return data, encoding
        if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
               and (data[2:4] != '\x00\x00'):
            encoding = 'utf-16be'
            data = data[2:]
        elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \
                 and (data[2:4] != '\x00\x00'):
            encoding = 'utf-16le'
            data = data[2:]
        elif data[:3] == b'\xef\xbb\xbf':
            encoding = 'utf-8'
            data = data[3:]
        elif data[:4] == b'\x00\x00\xfe\xff':
            encoding = 'utf-32be'
            data = data[4:]
        elif data[:4] == b'\xff\xfe\x00\x00':
            encoding = 'utf-32le'
            data = data[4:]
        return data, encoding

    @classmethod
    def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False):
        """Given a document, tries to find its declared encoding.

        An XML encoding is declared at the beginning of the document.

        An HTML encoding is declared in a <meta> tag, hopefully near the
        beginning of the document.

        :param markup: Some markup.
        :param is_html: If True, this markup is considered to be HTML. Otherwise
            it's assumed to be XML.
        :param search_entire_document: Since an encoding is supposed to declared near the beginning
            of the document, most of the time it's only necessary to search a few kilobytes of data.
            Set this to True to force this method to search the entire document.
        """
        if search_entire_document:
            xml_endpos = html_endpos = len(markup)
        else:
            xml_endpos = 1024
            html_endpos = max(2048, int(len(markup) * 0.05))

        if isinstance(markup, bytes):
            res = encoding_res[bytes]
        else:
            res = encoding_res[str]

        xml_re = res['xml']
        html_re = res['html']
        declared_encoding = None
        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
        if not declared_encoding_match and is_html:
            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
        if declared_encoding_match is not None:
            declared_encoding = declared_encoding_match.groups()[0]
        if declared_encoding:
            if isinstance(declared_encoding, bytes):
                declared_encoding = declared_encoding.decode('ascii', 'replace')
            return declared_encoding.lower()
        return None

class UnicodeDammit:
    """A class for detecting the encoding of a *ML document and
    converting it to a Unicode string. If the source encoding is
    windows-1252, can replace MS smart quotes with their HTML or XML
    equivalents."""

    # This dictionary maps commonly seen values for "charset" in HTML
    # meta tags to the corresponding Python codec names. It only covers
    # values that aren't in Python's aliases and can't be determined
    # by the heuristics in find_codec.
    CHARSET_ALIASES = {"macintosh": "mac-roman",
                       "x-sjis": "shift-jis"}

    ENCODINGS_WITH_SMART_QUOTES = [
        "windows-1252",
        "iso-8859-1",
        "iso-8859-2",
        ]

    def __init__(self, markup, known_definite_encodings=[],
                 smart_quotes_to=None, is_html=False, exclude_encodings=[],
                 user_encodings=None, override_encodings=None
    ):
        """Constructor.

        :param markup: A bytestring representing markup in an unknown encoding.

        :param known_definite_encodings: When determining the encoding
            of `markup`, these encodings will be tried first, in
            order. In HTML terms, this corresponds to the "known
            definite encoding" step defined here:
            https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding

        :param user_encodings: These encodings will be tried after the
            `known_definite_encodings` have been tried and failed, and
            after an attempt to sniff the encoding by looking at a
            byte order mark has failed. In HTML terms, this
            corresponds to the step "user has explicitly instructed
            the user agent to override the document's character
            encoding", defined here:
            https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding

        :param override_encodings: A deprecated alias for
            known_definite_encodings. Any encodings here will be tried
            immediately after the encodings in
            known_definite_encodings.

        :param smart_quotes_to: By default, Microsoft smart quotes will, like all other characters, be converted
           to Unicode characters. Setting this to 'ascii' will convert them to ASCII quotes instead.
           Setting it to 'xml' will convert them to XML entity references, and setting it to 'html'
           will convert them to HTML entity references.
        :param is_html: If True, this markup is considered to be HTML. Otherwise
            it's assumed to be XML.
        :param exclude_encodings: These encodings will not be considered, even
            if the sniffing code thinks they might make sense.

        """
        self.smart_quotes_to = smart_quotes_to
        self.tried_encodings = []
        self.contains_replacement_characters = False
        self.is_html = is_html
        self.log = logging.getLogger(__name__)
        self.detector = EncodingDetector(
            markup, known_definite_encodings, is_html, exclude_encodings,
            user_encodings, override_encodings
        )

        # Short-circuit if the data is in Unicode to begin with.
        if isinstance(markup, str) or markup == '':
            self.markup = markup
            self.unicode_markup = str(markup)
            self.original_encoding = None
            return

        # The encoding detector may have stripped a byte-order mark.
        # Use the stripped markup from this point on.
        self.markup = self.detector.markup

        u = None
        for encoding in self.detector.encodings:
            markup = self.detector.markup
            u = self._convert_from(encoding)
            if u is not None:
                break

        if not u:
            # None of the encodings worked. As an absolute last resort,
            # try them again with character replacement.

            for encoding in self.detector.encodings:
                if encoding != "ascii":
                    u = self._convert_from(encoding, "replace")
                if u is not None:
                    self.log.warning(
                            "Some characters could not be decoded, and were "
                            "replaced with REPLACEMENT CHARACTER."
                    )
                    self.contains_replacement_characters = True
                    break

        # If none of that worked, we could at this point force it to
        # ASCII, but that would destroy so much data that I think
        # giving up is better.
        self.unicode_markup = u
        if not u:
            self.original_encoding = None

    def _sub_ms_char(self, match):
        """Changes a MS smart quote character to an XML or HTML
        entity, or an ASCII character."""
        orig = match.group(1)
        if self.smart_quotes_to == 'ascii':
            sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
        else:
            sub = self.MS_CHARS.get(orig)
            if type(sub) == tuple:
                if self.smart_quotes_to == 'xml':
                    sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
                else:
                    sub = '&'.encode() + sub[0].encode() + ';'.encode()
            else:
                sub = sub.encode()
        return sub

    def _convert_from(self, proposed, errors="strict"):
        """Attempt to convert the markup to the proposed encoding.

        :param proposed: The name of a character encoding.
        """
        proposed = self.find_codec(proposed)
        if not proposed or (proposed, errors) in self.tried_encodings:
            return None
        self.tried_encodings.append((proposed, errors))
        markup = self.markup
        # Convert smart quotes to HTML if coming from an encoding
        # that might have them.
        if (self.smart_quotes_to is not None
            and proposed in self.ENCODINGS_WITH_SMART_QUOTES):
            smart_quotes_re = b"([\x80-\x9f])"
            smart_quotes_compiled = re.compile(smart_quotes_re)
            markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)

        try:
            #print("Trying to convert document to %s (errors=%s)" % (
            #    proposed, errors))
            u = self._to_unicode(markup, proposed, errors)
            self.markup = u
            self.original_encoding = proposed
        except Exception as e:
            #print("That didn't work!")
            #print(e)
            return None
        #print("Correct encoding: %s" % proposed)
        return self.markup

    def _to_unicode(self, data, encoding, errors="strict"):
        """Given a string and its encoding, decodes the string into Unicode.

        :param encoding: The name of an encoding.
        """
        return str(data, encoding, errors)

    @property
    def declared_html_encoding(self):
        """If the markup is an HTML document, returns the encoding declared _within_
        the document.
        """
        if not self.is_html:
            return None
        return self.detector.declared_encoding

    def find_codec(self, charset):
        """Convert the name of a character set to a codec name.

        :param charset: The name of a character set.
        :return: The name of a codec.
        """
        value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
               or (charset and self._codec(charset.replace("-", "")))
               or (charset and self._codec(charset.replace("-", "_")))
               or (charset and charset.lower())
               or charset
                )
        if value:
            return value.lower()
        return None

    def _codec(self, charset):
        if not charset:
            return charset
        codec = None
        try:
            codecs.lookup(charset)
            codec = charset
        except (LookupError, ValueError):
            pass
        return codec


    # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
    MS_CHARS = {b'\x80': ('euro', '20AC'),
                b'\x81': ' ',
                b'\x82': ('sbquo', '201A'),
                b'\x83': ('fnof', '192'),
                b'\x84': ('bdquo', '201E'),
                b'\x85': ('hellip', '2026'),
                b'\x86': ('dagger', '2020'),
                b'\x87': ('Dagger', '2021'),
                b'\x88': ('circ', '2C6'),
                b'\x89': ('permil', '2030'),
                b'\x8A': ('Scaron', '160'),
                b'\x8B': ('lsaquo', '2039'),
                b'\x8C': ('OElig', '152'),
                b'\x8D': '?',
                b'\x8E': ('#x17D', '17D'),
                b'\x8F': '?',
                b'\x90': '?',
                b'\x91': ('lsquo', '2018'),
                b'\x92': ('rsquo', '2019'),
                b'\x93': ('ldquo', '201C'),
                b'\x94': ('rdquo', '201D'),
                b'\x95': ('bull', '2022'),
                b'\x96': ('ndash', '2013'),
                b'\x97': ('mdash', '2014'),
                b'\x98': ('tilde', '2DC'),
                b'\x99': ('trade', '2122'),
                b'\x9a': ('scaron', '161'),
                b'\x9b': ('rsaquo', '203A'),
                b'\x9c': ('oelig', '153'),
                b'\x9d': '?',
                b'\x9e': ('#x17E', '17E'),
                b'\x9f': ('Yuml', ''),}

    # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
    # horrors like stripping diacritical marks to turn á into a, but also
    # contains non-horrors like turning “ into ".
    MS_CHARS_TO_ASCII = {
        b'\x80' : 'EUR',
        b'\x81' : ' ',
        b'\x82' : ',',
        b'\x83' : 'f',
        b'\x84' : ',,',
        b'\x85' : '...',
        b'\x86' : '+',
        b'\x87' : '++',
        b'\x88' : '^',
        b'\x89' : '%',
        b'\x8a' : 'S',
        b'\x8b' : '<',
        b'\x8c' : 'OE',
        b'\x8d' : '?',
        b'\x8e' : 'Z',
        b'\x8f' : '?',
        b'\x90' : '?',
        b'\x91' : "'",
        b'\x92' : "'",
        b'\x93' : '"',
        b'\x94' : '"',
        b'\x95' : '*',
        b'\x96' : '-',
        b'\x97' : '--',
        b'\x98' : '~',
        b'\x99' : '(TM)',
        b'\x9a' : 's',
        b'\x9b' : '>',
        b'\x9c' : 'oe',
        b'\x9d' : '?',
        b'\x9e' : 'z',
        b'\x9f' : 'Y',
        b'\xa0' : ' ',
        b'\xa1' : '!',
        b'\xa2' : 'c',
        b'\xa3' : 'GBP',
        b'\xa4' : '$', #This approximation is especially parochial--this is the
                       #generic currency symbol.
        b'\xa5' : 'YEN',
        b'\xa6' : '|',
        b'\xa7' : 'S',
        b'\xa8' : '..',
        b'\xa9' : '',
        b'\xaa' : '(th)',
        b'\xab' : '<<',
        b'\xac' : '!',
        b'\xad' : ' ',
        b'\xae' : '(R)',
        b'\xaf' : '-',
        b'\xb0' : 'o',
        b'\xb1' : '+-',
        b'\xb2' : '2',
        b'\xb3' : '3',
        b'\xb4' : ("'", 'acute'),
        b'\xb5' : 'u',
        b'\xb6' : 'P',
        b'\xb7' : '*',
        b'\xb8' : ',',
        b'\xb9' : '1',
        b'\xba' : '(th)',
        b'\xbb' : '>>',
        b'\xbc' : '1/4',
        b'\xbd' : '1/2',
        b'\xbe' : '3/4',
        b'\xbf' : '?',
        b'\xc0' : 'A',
        b'\xc1' : 'A',
        b'\xc2' : 'A',
        b'\xc3' : 'A',
        b'\xc4' : 'A',
        b'\xc5' : 'A',
        b'\xc6' : 'AE',
        b'\xc7' : 'C',
        b'\xc8' : 'E',
        b'\xc9' : 'E',
        b'\xca' : 'E',
        b'\xcb' : 'E',
        b'\xcc' : 'I',
        b'\xcd' : 'I',
        b'\xce' : 'I',
        b'\xcf' : 'I',
        b'\xd0' : 'D',
        b'\xd1' : 'N',
        b'\xd2' : 'O',
        b'\xd3' : 'O',
        b'\xd4' : 'O',
        b'\xd5' : 'O',
        b'\xd6' : 'O',
        b'\xd7' : '*',
        b'\xd8' : 'O',
        b'\xd9' : 'U',
        b'\xda' : 'U',
        b'\xdb' : 'U',
        b'\xdc' : 'U',
        b'\xdd' : 'Y',
        b'\xde' : 'b',
        b'\xdf' : 'B',
        b'\xe0' : 'a',
        b'\xe1' : 'a',
        b'\xe2' : 'a',
        b'\xe3' : 'a',
        b'\xe4' : 'a',
        b'\xe5' : 'a',
        b'\xe6' : 'ae',
        b'\xe7' : 'c',
        b'\xe8' : 'e',
        b'\xe9' : 'e',
        b'\xea' : 'e',
        b'\xeb' : 'e',
        b'\xec' : 'i',
        b'\xed' : 'i',
        b'\xee' : 'i',
        b'\xef' : 'i',
        b'\xf0' : 'o',
        b'\xf1' : 'n',
        b'\xf2' : 'o',
        b'\xf3' : 'o',
        b'\xf4' : 'o',
        b'\xf5' : 'o',
        b'\xf6' : 'o',
        b'\xf7' : '/',
        b'\xf8' : 'o',
        b'\xf9' : 'u',
        b'\xfa' : 'u',
        b'\xfb' : 'u',
        b'\xfc' : 'u',
        b'\xfd' : 'y',
        b'\xfe' : 'b',
        b'\xff' : 'y',
        }

    # A map used when removing rogue Windows-1252/ISO-8859-1
    # characters in otherwise UTF-8 documents.
    #
    # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
    # Windows-1252.
    WINDOWS_1252_TO_UTF8 = {
        0x80 : b'\xe2\x82\xac', # €
        0x82 : b'\xe2\x80\x9a', # ‚
        0x83 : b'\xc6\x92',     # ƒ
        0x84 : b'\xe2\x80\x9e', # „
        0x85 : b'\xe2\x80\xa6', # …
        0x86 : b'\xe2\x80\xa0', # †
        0x87 : b'\xe2\x80\xa1', # ‡
        0x88 : b'\xcb\x86',     # ˆ
        0x89 : b'\xe2\x80\xb0', # ‰
        0x8a : b'\xc5\xa0',     # Š
        0x8b : b'\xe2\x80\xb9', # ‹
        0x8c : b'\xc5\x92',     # Œ
        0x8e : b'\xc5\xbd',     # Ž
        0x91 : b'\xe2\x80\x98', # ‘
        0x92 : b'\xe2\x80\x99', # ’
        0x93 : b'\xe2\x80\x9c', # “
        0x94 : b'\xe2\x80\x9d', # ”
        0x95 : b'\xe2\x80\xa2', # •
        0x96 : b'\xe2\x80\x93', # –
        0x97 : b'\xe2\x80\x94', # —
        0x98 : b'\xcb\x9c',     # ˜
        0x99 : b'\xe2\x84\xa2', # ™
        0x9a : b'\xc5\xa1',     # š
        0x9b : b'\xe2\x80\xba', # ›
        0x9c : b'\xc5\x93',     # œ
        0x9e : b'\xc5\xbe',     # ž
        0x9f : b'\xc5\xb8',     # Ÿ
        0xa0 : b'\xc2\xa0',     #  
        0xa1 : b'\xc2\xa1',     # ¡
        0xa2 : b'\xc2\xa2',     # ¢
        0xa3 : b'\xc2\xa3',     # £
        0xa4 : b'\xc2\xa4',     # ¤
        0xa5 : b'\xc2\xa5',     # ¥
        0xa6 : b'\xc2\xa6',     # ¦
        0xa7 : b'\xc2\xa7',     # §
        0xa8 : b'\xc2\xa8',     # ¨
        0xa9 : b'\xc2\xa9',     # ©
        0xaa : b'\xc2\xaa',     # ª
        0xab : b'\xc2\xab',     # «
        0xac : b'\xc2\xac',     # ¬
        0xad : b'\xc2\xad',     # ­
        0xae : b'\xc2\xae',     # ®
        0xaf : b'\xc2\xaf',     # ¯
        0xb0 : b'\xc2\xb0',     # °
        0xb1 : b'\xc2\xb1',     # ±
        0xb2 : b'\xc2\xb2',     # ²
        0xb3 : b'\xc2\xb3',     # ³
        0xb4 : b'\xc2\xb4',     # ´
        0xb5 : b'\xc2\xb5',     # µ
        0xb6 : b'\xc2\xb6',     # ¶
        0xb7 : b'\xc2\xb7',     # ·
        0xb8 : b'\xc2\xb8',     # ¸
        0xb9 : b'\xc2\xb9',     # ¹
        0xba : b'\xc2\xba',     # º
        0xbb : b'\xc2\xbb',     # »
        0xbc : b'\xc2\xbc',     # ¼
        0xbd : b'\xc2\xbd',     # ½
        0xbe : b'\xc2\xbe',     # ¾
        0xbf : b'\xc2\xbf',     # ¿
        0xc0 : b'\xc3\x80',     # À
        0xc1 : b'\xc3\x81',     # Á
        0xc2 : b'\xc3\x82',     # Â
        0xc3 : b'\xc3\x83',     # Ã
        0xc4 : b'\xc3\x84',     # Ä
        0xc5 : b'\xc3\x85',     # Å
        0xc6 : b'\xc3\x86',     # Æ
        0xc7 : b'\xc3\x87',     # Ç
        0xc8 : b'\xc3\x88',     # È
        0xc9 : b'\xc3\x89',     # É
        0xca : b'\xc3\x8a',     # Ê
        0xcb : b'\xc3\x8b',     # Ë
        0xcc : b'\xc3\x8c',     # Ì
        0xcd : b'\xc3\x8d',     # Í
        0xce : b'\xc3\x8e',     # Î
        0xcf : b'\xc3\x8f',     # Ï
        0xd0 : b'\xc3\x90',     # Ð
        0xd1 : b'\xc3\x91',     # Ñ
        0xd2 : b'\xc3\x92',     # Ò
        0xd3 : b'\xc3\x93',     # Ó
        0xd4 : b'\xc3\x94',     # Ô
        0xd5 : b'\xc3\x95',     # Õ
        0xd6 : b'\xc3\x96',     # Ö
        0xd7 : b'\xc3\x97',     # ×
        0xd8 : b'\xc3\x98',     # Ø
        0xd9 : b'\xc3\x99',     # Ù
        0xda : b'\xc3\x9a',     # Ú
        0xdb : b'\xc3\x9b',     # Û
        0xdc : b'\xc3\x9c',     # Ü
        0xdd : b'\xc3\x9d',     # Ý
        0xde : b'\xc3\x9e',     # Þ
        0xdf : b'\xc3\x9f',     # ß
        0xe0 : b'\xc3\xa0',     # à
        0xe1 : b'\xa1',         # á
        0xe2 : b'\xc3\xa2',     # â
        0xe3 : b'\xc3\xa3',     # ã
        0xe4 : b'\xc3\xa4',     # ä
        0xe5 : b'\xc3\xa5',     # å
        0xe6 : b'\xc3\xa6',     # æ
        0xe7 : b'\xc3\xa7',     # ç
        0xe8 : b'\xc3\xa8',     # è
        0xe9 : b'\xc3\xa9',     # é
        0xea : b'\xc3\xaa',     # ê
        0xeb : b'\xc3\xab',     # ë
        0xec : b'\xc3\xac',     # ì
        0xed : b'\xc3\xad',     # í
        0xee : b'\xc3\xae',     # î
        0xef : b'\xc3\xaf',     # ï
        0xf0 : b'\xc3\xb0',     # ð
        0xf1 : b'\xc3\xb1',     # ñ
        0xf2 : b'\xc3\xb2',     # ò
        0xf3 : b'\xc3\xb3',     # ó
        0xf4 : b'\xc3\xb4',     # ô
        0xf5 : b'\xc3\xb5',     # õ
        0xf6 : b'\xc3\xb6',     # ö
        0xf7 : b'\xc3\xb7',     # ÷
        0xf8 : b'\xc3\xb8',     # ø
        0xf9 : b'\xc3\xb9',     # ù
        0xfa : b'\xc3\xba',     # ú
        0xfb : b'\xc3\xbb',     # û
        0xfc : b'\xc3\xbc',     # ü
        0xfd : b'\xc3\xbd',     # ý
        0xfe : b'\xc3\xbe',     # þ
        }

    MULTIBYTE_MARKERS_AND_SIZES = [
        (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
        (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
        (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
        ]

    FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
    LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]

    @classmethod
    def detwingle(cls, in_bytes, main_encoding="utf8",
                  embedded_encoding="windows-1252"):
        """Fix characters from one encoding embedded in some other encoding.

        Currently the only situation supported is Windows-1252 (or its
        subset ISO-8859-1), embedded in UTF-8.

        :param in_bytes: A bytestring that you suspect contains
            characters from multiple encodings. Note that this _must_
            be a bytestring. If you've already converted the document
            to Unicode, you're too late.
        :param main_encoding: The primary encoding of `in_bytes`.
        :param embedded_encoding: The encoding that was used to embed characters
            in the main document.
        :return: A bytestring in which `embedded_encoding`
          characters have been converted to their `main_encoding`
          equivalents.
        """
        if embedded_encoding.replace('_', '-').lower() not in (
            'windows-1252', 'windows_1252'):
            raise NotImplementedError(
                "Windows-1252 and ISO-8859-1 are the only currently supported "
                "embedded encodings.")

        if main_encoding.lower() not in ('utf8', 'utf-8'):
            raise NotImplementedError(
                "UTF-8 is the only currently supported main encoding.")

        byte_chunks = []

        chunk_start = 0
        pos = 0
        while pos < len(in_bytes):
            byte = in_bytes[pos]
            if not isinstance(byte, int):
                # Python 2.x
                byte = ord(byte)
            if (byte >= cls.FIRST_MULTIBYTE_MARKER
                and byte <= cls.LAST_MULTIBYTE_MARKER):
                # This is the start of a UTF-8 multibyte character. Skip
                # to the end.
                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
                    if byte >= start and byte <= end:
                        pos += size
                        break
            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
                # We found a Windows-1252 character!
                # Save the string up to this point as a chunk.
                byte_chunks.append(in_bytes[chunk_start:pos])

                # Now translate the Windows-1252 character into UTF-8
                # and add it as another, one-byte chunk.
                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
                pos += 1
                chunk_start = pos
            else:
                # Go on to the next character.
                pos += 1
        if chunk_start == 0:
            # The string is unchanged.
            return in_bytes
        else:
            # Store the final chunk.
            byte_chunks.append(in_bytes[chunk_start:])
        return b''.join(byte_chunks)


env_pyrecodes/lib/python3.9/site-packages/bs4/dammit.py
NikolaBlagojevic/pyrecodes
"""Diagnostic functions, mainly for use when doing tech support."""

# Use of this source code is governed by the MIT license.
__license__ = "MIT"

import cProfile
from io import BytesIO
from html.parser import HTMLParser
import bs4
from bs4 import BeautifulSoup, __version__
from bs4.builder import builder_registry

import os
import pstats
import random
import tempfile
import time
import traceback
import sys
import cProfile

def diagnose(data):
    """Diagnostic suite for isolating common problems.

    :param data: A string containing markup that needs to be explained.
    :return: None; diagnostics are printed to standard output.
    """
    print(("Diagnostic running on Beautiful Soup %s" % __version__))
    print(("Python version %s" % sys.version))

    basic_parsers = ["html.parser", "html5lib", "lxml"]
    for name in basic_parsers:
        for builder in builder_registry.builders:
            if name in builder.features:
                break
        else:
            basic_parsers.remove(name)
            print((
                "I noticed that %s is not installed. Installing it may help." %
                name))

    if 'lxml' in basic_parsers:
        basic_parsers.append("lxml-xml")
        try:
            from lxml import etree
            print(("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))))
        except ImportError as e:
            print(
                "lxml is not installed or couldn't be imported.")


    if 'html5lib' in basic_parsers:
        try:
            import html5lib
            print(("Found html5lib version %s" % html5lib.__version__))
        except ImportError as e:
            print(
                "html5lib is not installed or couldn't be imported.")

    if hasattr(data, 'read'):
        data = data.read()

    for parser in basic_parsers:
        print(("Trying to parse your markup with %s" % parser))
        success = False
        try:
            soup = BeautifulSoup(data, features=parser)
            success = True
        except Exception as e:
            print(("%s could not parse the markup." % parser))
            traceback.print_exc()
        if success:
            print(("Here's what %s did with the markup:" % parser))
            print((soup.prettify()))

        print(("-" * 80))

def lxml_trace(data, html=True, **kwargs):
    """Print out the lxml events that occur during parsing.

    This lets you see how lxml parses a document when no Beautiful
    Soup code is running. You can use this to determine whether
    an lxml-specific problem is in Beautiful Soup's lxml tree builders
    or in lxml itself.

    :param data: Some markup.
    :param html: If True, markup will be parsed with lxml's HTML parser.
       if False, lxml's XML parser will be used.
    """
    from lxml import etree
    recover = kwargs.pop('recover', True)
    if isinstance(data, str):
        data = data.encode("utf8")
    reader = BytesIO(data)
    for event, element in etree.iterparse(
        reader, html=html, recover=recover, **kwargs
    ):
        print(("%s, %4s, %s" % (event, element.tag, element.text)))

class AnnouncingParser(HTMLParser):
    """Subclass of HTMLParser that announces parse events, without doing
    anything else.

    You can use this to get a picture of how html.parser sees a given
    document. The easiest way to do this is to call `htmlparser_trace`.
    """

    def _p(self, s):
        print(s)

    def handle_starttag(self, name, attrs):
        self._p("%s START" % name)

    def handle_endtag(self, name):
        self._p("%s END" % name)

    def handle_data(self, data):
        self._p("%s DATA" % data)

    def handle_charref(self, name):
        self._p("%s CHARREF" % name)

    def handle_entityref(self, name):
        self._p("%s ENTITYREF" % name)

    def handle_comment(self, data):
        self._p("%s COMMENT" % data)

    def handle_decl(self, data):
        self._p("%s DECL" % data)

    def unknown_decl(self, data):
        self._p("%s UNKNOWN-DECL" % data)

    def handle_pi(self, data):
        self._p("%s PI" % data)

def htmlparser_trace(data):
    """Print out the HTMLParser events that occur during parsing.

    This lets you see how HTMLParser parses a document when no
    Beautiful Soup code is running.

    :param data: Some markup.
    """
    parser = AnnouncingParser()
    parser.feed(data)

_vowels = "aeiou"
_consonants = "bcdfghjklmnpqrstvwxyz"

def rword(length=5):
    "Generate a random word-like string."
    s = ''
    for i in range(length):
        if i % 2 == 0:
            t = _consonants
        else:
            t = _vowels
        s += random.choice(t)
    return s

def rsentence(length=4):
    "Generate a random sentence-like string."
    return " ".join(rword(random.randint(4,9)) for i in range(length))
        
def rdoc(num_elements=1000):
    """Randomly generate an invalid HTML document."""
    tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
    elements = []
    for i in range(num_elements):
        choice = random.randint(0,3)
        if choice == 0:
            # New tag.
            tag_name = random.choice(tag_names)
            elements.append("<%s>" % tag_name)
        elif choice == 1:
            elements.append(rsentence(random.randint(1,4)))
        elif choice == 2:
            # Close a tag.
            tag_name = random.choice(tag_names)
            elements.append("</%s>" % tag_name)
    return "<html>" + "\n".join(elements) + "</html>"

def benchmark_parsers(num_elements=100000):
    """Very basic head-to-head performance benchmark."""
    print(("Comparative parser benchmark on Beautiful Soup %s" % __version__))
    data = rdoc(num_elements)
    print(("Generated a large invalid HTML document (%d bytes)." % len(data)))
    
    for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
        success = False
        try:
            a = time.time()
            soup = BeautifulSoup(data, parser)
            b = time.time()
            success = True
        except Exception as e:
            print(("%s could not parse the markup." % parser))
            traceback.print_exc()
        if success:
            print(("BS4+%s parsed the markup in %.2fs." % (parser, b-a)))

    from lxml import etree
    a = time.time()
    etree.HTML(data)
    b = time.time()
    print(("Raw lxml parsed the markup in %.2fs." % (b-a)))

    import html5lib
    parser = html5lib.HTMLParser()
    a = time.time()
    parser.parse(data)
    b = time.time()
    print(("Raw html5lib parsed the markup in %.2fs." % (b-a)))

def profile(num_elements=100000, parser="lxml"):
    """Use Python's profiler on a randomly generated document."""
    filehandle = tempfile.NamedTemporaryFile()
    filename = filehandle.name

    data = rdoc(num_elements)
    vars = dict(bs4=bs4, data=data, parser=parser)
    cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename)

    stats = pstats.Stats(filename)
    # stats.strip_dirs()
    stats.sort_stats("cumulative")
    stats.print_stats('_html5lib|bs4', 50)

# If this file is run as a script, standard input is diagnosed.
if __name__ == '__main__':
    diagnose(sys.stdin.read())

env_pyrecodes/lib/python3.9/site-packages/bs4/diagnose.py
NikolaBlagojevic/pyrecodes
# Use of this source code is governed by the MIT license.
__license__ = "MIT"

try:
    from collections.abc import Callable # Python 3.6
except ImportError as e:
    from collections import Callable
import re
import sys
import warnings

from bs4.css import CSS
from bs4.formatter import (
    Formatter,
    HTMLFormatter,
    XMLFormatter,
)

DEFAULT_OUTPUT_ENCODING = "utf-8"

nonwhitespace_re = re.compile(r"\S+")

# NOTE: This isn't used as of 4.7.0. I'm leaving it for a little bit on
# the off chance someone imported it for their own use.
whitespace_re = re.compile(r"\s+")

def _alias(attr):
    """Alias one attribute name to another for backward compatibility"""
    @property
    def alias(self):
        return getattr(self, attr)

    @alias.setter
    def alias(self):
        return setattr(self, attr)
    return alias


# These encodings are recognized by Python (so PageElement.encode
# could theoretically support them) but XML and HTML don't recognize
# them (so they should not show up in an XML or HTML document as that
# document's encoding).
#
# If an XML document is encoded in one of these encodings, no encoding
# will be mentioned in the XML declaration. If an HTML document is
# encoded in one of these encodings, and the HTML document has a
# <meta> tag that mentions an encoding, the encoding will be given as
# the empty string.
#
# Source:
# https://docs.python.org/3/library/codecs.html#python-specific-encodings
PYTHON_SPECIFIC_ENCODINGS = set([
    "idna",
    "mbcs",
    "oem",
    "palmos",
    "punycode",
    "raw_unicode_escape",
    "undefined",
    "unicode_escape",
    "raw-unicode-escape",
    "unicode-escape",
    "string-escape",
    "string_escape",
])


class NamespacedAttribute(str):
    """A namespaced string (e.g. 'xml:lang') that remembers the namespace
    ('xml') and the name ('lang') that were used to create it.
    """

    def __new__(cls, prefix, name=None, namespace=None):
        if not name:
            # This is the default namespace. Its name "has no value"
            # per https://www.w3.org/TR/xml-names/#defaulting
            name = None

        if not name:
            obj = str.__new__(cls, prefix)
        elif not prefix:
            # Not really namespaced.
            obj = str.__new__(cls, name)
        else:
            obj = str.__new__(cls, prefix + ":" + name)
        obj.prefix = prefix
        obj.name = name
        obj.namespace = namespace
        return obj

class AttributeValueWithCharsetSubstitution(str):
    """A stand-in object for a character encoding specified in HTML."""

class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution):
    """A generic stand-in for the value of a meta tag's 'charset' attribute.

    When Beautiful Soup parses the markup '<meta charset="utf8">', the
    value of the 'charset' attribute will be one of these objects.
    """

    def __new__(cls, original_value):
        obj = str.__new__(cls, original_value)
        obj.original_value = original_value
        return obj

    def encode(self, encoding):
        """When an HTML document is being encoded to a given encoding, the
        value of a meta tag's 'charset' is the name of the encoding.
        """
        if encoding in PYTHON_SPECIFIC_ENCODINGS:
            return ''
        return encoding


class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution):
    """A generic stand-in for the value of a meta tag's 'content' attribute.

    When Beautiful Soup parses the markup:
     <meta http-equiv="content-type" content="text/html; charset=utf8">

    The value of the 'content' attribute will be one of these objects.
    """

    CHARSET_RE = re.compile(r"((^|;)\s*charset=)([^;]*)", re.M)

    def __new__(cls, original_value):
        match = cls.CHARSET_RE.search(original_value)
        if match is None:
            # No substitution necessary.
            return str.__new__(str, original_value)

        obj = str.__new__(cls, original_value)
        obj.original_value = original_value
        return obj

    def encode(self, encoding):
        if encoding in PYTHON_SPECIFIC_ENCODINGS:
            return ''
        def rewrite(match):
            return match.group(1) + encoding
        return self.CHARSET_RE.sub(rewrite, self.original_value)


class PageElement(object):
    """Contains the navigational information for some part of the page:
    that is, its current location in the parse tree.

    NavigableString, Tag, etc. are all subclasses of PageElement.
    """

    # In general, we can't tell just by looking at an element whether
    # it's contained in an XML document or an HTML document. But for
    # Tags (q.v.) we can store this information at parse time.
    known_xml = None

    def setup(self, parent=None, previous_element=None, next_element=None,
              previous_sibling=None, next_sibling=None):
        """Sets up the initial relations between this element and
        other elements.

        :param parent: The parent of this element.

        :param previous_element: The element parsed immediately before
            this one.

        :param next_element: The element parsed immediately before
            this one.

        :param previous_sibling: The most recently encountered element
            on the same level of the parse tree as this one.

        :param previous_sibling: The next element to be encountered
            on the same level of the parse tree as this one.
        """
        self.parent = parent

        self.previous_element = previous_element
        if previous_element is not None:
            self.previous_element.next_element = self

        self.next_element = next_element
        if self.next_element is not None:
            self.next_element.previous_element = self

        self.next_sibling = next_sibling
        if self.next_sibling is not None:
            self.next_sibling.previous_sibling = self

        if (previous_sibling is None
            and self.parent is not None and self.parent.contents):
            previous_sibling = self.parent.contents[-1]

        self.previous_sibling = previous_sibling
        if previous_sibling is not None:
            self.previous_sibling.next_sibling = self

    def format_string(self, s, formatter):
        """Format the given string using the given formatter.

        :param s: A string.
        :param formatter: A Formatter object, or a string naming one of the standard formatters.
        """
        if formatter is None:
            return s
        if not isinstance(formatter, Formatter):
            formatter = self.formatter_for_name(formatter)
        output = formatter.substitute(s)
        return output

    def formatter_for_name(self, formatter):
        """Look up or create a Formatter for the given identifier,
        if necessary.

        :param formatter: Can be a Formatter object (used as-is), a
            function (used as the entity substitution hook for an
            XMLFormatter or HTMLFormatter), or a string (used to look
            up an XMLFormatter or HTMLFormatter in the appropriate
            registry.
        """
        if isinstance(formatter, Formatter):
            return formatter
        if self._is_xml:
            c = XMLFormatter
        else:
            c = HTMLFormatter
        if isinstance(formatter, Callable):
            return c(entity_substitution=formatter)
        return c.REGISTRY[formatter]

    @property
    def _is_xml(self):
        """Is this element part of an XML tree or an HTML tree?

        This is used in formatter_for_name, when deciding whether an
        XMLFormatter or HTMLFormatter is more appropriate. It can be
        inefficient, but it should be called very rarely.
        """
        if self.known_xml is not None:
            # Most of the time we will have determined this when the
            # document is parsed.
            return self.known_xml

        # Otherwise, it's likely that this element was created by
        # direct invocation of the constructor from within the user's
        # Python code.
        if self.parent is None:
            # This is the top-level object. It should have .known_xml set
            # from tree creation. If not, take a guess--BS is usually
            # used on HTML markup.
            return getattr(self, 'is_xml', False)
        return self.parent._is_xml

    nextSibling = _alias("next_sibling")  # BS3
    previousSibling = _alias("previous_sibling")  # BS3

    default = object()
    def _all_strings(self, strip=False, types=default):
        """Yield all strings of certain classes, possibly stripping them.

        This is implemented differently in Tag and NavigableString.
        """
        raise NotImplementedError()

    @property
    def stripped_strings(self):
        """Yield all strings in this PageElement, stripping them first.

        :yield: A sequence of stripped strings.
        """
        for string in self._all_strings(True):
            yield string

    def get_text(self, separator="", strip=False,
                 types=default):
        """Get all child strings of this PageElement, concatenated using the
        given separator.

        :param separator: Strings will be concatenated using this separator.

        :param strip: If True, strings will be stripped before being
            concatenated.

        :param types: A tuple of NavigableString subclasses. Any
            strings of a subclass not found in this list will be
            ignored. Although there are exceptions, the default
            behavior in most cases is to consider only NavigableString
            and CData objects. That means no comments, processing
            instructions, etc.

        :return: A string.
        """
        return separator.join([s for s in self._all_strings(
                    strip, types=types)])
    getText = get_text
    text = property(get_text)

    def replace_with(self, *args):
        """Replace this PageElement with one or more PageElements, keeping the
        rest of the tree the same.

        :param args: One or more PageElements.
        :return: `self`, no longer part of the tree.
        """
        if self.parent is None:
            raise ValueError(
                "Cannot replace one element with another when the "
                "element to be replaced is not part of a tree.")
        if len(args) == 1 and args[0] is self:
            return
        if any(x is self.parent for x in args):
            raise ValueError("Cannot replace a Tag with its parent.")
        old_parent = self.parent
        my_index = self.parent.index(self)
        self.extract(_self_index=my_index)
        for idx, replace_with in enumerate(args, start=my_index):
            old_parent.insert(idx, replace_with)
        return self
    replaceWith = replace_with  # BS3

    def unwrap(self):
        """Replace this PageElement with its contents.

        :return: `self`, no longer part of the tree.
        """
        my_parent = self.parent
        if self.parent is None:
            raise ValueError(
                "Cannot replace an element with its contents when that"
                "element is not part of a tree.")
        my_index = self.parent.index(self)
        self.extract(_self_index=my_index)
        for child in reversed(self.contents[:]):
            my_parent.insert(my_index, child)
        return self
    replace_with_children = unwrap
    replaceWithChildren = unwrap  # BS3

    def wrap(self, wrap_inside):
        """Wrap this PageElement inside another one.

        :param wrap_inside: A PageElement.
        :return: `wrap_inside`, occupying the position in the tree that used
           to be occupied by `self`, and with `self` inside it.
        """
        me = self.replace_with(wrap_inside)
        wrap_inside.append(me)
        return wrap_inside

    def extract(self, _self_index=None):
        """Destructively rips this element out of the tree.

        :param _self_index: The location of this element in its parent's
           .contents, if known. Passing this in allows for a performance
           optimization.

        :return: `self`, no longer part of the tree.
        """
        if self.parent is not None:
            if _self_index is None:
                _self_index = self.parent.index(self)
            del self.parent.contents[_self_index]

        #Find the two elements that would be next to each other if
        #this element (and any children) hadn't been parsed. Connect
        #the two.
        last_child = self._last_descendant()
        next_element = last_child.next_element

        if (self.previous_element is not None and
            self.previous_element is not next_element):
            self.previous_element.next_element = next_element
        if next_element is not None and next_element is not self.previous_element:
            next_element.previous_element = self.previous_element
        self.previous_element = None
        last_child.next_element = None

        self.parent = None
        if (self.previous_sibling is not None
            and self.previous_sibling is not self.next_sibling):
            self.previous_sibling.next_sibling = self.next_sibling
        if (self.next_sibling is not None
            and self.next_sibling is not self.previous_sibling):
            self.next_sibling.previous_sibling = self.previous_sibling
        self.previous_sibling = self.next_sibling = None
        return self

    def _last_descendant(self, is_initialized=True, accept_self=True):
        """Finds the last element beneath this object to be parsed.

        :param is_initialized: Has `setup` been called on this PageElement
            yet?
        :param accept_self: Is `self` an acceptable answer to the question?
        """
        if is_initialized and self.next_sibling is not None:
            last_child = self.next_sibling.previous_element
        else:
            last_child = self
            while isinstance(last_child, Tag) and last_child.contents:
                last_child = last_child.contents[-1]
        if not accept_self and last_child is self:
            last_child = None
        return last_child
    # BS3: Not part of the API!
    _lastRecursiveChild = _last_descendant

    def insert(self, position, new_child):
        """Insert a new PageElement in the list of this PageElement's children.

        This works the same way as `list.insert`.

        :param position: The numeric position that should be occupied
           in `self.children` by the new PageElement.
        :param new_child: A PageElement.
        """
        if new_child is None:
            raise ValueError("Cannot insert None into a tag.")
        if new_child is self:
            raise ValueError("Cannot insert a tag into itself.")
        if (isinstance(new_child, str)
            and not isinstance(new_child, NavigableString)):
            new_child = NavigableString(new_child)

        from bs4 import BeautifulSoup
        if isinstance(new_child, BeautifulSoup):
            # We don't want to end up with a situation where one BeautifulSoup
            # object contains another. Insert the children one at a time.
            for subchild in list(new_child.contents):
                self.insert(position, subchild)
                position += 1
            return
        position = min(position, len(self.contents))
        if hasattr(new_child, 'parent') and new_child.parent is not None:
            # We're 'inserting' an element that's already one
            # of this object's children.
            if new_child.parent is self:
                current_index = self.index(new_child)
                if current_index < position:
                    # We're moving this element further down the list
                    # of this object's children. That means that when
                    # we extract this element, our target index will
                    # jump down one.
                    position -= 1
            new_child.extract()

        new_child.parent = self
        previous_child = None
        if position == 0:
            new_child.previous_sibling = None
            new_child.previous_element = self
        else:
            previous_child = self.contents[position - 1]
            new_child.previous_sibling = previous_child
            new_child.previous_sibling.next_sibling = new_child
            new_child.previous_element = previous_child._last_descendant(False)
        if new_child.previous_element is not None:
            new_child.previous_element.next_element = new_child

        new_childs_last_element = new_child._last_descendant(False)

        if position >= len(self.contents):
            new_child.next_sibling = None

            parent = self
            parents_next_sibling = None
            while parents_next_sibling is None and parent is not None:
                parents_next_sibling = parent.next_sibling
                parent = parent.parent
                if parents_next_sibling is not None:
                    # We found the element that comes next in the document.
                    break
            if parents_next_sibling is not None:
                new_childs_last_element.next_element = parents_next_sibling
            else:
                # The last element of this tag is the last element in
                # the document.
                new_childs_last_element.next_element = None
        else:
            next_child = self.contents[position]
            new_child.next_sibling = next_child
            if new_child.next_sibling is not None:
                new_child.next_sibling.previous_sibling = new_child
            new_childs_last_element.next_element = next_child

        if new_childs_last_element.next_element is not None:
            new_childs_last_element.next_element.previous_element = new_childs_last_element
        self.contents.insert(position, new_child)

    def append(self, tag):
        """Appends the given PageElement to the contents of this one.

        :param tag: A PageElement.
        """
        self.insert(len(self.contents), tag)

    def extend(self, tags):
        """Appends the given PageElements to this one's contents.

        :param tags: A list of PageElements. If a single Tag is
            provided instead, this PageElement's contents will be extended
            with that Tag's contents.
        """
        if isinstance(tags, Tag):
            tags = tags.contents
        if isinstance(tags, list):
            # Moving items around the tree may change their position in
            # the original list. Make a list that won't change.
            tags = list(tags)
        for tag in tags:
            self.append(tag)

    def insert_before(self, *args):
        """Makes the given element(s) the immediate predecessor of this one.

        All the elements will have the same parent, and the given elements
        will be immediately before this one.

        :param args: One or more PageElements.
        """
        parent = self.parent
        if parent is None:
            raise ValueError(
                "Element has no parent, so 'before' has no meaning.")
        if any(x is self for x in args):
                raise ValueError("Can't insert an element before itself.")
        for predecessor in args:
            # Extract first so that the index won't be screwed up if they
            # are siblings.
            if isinstance(predecessor, PageElement):
                predecessor.extract()
            index = parent.index(self)
            parent.insert(index, predecessor)

    def insert_after(self, *args):
        """Makes the given element(s) the immediate successor of this one.

        The elements will have the same parent, and the given elements
        will be immediately after this one.

        :param args: One or more PageElements.
        """
        # Do all error checking before modifying the tree.
        parent = self.parent
        if parent is None:
            raise ValueError(
                "Element has no parent, so 'after' has no meaning.")
        if any(x is self for x in args):
            raise ValueError("Can't insert an element after itself.")

        offset = 0
        for successor in args:
            # Extract first so that the index won't be screwed up if they
            # are siblings.
            if isinstance(successor, PageElement):
                successor.extract()
            index = parent.index(self)
            parent.insert(index+1+offset, successor)
            offset += 1

    def find_next(self, name=None, attrs={}, string=None, **kwargs):
        """Find the first PageElement that matches the given criteria and
        appears later in the document than this PageElement.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :kwargs: A dictionary of filters on attribute values.
        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        return self._find_one(self.find_all_next, name, attrs, string, **kwargs)
    findNext = find_next  # BS3

    def find_all_next(self, name=None, attrs={}, string=None, limit=None,
                    **kwargs):
        """Find all PageElements that match the given criteria and appear
        later in the document than this PageElement.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.
        :return: A ResultSet containing PageElements.
        """
        _stacklevel = kwargs.pop('_stacklevel', 2)
        return self._find_all(name, attrs, string, limit, self.next_elements,
                              _stacklevel=_stacklevel+1, **kwargs)
    findAllNext = find_all_next  # BS3

    def find_next_sibling(self, name=None, attrs={}, string=None, **kwargs):
        """Find the closest sibling to this PageElement that matches the
        given criteria and appears later in the document.

        All find_* methods take a common set of arguments. See the
        online documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :kwargs: A dictionary of filters on attribute values.
        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        return self._find_one(self.find_next_siblings, name, attrs, string,
                             **kwargs)
    findNextSibling = find_next_sibling  # BS3

    def find_next_siblings(self, name=None, attrs={}, string=None, limit=None,
                           **kwargs):
        """Find all siblings of this PageElement that match the given criteria
        and appear later in the document.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.
        :return: A ResultSet of PageElements.
        :rtype: bs4.element.ResultSet
        """
        _stacklevel = kwargs.pop('_stacklevel', 2)
        return self._find_all(
            name, attrs, string, limit,
            self.next_siblings, _stacklevel=_stacklevel+1, **kwargs
        )
    findNextSiblings = find_next_siblings   # BS3
    fetchNextSiblings = find_next_siblings  # BS2

    def find_previous(self, name=None, attrs={}, string=None, **kwargs):
        """Look backwards in the document from this PageElement and find the
        first PageElement that matches the given criteria.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :kwargs: A dictionary of filters on attribute values.
        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        return self._find_one(
            self.find_all_previous, name, attrs, string, **kwargs)
    findPrevious = find_previous  # BS3

    def find_all_previous(self, name=None, attrs={}, string=None, limit=None,
                        **kwargs):
        """Look backwards in the document from this PageElement and find all
        PageElements that match the given criteria.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.
        :return: A ResultSet of PageElements.
        :rtype: bs4.element.ResultSet
        """
        _stacklevel = kwargs.pop('_stacklevel', 2)
        return self._find_all(
            name, attrs, string, limit, self.previous_elements,
            _stacklevel=_stacklevel+1, **kwargs
        )
    findAllPrevious = find_all_previous  # BS3
    fetchPrevious = find_all_previous    # BS2

    def find_previous_sibling(self, name=None, attrs={}, string=None, **kwargs):
        """Returns the closest sibling to this PageElement that matches the
        given criteria and appears earlier in the document.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :kwargs: A dictionary of filters on attribute values.
        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        return self._find_one(self.find_previous_siblings, name, attrs, string,
                             **kwargs)
    findPreviousSibling = find_previous_sibling  # BS3

    def find_previous_siblings(self, name=None, attrs={}, string=None,
                               limit=None, **kwargs):
        """Returns all siblings to this PageElement that match the
        given criteria and appear earlier in the document.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.
        :return: A ResultSet of PageElements.
        :rtype: bs4.element.ResultSet
        """
        _stacklevel = kwargs.pop('_stacklevel', 2)
        return self._find_all(
            name, attrs, string, limit,
            self.previous_siblings, _stacklevel=_stacklevel+1, **kwargs
        )
    findPreviousSiblings = find_previous_siblings   # BS3
    fetchPreviousSiblings = find_previous_siblings  # BS2

    def find_parent(self, name=None, attrs={}, **kwargs):
        """Find the closest parent of this PageElement that matches the given
        criteria.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :kwargs: A dictionary of filters on attribute values.

        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        # NOTE: We can't use _find_one because findParents takes a different
        # set of arguments.
        r = None
        l = self.find_parents(name, attrs, 1, _stacklevel=3, **kwargs)
        if l:
            r = l[0]
        return r
    findParent = find_parent  # BS3

    def find_parents(self, name=None, attrs={}, limit=None, **kwargs):
        """Find all parents of this PageElement that match the given criteria.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.

        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        _stacklevel = kwargs.pop('_stacklevel', 2)
        return self._find_all(name, attrs, None, limit, self.parents,
                              _stacklevel=_stacklevel+1, **kwargs)
    findParents = find_parents   # BS3
    fetchParents = find_parents  # BS2

    @property
    def next(self):
        """The PageElement, if any, that was parsed just after this one.

        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        return self.next_element

    @property
    def previous(self):
        """The PageElement, if any, that was parsed just before this one.

        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        return self.previous_element

    #These methods do the real heavy lifting.

    def _find_one(self, method, name, attrs, string, **kwargs):
        r = None
        l = method(name, attrs, string, 1, _stacklevel=4, **kwargs)
        if l:
            r = l[0]
        return r

    def _find_all(self, name, attrs, string, limit, generator, **kwargs):
        "Iterates over a generator looking for things that match."
        _stacklevel = kwargs.pop('_stacklevel', 3)

        if string is None and 'text' in kwargs:
            string = kwargs.pop('text')
            warnings.warn(
                "The 'text' argument to find()-type methods is deprecated. Use 'string' instead.",
                DeprecationWarning, stacklevel=_stacklevel
            )

        if isinstance(name, SoupStrainer):
            strainer = name
        else:
            strainer = SoupStrainer(name, attrs, string, **kwargs)

        if string is None and not limit and not attrs and not kwargs:
            if name is True or name is None:
                # Optimization to find all tags.
                result = (element for element in generator
                          if isinstance(element, Tag))
                return ResultSet(strainer, result)
            elif isinstance(name, str):
                # Optimization to find all tags with a given name.
                if name.count(':') == 1:
                    # This is a name with a prefix. If this is a namespace-aware document,
                    # we need to match the local name against tag.name. If not,
                    # we need to match the fully-qualified name against tag.name.
                    prefix, local_name = name.split(':', 1)
                else:
                    prefix = None
                    local_name = name
                result = (element for element in generator
                          if isinstance(element, Tag)
                          and (
                              element.name == name
                          ) or (
                              element.name == local_name
                              and (prefix is None or element.prefix == prefix)
                          )
                )
                return ResultSet(strainer, result)
        results = ResultSet(strainer)
        while True:
            try:
                i = next(generator)
            except StopIteration:
                break
            if i:
                found = strainer.search(i)
                if found:
                    results.append(found)
                    if limit and len(results) >= limit:
                        break
        return results

    #These generators can be used to navigate starting from both
    #NavigableStrings and Tags.
    @property
    def next_elements(self):
        """All PageElements that were parsed after this one.

        :yield: A sequence of PageElements.
        """
        i = self.next_element
        while i is not None:
            yield i
            i = i.next_element

    @property
    def next_siblings(self):
        """All PageElements that are siblings of this one but were parsed
        later.

        :yield: A sequence of PageElements.
        """
        i = self.next_sibling
        while i is not None:
            yield i
            i = i.next_sibling

    @property
    def previous_elements(self):
        """All PageElements that were parsed before this one.

        :yield: A sequence of PageElements.
        """
        i = self.previous_element
        while i is not None:
            yield i
            i = i.previous_element

    @property
    def previous_siblings(self):
        """All PageElements that are siblings of this one but were parsed
        earlier.

        :yield: A sequence of PageElements.
        """
        i = self.previous_sibling
        while i is not None:
            yield i
            i = i.previous_sibling

    @property
    def parents(self):
        """All PageElements that are parents of this PageElement.

        :yield: A sequence of PageElements.
        """
        i = self.parent
        while i is not None:
            yield i
            i = i.parent

    @property
    def decomposed(self):
        """Check whether a PageElement has been decomposed.

        :rtype: bool
        """
        return getattr(self, '_decomposed', False) or False
   
    # Old non-property versions of the generators, for backwards
    # compatibility with BS3.
    def nextGenerator(self):
        return self.next_elements

    def nextSiblingGenerator(self):
        return self.next_siblings

    def previousGenerator(self):
        return self.previous_elements

    def previousSiblingGenerator(self):
        return self.previous_siblings

    def parentGenerator(self):
        return self.parents


class NavigableString(str, PageElement):
    """A Python Unicode string that is part of a parse tree.

    When Beautiful Soup parses the markup <b>penguin</b>, it will
    create a NavigableString for the string "penguin".
    """

    PREFIX = ''
    SUFFIX = ''

    def __new__(cls, value):
        """Create a new NavigableString.

        When unpickling a NavigableString, this method is called with
        the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be
        passed in to the superclass's __new__ or the superclass won't know
        how to handle non-ASCII characters.
        """
        if isinstance(value, str):
            u = str.__new__(cls, value)
        else:
            u = str.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
        u.setup()
        return u

    def __deepcopy__(self, memo, recursive=False):
        """A copy of a NavigableString has the same contents and class
        as the original, but it is not connected to the parse tree.

        :param recursive: This parameter is ignored; it's only defined
           so that NavigableString.__deepcopy__ implements the same
           signature as Tag.__deepcopy__.
        """
        return type(self)(self)

    def __copy__(self):
        """A copy of a NavigableString can only be a deep copy, because
        only one PageElement can occupy a given place in a parse tree.
        """
        return self.__deepcopy__({})

    def __getnewargs__(self):
        return (str(self),)

    def __getattr__(self, attr):
        """text.string gives you text. This is for backwards
        compatibility for Navigable*String, but for CData* it lets you
        get the string without the CData wrapper."""
        if attr == 'string':
            return self
        else:
            raise AttributeError(
                "'%s' object has no attribute '%s'" % (
                    self.__class__.__name__, attr))

    def output_ready(self, formatter="minimal"):
        """Run the string through the provided formatter.

        :param formatter: A Formatter object, or a string naming one of the standard formatters.
        """
        output = self.format_string(self, formatter)
        return self.PREFIX + output + self.SUFFIX

    @property
    def name(self):
        """Since a NavigableString is not a Tag, it has no .name.

        This property is implemented so that code like this doesn't crash
        when run on a mixture of Tag and NavigableString objects:
            [x.name for x in tag.children]
        """
        return None

    @name.setter
    def name(self, name):
        """Prevent NavigableString.name from ever being set."""
        raise AttributeError("A NavigableString cannot be given a name.")

    def _all_strings(self, strip=False, types=PageElement.default):
        """Yield all strings of certain classes, possibly stripping them.

        This makes it easy for NavigableString to implement methods
        like get_text() as conveniences, creating a consistent
        text-extraction API across all PageElements.

        :param strip: If True, all strings will be stripped before being
            yielded.

        :param types: A tuple of NavigableString subclasses. If this
            NavigableString isn't one of those subclasses, the
            sequence will be empty. By default, the subclasses
            considered are NavigableString and CData objects. That
            means no comments, processing instructions, etc.

        :yield: A sequence that either contains this string, or is empty.

        """
        if types is self.default:
            # This is kept in Tag because it's full of subclasses of
            # this class, which aren't defined until later in the file.
            types = Tag.DEFAULT_INTERESTING_STRING_TYPES

        # Do nothing if the caller is looking for specific types of
        # string, and we're of a different type.
        #
        # We check specific types instead of using isinstance(self,
        # types) because all of these classes subclass
        # NavigableString. Anyone who's using this feature probably
        # wants generic NavigableStrings but not other stuff.
        my_type = type(self)
        if types is not None:
            if isinstance(types, type):
                # Looking for a single type.
                if my_type is not types:
                    return
            elif my_type not in types:
                # Looking for one of a list of types.
                return

        value = self
        if strip:
            value = value.strip()
        if len(value) > 0:
            yield value
    strings = property(_all_strings)

class PreformattedString(NavigableString):
    """A NavigableString not subject to the normal formatting rules.

    This is an abstract class used for special kinds of strings such
    as comments (the Comment class) and CDATA blocks (the CData
    class).
    """

    PREFIX = ''
    SUFFIX = ''

    def output_ready(self, formatter=None):
        """Make this string ready for output by adding any subclass-specific
            prefix or suffix.

        :param formatter: A Formatter object, or a string naming one
            of the standard formatters. The string will be passed into the
            Formatter, but only to trigger any side effects: the return
            value is ignored.

        :return: The string, with any subclass-specific prefix and
           suffix added on.
        """
        if formatter is not None:
            ignore = self.format_string(self, formatter)
        return self.PREFIX + self + self.SUFFIX

class CData(PreformattedString):
    """A CDATA block."""
    PREFIX = '<![CDATA['
    SUFFIX = ']]>'

class ProcessingInstruction(PreformattedString):
    """A SGML processing instruction."""

    PREFIX = '<?'
    SUFFIX = '>'

class XMLProcessingInstruction(ProcessingInstruction):
    """An XML processing instruction."""
    PREFIX = '<?'
    SUFFIX = '?>'

class Comment(PreformattedString):
    """An HTML or XML comment."""
    PREFIX = '<!--'
    SUFFIX = '-->'


class Declaration(PreformattedString):
    """An XML declaration."""
    PREFIX = '<?'
    SUFFIX = '?>'


class Doctype(PreformattedString):
    """A document type declaration."""
    @classmethod
    def for_name_and_ids(cls, name, pub_id, system_id):
        """Generate an appropriate document type declaration for a given
        public ID and system ID.

        :param name: The name of the document's root element, e.g. 'html'.
        :param pub_id: The Formal Public Identifier for this document type,
            e.g. '-//W3C//DTD XHTML 1.1//EN'
        :param system_id: The system identifier for this document type,
            e.g. 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'

        :return: A Doctype.
        """
        value = name or ''
        if pub_id is not None:
            value += ' PUBLIC "%s"' % pub_id
            if system_id is not None:
                value += ' "%s"' % system_id
        elif system_id is not None:
            value += ' SYSTEM "%s"' % system_id

        return Doctype(value)

    PREFIX = '<!DOCTYPE '
    SUFFIX = '>\n'


class Stylesheet(NavigableString):
    """A NavigableString representing an stylesheet (probably
    CSS).

    Used to distinguish embedded stylesheets from textual content.
    """
    pass


class Script(NavigableString):
    """A NavigableString representing an executable script (probably
    Javascript).

    Used to distinguish executable code from textual content.
    """
    pass


class TemplateString(NavigableString):
    """A NavigableString representing a string found inside an HTML
    template embedded in a larger document.

    Used to distinguish such strings from the main body of the document.
    """
    pass


class RubyTextString(NavigableString):
    """A NavigableString representing the contents of the <rt> HTML
    element.

    https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rt-element

    Can be used to distinguish such strings from the strings they're
    annotating.
    """
    pass


class RubyParenthesisString(NavigableString):
    """A NavigableString representing the contents of the <rp> HTML
    element.

    https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rp-element
    """
    pass


class Tag(PageElement):
    """Represents an HTML or XML tag that is part of a parse tree, along
    with its attributes and contents.

    When Beautiful Soup parses the markup <b>penguin</b>, it will
    create a Tag object representing the <b> tag.
    """

    def __init__(self, parser=None, builder=None, name=None, namespace=None,
                 prefix=None, attrs=None, parent=None, previous=None,
                 is_xml=None, sourceline=None, sourcepos=None,
                 can_be_empty_element=None, cdata_list_attributes=None,
                 preserve_whitespace_tags=None,
                 interesting_string_types=None,
                 namespaces=None
    ):
        """Basic constructor.

        :param parser: A BeautifulSoup object.
        :param builder: A TreeBuilder.
        :param name: The name of the tag.
        :param namespace: The URI of this Tag's XML namespace, if any.
        :param prefix: The prefix for this Tag's XML namespace, if any.
        :param attrs: A dictionary of this Tag's attribute values.
        :param parent: The PageElement to use as this Tag's parent.
        :param previous: The PageElement that was parsed immediately before
            this tag.
        :param is_xml: If True, this is an XML tag. Otherwise, this is an
            HTML tag.
        :param sourceline: The line number where this tag was found in its
            source document.
        :param sourcepos: The character position within `sourceline` where this
            tag was found.
        :param can_be_empty_element: If True, this tag should be
            represented as <tag/>. If False, this tag should be represented
            as <tag></tag>.
        :param cdata_list_attributes: A list of attributes whose values should
            be treated as CDATA if they ever show up on this tag.
        :param preserve_whitespace_tags: A list of tag names whose contents
            should have their whitespace preserved.
        :param interesting_string_types: This is a NavigableString
            subclass or a tuple of them. When iterating over this
            Tag's strings in methods like Tag.strings or Tag.get_text,
            these are the types of strings that are interesting enough
            to be considered. The default is to consider
            NavigableString and CData the only interesting string
            subtypes.
        :param namespaces: A dictionary mapping currently active
            namespace prefixes to URIs. This can be used later to
            construct CSS selectors.
        """
        if parser is None:
            self.parser_class = None
        else:
            # We don't actually store the parser object: that lets extracted
            # chunks be garbage-collected.
            self.parser_class = parser.__class__
        if name is None:
            raise ValueError("No value provided for new tag's name.")
        self.name = name
        self.namespace = namespace
        self._namespaces = namespaces or {}
        self.prefix = prefix
        if ((not builder or builder.store_line_numbers)
            and (sourceline is not None or sourcepos is not None)):
            self.sourceline = sourceline
            self.sourcepos = sourcepos
        if attrs is None:
            attrs = {}
        elif attrs:
            if builder is not None and builder.cdata_list_attributes:
                attrs = builder._replace_cdata_list_attribute_values(
                    self.name, attrs)
            else:
                attrs = dict(attrs)
        else:
            attrs = dict(attrs)

        # If possible, determine ahead of time whether this tag is an
        # XML tag.
        if builder:
            self.known_xml = builder.is_xml
        else:
            self.known_xml = is_xml
        self.attrs = attrs
        self.contents = []
        self.setup(parent, previous)
        self.hidden = False

        if builder is None:
            # In the absence of a TreeBuilder, use whatever values were
            # passed in here. They're probably None, unless this is a copy of some
            # other tag.
            self.can_be_empty_element = can_be_empty_element
            self.cdata_list_attributes = cdata_list_attributes
            self.preserve_whitespace_tags = preserve_whitespace_tags
            self.interesting_string_types = interesting_string_types
        else:
            # Set up any substitutions for this tag, such as the charset in a META tag.
            builder.set_up_substitutions(self)

            # Ask the TreeBuilder whether this tag might be an empty-element tag.
            self.can_be_empty_element = builder.can_be_empty_element(name)

            # Keep track of the list of attributes of this tag that
            # might need to be treated as a list.
            #
            # For performance reasons, we store the whole data structure
            # rather than asking the question of every tag. Asking would
            # require building a new data structure every time, and
            # (unlike can_be_empty_element), we almost never need
            # to check this.
            self.cdata_list_attributes = builder.cdata_list_attributes

            # Keep track of the names that might cause this tag to be treated as a
            # whitespace-preserved tag.
            self.preserve_whitespace_tags = builder.preserve_whitespace_tags

            if self.name in builder.string_containers:
                # This sort of tag uses a special string container
                # subclass for most of its strings. When we ask the
                self.interesting_string_types = builder.string_containers[self.name]
            else:
                self.interesting_string_types = self.DEFAULT_INTERESTING_STRING_TYPES

    parserClass = _alias("parser_class")  # BS3

    def __deepcopy__(self, memo, recursive=True):
        """A deepcopy of a Tag is a new Tag, unconnected to the parse tree.
        Its contents are a copy of the old Tag's contents.
        """
        clone = self._clone()

        if recursive:
            # Clone this tag's descendants recursively, but without
            # making any recursive function calls.
            tag_stack = [clone]
            for event, element in self._event_stream(self.descendants):
                if event is Tag.END_ELEMENT_EVENT:
                    # Stop appending incoming Tags to the Tag that was
                    # just closed.
                    tag_stack.pop()
                else:
                    descendant_clone = element.__deepcopy__(
                        memo, recursive=False
                    )
                    # Add to its parent's .contents
                    tag_stack[-1].append(descendant_clone)

                    if event is Tag.START_ELEMENT_EVENT:
                        # Add the Tag itself to the stack so that its
                        # children will be .appended to it.
                        tag_stack.append(descendant_clone)
        return clone

    def __copy__(self):
        """A copy of a Tag must always be a deep copy, because a Tag's
        children can only have one parent at a time.
        """
        return self.__deepcopy__({})

    def _clone(self):
        """Create a new Tag just like this one, but with no
        contents and unattached to any parse tree.

        This is the first step in the deepcopy process.
        """
        clone = type(self)(
            None, self.builder, self.name, self.namespace,
            self.prefix, self.attrs, is_xml=self._is_xml,
            sourceline=self.sourceline, sourcepos=self.sourcepos,
            can_be_empty_element=self.can_be_empty_element,
            cdata_list_attributes=self.cdata_list_attributes,
            preserve_whitespace_tags=self.preserve_whitespace_tags,
            interesting_string_types=self.interesting_string_types
        )
        for attr in ('can_be_empty_element', 'hidden'):
            setattr(clone, attr, getattr(self, attr))
        return clone
    
    @property
    def is_empty_element(self):
        """Is this tag an empty-element tag? (aka a self-closing tag)

        A tag that has contents is never an empty-element tag.

        A tag that has no contents may or may not be an empty-element
        tag. It depends on the builder used to create the tag. If the
        builder has a designated list of empty-element tags, then only
        a tag whose name shows up in that list is considered an
        empty-element tag.

        If the builder has no designated list of empty-element tags,
        then any tag with no contents is an empty-element tag.
        """
        return len(self.contents) == 0 and self.can_be_empty_element
    isSelfClosing = is_empty_element  # BS3

    @property
    def string(self):
        """Convenience property to get the single string within this
        PageElement.

        TODO It might make sense to have NavigableString.string return
        itself.

        :return: If this element has a single string child, return
         value is that string. If this element has one child tag,
         return value is the 'string' attribute of the child tag,
         recursively. If this element is itself a string, has no
         children, or has more than one child, return value is None.
        """
        if len(self.contents) != 1:
            return None
        child = self.contents[0]
        if isinstance(child, NavigableString):
            return child
        return child.string

    @string.setter
    def string(self, string):
        """Replace this PageElement's contents with `string`."""
        self.clear()
        self.append(string.__class__(string))

    DEFAULT_INTERESTING_STRING_TYPES = (NavigableString, CData)
    def _all_strings(self, strip=False, types=PageElement.default):
        """Yield all strings of certain classes, possibly stripping them.

        :param strip: If True, all strings will be stripped before being
            yielded.

        :param types: A tuple of NavigableString subclasses. Any strings of
            a subclass not found in this list will be ignored. By
            default, the subclasses considered are the ones found in
            self.interesting_string_types. If that's not specified,
            only NavigableString and CData objects will be
            considered. That means no comments, processing
            instructions, etc.

        :yield: A sequence of strings.

        """
        if types is self.default:
            types = self.interesting_string_types

        for descendant in self.descendants:
            if (types is None and not isinstance(descendant, NavigableString)):
                continue
            descendant_type = type(descendant)
            if isinstance(types, type):
                if descendant_type is not types:
                    # We're not interested in strings of this type.
                    continue
            elif types is not None and descendant_type not in types:
                # We're not interested in strings of this type.
                continue
            if strip:
                descendant = descendant.strip()
                if len(descendant) == 0:
                    continue
            yield descendant
    strings = property(_all_strings)

    def decompose(self):
        """Recursively destroys this PageElement and its children.

        This element will be removed from the tree and wiped out; so
        will everything beneath it.

        The behavior of a decomposed PageElement is undefined and you
        should never use one for anything, but if you need to _check_
        whether an element has been decomposed, you can use the
        `decomposed` property.
        """
        self.extract()
        i = self
        while i is not None:
            n = i.next_element
            i.__dict__.clear()
            i.contents = []
            i._decomposed = True
            i = n

    def clear(self, decompose=False):
        """Wipe out all children of this PageElement by calling extract()
           on them.

        :param decompose: If this is True, decompose() (a more
            destructive method) will be called instead of extract().
        """
        if decompose:
            for element in self.contents[:]:
                if isinstance(element, Tag):
                    element.decompose()
                else:
                    element.extract()
        else:
            for element in self.contents[:]:
                element.extract()

    def smooth(self):
        """Smooth out this element's children by consolidating consecutive
        strings.

        This makes pretty-printed output look more natural following a
        lot of operations that modified the tree.
        """
        # Mark the first position of every pair of children that need
        # to be consolidated.  Do this rather than making a copy of
        # self.contents, since in most cases very few strings will be
        # affected.
        marked = []
        for i, a in enumerate(self.contents):
            if isinstance(a, Tag):
                # Recursively smooth children.
                a.smooth()
            if i == len(self.contents)-1:
                # This is the last item in .contents, and it's not a
                # tag. There's no chance it needs any work.
                continue
            b = self.contents[i+1]
            if (isinstance(a, NavigableString)
                and isinstance(b, NavigableString)
                and not isinstance(a, PreformattedString)
                and not isinstance(b, PreformattedString)
            ):
                marked.append(i)

        # Go over the marked positions in reverse order, so that
        # removing items from .contents won't affect the remaining
        # positions.
        for i in reversed(marked):
            a = self.contents[i]
            b = self.contents[i+1]
            b.extract()
            n = NavigableString(a+b)
            a.replace_with(n)

    def index(self, element):
        """Find the index of a child by identity, not value.

        Avoids issues with tag.contents.index(element) getting the
        index of equal elements.

        :param element: Look for this PageElement in `self.contents`.
        """
        for i, child in enumerate(self.contents):
            if child is element:
                return i
        raise ValueError("Tag.index: element not in tag")

    def get(self, key, default=None):
        """Returns the value of the 'key' attribute for the tag, or
        the value given for 'default' if it doesn't have that
        attribute."""
        return self.attrs.get(key, default)

    def get_attribute_list(self, key, default=None):
        """The same as get(), but always returns a list.

        :param key: The attribute to look for.
        :param default: Use this value if the attribute is not present
            on this PageElement.
        :return: A list of values, probably containing only a single
            value.
        """
        value = self.get(key, default)
        if not isinstance(value, list):
            value = [value]
        return value

    def has_attr(self, key):
        """Does this PageElement have an attribute with the given name?"""
        return key in self.attrs

    def __hash__(self):
        return str(self).__hash__()

    def __getitem__(self, key):
        """tag[key] returns the value of the 'key' attribute for the Tag,
        and throws an exception if it's not there."""
        return self.attrs[key]

    def __iter__(self):
        "Iterating over a Tag iterates over its contents."
        return iter(self.contents)

    def __len__(self):
        "The length of a Tag is the length of its list of contents."
        return len(self.contents)

    def __contains__(self, x):
        return x in self.contents

    def __bool__(self):
        "A tag is non-None even if it has no contents."
        return True

    def __setitem__(self, key, value):
        """Setting tag[key] sets the value of the 'key' attribute for the
        tag."""
        self.attrs[key] = value

    def __delitem__(self, key):
        "Deleting tag[key] deletes all 'key' attributes for the tag."
        self.attrs.pop(key, None)

    def __call__(self, *args, **kwargs):
        """Calling a Tag like a function is the same as calling its
        find_all() method. Eg. tag('a') returns a list of all the A tags
        found within this tag."""
        return self.find_all(*args, **kwargs)

    def __getattr__(self, tag):
        """Calling tag.subtag is the same as calling tag.find(name="subtag")"""
        #print("Getattr %s.%s" % (self.__class__, tag))
        if len(tag) > 3 and tag.endswith('Tag'):
            # BS3: soup.aTag -> "soup.find("a")
            tag_name = tag[:-3]
            warnings.warn(
                '.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag")' % dict(
                    name=tag_name
                ),
                DeprecationWarning, stacklevel=2
            )
            return self.find(tag_name)
        # We special case contents to avoid recursion.
        elif not tag.startswith("__") and not tag == "contents":
            return self.find(tag)
        raise AttributeError(
            "'%s' object has no attribute '%s'" % (self.__class__, tag))

    def __eq__(self, other):
        """Returns true iff this Tag has the same name, the same attributes,
        and the same contents (recursively) as `other`."""
        if self is other:
            return True
        if (not hasattr(other, 'name') or
            not hasattr(other, 'attrs') or
            not hasattr(other, 'contents') or
            self.name != other.name or
            self.attrs != other.attrs or
            len(self) != len(other)):
            return False
        for i, my_child in enumerate(self.contents):
            if my_child != other.contents[i]:
                return False
        return True

    def __ne__(self, other):
        """Returns true iff this Tag is not identical to `other`,
        as defined in __eq__."""
        return not self == other

    def __repr__(self, encoding="unicode-escape"):
        """Renders this PageElement as a string.

        :param encoding: The encoding to use (Python 2 only).
            TODO: This is now ignored and a warning should be issued
            if a value is provided.
        :return: A (Unicode) string.
        """
        # "The return value must be a string object", i.e. Unicode
        return self.decode()

    def __unicode__(self):
        """Renders this PageElement as a Unicode string."""
        return self.decode()

    __str__ = __repr__ = __unicode__

    def encode(self, encoding=DEFAULT_OUTPUT_ENCODING,
               indent_level=None, formatter="minimal",
               errors="xmlcharrefreplace"):
        """Render a bytestring representation of this PageElement and its
        contents.

        :param encoding: The destination encoding.
        :param indent_level: Each line of the rendering will be
           indented this many levels. (The formatter decides what a
           'level' means in terms of spaces or other characters
           output.) Used internally in recursive calls while
           pretty-printing.
        :param formatter: A Formatter object, or a string naming one of
            the standard formatters.
        :param errors: An error handling strategy such as
            'xmlcharrefreplace'. This value is passed along into
            encode() and its value should be one of the constants
            defined by Python.
        :return: A bytestring.

        """
        # Turn the data structure into Unicode, then encode the
        # Unicode.
        u = self.decode(indent_level, encoding, formatter)
        return u.encode(encoding, errors)

    def decode(self, indent_level=None,
               eventual_encoding=DEFAULT_OUTPUT_ENCODING,
               formatter="minimal",
               iterator=None):
        pieces = []
        # First off, turn a non-Formatter `formatter` into a Formatter
        # object. This will stop the lookup from happening over and
        # over again.
        if not isinstance(formatter, Formatter):
            formatter = self.formatter_for_name(formatter)

        if indent_level is True:
            indent_level = 0

        # The currently active tag that put us into string literal
        # mode. Until this element is closed, children will be treated
        # as string literals and not pretty-printed. String literal
        # mode is turned on immediately after this tag begins, and
        # turned off immediately before it's closed. This means there
        # will be whitespace before and after the tag itself.
        string_literal_tag = None

        for event, element in self._event_stream(iterator):
            if event in (Tag.START_ELEMENT_EVENT, Tag.EMPTY_ELEMENT_EVENT):
                piece = element._format_tag(
                    eventual_encoding, formatter, opening=True
                )
            elif event is Tag.END_ELEMENT_EVENT:
                piece = element._format_tag(
                    eventual_encoding, formatter, opening=False
                )
                if indent_level is not None:
                    indent_level -= 1
            else:
                piece = element.output_ready(formatter)

            # Now we need to apply the 'prettiness' -- extra
            # whitespace before and/or after this tag. This can get
            # complicated because certain tags, like <pre> and
            # <script>, can't be prettified, since adding whitespace would
            # change the meaning of the content.

            # The default behavior is to add whitespace before and
            # after an element when string literal mode is off, and to
            # leave things as they are when string literal mode is on.
            if string_literal_tag:
                indent_before = indent_after = False
            else:
                indent_before = indent_after = True

            # The only time the behavior is more complex than that is
            # when we encounter an opening or closing tag that might
            # put us into or out of string literal mode.
            if (event is Tag.START_ELEMENT_EVENT
                and not string_literal_tag
                and not element._should_pretty_print()):
                    # We are about to enter string literal mode. Add
                    # whitespace before this tag, but not after. We
                    # will stay in string literal mode until this tag
                    # is closed.
                    indent_before = True
                    indent_after = False
                    string_literal_tag = element
            elif (event is Tag.END_ELEMENT_EVENT
                  and element is string_literal_tag):
                # We are about to exit string literal mode by closing
                # the tag that sent us into that mode. Add whitespace
                # after this tag, but not before.
                indent_before = False
                indent_after = True
                string_literal_tag = None

            # Now we know whether to add whitespace before and/or
            # after this element.
            if indent_level is not None:
                if (indent_before or indent_after):
                    if isinstance(element, NavigableString):
                        piece = piece.strip()
                    if piece:
                        piece = self._indent_string(
                            piece, indent_level, formatter,
                            indent_before, indent_after
                        )
                if event == Tag.START_ELEMENT_EVENT:
                    indent_level += 1
            pieces.append(piece)
        return "".join(pieces)

    # Names for the different events yielded by _event_stream
    START_ELEMENT_EVENT = object()
    END_ELEMENT_EVENT = object()
    EMPTY_ELEMENT_EVENT = object()
    STRING_ELEMENT_EVENT = object()

    def _event_stream(self, iterator=None):
        """Yield a sequence of events that can be used to reconstruct the DOM
        for this element.

        This lets us recreate the nested structure of this element
        (e.g. when formatting it as a string) without using recursive
        method calls.

        This is similar in concept to the SAX API, but it's a simpler
        interface designed for internal use. The events are different
        from SAX and the arguments associated with the events are Tags
        and other Beautiful Soup objects.

        :param iterator: An alternate iterator to use when traversing
         the tree.
        """
        tag_stack = []

        iterator = iterator or self.self_and_descendants

        for c in iterator:
            # If the parent of the element we're about to yield is not
            # the tag currently on the stack, it means that the tag on
            # the stack closed before this element appeared.
            while tag_stack and c.parent != tag_stack[-1]:
                now_closed_tag = tag_stack.pop()
                yield Tag.END_ELEMENT_EVENT, now_closed_tag

            if isinstance(c, Tag):
                if c.is_empty_element:
                    yield Tag.EMPTY_ELEMENT_EVENT, c
                else:
                    yield Tag.START_ELEMENT_EVENT, c
                    tag_stack.append(c)
                    continue
            else:
                yield Tag.STRING_ELEMENT_EVENT, c

        while tag_stack:
            now_closed_tag = tag_stack.pop()
            yield Tag.END_ELEMENT_EVENT, now_closed_tag

    def _indent_string(self, s, indent_level, formatter,
                       indent_before, indent_after):
        """Add indentation whitespace before and/or after a string.

        :param s: The string to amend with whitespace.
        :param indent_level: The indentation level; affects how much
           whitespace goes before the string.
        :param indent_before: Whether or not to add whitespace
           before the string.
        :param indent_after: Whether or not to add whitespace
           (a newline) after the string.
        """
        space_before = ''
        if indent_before and indent_level:
            space_before = (formatter.indent * indent_level)

        space_after = ''
        if indent_after:
            space_after = "\n"

        return space_before + s + space_after

    def _format_tag(self, eventual_encoding, formatter, opening):
        # A tag starts with the < character (see below).

        # Then the / character, if this is a closing tag.
        closing_slash = ''
        if not opening:
            closing_slash = '/'

        # Then an optional namespace prefix.
        prefix = ''
        if self.prefix:
            prefix = self.prefix + ":"

        # Then a list of attribute values, if this is an opening tag.
        attribute_string = ''
        if opening:
            attributes = formatter.attributes(self)
            attrs = []
            for key, val in attributes:
                if val is None:
                    decoded = key
                else:
                    if isinstance(val, list) or isinstance(val, tuple):
                        val = ' '.join(val)
                    elif not isinstance(val, str):
                        val = str(val)
                    elif (
                            isinstance(val, AttributeValueWithCharsetSubstitution)
                            and eventual_encoding is not None
                    ):
                        val = val.encode(eventual_encoding)

                    text = formatter.attribute_value(val)
                    decoded = (
                        str(key) + '='
                        + formatter.quoted_attribute_value(text))
                attrs.append(decoded)
            if attrs:
                attribute_string = ' ' + ' '.join(attrs)

        # Then an optional closing slash (for a void element in an
        # XML document).
        void_element_closing_slash = ''
        if self.is_empty_element:
            void_element_closing_slash = formatter.void_element_close_prefix or ''

        # Put it all together.
        return '<' + closing_slash + prefix + self.name + attribute_string + void_element_closing_slash + '>'

    def _should_pretty_print(self, indent_level=1):
        """Should this tag be pretty-printed?

        Most of them should, but some (such as <pre> in HTML
        documents) should not.
        """
        return (
            indent_level is not None
            and (
                not self.preserve_whitespace_tags
                or self.name not in self.preserve_whitespace_tags
            )
        )

    def prettify(self, encoding=None, formatter="minimal"):
        """Pretty-print this PageElement as a string.

        :param encoding: The eventual encoding of the string. If this is None,
            a Unicode string will be returned.
        :param formatter: A Formatter object, or a string naming one of
            the standard formatters.
        :return: A Unicode string (if encoding==None) or a bytestring
            (otherwise).
        """
        if encoding is None:
            return self.decode(True, formatter=formatter)
        else:
            return self.encode(encoding, True, formatter=formatter)

    def decode_contents(self, indent_level=None,
                       eventual_encoding=DEFAULT_OUTPUT_ENCODING,
                       formatter="minimal"):
        """Renders the contents of this tag as a Unicode string.

        :param indent_level: Each line of the rendering will be
           indented this many levels. (The formatter decides what a
           'level' means in terms of spaces or other characters
           output.) Used internally in recursive calls while
           pretty-printing.

        :param eventual_encoding: The tag is destined to be
           encoded into this encoding. decode_contents() is _not_
           responsible for performing that encoding. This information
           is passed in so that it can be substituted in if the
           document contains a <META> tag that mentions the document's
           encoding.

        :param formatter: A Formatter object, or a string naming one of
            the standard Formatters.

        """
        return self.decode(indent_level, eventual_encoding, formatter,
                           iterator=self.descendants)

    def encode_contents(
        self, indent_level=None, encoding=DEFAULT_OUTPUT_ENCODING,
        formatter="minimal"):
        """Renders the contents of this PageElement as a bytestring.

        :param indent_level: Each line of the rendering will be
           indented this many levels. (The formatter decides what a
           'level' means in terms of spaces or other characters
           output.) Used internally in recursive calls while
           pretty-printing.

        :param eventual_encoding: The bytestring will be in this encoding.

        :param formatter: A Formatter object, or a string naming one of
            the standard Formatters.

        :return: A bytestring.
        """
        contents = self.decode_contents(indent_level, encoding, formatter)
        return contents.encode(encoding)

    # Old method for BS3 compatibility
    def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
                       prettyPrint=False, indentLevel=0):
        """Deprecated method for BS3 compatibility."""
        if not prettyPrint:
            indentLevel = None
        return self.encode_contents(
            indent_level=indentLevel, encoding=encoding)

    #Soup methods

    def find(self, name=None, attrs={}, recursive=True, string=None,
             **kwargs):
        """Look in the children of this PageElement and find the first
        PageElement that matches the given criteria.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param recursive: If this is True, find() will perform a
            recursive search of this PageElement's children. Otherwise,
            only the direct children will be considered.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.
        :return: A PageElement.
        :rtype: bs4.element.Tag | bs4.element.NavigableString
        """
        r = None
        l = self.find_all(name, attrs, recursive, string, 1, _stacklevel=3,
                          **kwargs)
        if l:
            r = l[0]
        return r
    findChild = find #BS2

    def find_all(self, name=None, attrs={}, recursive=True, string=None,
                 limit=None, **kwargs):
        """Look in the children of this PageElement and find all
        PageElements that match the given criteria.

        All find_* methods take a common set of arguments. See the online
        documentation for detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param recursive: If this is True, find_all() will perform a
            recursive search of this PageElement's children. Otherwise,
            only the direct children will be considered.
        :param limit: Stop looking after finding this many results.
        :kwargs: A dictionary of filters on attribute values.
        :return: A ResultSet of PageElements.
        :rtype: bs4.element.ResultSet
        """
        generator = self.descendants
        if not recursive:
            generator = self.children
        _stacklevel = kwargs.pop('_stacklevel', 2)
        return self._find_all(name, attrs, string, limit, generator,
                              _stacklevel=_stacklevel+1, **kwargs)
    findAll = find_all       # BS3
    findChildren = find_all  # BS2

    #Generator methods
    @property
    def children(self):
        """Iterate over all direct children of this PageElement.

        :yield: A sequence of PageElements.
        """
        # return iter() to make the purpose of the method clear
        return iter(self.contents)  # XXX This seems to be untested.

    @property
    def self_and_descendants(self):
        """Iterate over this PageElement and its children in a
        breadth-first sequence.

        :yield: A sequence of PageElements.
        """
        if not self.hidden:
            yield self
        for i in self.descendants:
            yield i

    @property
    def descendants(self):
        """Iterate over all children of this PageElement in a
        breadth-first sequence.

        :yield: A sequence of PageElements.
        """
        if not len(self.contents):
            return
        stopNode = self._last_descendant().next_element
        current = self.contents[0]
        while current is not stopNode:
            yield current
            current = current.next_element

    # CSS selector code
    def select_one(self, selector, namespaces=None, **kwargs):
        """Perform a CSS selection operation on the current element.

        :param selector: A CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
           used in the CSS selector to namespace URIs. By default,
           Beautiful Soup will use the prefixes it encountered while
           parsing the document.

        :param kwargs: Keyword arguments to be passed into Soup Sieve's
           soupsieve.select() method.

        :return: A Tag.
        :rtype: bs4.element.Tag
        """
        return self.css.select_one(selector, namespaces, **kwargs)

    def select(self, selector, namespaces=None, limit=None, **kwargs):
        """Perform a CSS selection operation on the current element.

        This uses the SoupSieve library.

        :param selector: A string containing a CSS selector.

        :param namespaces: A dictionary mapping namespace prefixes
           used in the CSS selector to namespace URIs. By default,
           Beautiful Soup will use the prefixes it encountered while
           parsing the document.

        :param limit: After finding this number of results, stop looking.

        :param kwargs: Keyword arguments to be passed into SoupSieve's
           soupsieve.select() method.

        :return: A ResultSet of Tags.
        :rtype: bs4.element.ResultSet
        """
        return self.css.select(selector, namespaces, limit, **kwargs)

    @property
    def css(self):
        """Return an interface to the CSS selector API."""
        return CSS(self)

    # Old names for backwards compatibility
    def childGenerator(self):
        """Deprecated generator."""
        return self.children

    def recursiveChildGenerator(self):
        """Deprecated generator."""
        return self.descendants

    def has_key(self, key):
        """Deprecated method. This was kind of misleading because has_key()
        (attributes) was different from __in__ (contents).

        has_key() is gone in Python 3, anyway.
        """
        warnings.warn(
            'has_key is deprecated. Use has_attr(key) instead.',
            DeprecationWarning, stacklevel=2
        )
        return self.has_attr(key)

# Next, a couple classes to represent queries and their results.
class SoupStrainer(object):
    """Encapsulates a number of ways of matching a markup element (tag or
    string).

    This is primarily used to underpin the find_* methods, but you can
    create one yourself and pass it in as `parse_only` to the
    `BeautifulSoup` constructor, to parse a subset of a large
    document.
    """

    def __init__(self, name=None, attrs={}, string=None, **kwargs):
        """Constructor.

        The SoupStrainer constructor takes the same arguments passed
        into the find_* methods. See the online documentation for
        detailed explanations.

        :param name: A filter on tag name.
        :param attrs: A dictionary of filters on attribute values.
        :param string: A filter for a NavigableString with specific text.
        :kwargs: A dictionary of filters on attribute values.
        """
        if string is None and 'text' in kwargs:
            string = kwargs.pop('text')
            warnings.warn(
                "The 'text' argument to the SoupStrainer constructor is deprecated. Use 'string' instead.",
                DeprecationWarning, stacklevel=2
            )

        self.name = self._normalize_search_value(name)
        if not isinstance(attrs, dict):
            # Treat a non-dict value for attrs as a search for the 'class'
            # attribute.
            kwargs['class'] = attrs
            attrs = None

        if 'class_' in kwargs:
            # Treat class_="foo" as a search for the 'class'
            # attribute, overriding any non-dict value for attrs.
            kwargs['class'] = kwargs['class_']
            del kwargs['class_']

        if kwargs:
            if attrs:
                attrs = attrs.copy()
                attrs.update(kwargs)
            else:
                attrs = kwargs
        normalized_attrs = {}
        for key, value in list(attrs.items()):
            normalized_attrs[key] = self._normalize_search_value(value)

        self.attrs = normalized_attrs
        self.string = self._normalize_search_value(string)

        # DEPRECATED but just in case someone is checking this.
        self.text = self.string

    def _normalize_search_value(self, value):
        # Leave it alone if it's a Unicode string, a callable, a
        # regular expression, a boolean, or None.
        if (isinstance(value, str) or isinstance(value, Callable) or hasattr(value, 'match')
            or isinstance(value, bool) or value is None):
            return value

        # If it's a bytestring, convert it to Unicode, treating it as UTF-8.
        if isinstance(value, bytes):
            return value.decode("utf8")

        # If it's listlike, convert it into a list of strings.
        if hasattr(value, '__iter__'):
            new_value = []
            for v in value:
                if (hasattr(v, '__iter__') and not isinstance(v, bytes)
                    and not isinstance(v, str)):
                    # This is almost certainly the user's mistake. In the
                    # interests of avoiding infinite loops, we'll let
                    # it through as-is rather than doing a recursive call.
                    new_value.append(v)
                else:
                    new_value.append(self._normalize_search_value(v))
            return new_value

        # Otherwise, convert it into a Unicode string.
        # The unicode(str()) thing is so this will do the same thing on Python 2
        # and Python 3.
        return str(str(value))

    def __str__(self):
        """A human-readable representation of this SoupStrainer."""
        if self.string:
            return self.string
        else:
            return "%s|%s" % (self.name, self.attrs)

    def search_tag(self, markup_name=None, markup_attrs={}):
        """Check whether a Tag with the given name and attributes would
        match this SoupStrainer.

        Used prospectively to decide whether to even bother creating a Tag
        object.

        :param markup_name: A tag name as found in some markup.
        :param markup_attrs: A dictionary of attributes as found in some markup.

        :return: True if the prospective tag would match this SoupStrainer;
            False otherwise.
        """
        found = None
        markup = None
        if isinstance(markup_name, Tag):
            markup = markup_name
            markup_attrs = markup

        if isinstance(self.name, str):
            # Optimization for a very common case where the user is
            # searching for a tag with one specific name, and we're
            # looking at a tag with a different name.
            if markup and not markup.prefix and self.name != markup.name:
                 return False

        call_function_with_tag_data = (
            isinstance(self.name, Callable)
            and not isinstance(markup_name, Tag))

        if ((not self.name)
            or call_function_with_tag_data
            or (markup and self._matches(markup, self.name))
            or (not markup and self._matches(markup_name, self.name))):
            if call_function_with_tag_data:
                match = self.name(markup_name, markup_attrs)
            else:
                match = True
                markup_attr_map = None
                for attr, match_against in list(self.attrs.items()):
                    if not markup_attr_map:
                        if hasattr(markup_attrs, 'get'):
                            markup_attr_map = markup_attrs
                        else:
                            markup_attr_map = {}
                            for k, v in markup_attrs:
                                markup_attr_map[k] = v
                    attr_value = markup_attr_map.get(attr)
                    if not self._matches(attr_value, match_against):
                        match = False
                        break
            if match:
                if markup:
                    found = markup
                else:
                    found = markup_name
        if found and self.string and not self._matches(found.string, self.string):
            found = None
        return found

    # For BS3 compatibility.
    searchTag = search_tag

    def search(self, markup):
        """Find all items in `markup` that match this SoupStrainer.

        Used by the core _find_all() method, which is ultimately
        called by all find_* methods.

        :param markup: A PageElement or a list of them.
        """
        # print('looking for %s in %s' % (self, markup))
        found = None
        # If given a list of items, scan it for a text element that
        # matches.
        if hasattr(markup, '__iter__') and not isinstance(markup, (Tag, str)):
            for element in markup:
                if isinstance(element, NavigableString) \
                       and self.search(element):
                    found = element
                    break
        # If it's a Tag, make sure its name or attributes match.
        # Don't bother with Tags if we're searching for text.
        elif isinstance(markup, Tag):
            if not self.string or self.name or self.attrs:
                found = self.search_tag(markup)
        # If it's text, make sure the text matches.
        elif isinstance(markup, NavigableString) or \
                 isinstance(markup, str):
            if not self.name and not self.attrs and self._matches(markup, self.string):
                found = markup
        else:
            raise Exception(
                "I don't know how to match against a %s" % markup.__class__)
        return found

    def _matches(self, markup, match_against, already_tried=None):
        # print(u"Matching %s against %s" % (markup, match_against))
        result = False
        if isinstance(markup, list) or isinstance(markup, tuple):
            # This should only happen when searching a multi-valued attribute
            # like 'class'.
            for item in markup:
                if self._matches(item, match_against):
                    return True
            # We didn't match any particular value of the multivalue
            # attribute, but maybe we match the attribute value when
            # considered as a string.
            if self._matches(' '.join(markup), match_against):
                return True
            return False

        if match_against is True:
            # True matches any non-None value.
            return markup is not None

        if isinstance(match_against, Callable):
            return match_against(markup)

        # Custom callables take the tag as an argument, but all
        # other ways of matching match the tag name as a string.
        original_markup = markup
        if isinstance(markup, Tag):
            markup = markup.name

        # Ensure that `markup` is either a Unicode string, or None.
        markup = self._normalize_search_value(markup)

        if markup is None:
            # None matches None, False, an empty string, an empty list, and so on.
            return not match_against

        if (hasattr(match_against, '__iter__')
            and not isinstance(match_against, str)):
            # We're asked to match against an iterable of items.
            # The markup must be match at least one item in the
            # iterable. We'll try each one in turn.
            #
            # To avoid infinite recursion we need to keep track of
            # items we've already seen.
            if not already_tried:
                already_tried = set()
            for item in match_against:
                if item.__hash__:
                    key = item
                else:
                    key = id(item)
                if key in already_tried:
                    continue
                else:
                    already_tried.add(key)
                    if self._matches(original_markup, item, already_tried):
                        return True
            else:
                return False

        # Beyond this point we might need to run the test twice: once against
        # the tag's name and once against its prefixed name.
        match = False

        if not match and isinstance(match_against, str):
            # Exact string match
            match = markup == match_against

        if not match and hasattr(match_against, 'search'):
            # Regexp match
            return match_against.search(markup)

        if (not match
            and isinstance(original_markup, Tag)
            and original_markup.prefix):
            # Try the whole thing again with the prefixed tag name.
            return self._matches(
                original_markup.prefix + ':' + original_markup.name, match_against
            )

        return match


class ResultSet(list):
    """A ResultSet is just a list that keeps track of the SoupStrainer
    that created it."""
    def __init__(self, source, result=()):
        """Constructor.

        :param source: A SoupStrainer.
        :param result: A list of PageElements.
        """
        super(ResultSet, self).__init__(result)
        self.source = source

    def __getattr__(self, key):
        """Raise a helpful exception to explain a common code fix."""
        raise AttributeError(
            "ResultSet object has no attribute '%s'. You're probably treating a list of elements like a single element. Did you call find_all() when you meant to call find()?" % key
        )

env_pyrecodes/lib/python3.9/site-packages/bs4/element.py
NikolaBlagojevic/pyrecodes
# encoding: utf-8
"""Helper classes for tests."""

# Use of this source code is governed by the MIT license.
__license__ = "MIT"

import pickle
import copy
import functools
import warnings
import pytest
from bs4 import BeautifulSoup
from bs4.element import (
    CharsetMetaAttributeValue,
    Comment,
    ContentMetaAttributeValue,
    Doctype,
    PYTHON_SPECIFIC_ENCODINGS,
    SoupStrainer,
    Script,
    Stylesheet,
    Tag
)

from bs4.builder import (
    DetectsXMLParsedAsHTML,
    HTMLParserTreeBuilder,
    XMLParsedAsHTMLWarning,
)
default_builder = HTMLParserTreeBuilder

# Some tests depend on specific third-party libraries. We use
# @pytest.mark.skipIf on the following conditionals to skip them
# if the libraries are not installed.
try:
    from soupsieve import SelectorSyntaxError
    SOUP_SIEVE_PRESENT = True
except ImportError:
    SOUP_SIEVE_PRESENT = False

try:
    import html5lib
    HTML5LIB_PRESENT = True
except ImportError:
    HTML5LIB_PRESENT = False

try:
    import lxml.etree
    LXML_PRESENT = True
    LXML_VERSION = lxml.etree.LXML_VERSION
except ImportError:
    LXML_PRESENT = False
    LXML_VERSION = (0,)

BAD_DOCUMENT = """A bare string
<!DOCTYPE xsl:stylesheet SYSTEM "htmlent.dtd">
<!DOCTYPE xsl:stylesheet PUBLIC "htmlent.dtd">
<div><![CDATA[A CDATA section where it doesn't belong]]></div>
<div><svg><![CDATA[HTML5 does allow CDATA sections in SVG]]></svg></div>
<div>A <meta> tag</div>
<div>A <br> tag that supposedly has contents.</br></div>
<div>AT&T</div>
<div><textarea>Within a textarea, markup like <b> tags and <&<&amp; should be treated as literal</textarea></div>
<div><script>if (i < 2) { alert("<b>Markup within script tags should be treated as literal.</b>"); }</script></div>
<div>This numeric entity is missing the final semicolon: <x t="pi&#241ata"></div>
<div><a href="http://example.com/</a> that attribute value never got closed</div>
<div><a href="foo</a>, </a><a href="bar">that attribute value was closed by the subsequent tag</a></div>
<! This document starts with a bogus declaration ><div>a</div>
<div>This document contains <!an incomplete declaration <div>(do you see it?)</div>
<div>This document ends with <!an incomplete declaration
<div><a style={height:21px;}>That attribute value was bogus</a></div>
<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">The doctype is invalid because it contains extra whitespace
<div><table><td nowrap>That boolean attribute had no value</td></table></div>
<div>Here's a nonexistent entity: &#foo; (do you see it?)</div>
<div>This document ends before the entity finishes: &gt
<div><p>Paragraphs shouldn't contain block display elements, but this one does: <dl><dt>you see?</dt></p>
<b b="20" a="1" b="10" a="2" a="3" a="4">Multiple values for the same attribute.</b>
<div><table><tr><td>Here's a table</td></tr></table></div>
<div><table id="1"><tr><td>Here's a nested table:<table id="2"><tr><td>foo</td></tr></table></td></div>
<div>This tag contains nothing but whitespace: <b>    </b></div>
<div><blockquote><p><b>This p tag is cut off by</blockquote></p>the end of the blockquote tag</div>
<div><table><div>This table contains bare markup</div></table></div>
<div><div id="1">\n <a href="link1">This link is never closed.\n</div>\n<div id="2">\n <div id="3">\n   <a href="link2">This link is closed.</a>\n  </div>\n</div></div>
<div>This document contains a <!DOCTYPE surprise>surprise doctype</div>
<div><a><B><Cd><EFG>Mixed case tags are folded to lowercase</efg></CD></b></A></div>
<div><our\u2603>Tag name contains Unicode characters</our\u2603></div>
<div><a \u2603="snowman">Attribute name contains Unicode characters</a></div>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
"""


class SoupTest(object):

    @property
    def default_builder(self):
        return default_builder

    def soup(self, markup, **kwargs):
        """Build a Beautiful Soup object from markup."""
        builder = kwargs.pop('builder', self.default_builder)
        return BeautifulSoup(markup, builder=builder, **kwargs)

    def document_for(self, markup, **kwargs):
        """Turn an HTML fragment into a document.

        The details depend on the builder.
        """
        return self.default_builder(**kwargs).test_fragment_to_document(markup)
   
    def assert_soup(self, to_parse, compare_parsed_to=None):
        """Parse some markup using Beautiful Soup and verify that
        the output markup is as expected.
        """
        builder = self.default_builder
        obj = BeautifulSoup(to_parse, builder=builder)
        if compare_parsed_to is None:
            compare_parsed_to = to_parse

        # Verify that the documents come out the same.
        assert obj.decode() == self.document_for(compare_parsed_to)

        # Also run some checks on the BeautifulSoup object itself:

        # Verify that every tag that was opened was eventually closed.

        # There are no tags in the open tag counter.
        assert all(v==0 for v in list(obj.open_tag_counter.values()))

        # The only tag in the tag stack is the one for the root
        # document.
        assert [obj.ROOT_TAG_NAME] == [x.name for x in obj.tagStack]

    assertSoupEquals = assert_soup
        
    def assertConnectedness(self, element):
        """Ensure that next_element and previous_element are properly
        set for all descendants of the given element.
        """
        earlier = None
        for e in element.descendants:
            if earlier:
                assert e == earlier.next_element
                assert earlier == e.previous_element
            earlier = e

    def linkage_validator(self, el, _recursive_call=False):
        """Ensure proper linkage throughout the document."""
        descendant = None
        # Document element should have no previous element or previous sibling.
        # It also shouldn't have a next sibling.
        if el.parent is None:
            assert el.previous_element is None,\
                "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
                    el, el.previous_element, None
                )
            assert el.previous_sibling is None,\
                "Bad previous_sibling\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
                    el, el.previous_sibling, None
                )
            assert el.next_sibling is None,\
                "Bad next_sibling\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format(
                    el, el.next_sibling, None
                )

        idx = 0
        child = None
        last_child = None
        last_idx = len(el.contents) - 1
        for child in el.contents:
            descendant = None

            # Parent should link next element to their first child
            # That child should have no previous sibling
            if idx == 0:
                if el.parent is not None:
                    assert el.next_element is child,\
                       "Bad next_element\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format(
                            el, el.next_element, child
                        )
                    assert child.previous_element is el,\
                       "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
                            child, child.previous_element, el
                        )
                    assert child.previous_sibling is None,\
                       "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED: {}".format(
                            child, child.previous_sibling, None
                        )

            # If not the first child, previous index should link as sibling to this index
            # Previous element should match the last index or the last bubbled up descendant
            else:
                assert child.previous_sibling is el.contents[idx - 1],\
                    "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED {}".format(
                        child, child.previous_sibling, el.contents[idx - 1]
                    )
                assert el.contents[idx - 1].next_sibling is child,\
                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
                        el.contents[idx - 1], el.contents[idx - 1].next_sibling, child
                    )

                if last_child is not None:
                    assert child.previous_element is last_child,\
                        "Bad previous_element\nNODE: {}\nPREV {}\nEXPECTED {}\nCONTENTS {}".format(
                            child, child.previous_element, last_child, child.parent.contents
                        )
                    assert last_child.next_element is child,\
                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
                            last_child, last_child.next_element, child
                        )

            if isinstance(child, Tag) and child.contents:
                descendant = self.linkage_validator(child, True)
                # A bubbled up descendant should have no next siblings
                assert descendant.next_sibling is None,\
                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
                        descendant, descendant.next_sibling, None
                    )

            # Mark last child as either the bubbled up descendant or the current child
            if descendant is not None:
                last_child = descendant
            else:
                last_child = child

            # If last child, there are non next siblings
            if idx == last_idx:
                assert child.next_sibling is None,\
                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
                        child, child.next_sibling, None
                    )
            idx += 1

        child = descendant if descendant is not None else child
        if child is None:
            child = el

        if not _recursive_call and child is not None:
            target = el
            while True:
                if target is None:
                    assert child.next_element is None, \
                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
                            child, child.next_element, None
                        )
                    break
                elif target.next_sibling is not None:
                    assert child.next_element is target.next_sibling, \
                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
                            child, child.next_element, target.next_sibling
                        )
                    break
                target = target.parent

            # We are done, so nothing to return
            return None
        else:
            # Return the child to the recursive caller
            return child

    def assert_selects(self, tags, should_match):
        """Make sure that the given tags have the correct text.

        This is used in tests that define a bunch of tags, each
        containing a single string, and then select certain strings by
        some mechanism.
        """
        assert [tag.string for tag in tags] == should_match

    def assert_selects_ids(self, tags, should_match):
        """Make sure that the given tags have the correct IDs.

        This is used in tests that define a bunch of tags, each
        containing a single string, and then select certain strings by
        some mechanism.
        """
        assert [tag['id'] for tag in tags] == should_match


class TreeBuilderSmokeTest(object):
    # Tests that are common to HTML and XML tree builders.

    @pytest.mark.parametrize(
        "multi_valued_attributes",
        [None, {}, dict(b=['class']), {'*': ['notclass']}]
    )
    def test_attribute_not_multi_valued(self, multi_valued_attributes):
        markup = '<html xmlns="http://www.w3.org/1999/xhtml"><a class="a b c"></html>'
        soup = self.soup(markup, multi_valued_attributes=multi_valued_attributes)
        assert soup.a['class'] == 'a b c'

    @pytest.mark.parametrize(
        "multi_valued_attributes", [dict(a=['class']), {'*': ['class']}]
    )
    def test_attribute_multi_valued(self, multi_valued_attributes):
        markup = '<a class="a b c">'
        soup = self.soup(
            markup, multi_valued_attributes=multi_valued_attributes
        )
        assert soup.a['class'] == ['a', 'b', 'c']

    def test_invalid_doctype(self):
        markup = '<![if word]>content<![endif]>'
        markup = '<!DOCTYPE html]ff>'
        soup = self.soup(markup)

class HTMLTreeBuilderSmokeTest(TreeBuilderSmokeTest):

    """A basic test of a treebuilder's competence.

    Any HTML treebuilder, present or future, should be able to pass
    these tests. With invalid markup, there's room for interpretation,
    and different parsers can handle it differently. But with the
    markup in these tests, there's not much room for interpretation.
    """

    def test_empty_element_tags(self):
        """Verify that all HTML4 and HTML5 empty element (aka void element) tags
        are handled correctly.
        """
        for name in [
                'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
                'spacer', 'frame'
        ]:
            soup = self.soup("")
            new_tag = soup.new_tag(name)
            assert new_tag.is_empty_element == True

    def test_special_string_containers(self):
        soup = self.soup(
            "<style>Some CSS</style><script>Some Javascript</script>"
        )
        assert isinstance(soup.style.string, Stylesheet)
        assert isinstance(soup.script.string, Script)

        soup = self.soup(
            "<style><!--Some CSS--></style>"
        )
        assert isinstance(soup.style.string, Stylesheet)
        # The contents of the style tag resemble an HTML comment, but
        # it's not treated as a comment.
        assert soup.style.string == "<!--Some CSS-->"
        assert isinstance(soup.style.string, Stylesheet)
        
    def test_pickle_and_unpickle_identity(self):
        # Pickling a tree, then unpickling it, yields a tree identical
        # to the original.
        tree = self.soup("<a><b>foo</a>")
        dumped = pickle.dumps(tree, 2)
        loaded = pickle.loads(dumped)
        assert loaded.__class__ == BeautifulSoup
        assert loaded.decode() == tree.decode()

    def assertDoctypeHandled(self, doctype_fragment):
        """Assert that a given doctype string is handled correctly."""
        doctype_str, soup = self._document_with_doctype(doctype_fragment)

        # Make sure a Doctype object was created.
        doctype = soup.contents[0]
        assert doctype.__class__ == Doctype
        assert doctype == doctype_fragment
        assert soup.encode("utf8")[:len(doctype_str)] == doctype_str

        # Make sure that the doctype was correctly associated with the
        # parse tree and that the rest of the document parsed.
        assert soup.p.contents[0] == 'foo'

    def _document_with_doctype(self, doctype_fragment, doctype_string="DOCTYPE"):
        """Generate and parse a document with the given doctype."""
        doctype = '<!%s %s>' % (doctype_string, doctype_fragment)
        markup = doctype + '\n<p>foo</p>'
        soup = self.soup(markup)
        return doctype.encode("utf8"), soup

    def test_normal_doctypes(self):
        """Make sure normal, everyday HTML doctypes are handled correctly."""
        self.assertDoctypeHandled("html")
        self.assertDoctypeHandled(
            'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')

    def test_empty_doctype(self):
        soup = self.soup("<!DOCTYPE>")
        doctype = soup.contents[0]
        assert "" == doctype.strip()

    def test_mixed_case_doctype(self):
        # A lowercase or mixed-case doctype becomes a Doctype.
        for doctype_fragment in ("doctype", "DocType"):
            doctype_str, soup = self._document_with_doctype(
                "html", doctype_fragment
            )

            # Make sure a Doctype object was created and that the DOCTYPE
            # is uppercase.
            doctype = soup.contents[0]
            assert doctype.__class__ == Doctype
            assert doctype == "html"
            assert soup.encode("utf8")[:len(doctype_str)] == b"<!DOCTYPE html>"

            # Make sure that the doctype was correctly associated with the
            # parse tree and that the rest of the document parsed.
            assert soup.p.contents[0] == 'foo'
        
    def test_public_doctype_with_url(self):
        doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
        self.assertDoctypeHandled(doctype)

    def test_system_doctype(self):
        self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')

    def test_namespaced_system_doctype(self):
        # We can handle a namespaced doctype with a system ID.
        self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')

    def test_namespaced_public_doctype(self):
        # Test a namespaced doctype with a public id.
        self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')

    def test_real_xhtml_document(self):
        """A real XHTML document should come out more or less the same as it went in."""
        markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>Hello.</title></head>
<body>Goodbye.</body>
</html>"""
        with warnings.catch_warnings(record=True) as w:
            soup = self.soup(markup)
        assert soup.encode("utf-8").replace(b"\n", b"") == markup.replace(b"\n", b"")

        # No warning was issued about parsing an XML document as HTML,
        # because XHTML is both.
        assert w == []


    def test_namespaced_html(self):
        # When a namespaced XML document is parsed as HTML it should
        # be treated as HTML with weird tag names.
        markup = b"""<ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>"""
        with warnings.catch_warnings(record=True) as w:
            soup = self.soup(markup)

        assert 2 == len(soup.find_all("ns1:foo"))
            
        # n.b. no "you're parsing XML as HTML" warning was given
        # because there was no XML declaration.
        assert [] == w

    def test_detect_xml_parsed_as_html(self):
        # A warning is issued when parsing an XML document as HTML,
        # but basic stuff should still work.
        markup = b"""<?xml version="1.0" encoding="utf-8"?><tag>string</tag>"""
        with warnings.catch_warnings(record=True) as w:
            soup = self.soup(markup)
            assert soup.tag.string == 'string'
        [warning] = w
        assert isinstance(warning.message, XMLParsedAsHTMLWarning)
        assert str(warning.message) == XMLParsedAsHTMLWarning.MESSAGE

        # NOTE: the warning is not issued if the document appears to
        # be XHTML (tested with test_real_xhtml_document in the
        # superclass) or if there is no XML declaration (tested with
        # test_namespaced_html in the superclass).
        
    def test_processing_instruction(self):
        # We test both Unicode and bytestring to verify that
        # process_markup correctly sets processing_instruction_class
        # even when the markup is already Unicode and there is no
        # need to process anything.
        markup = """<?PITarget PIContent?>"""
        soup = self.soup(markup)
        assert markup == soup.decode()

        markup = b"""<?PITarget PIContent?>"""
        soup = self.soup(markup)
        assert markup == soup.encode("utf8")

    def test_deepcopy(self):
        """Make sure you can copy the tree builder.

        This is important because the builder is part of a
        BeautifulSoup object, and we want to be able to copy that.
        """
        copy.deepcopy(self.default_builder)

    def test_p_tag_is_never_empty_element(self):
        """A <p> tag is never designated as an empty-element tag.

        Even if the markup shows it as an empty-element tag, it
        shouldn't be presented that way.
        """
        soup = self.soup("<p/>")
        assert not soup.p.is_empty_element
        assert str(soup.p) == "<p></p>"

    def test_unclosed_tags_get_closed(self):
        """A tag that's not closed by the end of the document should be closed.

        This applies to all tags except empty-element tags.
        """
        self.assert_soup("<p>", "<p></p>")
        self.assert_soup("<b>", "<b></b>")

        self.assert_soup("<br>", "<br/>")

    def test_br_is_always_empty_element_tag(self):
        """A <br> tag is designated as an empty-element tag.

        Some parsers treat <br></br> as one <br/> tag, some parsers as
        two tags, but it should always be an empty-element tag.
        """
        soup = self.soup("<br></br>")
        assert soup.br.is_empty_element
        assert str(soup.br) == "<br/>"

    def test_nested_formatting_elements(self):
        self.assert_soup("<em><em></em></em>")

    def test_double_head(self):
        html = '''<!DOCTYPE html>
<html>
<head>
<title>Ordinary HEAD element test</title>
</head>
<script type="text/javascript">
alert("Help!");
</script>
<body>
Hello, world!
</body>
</html>
'''
        soup = self.soup(html)
        assert "text/javascript" == soup.find('script')['type']

    def test_comment(self):
        # Comments are represented as Comment objects.
        markup = "<p>foo<!--foobar-->baz</p>"
        self.assert_soup(markup)

        soup = self.soup(markup)
        comment = soup.find(string="foobar")
        assert comment.__class__ == Comment

        # The comment is properly integrated into the tree.
        foo = soup.find(string="foo")
        assert comment == foo.next_element
        baz = soup.find(string="baz")
        assert comment == baz.previous_element

    def test_preserved_whitespace_in_pre_and_textarea(self):
        """Whitespace must be preserved in <pre> and <textarea> tags,
        even if that would mean not prettifying the markup.
        """
        pre_markup = "<pre>a   z</pre>\n"
        textarea_markup = "<textarea> woo\nwoo  </textarea>\n"
        self.assert_soup(pre_markup)
        self.assert_soup(textarea_markup)

        soup = self.soup(pre_markup)
        assert soup.pre.prettify() == pre_markup

        soup = self.soup(textarea_markup)
        assert soup.textarea.prettify() == textarea_markup

        soup = self.soup("<textarea></textarea>")
        assert soup.textarea.prettify() == "<textarea></textarea>\n"

    def test_nested_inline_elements(self):
        """Inline elements can be nested indefinitely."""
        b_tag = "<b>Inside a B tag</b>"
        self.assert_soup(b_tag)

        nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
        self.assert_soup(nested_b_tag)

        double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
        self.assert_soup(nested_b_tag)

    def test_nested_block_level_elements(self):
        """Block elements can be nested."""
        soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
        blockquote = soup.blockquote
        assert blockquote.p.b.string == 'Foo'
        assert blockquote.b.string == 'Foo'

    def test_correctly_nested_tables(self):
        """One table can go inside another one."""
        markup = ('<table id="1">'
                  '<tr>'
                  "<td>Here's another table:"
                  '<table id="2">'
                  '<tr><td>foo</td></tr>'
                  '</table></td>')

        self.assert_soup(
            markup,
            '<table id="1"><tr><td>Here\'s another table:'
            '<table id="2"><tr><td>foo</td></tr></table>'
            '</td></tr></table>')

        self.assert_soup(
            "<table><thead><tr><td>Foo</td></tr></thead>"
            "<tbody><tr><td>Bar</td></tr></tbody>"
            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")

    def test_multivalued_attribute_with_whitespace(self):
        # Whitespace separating the values of a multi-valued attribute
        # should be ignored.

        markup = '<div class=" foo bar	 "></a>'
        soup = self.soup(markup)
        assert ['foo', 'bar'] == soup.div['class']

        # If you search by the literal name of the class it's like the whitespace
        # wasn't there.
        assert soup.div == soup.find('div', class_="foo bar")
        
    def test_deeply_nested_multivalued_attribute(self):
        # html5lib can set the attributes of the same tag many times
        # as it rearranges the tree. This has caused problems with
        # multivalued attributes.
        markup = '<table><div><div class="css"></div></div></table>'
        soup = self.soup(markup)
        assert ["css"] == soup.div.div['class']

    def test_multivalued_attribute_on_html(self):
        # html5lib uses a different API to set the attributes ot the
        # <html> tag. This has caused problems with multivalued
        # attributes.
        markup = '<html class="a b"></html>'
        soup = self.soup(markup)
        assert ["a", "b"] == soup.html['class']

    def test_angle_brackets_in_attribute_values_are_escaped(self):
        self.assert_soup('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')

    def test_strings_resembling_character_entity_references(self):
        # "&T" and "&p" look like incomplete character entities, but they are
        # not.
        self.assert_soup(
            "<p>&bull; AT&T is in the s&p 500</p>",
            "<p>\u2022 AT&amp;T is in the s&amp;p 500</p>"
        )

    def test_apos_entity(self):
        self.assert_soup(
            "<p>Bob&apos;s Bar</p>",
            "<p>Bob's Bar</p>",
        )
        
    def test_entities_in_foreign_document_encoding(self):
        # &#147; and &#148; are invalid numeric entities referencing
        # Windows-1252 characters. &#45; references a character common
        # to Windows-1252 and Unicode, and &#9731; references a
        # character only found in Unicode.
        #
        # All of these entities should be converted to Unicode
        # characters.
        markup = "<p>&#147;Hello&#148; &#45;&#9731;</p>"
        soup = self.soup(markup)
        assert "“Hello” -☃" == soup.p.string
        
    def test_entities_in_attributes_converted_to_unicode(self):
        expect = '<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
        self.assert_soup('<p id="pi&#241;ata"></p>', expect)
        self.assert_soup('<p id="pi&#xf1;ata"></p>', expect)
        self.assert_soup('<p id="pi&#Xf1;ata"></p>', expect)
        self.assert_soup('<p id="pi&ntilde;ata"></p>', expect)

    def test_entities_in_text_converted_to_unicode(self):
        expect = '<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
        self.assert_soup("<p>pi&#241;ata</p>", expect)
        self.assert_soup("<p>pi&#xf1;ata</p>", expect)
        self.assert_soup("<p>pi&#Xf1;ata</p>", expect)
        self.assert_soup("<p>pi&ntilde;ata</p>", expect)

    def test_quot_entity_converted_to_quotation_mark(self):
        self.assert_soup("<p>I said &quot;good day!&quot;</p>",
                              '<p>I said "good day!"</p>')

    def test_out_of_range_entity(self):
        expect = "\N{REPLACEMENT CHARACTER}"
        self.assert_soup("&#10000000000000;", expect)
        self.assert_soup("&#x10000000000000;", expect)
        self.assert_soup("&#1000000000;", expect)
       
    def test_multipart_strings(self):
        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
        soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
        assert "p" == soup.h2.string.next_element.name
        assert "p" == soup.p.name
        self.assertConnectedness(soup)

    def test_empty_element_tags(self):
        """Verify consistent handling of empty-element tags,
        no matter how they come in through the markup.
        """
        self.assert_soup('<br/><br/><br/>', "<br/><br/><br/>")
        self.assert_soup('<br /><br /><br />', "<br/><br/><br/>")
        
    def test_head_tag_between_head_and_body(self):
        "Prevent recurrence of a bug in the html5lib treebuilder."
        content = """<html><head></head>
  <link></link>
  <body>foo</body>
</html>
"""
        soup = self.soup(content)
        assert soup.html.body is not None
        self.assertConnectedness(soup)

    def test_multiple_copies_of_a_tag(self):
        "Prevent recurrence of a bug in the html5lib treebuilder."
        content = """<!DOCTYPE html>
<html>
 <body>
   <article id="a" >
   <div><a href="1"></div>
   <footer>
     <a href="2"></a>
   </footer>
  </article>
  </body>
</html>
"""
        soup = self.soup(content)
        self.assertConnectedness(soup.article)

    def test_basic_namespaces(self):
        """Parsers don't need to *understand* namespaces, but at the
        very least they should not choke on namespaces or lose
        data."""

        markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
        soup = self.soup(markup)
        assert markup == soup.encode()
        html = soup.html
        assert 'http://www.w3.org/1999/xhtml' == soup.html['xmlns']
        assert 'http://www.w3.org/1998/Math/MathML' == soup.html['xmlns:mathml']
        assert 'http://www.w3.org/2000/svg' == soup.html['xmlns:svg']

    def test_multivalued_attribute_value_becomes_list(self):
        markup = b'<a class="foo bar">'
        soup = self.soup(markup)
        assert ['foo', 'bar'] == soup.a['class']
        
    #
    # Generally speaking, tests below this point are more tests of
    # Beautiful Soup than tests of the tree builders. But parsers are
    # weird, so we run these tests separately for every tree builder
    # to detect any differences between them.
    #

    def test_can_parse_unicode_document(self):
        # A seemingly innocuous document... but it's in Unicode! And
        # it contains characters that can't be represented in the
        # encoding found in the  declaration! The horror!
        markup = '<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
        soup = self.soup(markup)
        assert 'Sacr\xe9 bleu!' == soup.body.string

    def test_soupstrainer(self):
        """Parsers should be able to work with SoupStrainers."""
        strainer = SoupStrainer("b")
        soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
                         parse_only=strainer)
        assert soup.decode() == "<b>bold</b>"

    def test_single_quote_attribute_values_become_double_quotes(self):
        self.assert_soup("<foo attr='bar'></foo>",
                              '<foo attr="bar"></foo>')

    def test_attribute_values_with_nested_quotes_are_left_alone(self):
        text = """<foo attr='bar "brawls" happen'>a</foo>"""
        self.assert_soup(text)

    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
        text = """<foo attr='bar "brawls" happen'>a</foo>"""
        soup = self.soup(text)
        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
        self.assert_soup(
            soup.foo.decode(),
            """<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")

    def test_ampersand_in_attribute_value_gets_escaped(self):
        self.assert_soup('<this is="really messed up & stuff"></this>',
                              '<this is="really messed up &amp; stuff"></this>')

        self.assert_soup(
            '<a href="http://example.org?a=1&b=2;3">foo</a>',
            '<a href="http://example.org?a=1&amp;b=2;3">foo</a>')

    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
        self.assert_soup('<a href="http://example.org?a=1&amp;b=2;3"></a>')

    def test_entities_in_strings_converted_during_parsing(self):
        # Both XML and HTML entities are converted to Unicode characters
        # during parsing.
        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
        self.assert_soup(text, expected)

    def test_smart_quotes_converted_on_the_way_in(self):
        # Microsoft smart quotes are converted to Unicode characters during
        # parsing.
        quote = b"<p>\x91Foo\x92</p>"
        soup = self.soup(quote)
        assert soup.p.string == "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}"

    def test_non_breaking_spaces_converted_on_the_way_in(self):
        soup = self.soup("<a>&nbsp;&nbsp;</a>")
        assert soup.a.string == "\N{NO-BREAK SPACE}" * 2

    def test_entities_converted_on_the_way_out(self):
        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
        soup = self.soup(text)
        assert soup.p.encode("utf-8") == expected

    def test_real_iso_8859_document(self):
        # Smoke test of interrelated functionality, using an
        # easy-to-understand document.

        # Here it is in Unicode. Note that it claims to be in ISO-8859-1.
        unicode_html = '<html><head><meta content="text/html; charset=ISO-8859-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'

        # That's because we're going to encode it into ISO-8859-1,
        # and use that to test.
        iso_latin_html = unicode_html.encode("iso-8859-1")

        # Parse the ISO-8859-1 HTML.
        soup = self.soup(iso_latin_html)

        # Encode it to UTF-8.
        result = soup.encode("utf-8")

        # What do we expect the result to look like? Well, it would
        # look like unicode_html, except that the META tag would say
        # UTF-8 instead of ISO-8859-1.
        expected = unicode_html.replace("ISO-8859-1", "utf-8")

        # And, of course, it would be in UTF-8, not Unicode.
        expected = expected.encode("utf-8")

        # Ta-da!
        assert result == expected

    def test_real_shift_jis_document(self):
        # Smoke test to make sure the parser can handle a document in
        # Shift-JIS encoding, without choking.
        shift_jis_html = (
            b'<html><head></head><body><pre>'
            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
            b'</pre></body></html>')
        unicode_html = shift_jis_html.decode("shift-jis")
        soup = self.soup(unicode_html)

        # Make sure the parse tree is correctly encoded to various
        # encodings.
        assert soup.encode("utf-8") == unicode_html.encode("utf-8")
        assert soup.encode("euc_jp") == unicode_html.encode("euc_jp")

    def test_real_hebrew_document(self):
        # A real-world test to make sure we can convert ISO-8859-9 (a
        # Hebrew encoding) to UTF-8.
        hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
        soup = self.soup(
            hebrew_document, from_encoding="iso8859-8")
        # Some tree builders call it iso8859-8, others call it iso-8859-9.
        # That's not a difference we really care about.
        assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
        assert soup.encode('utf-8') == (
            hebrew_document.decode("iso8859-8").encode("utf-8")
        )

    def test_meta_tag_reflects_current_encoding(self):
        # Here's the <meta> tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('<meta content="text/html; charset=x-sjis" '
                    'http-equiv="Content-type"/>')

        # Here's a document incorporating that meta tag.
        shift_jis_html = (
            '<html><head>\n%s\n'
            '<meta http-equiv="Content-language" content="ja"/>'
            '</head><body>Shift-JIS markup goes here.') % meta_tag
        soup = self.soup(shift_jis_html)

        # Parse the document, and the charset is seemingly unaffected.
        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
        content = parsed_meta['content']
        assert 'text/html; charset=x-sjis' == content

        # But that value is actually a ContentMetaAttributeValue object.
        assert isinstance(content, ContentMetaAttributeValue)

        # And it will take on a value that reflects its current
        # encoding.
        assert 'text/html; charset=utf8' == content.encode("utf8")

        # For the rest of the story, see TestSubstitutions in
        # test_tree.py.

    def test_html5_style_meta_tag_reflects_current_encoding(self):
        # Here's the <meta> tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('<meta id="encoding" charset="x-sjis" />')

        # Here's a document incorporating that meta tag.
        shift_jis_html = (
            '<html><head>\n%s\n'
            '<meta http-equiv="Content-language" content="ja"/>'
            '</head><body>Shift-JIS markup goes here.') % meta_tag
        soup = self.soup(shift_jis_html)

        # Parse the document, and the charset is seemingly unaffected.
        parsed_meta = soup.find('meta', id="encoding")
        charset = parsed_meta['charset']
        assert 'x-sjis' == charset

        # But that value is actually a CharsetMetaAttributeValue object.
        assert isinstance(charset, CharsetMetaAttributeValue)

        # And it will take on a value that reflects its current
        # encoding.
        assert 'utf8' == charset.encode("utf8")

    def test_python_specific_encodings_not_used_in_charset(self):
        # You can encode an HTML document using a Python-specific
        # encoding, but that encoding won't be mentioned _inside_ the
        # resulting document. Instead, the document will appear to
        # have no encoding.
        for markup in [
            b'<meta charset="utf8"></head>'
            b'<meta id="encoding" charset="utf-8" />'
        ]:
            soup = self.soup(markup)
            for encoding in PYTHON_SPECIFIC_ENCODINGS:
                if encoding in (
                    'idna', 'mbcs', 'oem', 'undefined',
                    'string_escape', 'string-escape'
                ):
                    # For one reason or another, these will raise an
                    # exception if we actually try to use them, so don't
                    # bother.
                    continue
                encoded = soup.encode(encoding)
                assert b'meta charset=""' in encoded
                assert encoding.encode("ascii") not in encoded
        
    def test_tag_with_no_attributes_can_have_attributes_added(self):
        data = self.soup("<a>text</a>")
        data.a['foo'] = 'bar'
        assert '<a foo="bar">text</a>' == data.a.decode()

    def test_closing_tag_with_no_opening_tag(self):
        # Without BeautifulSoup.open_tag_counter, the </span> tag will
        # cause _popToTag to be called over and over again as we look
        # for a <span> tag that wasn't there. The result is that 'text2'
        # will show up outside the body of the document.
        soup = self.soup("<body><div><p>text1</p></span>text2</div></body>")
        assert "<body><div><p>text1</p>text2</div></body>" == soup.body.decode()
        
    def test_worst_case(self):
        """Test the worst case (currently) for linking issues."""

        soup = self.soup(BAD_DOCUMENT)
        self.linkage_validator(soup)


class XMLTreeBuilderSmokeTest(TreeBuilderSmokeTest):

    def test_pickle_and_unpickle_identity(self):
        # Pickling a tree, then unpickling it, yields a tree identical
        # to the original.
        tree = self.soup("<a><b>foo</a>")
        dumped = pickle.dumps(tree, 2)
        loaded = pickle.loads(dumped)
        assert loaded.__class__ == BeautifulSoup
        assert loaded.decode() == tree.decode()

    def test_docstring_generated(self):
        soup = self.soup("<root/>")
        assert soup.encode() == b'<?xml version="1.0" encoding="utf-8"?>\n<root/>'

    def test_xml_declaration(self):
        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<foo/>"""
        soup = self.soup(markup)
        assert markup == soup.encode("utf8")

    def test_python_specific_encodings_not_used_in_xml_declaration(self):
        # You can encode an XML document using a Python-specific
        # encoding, but that encoding won't be mentioned _inside_ the
        # resulting document.
        markup = b"""<?xml version="1.0"?>\n<foo/>"""
        soup = self.soup(markup)
        for encoding in PYTHON_SPECIFIC_ENCODINGS:
            if encoding in (
                'idna', 'mbcs', 'oem', 'undefined',
                'string_escape', 'string-escape'
            ):
                # For one reason or another, these will raise an
                # exception if we actually try to use them, so don't
                # bother.
                continue
            encoded = soup.encode(encoding)
            assert b'<?xml version="1.0"?>' in encoded
            assert encoding.encode("ascii") not in encoded

    def test_processing_instruction(self):
        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<?PITarget PIContent?>"""
        soup = self.soup(markup)
        assert markup == soup.encode("utf8")

    def test_real_xhtml_document(self):
        """A real XHTML document should come out *exactly* the same as it went in."""
        markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>Hello.</title></head>
<body>Goodbye.</body>
</html>"""
        soup = self.soup(markup)
        assert soup.encode("utf-8") == markup
       
    def test_nested_namespaces(self):
        doc = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<parent xmlns="http://ns1/">
<child xmlns="http://ns2/" xmlns:ns3="http://ns3/">
<grandchild ns3:attr="value" xmlns="http://ns4/"/>
</child>
</parent>"""
        soup = self.soup(doc)
        assert doc == soup.encode()
        
    def test_formatter_processes_script_tag_for_xml_documents(self):
        doc = """
  <script type="text/javascript">
  </script>
"""
        soup = BeautifulSoup(doc, "lxml-xml")
        # lxml would have stripped this while parsing, but we can add
        # it later.
        soup.script.string = 'console.log("< < hey > > ");'
        encoded = soup.encode()
        assert b"&lt; &lt; hey &gt; &gt;" in encoded

    def test_can_parse_unicode_document(self):
        markup = '<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
        soup = self.soup(markup)
        assert 'Sacr\xe9 bleu!' == soup.root.string

    def test_can_parse_unicode_document_begining_with_bom(self):
        markup = '\N{BYTE ORDER MARK}<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
        soup = self.soup(markup)
        assert 'Sacr\xe9 bleu!' == soup.root.string
        
    def test_popping_namespaced_tag(self):
        markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
        soup = self.soup(markup)
        assert str(soup.rss) == markup

    def test_docstring_includes_correct_encoding(self):
        soup = self.soup("<root/>")
        assert soup.encode("latin1") == b'<?xml version="1.0" encoding="latin1"?>\n<root/>'

    def test_large_xml_document(self):
        """A large XML document should come out the same as it went in."""
        markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
                  + b'0' * (2**12)
                  + b'</root>')
        soup = self.soup(markup)
        assert soup.encode("utf-8") == markup

    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
        self.assert_soup("<p>", "<p/>")
        self.assert_soup("<p>foo</p>")

    def test_namespaces_are_preserved(self):
        markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
        soup = self.soup(markup)
        root = soup.root
        assert "http://example.com/" == root['xmlns:a']
        assert "http://example.net/" == root['xmlns:b']

    def test_closing_namespaced_tag(self):
        markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
        soup = self.soup(markup)
        assert str(soup.p) == markup

    def test_namespaced_attributes(self):
        markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
        soup = self.soup(markup)
        assert str(soup.foo) == markup

    def test_namespaced_attributes_xml_namespace(self):
        markup = '<foo xml:lang="fr">bar</foo>'
        soup = self.soup(markup)
        assert str(soup.foo) == markup

    def test_find_by_prefixed_name(self):
        doc = """<?xml version="1.0" encoding="utf-8"?>
<Document xmlns="http://example.com/ns0"
    xmlns:ns1="http://example.com/ns1"
    xmlns:ns2="http://example.com/ns2"
    <ns1:tag>foo</ns1:tag>
    <ns1:tag>bar</ns1:tag>
    <ns2:tag key="value">baz</ns2:tag>
</Document>
"""
        soup = self.soup(doc)

        # There are three <tag> tags.
        assert 3 == len(soup.find_all('tag'))

        # But two of them are ns1:tag and one of them is ns2:tag.
        assert 2 == len(soup.find_all('ns1:tag'))
        assert 1 == len(soup.find_all('ns2:tag'))
        
        assert 1, len(soup.find_all('ns2:tag', key='value'))
        assert 3, len(soup.find_all(['ns1:tag', 'ns2:tag']))
        
    def test_copy_tag_preserves_namespace(self):
        xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:document xmlns:w="http://example.com/ns0"/>"""
    
        soup = self.soup(xml)
        tag = soup.document
        duplicate = copy.copy(tag)

        # The two tags have the same namespace prefix.
        assert tag.prefix == duplicate.prefix

    def test_worst_case(self):
        """Test the worst case (currently) for linking issues."""

        soup = self.soup(BAD_DOCUMENT)
        self.linkage_validator(soup)


class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
    """Smoke test for a tree builder that supports HTML5."""

    def test_real_xhtml_document(self):
        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
        # XHTML documents in any particular way.
        pass

    def test_html_tags_have_namespace(self):
        markup = "<a>"
        soup = self.soup(markup)
        assert "http://www.w3.org/1999/xhtml" == soup.a.namespace

    def test_svg_tags_have_namespace(self):
        markup = '<svg><circle/></svg>'
        soup = self.soup(markup)
        namespace = "http://www.w3.org/2000/svg"
        assert namespace == soup.svg.namespace
        assert namespace == soup.circle.namespace


    def test_mathml_tags_have_namespace(self):
        markup = '<math><msqrt>5</msqrt></math>'
        soup = self.soup(markup)
        namespace = 'http://www.w3.org/1998/Math/MathML'
        assert namespace == soup.math.namespace
        assert namespace == soup.msqrt.namespace

    def test_xml_declaration_becomes_comment(self):
        markup = '<?xml version="1.0" encoding="utf-8"?><html></html>'
        soup = self.soup(markup)
        assert isinstance(soup.contents[0], Comment)
        assert soup.contents[0] == '?xml version="1.0" encoding="utf-8"?'
        assert "html" == soup.contents[0].next_element.name

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/__init__.py
NikolaBlagojevic/pyrecodes
import pytest
from unittest.mock import patch
from bs4.builder import DetectsXMLParsedAsHTML

class TestDetectsXMLParsedAsHTML(object):

    @pytest.mark.parametrize(
        "markup,looks_like_xml",
        [("No xml declaration", False),
         ("<html>obviously HTML</html", False),
         ("<?xml ><html>Actually XHTML</html>", False),
         ("<?xml>            <    html>Tricky XHTML</html>", False),
         ("<?xml ><no-html-tag>", True),
        ]
    )
    def test_warn_if_markup_looks_like_xml(self, markup, looks_like_xml):
        # Test of our ability to guess at whether markup looks XML-ish
        # _and_ not HTML-ish.
        with patch('bs4.builder.DetectsXMLParsedAsHTML._warn') as mock:
            for data in markup, markup.encode('utf8'):
                result = DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(
                    data
                )
                assert result == looks_like_xml
                if looks_like_xml:
                    assert mock.called
                else:
                    assert not mock.called
                mock.reset_mock()

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_builder.py
NikolaBlagojevic/pyrecodes
"""Tests of the builder registry."""

import pytest
import warnings

from bs4 import BeautifulSoup
from bs4.builder import (
    builder_registry as registry,
    HTMLParserTreeBuilder,
    TreeBuilderRegistry,
)

from . import (
    HTML5LIB_PRESENT,
    LXML_PRESENT,
)

if HTML5LIB_PRESENT:
    from bs4.builder import HTML5TreeBuilder

if LXML_PRESENT:
    from bs4.builder import (
        LXMLTreeBuilderForXML,
        LXMLTreeBuilder,
        )


# TODO: Split out the lxml and html5lib tests into their own classes
# and gate with pytest.mark.skipIf.
class TestBuiltInRegistry(object):
    """Test the built-in registry with the default builders registered."""

    def test_combination(self):
        assert registry.lookup('strict', 'html') == HTMLParserTreeBuilder
        if LXML_PRESENT:
            assert registry.lookup('fast', 'html') == LXMLTreeBuilder
            assert registry.lookup('permissive', 'xml') == LXMLTreeBuilderForXML
        if HTML5LIB_PRESENT:
            assert registry.lookup('html5lib', 'html') == HTML5TreeBuilder

    def test_lookup_by_markup_type(self):
        if LXML_PRESENT:
            assert registry.lookup('html') == LXMLTreeBuilder
            assert registry.lookup('xml') == LXMLTreeBuilderForXML
        else:
            assert registry.lookup('xml') == None
            if HTML5LIB_PRESENT:
                assert registry.lookup('html') == HTML5TreeBuilder
            else:
                assert registry.lookup('html') == HTMLParserTreeBuilder

    def test_named_library(self):
        if LXML_PRESENT:
            assert registry.lookup('lxml', 'xml') == LXMLTreeBuilderForXML
            assert registry.lookup('lxml', 'html') == LXMLTreeBuilder
        if HTML5LIB_PRESENT:
            assert registry.lookup('html5lib') == HTML5TreeBuilder

        assert registry.lookup('html.parser') == HTMLParserTreeBuilder

    def test_beautifulsoup_constructor_does_lookup(self):

        with warnings.catch_warnings(record=True) as w:
            # This will create a warning about not explicitly
            # specifying a parser, but we'll ignore it.

            # You can pass in a string.
            BeautifulSoup("", features="html")
            # Or a list of strings.
            BeautifulSoup("", features=["html", "fast"])
            pass
            
        # You'll get an exception if BS can't find an appropriate
        # builder.
        with pytest.raises(ValueError):
            BeautifulSoup("", features="no-such-feature")

class TestRegistry(object):
    """Test the TreeBuilderRegistry class in general."""

    def setup_method(self):
        self.registry = TreeBuilderRegistry()

    def builder_for_features(self, *feature_list):
        cls = type('Builder_' + '_'.join(feature_list),
                   (object,), {'features' : feature_list})

        self.registry.register(cls)
        return cls

    def test_register_with_no_features(self):
        builder = self.builder_for_features()

        # Since the builder advertises no features, you can't find it
        # by looking up features.
        assert self.registry.lookup('foo') is None

        # But you can find it by doing a lookup with no features, if
        # this happens to be the only registered builder.
        assert self.registry.lookup() == builder

    def test_register_with_features_makes_lookup_succeed(self):
        builder = self.builder_for_features('foo', 'bar')
        assert self.registry.lookup('foo') is builder
        assert self.registry.lookup('bar') is builder

    def test_lookup_fails_when_no_builder_implements_feature(self):
        builder = self.builder_for_features('foo', 'bar')
        assert self.registry.lookup('baz') is None

    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
        builder1 = self.builder_for_features('foo')
        builder2 = self.builder_for_features('bar')
        assert self.registry.lookup() == builder2

    def test_lookup_fails_when_no_tree_builders_registered(self):
        assert self.registry.lookup() is None

    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
        has_one = self.builder_for_features('foo')
        has_the_other = self.builder_for_features('bar')
        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
        lacks_one = self.builder_for_features('bar')
        has_the_other = self.builder_for_features('foo')

        # There are two builders featuring 'foo' and 'bar', but
        # the one that also features 'quux' was registered later.
        assert self.registry.lookup('foo', 'bar') == has_both_late

        # There is only one builder featuring 'foo', 'bar', and 'baz'.
        assert self.registry.lookup('foo', 'bar', 'baz') == has_both_early

    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
        builder1 = self.builder_for_features('foo', 'bar')
        builder2 = self.builder_for_features('foo', 'baz')
        assert self.registry.lookup('bar', 'baz') is None

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_builder_registry.py
NikolaBlagojevic/pyrecodes
import pytest
import types
from unittest.mock import MagicMock

from bs4 import (
    CSS,
    BeautifulSoup,
    ResultSet,
)

from . import (
    SoupTest,
    SOUP_SIEVE_PRESENT,
)

if SOUP_SIEVE_PRESENT:
    from soupsieve import SelectorSyntaxError


@pytest.mark.skipif(not SOUP_SIEVE_PRESENT, reason="Soup Sieve not installed")
class TestCSSSelectors(SoupTest):
    """Test basic CSS selector functionality.

    This functionality is implemented in soupsieve, which has a much
    more comprehensive test suite, so this is basically an extra check
    that soupsieve works as expected.
    """

    HTML = """
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<title>The title</title>
<link rel="stylesheet" href="blah.css" type="text/css" id="l1">
</head>
<body>
<custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag>
<div id="main" class="fancy">
<div id="inner">
<h1 id="header1">An H1</h1>
<p>Some text</p>
<p class="onep" id="p1">Some more text</p>
<h2 id="header2">An H2</h2>
<p class="class1 class2 class3" id="pmulti">Another</p>
<a href="http://bob.example.org/" rel="friend met" id="bob">Bob</a>
<h2 id="header3">Another H2</h2>
<a id="me" href="http://simonwillison.net/" rel="me">me</a>
<span class="s1">
<a href="#" id="s1a1">span1a1</a>
<a href="#" id="s1a2">span1a2 <span id="s1a2s1">test</span></a>
<span class="span2">
<a href="#" id="s2a1">span2a1</a>
</span>
<span class="span3"></span>
<custom-dashed-tag class="dashed" id="dash2"/>
<div data-tag="dashedvalue" id="data1"/>
</span>
</div>
<x id="xid">
<z id="zida"/>
<z id="zidab"/>
<z id="zidac"/>
</x>
<y id="yid">
<z id="zidb"/>
</y>
<p lang="en" id="lang-en">English</p>
<p lang="en-gb" id="lang-en-gb">English UK</p>
<p lang="en-us" id="lang-en-us">English US</p>
<p lang="fr" id="lang-fr">French</p>
</div>

<div id="footer">
</div>
"""

    def setup_method(self):
        self.soup = BeautifulSoup(self.HTML, 'html.parser')

    def assert_selects(self, selector, expected_ids, **kwargs):
        results = self.soup.select(selector, **kwargs)
        assert isinstance(results, ResultSet)
        el_ids = [el['id'] for el in results]
        el_ids.sort()
        expected_ids.sort()
        assert expected_ids == el_ids, "Selector %s, expected [%s], got [%s]" % (
                selector, ', '.join(expected_ids), ', '.join(el_ids)
        )

    assertSelect = assert_selects

    def assert_select_multiple(self, *tests):
        for selector, expected_ids in tests:
            self.assert_selects(selector, expected_ids)

    def test_precompiled(self):
        sel = self.soup.css.compile('div')

        els = self.soup.select(sel)
        assert len(els) == 4
        for div in els:
            assert div.name == 'div'

        el = self.soup.select_one(sel)
        assert 'main' == el['id']

    def test_one_tag_one(self):
        els = self.soup.select('title')
        assert len(els) == 1
        assert els[0].name == 'title'
        assert els[0].contents == ['The title']

    def test_one_tag_many(self):
        els = self.soup.select('div')
        assert len(els) == 4
        for div in els:
            assert div.name == 'div'

        el = self.soup.select_one('div')
        assert 'main' == el['id']

    def test_select_one_returns_none_if_no_match(self):
        match = self.soup.select_one('nonexistenttag')
        assert None == match


    def test_tag_in_tag_one(self):
        els = self.soup.select('div div')
        self.assert_selects('div div', ['inner', 'data1'])

    def test_tag_in_tag_many(self):
        for selector in ('html div', 'html body div', 'body div'):
            self.assert_selects(selector, ['data1', 'main', 'inner', 'footer'])


    def test_limit(self):
        self.assert_selects('html div', ['main'], limit=1)
        self.assert_selects('html body div', ['inner', 'main'], limit=2)
        self.assert_selects('body div', ['data1', 'main', 'inner', 'footer'],
                           limit=10)

    def test_tag_no_match(self):
        assert len(self.soup.select('del')) == 0

    def test_invalid_tag(self):
        with pytest.raises(SelectorSyntaxError):
            self.soup.select('tag%t')

    def test_select_dashed_tag_ids(self):
        self.assert_selects('custom-dashed-tag', ['dash1', 'dash2'])

    def test_select_dashed_by_id(self):
        dashed = self.soup.select('custom-dashed-tag[id=\"dash2\"]')
        assert dashed[0].name == 'custom-dashed-tag'
        assert dashed[0]['id'] == 'dash2'

    def test_dashed_tag_text(self):
        assert self.soup.select('body > custom-dashed-tag')[0].text == 'Hello there.'

    def test_select_dashed_matches_find_all(self):
        assert self.soup.select('custom-dashed-tag') == self.soup.find_all('custom-dashed-tag')

    def test_header_tags(self):
        self.assert_select_multiple(
            ('h1', ['header1']),
            ('h2', ['header2', 'header3']),
        )

    def test_class_one(self):
        for selector in ('.onep', 'p.onep', 'html p.onep'):
            els = self.soup.select(selector)
            assert len(els) == 1
            assert els[0].name == 'p'
            assert els[0]['class'] == ['onep']

    def test_class_mismatched_tag(self):
        els = self.soup.select('div.onep')
        assert len(els) == 0

    def test_one_id(self):
        for selector in ('div#inner', '#inner', 'div div#inner'):
            self.assert_selects(selector, ['inner'])

    def test_bad_id(self):
        els = self.soup.select('#doesnotexist')
        assert len(els) == 0

    def test_items_in_id(self):
        els = self.soup.select('div#inner p')
        assert len(els) == 3
        for el in els:
            assert el.name == 'p'
        assert els[1]['class'] == ['onep']
        assert not els[0].has_attr('class')

    def test_a_bunch_of_emptys(self):
        for selector in ('div#main del', 'div#main div.oops', 'div div#main'):
            assert len(self.soup.select(selector)) == 0

    def test_multi_class_support(self):
        for selector in ('.class1', 'p.class1', '.class2', 'p.class2',
            '.class3', 'p.class3', 'html p.class2', 'div#inner .class2'):
            self.assert_selects(selector, ['pmulti'])

    def test_multi_class_selection(self):
        for selector in ('.class1.class3', '.class3.class2',
                         '.class1.class2.class3'):
            self.assert_selects(selector, ['pmulti'])

    def test_child_selector(self):
        self.assert_selects('.s1 > a', ['s1a1', 's1a2'])
        self.assert_selects('.s1 > a span', ['s1a2s1'])

    def test_child_selector_id(self):
        self.assert_selects('.s1 > a#s1a2 span', ['s1a2s1'])

    def test_attribute_equals(self):
        self.assert_select_multiple(
            ('p[class="onep"]', ['p1']),
            ('p[id="p1"]', ['p1']),
            ('[class="onep"]', ['p1']),
            ('[id="p1"]', ['p1']),
            ('link[rel="stylesheet"]', ['l1']),
            ('link[type="text/css"]', ['l1']),
            ('link[href="blah.css"]', ['l1']),
            ('link[href="no-blah.css"]', []),
            ('[rel="stylesheet"]', ['l1']),
            ('[type="text/css"]', ['l1']),
            ('[href="blah.css"]', ['l1']),
            ('[href="no-blah.css"]', []),
            ('p[href="no-blah.css"]', []),
            ('[href="no-blah.css"]', []),
        )

    def test_attribute_tilde(self):
        self.assert_select_multiple(
            ('p[class~="class1"]', ['pmulti']),
            ('p[class~="class2"]', ['pmulti']),
            ('p[class~="class3"]', ['pmulti']),
            ('[class~="class1"]', ['pmulti']),
            ('[class~="class2"]', ['pmulti']),
            ('[class~="class3"]', ['pmulti']),
            ('a[rel~="friend"]', ['bob']),
            ('a[rel~="met"]', ['bob']),
            ('[rel~="friend"]', ['bob']),
            ('[rel~="met"]', ['bob']),
        )

    def test_attribute_startswith(self):
        self.assert_select_multiple(
            ('[rel^="style"]', ['l1']),
            ('link[rel^="style"]', ['l1']),
            ('notlink[rel^="notstyle"]', []),
            ('[rel^="notstyle"]', []),
            ('link[rel^="notstyle"]', []),
            ('link[href^="bla"]', ['l1']),
            ('a[href^="http://"]', ['bob', 'me']),
            ('[href^="http://"]', ['bob', 'me']),
            ('[id^="p"]', ['pmulti', 'p1']),
            ('[id^="m"]', ['me', 'main']),
            ('div[id^="m"]', ['main']),
            ('a[id^="m"]', ['me']),
            ('div[data-tag^="dashed"]', ['data1'])
        )

    def test_attribute_endswith(self):
        self.assert_select_multiple(
            ('[href$=".css"]', ['l1']),
            ('link[href$=".css"]', ['l1']),
            ('link[id$="1"]', ['l1']),
            ('[id$="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1', 'dash1']),
            ('div[id$="1"]', ['data1']),
            ('[id$="noending"]', []),
        )

    def test_attribute_contains(self):
        self.assert_select_multiple(
            # From test_attribute_startswith
            ('[rel*="style"]', ['l1']),
            ('link[rel*="style"]', ['l1']),
            ('notlink[rel*="notstyle"]', []),
            ('[rel*="notstyle"]', []),
            ('link[rel*="notstyle"]', []),
            ('link[href*="bla"]', ['l1']),
            ('[href*="http://"]', ['bob', 'me']),
            ('[id*="p"]', ['pmulti', 'p1']),
            ('div[id*="m"]', ['main']),
            ('a[id*="m"]', ['me']),
            # From test_attribute_endswith
            ('[href*=".css"]', ['l1']),
            ('link[href*=".css"]', ['l1']),
            ('link[id*="1"]', ['l1']),
            ('[id*="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1', 'dash1']),
            ('div[id*="1"]', ['data1']),
            ('[id*="noending"]', []),
            # New for this test
            ('[href*="."]', ['bob', 'me', 'l1']),
            ('a[href*="."]', ['bob', 'me']),
            ('link[href*="."]', ['l1']),
            ('div[id*="n"]', ['main', 'inner']),
            ('div[id*="nn"]', ['inner']),
            ('div[data-tag*="edval"]', ['data1'])
        )

    def test_attribute_exact_or_hypen(self):
        self.assert_select_multiple(
            ('p[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
            ('[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
            ('p[lang|="fr"]', ['lang-fr']),
            ('p[lang|="gb"]', []),
        )

    def test_attribute_exists(self):
        self.assert_select_multiple(
            ('[rel]', ['l1', 'bob', 'me']),
            ('link[rel]', ['l1']),
            ('a[rel]', ['bob', 'me']),
            ('[lang]', ['lang-en', 'lang-en-gb', 'lang-en-us', 'lang-fr']),
            ('p[class]', ['p1', 'pmulti']),
            ('[blah]', []),
            ('p[blah]', []),
            ('div[data-tag]', ['data1'])
        )

    def test_quoted_space_in_selector_name(self):
        html = """<div style="display: wrong">nope</div>
        <div style="display: right">yes</div>
        """
        soup = BeautifulSoup(html, 'html.parser')
        [chosen] = soup.select('div[style="display: right"]')
        assert "yes" == chosen.string

    def test_unsupported_pseudoclass(self):
        with pytest.raises(NotImplementedError):
            self.soup.select("a:no-such-pseudoclass")

        with pytest.raises(SelectorSyntaxError):
            self.soup.select("a:nth-of-type(a)")

    def test_nth_of_type(self):
        # Try to select first paragraph
        els = self.soup.select('div#inner p:nth-of-type(1)')
        assert len(els) == 1
        assert els[0].string == 'Some text'

        # Try to select third paragraph
        els = self.soup.select('div#inner p:nth-of-type(3)')
        assert len(els) == 1
        assert els[0].string == 'Another'

        # Try to select (non-existent!) fourth paragraph
        els = self.soup.select('div#inner p:nth-of-type(4)')
        assert len(els) == 0

        # Zero will select no tags.
        els = self.soup.select('div p:nth-of-type(0)')
        assert len(els) == 0

    def test_nth_of_type_direct_descendant(self):
        els = self.soup.select('div#inner > p:nth-of-type(1)')
        assert len(els) == 1
        assert els[0].string == 'Some text'

    def test_id_child_selector_nth_of_type(self):
        self.assert_selects('#inner > p:nth-of-type(2)', ['p1'])

    def test_select_on_element(self):
        # Other tests operate on the tree; this operates on an element
        # within the tree.
        inner = self.soup.find("div", id="main")
        selected = inner.select("div")
        # The <div id="inner"> tag was selected. The <div id="footer">
        # tag was not.
        self.assert_selects_ids(selected, ['inner', 'data1'])

    def test_overspecified_child_id(self):
        self.assert_selects(".fancy #inner", ['inner'])
        self.assert_selects(".normal #inner", [])

    def test_adjacent_sibling_selector(self):
        self.assert_selects('#p1 + h2', ['header2'])
        self.assert_selects('#p1 + h2 + p', ['pmulti'])
        self.assert_selects('#p1 + #header2 + .class1', ['pmulti'])
        assert [] == self.soup.select('#p1 + p')

    def test_general_sibling_selector(self):
        self.assert_selects('#p1 ~ h2', ['header2', 'header3'])
        self.assert_selects('#p1 ~ #header2', ['header2'])
        self.assert_selects('#p1 ~ h2 + a', ['me'])
        self.assert_selects('#p1 ~ h2 + [rel="me"]', ['me'])
        assert [] == self.soup.select('#inner ~ h2')

    def test_dangling_combinator(self):
        with pytest.raises(SelectorSyntaxError):
            self.soup.select('h1 >')

    def test_sibling_combinator_wont_select_same_tag_twice(self):
        self.assert_selects('p[lang] ~ p', ['lang-en-gb', 'lang-en-us', 'lang-fr'])

    # Test the selector grouping operator (the comma)
    def test_multiple_select(self):
        self.assert_selects('x, y', ['xid', 'yid'])

    def test_multiple_select_with_no_space(self):
        self.assert_selects('x,y', ['xid', 'yid'])

    def test_multiple_select_with_more_space(self):
        self.assert_selects('x,    y', ['xid', 'yid'])

    def test_multiple_select_duplicated(self):
        self.assert_selects('x, x', ['xid'])

    def test_multiple_select_sibling(self):
        self.assert_selects('x, y ~ p[lang=fr]', ['xid', 'lang-fr'])

    def test_multiple_select_tag_and_direct_descendant(self):
        self.assert_selects('x, y > z', ['xid', 'zidb'])

    def test_multiple_select_direct_descendant_and_tags(self):
        self.assert_selects('div > x, y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac'])

    def test_multiple_select_indirect_descendant(self):
        self.assert_selects('div x,y,  z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac'])

    def test_invalid_multiple_select(self):
        with pytest.raises(SelectorSyntaxError):
            self.soup.select(',x, y')
        with pytest.raises(SelectorSyntaxError):
            self.soup.select('x,,y')

    def test_multiple_select_attrs(self):
        self.assert_selects('p[lang=en], p[lang=en-gb]', ['lang-en', 'lang-en-gb'])

    def test_multiple_select_ids(self):
        self.assert_selects('x, y > z[id=zida], z[id=zidab], z[id=zidb]', ['xid', 'zidb', 'zidab'])

    def test_multiple_select_nested(self):
        self.assert_selects('body > div > x, y > z', ['xid', 'zidb'])

    def test_select_duplicate_elements(self):
        # When markup contains duplicate elements, a multiple select
        # will find all of them.
        markup = '<div class="c1"/><div class="c2"/><div class="c1"/>'
        soup = BeautifulSoup(markup, 'html.parser')
        selected = soup.select(".c1, .c2")
        assert 3 == len(selected)

        # Verify that find_all finds the same elements, though because
        # of an implementation detail it finds them in a different
        # order.
        for element in soup.find_all(class_=['c1', 'c2']):
            assert element in selected

    def test_closest(self):
        inner = self.soup.find("div", id="inner")
        closest = inner.css.closest("div[id=main]")
        assert closest == self.soup.find("div", id="main")

    def test_match(self):
        inner = self.soup.find("div", id="inner")
        main = self.soup.find("div", id="main")
        assert inner.css.match("div[id=main]") == False
        assert main.css.match("div[id=main]") == True

    def test_iselect(self):
        gen = self.soup.css.iselect("h2")
        assert isinstance(gen, types.GeneratorType)
        [header2, header3] = gen
        assert header2['id'] == 'header2'
        assert header3['id'] == 'header3'

    def test_filter(self):
        inner = self.soup.find("div", id="inner")
        results = inner.css.filter("h2")
        assert len(inner.css.filter("h2")) == 2

        results = inner.css.filter("h2[id=header3]")
        assert isinstance(results, ResultSet)
        [result] = results
        assert result['id'] == 'header3'

    def test_escape(self):
        m = self.soup.css.escape
        assert m(".foo#bar") == '\\.foo\\#bar'
        assert m("()[]{}") == '\\(\\)\\[\\]\\{\\}'
        assert m(".foo") == self.soup.css.escape(".foo")

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_css.py
NikolaBlagojevic/pyrecodes
# encoding: utf-8
import pytest
import logging
import bs4
from bs4 import BeautifulSoup
from bs4.dammit import (
    EntitySubstitution,
    EncodingDetector,
    UnicodeDammit,
)

class TestUnicodeDammit(object):
    """Standalone tests of UnicodeDammit."""

    def test_unicode_input(self):
        markup = "I'm already Unicode! \N{SNOWMAN}"
        dammit = UnicodeDammit(markup)
        assert dammit.unicode_markup == markup

    @pytest.mark.parametrize(
        "smart_quotes_to,expect_converted",
        [(None, "\u2018\u2019\u201c\u201d"),
         ("xml", "&#x2018;&#x2019;&#x201C;&#x201D;"),
         ("html", "&lsquo;&rsquo;&ldquo;&rdquo;"),
         ("ascii", "''" + '""'),
        ]
    )
    def test_smart_quotes_to(self, smart_quotes_to, expect_converted):
        """Verify the functionality of the smart_quotes_to argument
        to the UnicodeDammit constructor."""
        markup = b"<foo>\x91\x92\x93\x94</foo>"
        converted = UnicodeDammit(
            markup, known_definite_encodings=["windows-1252"],
            smart_quotes_to=smart_quotes_to
        ).unicode_markup
        assert converted == "<foo>{}</foo>".format(expect_converted)
        
    def test_detect_utf8(self):
        utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83"
        dammit = UnicodeDammit(utf8)
        assert dammit.original_encoding.lower() == 'utf-8'
        assert dammit.unicode_markup == 'Sacr\xe9 bleu! \N{SNOWMAN}'

    def test_convert_hebrew(self):
        hebrew = b"\xed\xe5\xec\xf9"
        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
        assert dammit.original_encoding.lower() == 'iso-8859-8'
        assert dammit.unicode_markup == '\u05dd\u05d5\u05dc\u05e9'

    def test_dont_see_smart_quotes_where_there_are_none(self):
        utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
        dammit = UnicodeDammit(utf_8)
        assert dammit.original_encoding.lower() == 'utf-8'
        assert dammit.unicode_markup.encode("utf-8") == utf_8

    def test_ignore_inappropriate_codecs(self):
        utf8_data = "Räksmörgås".encode("utf-8")
        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
        assert dammit.original_encoding.lower() == 'utf-8'

    def test_ignore_invalid_codecs(self):
        utf8_data = "Räksmörgås".encode("utf-8")
        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
            dammit = UnicodeDammit(utf8_data, [bad_encoding])
            assert dammit.original_encoding.lower() == 'utf-8'

    def test_exclude_encodings(self):
        # This is UTF-8.
        utf8_data = "Räksmörgås".encode("utf-8")

        # But if we exclude UTF-8 from consideration, the guess is
        # Windows-1252.
        dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
        assert dammit.original_encoding.lower() == 'windows-1252'

        # And if we exclude that, there is no valid guess at all.
        dammit = UnicodeDammit(
            utf8_data, exclude_encodings=["utf-8", "windows-1252"])
        assert dammit.original_encoding == None

class TestEncodingDetector(object):
        
    def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
        detected = EncodingDetector(
            b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
        encodings = list(detected.encodings)
        assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings

    def test_detect_html5_style_meta_tag(self):

        for data in (
            b'<html><meta charset="euc-jp" /></html>',
            b"<html><meta charset='euc-jp' /></html>",
            b"<html><meta charset=euc-jp /></html>",
            b"<html><meta charset=euc-jp/></html>"):
            dammit = UnicodeDammit(data, is_html=True)
            assert "euc-jp" == dammit.original_encoding

    def test_last_ditch_entity_replacement(self):
        # This is a UTF-8 document that contains bytestrings
        # completely incompatible with UTF-8 (ie. encoded with some other
        # encoding).
        #
        # Since there is no consistent encoding for the document,
        # Unicode, Dammit will eventually encode the document as UTF-8
        # and encode the incompatible characters as REPLACEMENT
        # CHARACTER.
        #
        # If chardet is installed, it will detect that the document
        # can be converted into ISO-8859-1 without errors. This happens
        # to be the wrong encoding, but it is a consistent encoding, so the
        # code we're testing here won't run.
        #
        # So we temporarily disable chardet if it's present.
        doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?>
<html><b>\330\250\330\252\330\261</b>
<i>\310\322\321\220\312\321\355\344</i></html>"""
        chardet = bs4.dammit.chardet_dammit
        logging.disable(logging.WARNING)
        try:
            def noop(str):
                return None
            bs4.dammit.chardet_dammit = noop
            dammit = UnicodeDammit(doc)
            assert True == dammit.contains_replacement_characters
            assert "\ufffd" in dammit.unicode_markup

            soup = BeautifulSoup(doc, "html.parser")
            assert soup.contains_replacement_characters
        finally:
            logging.disable(logging.NOTSET)
            bs4.dammit.chardet_dammit = chardet

    def test_byte_order_mark_removed(self):
        # A document written in UTF-16LE will have its byte order marker stripped.
        data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
        dammit = UnicodeDammit(data)
        assert "<a>áé</a>" == dammit.unicode_markup
        assert "utf-16le" == dammit.original_encoding
       
    def test_known_definite_versus_user_encodings(self):
        # The known_definite_encodings are used before sniffing the
        # byte-order mark; the user_encodings are used afterwards.

        # Here's a document in UTF-16LE.
        data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
        dammit = UnicodeDammit(data)

        # We can process it as UTF-16 by passing it in as a known
        # definite encoding.
        before = UnicodeDammit(data, known_definite_encodings=["utf-16"])
        assert "utf-16" == before.original_encoding
        
        # If we pass UTF-18 as a user encoding, it's not even
        # tried--the encoding sniffed from the byte-order mark takes
        # precedence.
        after = UnicodeDammit(data, user_encodings=["utf-8"])
        assert "utf-16le" == after.original_encoding
        assert ["utf-16le"] == [x[0] for x in dammit.tried_encodings]
        
        # Here's a document in ISO-8859-8.
        hebrew = b"\xed\xe5\xec\xf9"
        dammit = UnicodeDammit(hebrew, known_definite_encodings=["utf-8"],
                               user_encodings=["iso-8859-8"])
        
        # The known_definite_encodings don't work, BOM sniffing does
        # nothing (it only works for a few UTF encodings), but one of
        # the user_encodings does work.
        assert "iso-8859-8" == dammit.original_encoding
        assert ["utf-8", "iso-8859-8"] == [x[0] for x in dammit.tried_encodings]
        
    def test_deprecated_override_encodings(self):
        # override_encodings is a deprecated alias for
        # known_definite_encodings.
        hebrew = b"\xed\xe5\xec\xf9"
        dammit = UnicodeDammit(
            hebrew,
            known_definite_encodings=["shift-jis"],
            override_encodings=["utf-8"],
            user_encodings=["iso-8859-8"],
        )
        assert "iso-8859-8" == dammit.original_encoding

        # known_definite_encodings and override_encodings were tried
        # before user_encodings.
        assert ["shift-jis", "utf-8", "iso-8859-8"] == (
            [x[0] for x in dammit.tried_encodings]
        )

    def test_detwingle(self):
        # Here's a UTF8 document.
        utf8 = ("\N{SNOWMAN}" * 3).encode("utf8")

        # Here's a Windows-1252 document.
        windows_1252 = (
            "\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
            "\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")

        # Through some unholy alchemy, they've been stuck together.
        doc = utf8 + windows_1252 + utf8

        # The document can't be turned into UTF-8:
        with pytest.raises(UnicodeDecodeError):
            doc.decode("utf8")

        # Unicode, Dammit thinks the whole document is Windows-1252,
        # and decodes it into "â˜ƒâ˜ƒâ˜ƒ“Hi, I like Windows!”â˜ƒâ˜ƒâ˜ƒ"

        # But if we run it through fix_embedded_windows_1252, it's fixed:
        fixed = UnicodeDammit.detwingle(doc)
        assert "☃☃☃“Hi, I like Windows!”☃☃☃" == fixed.decode("utf8")

    def test_detwingle_ignores_multibyte_characters(self):
        # Each of these characters has a UTF-8 representation ending
        # in \x93. \x93 is a smart quote if interpreted as
        # Windows-1252. But our code knows to skip over multibyte
        # UTF-8 characters, so they'll survive the process unscathed.
        for tricky_unicode_char in (
            "\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
            "\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
            "\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
            ):
            input = tricky_unicode_char.encode("utf8")
            assert input.endswith(b'\x93')
            output = UnicodeDammit.detwingle(input)
            assert output == input

    def test_find_declared_encoding(self):
        # Test our ability to find a declared encoding inside an
        # XML or HTML document.
        #
        # Even if the document comes in as Unicode, it may be
        # interesting to know what encoding was claimed
        # originally.

        html_unicode = '<html><head><meta charset="utf-8"></head></html>'
        html_bytes = html_unicode.encode("ascii")

        xml_unicode= '<?xml version="1.0" encoding="ISO-8859-1" ?>'
        xml_bytes = xml_unicode.encode("ascii")

        m = EncodingDetector.find_declared_encoding
        assert m(html_unicode, is_html=False) is None
        assert "utf-8" == m(html_unicode, is_html=True)
        assert "utf-8" == m(html_bytes, is_html=True)

        assert "iso-8859-1" == m(xml_unicode)
        assert "iso-8859-1" == m(xml_bytes)

        # Normally, only the first few kilobytes of a document are checked for
        # an encoding.
        spacer = b' ' * 5000
        assert m(spacer + html_bytes) is None
        assert m(spacer + xml_bytes) is None

        # But you can tell find_declared_encoding to search an entire
        # HTML document.
        assert (
            m(spacer + html_bytes, is_html=True, search_entire_document=True)
            == "utf-8"
        )

        # The XML encoding declaration has to be the very first thing
        # in the document. We'll allow whitespace before the document
        # starts, but nothing else.
        assert m(xml_bytes, search_entire_document=True) == "iso-8859-1"
        assert m(b' ' + xml_bytes, search_entire_document=True) == "iso-8859-1"
        assert m(b'a' + xml_bytes, search_entire_document=True) is None


class TestEntitySubstitution(object):
    """Standalone tests of the EntitySubstitution class."""
    def setup_method(self):
        self.sub = EntitySubstitution


    @pytest.mark.parametrize(
        "original,substituted",
        [
            # Basic case. Unicode characters corresponding to named
            # HTML entites are substituted; others are not.
            ("foo\u2200\N{SNOWMAN}\u00f5bar",
             "foo&forall;\N{SNOWMAN}&otilde;bar"),

            # MS smart quotes are a common source of frustration, so we
            # give them a special test.
            ('‘’foo“”', "&lsquo;&rsquo;foo&ldquo;&rdquo;"),           
        ]
    )
    def test_substitute_html(self, original, substituted):
        assert self.sub.substitute_html(original) == substituted
        
    def test_html5_entity(self):
        for entity, u in (
            # A few spot checks of our ability to recognize
            # special character sequences and convert them
            # to named entities.
            ('&models;', '\u22a7'),
            ('&Nfr;', '\U0001d511'),
            ('&ngeqq;', '\u2267\u0338'),
            ('&not;', '\xac'),
            ('&Not;', '\u2aec'),
                
            # We _could_ convert | to &verbarr;, but we don't, because
            # | is an ASCII character.
            ('|' '|'),

            # Similarly for the fj ligature, which we could convert to
            # &fjlig;, but we don't.
            ("fj", "fj"),

            # We do convert _these_ ASCII characters to HTML entities,
            # because that's required to generate valid HTML.
            ('&gt;', '>'),
            ('&lt;', '<'),
            ('&amp;', '&'),
        ):
            template = '3 %s 4'
            raw = template % u
            with_entities = template % entity
            assert self.sub.substitute_html(raw) == with_entities
            
    def test_html5_entity_with_variation_selector(self):
        # Some HTML5 entities correspond either to a single-character
        # Unicode sequence _or_ to the same character plus U+FE00,
        # VARIATION SELECTOR 1. We can handle this.
        data = "fjords \u2294 penguins"
        markup = "fjords &sqcup; penguins"
        assert self.sub.substitute_html(data) == markup

        data = "fjords \u2294\ufe00 penguins"
        markup = "fjords &sqcups; penguins"
        assert self.sub.substitute_html(data) == markup
        
    def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
        s = 'Welcome to "my bar"'
        assert self.sub.substitute_xml(s, False) == s

    def test_xml_attribute_quoting_normally_uses_double_quotes(self):
        assert self.sub.substitute_xml("Welcome", True) == '"Welcome"'
        assert self.sub.substitute_xml("Bob's Bar", True) == '"Bob\'s Bar"'

    def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
        s = 'Welcome to "my bar"'
        assert self.sub.substitute_xml(s, True) == "'Welcome to \"my bar\"'"

    def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
        s = 'Welcome to "Bob\'s Bar"'
        assert self.sub.substitute_xml(s, True) == '"Welcome to &quot;Bob\'s Bar&quot;"'

    def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
        quoted = 'Welcome to "Bob\'s Bar"'
        assert self.sub.substitute_xml(quoted) == quoted

    def test_xml_quoting_handles_angle_brackets(self):
        assert self.sub.substitute_xml("foo<bar>") == "foo&lt;bar&gt;"

    def test_xml_quoting_handles_ampersands(self):
        assert self.sub.substitute_xml("AT&T") == "AT&amp;T"

    def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
        assert self.sub.substitute_xml("&Aacute;T&T") == "&amp;Aacute;T&amp;T"

    def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
        assert self.sub.substitute_xml_containing_entities("&Aacute;T&T") == "&Aacute;T&amp;T"
       
    def test_quotes_not_html_substituted(self):
        """There's no need to do this except inside attribute values."""
        text = 'Bob\'s "bar"'
        assert self.sub.substitute_html(text) == text

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_dammit.py
NikolaBlagojevic/pyrecodes
"Test harness for doctests."

# TODO: Pretty sure this isn't used and should be deleted.

# pylint: disable-msg=E0611,W0142

__metaclass__ = type
__all__ = [
    'additional_tests',
    ]

import atexit
import doctest
import os
#from pkg_resources import (
#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
import unittest

DOCTEST_FLAGS = (
    doctest.ELLIPSIS |
    doctest.NORMALIZE_WHITESPACE |
    doctest.REPORT_NDIFF)


# def additional_tests():
#     "Run the doc tests (README.txt and docs/*, if any exist)"
#     doctest_files = [
#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
#     if resource_exists('bs4', 'docs'):
#         for name in resource_listdir('bs4', 'docs'):
#             if name.endswith('.txt'):
#                 doctest_files.append(
#                     os.path.abspath(
#                         resource_filename('bs4', 'docs/%s' % name)))
#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
#     atexit.register(cleanup_resources)
#     return unittest.TestSuite((
#         doctest.DocFileSuite(*doctest_files, **kwargs)))

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_docs.py
NikolaBlagojevic/pyrecodes
"""Tests of classes in element.py.

The really big classes -- Tag, PageElement, and NavigableString --
are tested in separate files.
"""

from bs4.element import (
    CharsetMetaAttributeValue,
    ContentMetaAttributeValue,
    NamespacedAttribute,
)
from . import SoupTest


class TestNamedspacedAttribute(object):

    def test_name_may_be_none_or_missing(self):
        a = NamespacedAttribute("xmlns", None)
        assert a == "xmlns"

        a = NamespacedAttribute("xmlns", "")
        assert a == "xmlns"

        a = NamespacedAttribute("xmlns")
        assert a == "xmlns"
        
    def test_namespace_may_be_none_or_missing(self):
        a = NamespacedAttribute(None, "tag")
        assert a == "tag"
        
        a = NamespacedAttribute("", "tag")
        assert a == "tag"
        
    def test_attribute_is_equivalent_to_colon_separated_string(self):
        a = NamespacedAttribute("a", "b")
        assert "a:b" == a

    def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
        a = NamespacedAttribute("a", "b", "c")
        b = NamespacedAttribute("a", "b", "c")
        assert a == b

        # The actual namespace is not considered.
        c = NamespacedAttribute("a", "b", None)
        assert a == c

        # But name and prefix are important.
        d = NamespacedAttribute("a", "z", "c")
        assert a != d

        e = NamespacedAttribute("z", "b", "c")
        assert a != e


class TestAttributeValueWithCharsetSubstitution(object):
    """Certain attributes are designed to have the charset of the
    final document substituted into their value.
    """
    
    def test_content_meta_attribute_value(self):
        # The value of a CharsetMetaAttributeValue is whatever
        # encoding the string is in.
        value = CharsetMetaAttributeValue("euc-jp")
        assert "euc-jp" == value
        assert "euc-jp" == value.original_value
        assert "utf8" == value.encode("utf8")
        assert "ascii" == value.encode("ascii")

    def test_content_meta_attribute_value(self):
        value = ContentMetaAttributeValue("text/html; charset=euc-jp")
        assert "text/html; charset=euc-jp" == value
        assert "text/html; charset=euc-jp" == value.original_value
        assert "text/html; charset=utf8" == value.encode("utf8")
        assert "text/html; charset=ascii" == value.encode("ascii")

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_element.py
NikolaBlagojevic/pyrecodes
import pytest

from bs4.element import Tag
from bs4.formatter import (
    Formatter,
    HTMLFormatter,
    XMLFormatter,
)
from . import SoupTest

class TestFormatter(SoupTest):

    def test_default_attributes(self):
        # Test the default behavior of Formatter.attributes().
        formatter = Formatter()
        tag = Tag(name="tag")
        tag['b'] = 1
        tag['a'] = 2

        # Attributes come out sorted by name. In Python 3, attributes
        # normally come out of a dictionary in the order they were
        # added.
        assert [('a', 2), ('b', 1)] == formatter.attributes(tag)

        # This works even if Tag.attrs is None, though this shouldn't
        # normally happen.
        tag.attrs = None
        assert [] == formatter.attributes(tag)

        assert ' ' == formatter.indent
        
    def test_sort_attributes(self):
        # Test the ability to override Formatter.attributes() to,
        # e.g., disable the normal sorting of attributes.
        class UnsortedFormatter(Formatter):
            def attributes(self, tag):
                self.called_with = tag
                for k, v in sorted(tag.attrs.items()):
                    if k == 'ignore':
                        continue
                    yield k,v

        soup = self.soup('<p cval="1" aval="2" ignore="ignored"></p>')
        formatter = UnsortedFormatter()
        decoded = soup.decode(formatter=formatter)

        # attributes() was called on the <p> tag. It filtered out one
        # attribute and sorted the other two.
        assert formatter.called_with == soup.p
        assert '<p aval="2" cval="1"></p>' == decoded

    def test_empty_attributes_are_booleans(self):
        # Test the behavior of empty_attributes_are_booleans as well
        # as which Formatters have it enabled.
        
        for name in ('html', 'minimal', None):
            formatter = HTMLFormatter.REGISTRY[name]
            assert False == formatter.empty_attributes_are_booleans

        formatter = XMLFormatter.REGISTRY[None]
        assert False == formatter.empty_attributes_are_booleans

        formatter = HTMLFormatter.REGISTRY['html5']
        assert True == formatter.empty_attributes_are_booleans

        # Verify that the constructor sets the value.
        formatter = Formatter(empty_attributes_are_booleans=True)
        assert True == formatter.empty_attributes_are_booleans

        # Now demonstrate what it does to markup.
        for markup in (
                "<option selected></option>",
                '<option selected=""></option>'
        ):
            soup = self.soup(markup)
            for formatter in ('html', 'minimal', 'xml', None):
                assert b'<option selected=""></option>' == soup.option.encode(formatter='html')
                assert b'<option selected></option>' == soup.option.encode(formatter='html5')

    @pytest.mark.parametrize(
        "indent,expect",
        [
            (None, '<a>\n<b>\ntext\n</b>\n</a>\n'),
            (-1, '<a>\n<b>\ntext\n</b>\n</a>\n'),
            (0, '<a>\n<b>\ntext\n</b>\n</a>\n'),
            ("", '<a>\n<b>\ntext\n</b>\n</a>\n'),

            (1, '<a>\n <b>\n  text\n </b>\n</a>\n'),
            (2, '<a>\n  <b>\n    text\n  </b>\n</a>\n'),

            ("\t", '<a>\n\t<b>\n\t\ttext\n\t</b>\n</a>\n'),
            ('abc', '<a>\nabc<b>\nabcabctext\nabc</b>\n</a>\n'),

            # Some invalid inputs -- the default behavior is used.
            (object(), '<a>\n <b>\n  text\n </b>\n</a>\n'),
            (b'bytes', '<a>\n <b>\n  text\n </b>\n</a>\n'),
        ]
    )
    def test_indent(self, indent, expect):
        # Pretty-print a tree with a Formatter set to
        # indent in a certain way and verify the results.
        soup = self.soup("<a><b>text</b></a>")
        formatter = Formatter(indent=indent)
        assert soup.prettify(formatter=formatter) == expect

        # Pretty-printing only happens with prettify(), not
        # encode().
        assert soup.encode(formatter=formatter) != expect
        
    def test_default_indent_value(self):
        formatter = Formatter()
        assert formatter.indent == ' '


env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_formatter.py
NikolaBlagojevic/pyrecodes
"""This file contains test cases reported by third parties using
fuzzing tools, primarily from Google's oss-fuzz project. Some of these
represent real problems with Beautiful Soup, but many are problems in
libraries that Beautiful Soup depends on, and many of the test cases
represent different ways of triggering the same problem.

Grouping these test cases together makes it easy to see which test
cases represent the same problem, and puts the test cases in close
proximity to code that can trigger the problems.
"""
import os
import pytest
from bs4 import (
    BeautifulSoup,
    ParserRejectedMarkup,
)

class TestFuzz(object):

    # Test case markup files from fuzzers are given this extension so
    # they can be included in builds.
    TESTCASE_SUFFIX = ".testcase"

    # This class of error has been fixed by catching a less helpful
    # exception from html.parser and raising ParserRejectedMarkup
    # instead.
    @pytest.mark.parametrize(
        "filename", [
            "clusterfuzz-testcase-minimized-bs4_fuzzer-5703933063462912",
        ]
    )
    def test_rejected_markup(self, filename):
        markup = self.__markup(filename)
        with pytest.raises(ParserRejectedMarkup):
            BeautifulSoup(markup, 'html.parser')

    # This class of error has to do with very deeply nested documents
    # which overflow the Python call stack when the tree is converted
    # to a string. This is an issue with Beautiful Soup which was fixed
    # as part of [bug=1471755].
    @pytest.mark.parametrize(
        "filename", [
            "clusterfuzz-testcase-minimized-bs4_fuzzer-5984173902397440",
            "clusterfuzz-testcase-minimized-bs4_fuzzer-5167584867909632",
            "clusterfuzz-testcase-minimized-bs4_fuzzer-6124268085182464",
            "clusterfuzz-testcase-minimized-bs4_fuzzer-6450958476902400",
        ]
    )
    def test_deeply_nested_document(self, filename):
        # Parsing the document and encoding it back to a string is
        # sufficient to demonstrate that the overflow problem has
        # been fixed.
        markup = self.__markup(filename)
        BeautifulSoup(markup, 'html.parser').encode()

    # This class of error represents problems with html5lib's parser,
    # not Beautiful Soup. I use
    # https://github.com/html5lib/html5lib-python/issues/568 to notify
    # the html5lib developers of these issues.
    @pytest.mark.skip("html5lib problems")
    @pytest.mark.parametrize(
        "filename", [
            # b"""ÿ<!DOCTyPEV PUBLIC'''Ð'"""
            "clusterfuzz-testcase-minimized-bs4_fuzzer-4818336571064320",

            # b')<a><math><TR><a><mI><a><p><a>'
            "clusterfuzz-testcase-minimized-bs4_fuzzer-4999465949331456",

            # b'-<math><sElect><mi><sElect><sElect>'
            "clusterfuzz-testcase-minimized-bs4_fuzzer-5843991618256896",

            # b'ñ<table><svg><html>'
            "clusterfuzz-testcase-minimized-bs4_fuzzer-6241471367348224",

            # <TABLE>, some ^@ characters, some <math> tags.
            "clusterfuzz-testcase-minimized-bs4_fuzzer-6600557255327744",

            # Nested table
            "crash-0d306a50c8ed8bcd0785b67000fcd5dea1d33f08"
        ]
    )
    def test_html5lib_parse_errors(self, filename):
        markup = self.__markup(filename)
        print(BeautifulSoup(markup, 'html5lib').encode())

    def __markup(self, filename):
        if not filename.endswith(self.TESTCASE_SUFFIX):
            filename += self.TESTCASE_SUFFIX
        this_dir = os.path.split(__file__)[0]
        path = os.path.join(this_dir, 'fuzz', filename)
        return open(path, 'rb').read()

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_fuzz.py
NikolaBlagojevic/pyrecodes
"""Tests to ensure that the html5lib tree builder generates good trees."""

import pytest
import warnings

from bs4 import BeautifulSoup
from bs4.element import SoupStrainer
from . import (
    HTML5LIB_PRESENT,
    HTML5TreeBuilderSmokeTest,
    SoupTest,
)

@pytest.mark.skipif(
    not HTML5LIB_PRESENT,
    reason="html5lib seems not to be present, not testing its tree builder."
)
class TestHTML5LibBuilder(SoupTest, HTML5TreeBuilderSmokeTest):
    """See ``HTML5TreeBuilderSmokeTest``."""

    @property
    def default_builder(self):
        from bs4.builder import HTML5TreeBuilder
        return HTML5TreeBuilder

    def test_soupstrainer(self):
        # The html5lib tree builder does not support SoupStrainers.
        strainer = SoupStrainer("b")
        markup = "<p>A <b>bold</b> statement.</p>"
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulSoup(markup, "html5lib", parse_only=strainer)
        assert soup.decode() == self.document_for(markup)

        [warning] = w
        assert warning.filename == __file__
        assert "the html5lib tree builder doesn't support parse_only" in str(warning.message)

    def test_correctly_nested_tables(self):
        """html5lib inserts <tbody> tags where other parsers don't."""
        markup = ('<table id="1">'
                  '<tr>'
                  "<td>Here's another table:"
                  '<table id="2">'
                  '<tr><td>foo</td></tr>'
                  '</table></td>')

        self.assert_soup(
            markup,
            '<table id="1"><tbody><tr><td>Here\'s another table:'
            '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
            '</td></tr></tbody></table>')

        self.assert_soup(
            "<table><thead><tr><td>Foo</td></tr></thead>"
            "<tbody><tr><td>Bar</td></tr></tbody>"
            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")

    def test_xml_declaration_followed_by_doctype(self):
        markup = '''<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html>
<html>
  <head>
  </head>
  <body>
   <p>foo</p>
  </body>
</html>'''
        soup = self.soup(markup)
        # Verify that we can reach the <p> tag; this means the tree is connected.
        assert b"<p>foo</p>" == soup.p.encode()

    def test_reparented_markup(self):
        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>'
        soup = self.soup(markup)
        assert "<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>" == soup.body.decode()
        assert 2 == len(soup.find_all('p'))


    def test_reparented_markup_ends_with_whitespace(self):
        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n'
        soup = self.soup(markup)
        assert "<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>" == soup.body.decode()
        assert 2 == len(soup.find_all('p'))

    def test_reparented_markup_containing_identical_whitespace_nodes(self):
        """Verify that we keep the two whitespace nodes in this
        document distinct when reparenting the adjacent <tbody> tags.
        """
        markup = '<table> <tbody><tbody><ims></tbody> </table>'
        soup = self.soup(markup)
        space1, space2 = soup.find_all(string=' ')
        tbody1, tbody2 = soup.find_all('tbody')
        assert space1.next_element is tbody1
        assert tbody2.next_element is space2

    def test_reparented_markup_containing_children(self):
        markup = '<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>'
        soup = self.soup(markup)
        noscript = soup.noscript
        assert "target" == noscript.next_element
        target = soup.find(string='target')

        # The 'aftermath' string was duplicated; we want the second one.
        final_aftermath = soup.find_all(string='aftermath')[-1]

        # The <noscript> tag was moved beneath a copy of the <a> tag,
        # but the 'target' string within is still connected to the
        # (second) 'aftermath' string.
        assert final_aftermath == target.next_element
        assert target == final_aftermath.previous_element
        
    def test_processing_instruction(self):
        """Processing instructions become comments."""
        markup = b"""<?PITarget PIContent?>"""
        soup = self.soup(markup)
        assert str(soup).startswith("<!--?PITarget PIContent?-->")

    def test_cloned_multivalue_node(self):
        markup = b"""<a class="my_class"><p></a>"""
        soup = self.soup(markup)
        a1, a2 = soup.find_all('a')
        assert a1 == a2
        assert a1 is not a2

    def test_foster_parenting(self):
        markup = b"""<table><td></tbody>A"""
        soup = self.soup(markup)
        assert "<body>A<table><tbody><tr><td></td></tr></tbody></table></body>" == soup.body.decode()

    def test_extraction(self):
        """
        Test that extraction does not destroy the tree.

        https://bugs.launchpad.net/beautifulsoup/+bug/1782928
        """

        markup = """
<html><head></head>
<style>
</style><script></script><body><p>hello</p></body></html>
"""
        soup = self.soup(markup)
        [s.extract() for s in soup('script')]
        [s.extract() for s in soup('style')]

        assert len(soup.find_all("p")) == 1

    def test_empty_comment(self):
        """
        Test that empty comment does not break structure.

        https://bugs.launchpad.net/beautifulsoup/+bug/1806598
        """

        markup = """
<html>
<body>
<form>
<!----><input type="text">
</form>
</body>
</html>
"""
        soup = self.soup(markup)
        inputs = []
        for form in soup.find_all('form'):
            inputs.extend(form.find_all('input'))
        assert len(inputs) == 1

    def test_tracking_line_numbers(self):
        # The html.parser TreeBuilder keeps track of line number and
        # position of each element.
        markup = "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
        soup = self.soup(markup)
        assert 2 == soup.p.sourceline
        assert 5 == soup.p.sourcepos
        assert "sourceline" == soup.p.find('sourceline').name

        # You can deactivate this behavior.
        soup = self.soup(markup, store_line_numbers=False)
        assert "sourceline" == soup.p.sourceline.name
        assert "sourcepos" == soup.p.sourcepos.name

    def test_special_string_containers(self):
        # The html5lib tree builder doesn't support this standard feature,
        # because there's no way of knowing, when a string is created,
        # where in the tree it will eventually end up.
        pass

    def test_html5_attributes(self):
        # The html5lib TreeBuilder can convert any entity named in
        # the HTML5 spec to a sequence of Unicode characters, and
        # convert those Unicode characters to a (potentially
        # different) named entity on the way out.
        #
        # This is a copy of the same test from
        # HTMLParserTreeBuilderSmokeTest.  It's not in the superclass
        # because the lxml HTML TreeBuilder _doesn't_ work this way.
        for input_element, output_unicode, output_element in (
                ("&RightArrowLeftArrow;", '\u21c4', b'&rlarr;'),
                ('&models;', '\u22a7', b'&models;'),
                ('&Nfr;', '\U0001d511', b'&Nfr;'),
                ('&ngeqq;', '\u2267\u0338', b'&ngeqq;'),
                ('&not;', '\xac', b'&not;'),
                ('&Not;', '\u2aec', b'&Not;'),
                ('&quot;', '"', b'"'),
                ('&there4;', '\u2234', b'&there4;'),
                ('&Therefore;', '\u2234', b'&there4;'),
                ('&therefore;', '\u2234', b'&there4;'),
                ("&fjlig;", 'fj', b'fj'),                
                ("&sqcup;", '\u2294', b'&sqcup;'),
                ("&sqcups;", '\u2294\ufe00', b'&sqcups;'),
                ("&apos;", "'", b"'"),
                ("&verbar;", "|", b"|"),
        ):
            markup = '<div>%s</div>' % input_element
            div = self.soup(markup).div
            without_element = div.encode()
            expect = b"<div>%s</div>" % output_unicode.encode("utf8")
            assert without_element == expect

            with_element = div.encode(formatter="html")
            expect = b"<div>%s</div>" % output_element
            assert with_element == expect

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_html5lib.py
NikolaBlagojevic/pyrecodes
"""Tests to ensure that the html.parser tree builder generates good
trees."""

from pdb import set_trace
import pickle
import pytest
import warnings
from bs4.builder import (
    HTMLParserTreeBuilder,
    ParserRejectedMarkup,
    XMLParsedAsHTMLWarning,
)
from bs4.builder._htmlparser import BeautifulSoupHTMLParser
from . import SoupTest, HTMLTreeBuilderSmokeTest

class TestHTMLParserTreeBuilder(SoupTest, HTMLTreeBuilderSmokeTest):

    default_builder = HTMLParserTreeBuilder

    def test_rejected_input(self):
        # Python's html.parser will occasionally reject markup,
        # especially when there is a problem with the initial DOCTYPE
        # declaration. Different versions of Python sound the alarm in
        # different ways, but Beautiful Soup consistently raises
        # errors as ParserRejectedMarkup exceptions.
        bad_markup = [
            # https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28873
            # https://github.com/guidovranken/python-library-fuzzers/blob/master/corp-html/519e5b4269a01185a0d5e76295251921da2f0700
            # https://github.com/python/cpython/issues/81928
            b'\n<![\xff\xfe\xfe\xcd\x00',

            #https://github.com/guidovranken/python-library-fuzzers/blob/master/corp-html/de32aa55785be29bbc72a1a8e06b00611fb3d9f8
            # https://github.com/python/cpython/issues/78661
            #
            b'<![n\x00',
            b"<![UNKNOWN[]]>",
        ]
        for markup in bad_markup:
            with pytest.raises(ParserRejectedMarkup):
                soup = self.soup(markup)
    
    def test_namespaced_system_doctype(self):
        # html.parser can't handle namespaced doctypes, so skip this one.
        pass

    def test_namespaced_public_doctype(self):
        # html.parser can't handle namespaced doctypes, so skip this one.
        pass

    def test_builder_is_pickled(self):
        """Unlike most tree builders, HTMLParserTreeBuilder and will
        be restored after pickling.
        """
        tree = self.soup("<a><b>foo</a>")
        dumped = pickle.dumps(tree, 2)
        loaded = pickle.loads(dumped)
        assert isinstance(loaded.builder, type(tree.builder))

    def test_redundant_empty_element_closing_tags(self):
        self.assert_soup('<br></br><br></br><br></br>', "<br/><br/><br/>")
        self.assert_soup('</br></br></br>', "")

    def test_empty_element(self):
        # This verifies that any buffered data present when the parser
        # finishes working is handled.
        self.assert_soup("foo &# bar", "foo &amp;# bar")

    def test_tracking_line_numbers(self):
        # The html.parser TreeBuilder keeps track of line number and
        # position of each element.
        markup = "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
        soup = self.soup(markup)
        assert 2 == soup.p.sourceline
        assert 3 == soup.p.sourcepos
        assert "sourceline" == soup.p.find('sourceline').name

        # You can deactivate this behavior.
        soup = self.soup(markup, store_line_numbers=False)
        assert "sourceline" == soup.p.sourceline.name
        assert "sourcepos" == soup.p.sourcepos.name

    def test_on_duplicate_attribute(self):
        # The html.parser tree builder has a variety of ways of
        # handling a tag that contains the same attribute multiple times.

        markup = '<a class="cls" href="url1" href="url2" href="url3" id="id">'

        # If you don't provide any particular value for
        # on_duplicate_attribute, later values replace earlier values.
        soup = self.soup(markup)
        assert "url3" == soup.a['href']
        assert ["cls"] == soup.a['class']
        assert "id" == soup.a['id']
        
        # You can also get this behavior explicitly.
        def assert_attribute(on_duplicate_attribute, expected):
            soup = self.soup(
                markup, on_duplicate_attribute=on_duplicate_attribute
            )
            assert expected == soup.a['href']

            # Verify that non-duplicate attributes are treated normally.
            assert ["cls"] == soup.a['class']
            assert "id" == soup.a['id']
        assert_attribute(None, "url3")
        assert_attribute(BeautifulSoupHTMLParser.REPLACE, "url3")

        # You can ignore subsequent values in favor of the first.
        assert_attribute(BeautifulSoupHTMLParser.IGNORE, "url1")

        # And you can pass in a callable that does whatever you want.
        def accumulate(attrs, key, value):
            if not isinstance(attrs[key], list):
                attrs[key] = [attrs[key]]
            attrs[key].append(value)
        assert_attribute(accumulate, ["url1", "url2", "url3"])            

    def test_html5_attributes(self):
        # The html.parser TreeBuilder can convert any entity named in
        # the HTML5 spec to a sequence of Unicode characters, and
        # convert those Unicode characters to a (potentially
        # different) named entity on the way out.
        for input_element, output_unicode, output_element in (
                ("&RightArrowLeftArrow;", '\u21c4', b'&rlarr;'),
                ('&models;', '\u22a7', b'&models;'),
                ('&Nfr;', '\U0001d511', b'&Nfr;'),
                ('&ngeqq;', '\u2267\u0338', b'&ngeqq;'),
                ('&not;', '\xac', b'&not;'),
                ('&Not;', '\u2aec', b'&Not;'),
                ('&quot;', '"', b'"'),
                ('&there4;', '\u2234', b'&there4;'),
                ('&Therefore;', '\u2234', b'&there4;'),
                ('&therefore;', '\u2234', b'&there4;'),
                ("&fjlig;", 'fj', b'fj'),                
                ("&sqcup;", '\u2294', b'&sqcup;'),
                ("&sqcups;", '\u2294\ufe00', b'&sqcups;'),
                ("&apos;", "'", b"'"),
                ("&verbar;", "|", b"|"),
        ):
            markup = '<div>%s</div>' % input_element
            div = self.soup(markup).div
            without_element = div.encode()
            expect = b"<div>%s</div>" % output_unicode.encode("utf8")
            assert without_element == expect

            with_element = div.encode(formatter="html")
            expect = b"<div>%s</div>" % output_element
            assert with_element == expect

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_htmlparser.py
NikolaBlagojevic/pyrecodes
"""Tests to ensure that the lxml tree builder generates good trees."""

import pickle
import pytest
import re
import warnings
from . import LXML_PRESENT, LXML_VERSION

if LXML_PRESENT:
    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML

from bs4 import (
    BeautifulSoup,
    BeautifulStoneSoup,
    )
from bs4.element import Comment, Doctype, SoupStrainer
from . import (
    HTMLTreeBuilderSmokeTest,
    XMLTreeBuilderSmokeTest,
    SOUP_SIEVE_PRESENT,
    SoupTest,
)

@pytest.mark.skipif(
    not LXML_PRESENT,
    reason="lxml seems not to be present, not testing its tree builder."
)
class TestLXMLTreeBuilder(SoupTest, HTMLTreeBuilderSmokeTest):
    """See ``HTMLTreeBuilderSmokeTest``."""

    @property
    def default_builder(self):
        return LXMLTreeBuilder

    def test_out_of_range_entity(self):
        self.assert_soup(
            "<p>foo&#10000000000000;bar</p>", "<p>foobar</p>")
        self.assert_soup(
            "<p>foo&#x10000000000000;bar</p>", "<p>foobar</p>")
        self.assert_soup(
            "<p>foo&#1000000000;bar</p>", "<p>foobar</p>")
        
    def test_entities_in_foreign_document_encoding(self):
        # We can't implement this case correctly because by the time we
        # hear about markup like "&#147;", it's been (incorrectly) converted into
        # a string like u'\x93'
        pass
        
    # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
    # test if an old version of lxml is installed.

    @pytest.mark.skipif(
        not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
        reason="Skipping doctype test for old version of lxml to avoid segfault."
    )
    def test_empty_doctype(self):
        soup = self.soup("<!DOCTYPE>")
        doctype = soup.contents[0]
        assert "" == doctype.strip()

    def test_beautifulstonesoup_is_xml_parser(self):
        # Make sure that the deprecated BSS class uses an xml builder
        # if one is installed.
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulStoneSoup("<b />")
        assert "<b/>" == str(soup.b)
        [warning] = w
        assert warning.filename == __file__
        assert "BeautifulStoneSoup class is deprecated" in str(warning.message)

    def test_tracking_line_numbers(self):
        # The lxml TreeBuilder cannot keep track of line numbers from
        # the original markup. Even if you ask for line numbers, we
        # don't have 'em.
        #
        # This means that if you have a tag like <sourceline> or
        # <sourcepos>, attribute access will find it rather than
        # giving you a numeric answer.
        soup = self.soup(
            "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>",
            store_line_numbers=True
        )
        assert "sourceline" == soup.p.sourceline.name
        assert "sourcepos" == soup.p.sourcepos.name
        
@pytest.mark.skipif(
    not LXML_PRESENT,
    reason="lxml seems not to be present, not testing its XML tree builder."
)
class TestLXMLXMLTreeBuilder(SoupTest, XMLTreeBuilderSmokeTest):
    """See ``HTMLTreeBuilderSmokeTest``."""

    @property
    def default_builder(self):
        return LXMLTreeBuilderForXML

    def test_namespace_indexing(self):
        soup = self.soup(
            '<?xml version="1.1"?>\n'
            '<root>'
            '<tag xmlns="http://unprefixed-namespace.com">content</tag>'
            '<prefix:tag2 xmlns:prefix="http://prefixed-namespace.com">content</prefix:tag2>'
            '<prefix2:tag3 xmlns:prefix2="http://another-namespace.com">'
            '<subtag xmlns="http://another-unprefixed-namespace.com">'
            '<subsubtag xmlns="http://yet-another-unprefixed-namespace.com">'
            '</prefix2:tag3>'
            '</root>'
        )

        # The BeautifulSoup object includes every namespace prefix
        # defined in the entire document. This is the default set of
        # namespaces used by soupsieve.
        #
        # Un-prefixed namespaces are not included, and if a given
        # prefix is defined twice, only the first prefix encountered
        # in the document shows up here.
        assert soup._namespaces == {
            'xml': 'http://www.w3.org/XML/1998/namespace',
            'prefix': 'http://prefixed-namespace.com',
            'prefix2': 'http://another-namespace.com'
        }

        # A Tag object includes only the namespace prefixes
        # that were in scope when it was parsed.

        # We do not track un-prefixed namespaces as we can only hold
        # one (the first one), and it will be recognized as the
        # default namespace by soupsieve, even when operating from a
        # tag with a different un-prefixed namespace.
        assert soup.tag._namespaces == {
            'xml': 'http://www.w3.org/XML/1998/namespace',
        }

        assert soup.tag2._namespaces == {
            'prefix': 'http://prefixed-namespace.com',
            'xml': 'http://www.w3.org/XML/1998/namespace',
        }

        assert soup.subtag._namespaces == {
            'prefix2': 'http://another-namespace.com',
            'xml': 'http://www.w3.org/XML/1998/namespace',
        }

        assert soup.subsubtag._namespaces == {
            'prefix2': 'http://another-namespace.com',
            'xml': 'http://www.w3.org/XML/1998/namespace',
        }


    @pytest.mark.skipif(
        not SOUP_SIEVE_PRESENT, reason="Soup Sieve not installed"
    )
    def test_namespace_interaction_with_select_and_find(self):
        # Demonstrate how namespaces interact with select* and
        # find* methods.
        
        soup = self.soup(
            '<?xml version="1.1"?>\n'
            '<root>'
            '<tag xmlns="http://unprefixed-namespace.com">content</tag>'
            '<prefix:tag2 xmlns:prefix="http://prefixed-namespace.com">content</tag>'
            '<subtag xmlns:prefix="http://another-namespace-same-prefix.com">'
             '<prefix:tag3>'
            '</subtag>'
            '</root>'
        )

        # soupselect uses namespace URIs.
        assert soup.select_one('tag').name == 'tag'
        assert soup.select_one('prefix|tag2').name == 'tag2'

        # If a prefix is declared more than once, only the first usage
        # is registered with the BeautifulSoup object.
        assert soup.select_one('prefix|tag3') is None

        # But you can always explicitly specify a namespace dictionary.
        assert soup.select_one(
            'prefix|tag3', namespaces=soup.subtag._namespaces
        ).name == 'tag3'

        # And a Tag (as opposed to the BeautifulSoup object) will
        # have a set of default namespaces scoped to that Tag.
        assert soup.subtag.select_one('prefix|tag3').name=='tag3'

        # the find() methods aren't fully namespace-aware; they just
        # look at prefixes.
        assert soup.find('tag').name == 'tag'
        assert soup.find('prefix:tag2').name == 'tag2'
        assert soup.find('prefix:tag3').name == 'tag3'
        assert soup.subtag.find('prefix:tag3').name == 'tag3'

    def test_pickle_restores_builder(self):
        # The lxml TreeBuilder is not picklable, so when unpickling
        # a document created with it, a new TreeBuilder of the
        # appropriate class is created.
        soup = self.soup("<a>some markup</a>")
        assert isinstance(soup.builder, self.default_builder)
        pickled = pickle.dumps(soup)
        unpickled = pickle.loads(pickled)

        assert "some markup" == unpickled.a.string
        assert unpickled.builder != soup.builder
        assert isinstance(unpickled.builder, self.default_builder)

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_lxml.py
NikolaBlagojevic/pyrecodes
import pytest

from bs4.element import (
    CData,
    Comment,
    Declaration,
    Doctype,
    NavigableString,
    RubyParenthesisString,
    RubyTextString,
    Script,
    Stylesheet,
    TemplateString,
)

from . import SoupTest

class TestNavigableString(SoupTest):

    def test_text_acquisition_methods(self):
        # These methods are intended for use against Tag, but they
        # work on NavigableString as well,
        
        s = NavigableString("fee ")
        cdata = CData("fie ")
        comment = Comment("foe ")

        assert "fee " == s.get_text()
        assert "fee" == s.get_text(strip=True)
        assert ["fee "] == list(s.strings)
        assert ["fee"] == list(s.stripped_strings)
        assert ["fee "] == list(s._all_strings())

        assert "fie " == cdata.get_text()
        assert "fie" == cdata.get_text(strip=True)
        assert ["fie "] == list(cdata.strings)
        assert ["fie"] == list(cdata.stripped_strings)
        assert ["fie "] == list(cdata._all_strings())
        
        # Since a Comment isn't normally considered 'text',
        # these methods generally do nothing.
        assert "" == comment.get_text()
        assert [] == list(comment.strings)
        assert [] == list(comment.stripped_strings)
        assert [] == list(comment._all_strings())

        # Unless you specifically say that comments are okay.
        assert "foe" == comment.get_text(strip=True, types=Comment)
        assert "foe " == comment.get_text(types=(Comment, NavigableString))

    def test_string_has_immutable_name_property(self):
        # string.name is defined as None and can't be modified
        string = self.soup("s").string
        assert None == string.name
        with pytest.raises(AttributeError):
            string.name = 'foo'

class TestNavigableStringSubclasses(SoupTest):

    def test_cdata(self):
        # None of the current builders turn CDATA sections into CData
        # objects, but you can create them manually.
        soup = self.soup("")
        cdata = CData("foo")
        soup.insert(1, cdata)
        assert str(soup) == "<![CDATA[foo]]>"
        assert soup.find(string="foo") == "foo"
        assert soup.contents[0] == "foo"

    def test_cdata_is_never_formatted(self):
        """Text inside a CData object is passed into the formatter.

        But the return value is ignored.
        """

        self.count = 0
        def increment(*args):
            self.count += 1
            return "BITTER FAILURE"

        soup = self.soup("")
        cdata = CData("<><><>")
        soup.insert(1, cdata)
        assert b"<![CDATA[<><><>]]>" == soup.encode(formatter=increment)
        assert 1 == self.count

    def test_doctype_ends_in_newline(self):
        # Unlike other NavigableString subclasses, a DOCTYPE always ends
        # in a newline.
        doctype = Doctype("foo")
        soup = self.soup("")
        soup.insert(1, doctype)
        assert soup.encode() == b"<!DOCTYPE foo>\n"

    def test_declaration(self):
        d = Declaration("foo")
        assert "<?foo?>" == d.output_ready()

    def test_default_string_containers(self):
        # In some cases, we use different NavigableString subclasses for
        # the same text in different tags.
        soup = self.soup(
            "<div>text</div><script>text</script><style>text</style>"
        )
        assert [NavigableString, Script, Stylesheet] == [
            x.__class__ for x in soup.find_all(string=True)
        ]

        # The TemplateString is a little unusual because it's generally found
        # _inside_ children of a <template> element, not a direct child of the
        # <template> element.
        soup = self.soup(
            "<template>Some text<p>In a tag</p></template>Some text outside"
        )
        assert all(
            isinstance(x, TemplateString)
            for x in soup.template._all_strings(types=None)
        )
        
        # Once the <template> tag closed, we went back to using
        # NavigableString.
        outside = soup.template.next_sibling
        assert isinstance(outside, NavigableString)
        assert not isinstance(outside, TemplateString)

        # The TemplateString is also unusual because it can contain
        # NavigableString subclasses of _other_ types, such as
        # Comment.
        markup = b"<template>Some text<p>In a tag</p><!--with a comment--></template>"
        soup = self.soup(markup)
        assert markup == soup.template.encode("utf8")

    def test_ruby_strings(self):
        markup = "<ruby>漢 <rp>(</rp><rt>kan</rt><rp>)</rp> 字 <rp>(</rp><rt>ji</rt><rp>)</rp></ruby>"
        soup = self.soup(markup)
        assert isinstance(soup.rp.string, RubyParenthesisString)
        assert isinstance(soup.rt.string, RubyTextString)

        # Just as a demo, here's what this means for get_text usage.
        assert "漢字" == soup.get_text(strip=True)
        assert "漢(kan)字(ji)" == soup.get_text(
            strip=True,
            types=(NavigableString, RubyTextString, RubyParenthesisString)
        )

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_navigablestring.py
NikolaBlagojevic/pyrecodes
"""Tests of the bs4.element.PageElement class"""
import copy
import pickle
import pytest
import sys

from bs4 import BeautifulSoup
from bs4.element import (
    Comment,
    ResultSet,
    SoupStrainer,
)
from . import (
    SoupTest,
)

class TestEncoding(SoupTest):
    """Test the ability to encode objects into strings."""

    def test_unicode_string_can_be_encoded(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        assert soup.b.string.encode("utf-8") == "\N{SNOWMAN}".encode("utf-8")

    def test_tag_containing_unicode_string_can_be_encoded(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        assert soup.b.encode("utf-8") == html.encode("utf-8")

    def test_encoding_substitutes_unrecognized_characters_by_default(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        assert soup.b.encode("ascii") == b"<b>&#9731;</b>"

    def test_encoding_can_be_made_strict(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        with pytest.raises(UnicodeEncodeError):
            soup.encode("ascii", errors="strict")

    def test_decode_contents(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        assert "\N{SNOWMAN}" == soup.b.decode_contents()

    def test_encode_contents(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        assert "\N{SNOWMAN}".encode("utf8") == soup.b.encode_contents(
            encoding="utf8"
        )
        
    def test_encode_deeply_nested_document(self):
        # This test verifies that encoding a string doesn't involve
        # any recursive function calls. If it did, this test would
        # overflow the Python interpreter stack.
        limit = sys.getrecursionlimit() + 1
        markup = "<span>" * limit
        soup = self.soup(markup)
        encoded = soup.encode()
        assert limit == encoded.count(b"<span>")

    def test_deprecated_renderContents(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        soup.renderContents()
        assert "\N{SNOWMAN}".encode("utf8") == soup.b.renderContents()

    def test_repr(self):
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        assert html == repr(soup)

        
class TestFormatters(SoupTest):
    """Test the formatting feature, used by methods like decode() and
    prettify(), and the formatters themselves.
    """
    
    def test_default_formatter_is_minimal(self):
        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        soup = self.soup(markup)
        decoded = soup.decode(formatter="minimal")
        # The < is converted back into &lt; but the e-with-acute is left alone.
        assert decoded == self.document_for(
                "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        )

    def test_formatter_html(self):
        markup = "<br><b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        soup = self.soup(markup)
        decoded = soup.decode(formatter="html")
        assert decoded == self.document_for(
            "<br/><b>&lt;&lt;Sacr&eacute; bleu!&gt;&gt;</b>"
        )

    def test_formatter_html5(self):
        markup = "<br><b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        soup = self.soup(markup)
        decoded = soup.decode(formatter="html5")
        assert decoded == self.document_for(
            "<br><b>&lt;&lt;Sacr&eacute; bleu!&gt;&gt;</b>"
        )
        
    def test_formatter_minimal(self):
        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        soup = self.soup(markup)
        decoded = soup.decode(formatter="minimal")
        # The < is converted back into &lt; but the e-with-acute is left alone.
        assert decoded == self.document_for(
                "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        )

    def test_formatter_null(self):
        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
        soup = self.soup(markup)
        decoded = soup.decode(formatter=None)
        # Neither the angle brackets nor the e-with-acute are converted.
        # This is not valid HTML, but it's what the user wanted.
        assert decoded == self.document_for(
            "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>"
        )

    def test_formatter_custom(self):
        markup = "<b>&lt;foo&gt;</b><b>bar</b><br/>"
        soup = self.soup(markup)
        decoded = soup.decode(formatter = lambda x: x.upper())
        # Instead of normal entity conversion code, the custom
        # callable is called on every string.
        assert decoded == self.document_for("<b><FOO></b><b>BAR</b><br/>")

    def test_formatter_is_run_on_attribute_values(self):
        markup = '<a href="http://a.com?a=b&c=é">e</a>'
        soup = self.soup(markup)
        a = soup.a

        expect_minimal = '<a href="http://a.com?a=b&amp;c=é">e</a>'

        assert expect_minimal == a.decode()
        assert expect_minimal == a.decode(formatter="minimal")

        expect_html = '<a href="http://a.com?a=b&amp;c=&eacute;">e</a>'
        assert expect_html == a.decode(formatter="html")

        assert markup == a.decode(formatter=None)
        expect_upper = '<a href="HTTP://A.COM?A=B&C=É">E</a>'
        assert expect_upper == a.decode(formatter=lambda x: x.upper())

    def test_formatter_skips_script_tag_for_html_documents(self):
        doc = """
  <script type="text/javascript">
   console.log("< < hey > > ");
  </script>
"""
        encoded = BeautifulSoup(doc, 'html.parser').encode()
        assert b"< < hey > >" in encoded

    def test_formatter_skips_style_tag_for_html_documents(self):
        doc = """
  <style type="text/css">
   console.log("< < hey > > ");
  </style>
"""
        encoded = BeautifulSoup(doc, 'html.parser').encode()
        assert b"< < hey > >" in encoded

    def test_prettify_leaves_preformatted_text_alone(self):
        soup = self.soup("<div>  foo  <pre>  \tbar\n  \n  </pre>  baz  <textarea> eee\nfff\t</textarea></div>")
        # Everything outside the <pre> tag is reformatted, but everything
        # inside is left alone.
        assert '<div>\n foo\n <pre>  \tbar\n  \n  </pre>\n baz\n <textarea> eee\nfff\t</textarea>\n</div>\n' == soup.div.prettify()

    def test_prettify_handles_nested_string_literal_tags(self):
        # Most of this markup is inside a <pre> tag, so prettify()
        # only does three things to it:
        # 1. Add a newline and a space between the <div> and the <pre>
        # 2. Add a newline after the </pre>
        # 3. Add a newline at the end.
        #
        # The contents of the <pre> tag are left completely alone.  In
        # particular, we don't start adding whitespace again once we
        # encounter the first </pre> tag, because we know it's not
        # the one that put us into string literal mode.
        markup = """<div><pre><code>some
<script><pre>code</pre></script> for you 
</code></pre></div>"""

        expect = """<div>
 <pre><code>some
<script><pre>code</pre></script> for you 
</code></pre>
</div>
"""
        soup = self.soup(markup)
        assert expect == soup.div.prettify()

    def test_prettify_accepts_formatter_function(self):
        soup = BeautifulSoup("<html><body>foo</body></html>", 'html.parser')
        pretty = soup.prettify(formatter = lambda x: x.upper())
        assert "FOO" in pretty

    def test_prettify_outputs_unicode_by_default(self):
        soup = self.soup("<a></a>")
        assert str == type(soup.prettify())

    def test_prettify_can_encode_data(self):
        soup = self.soup("<a></a>")
        assert bytes == type(soup.prettify("utf-8"))

    def test_html_entity_substitution_off_by_default(self):
        markup = "<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>"
        soup = self.soup(markup)
        encoded = soup.b.encode("utf-8")
        assert encoded == markup.encode('utf-8')

    def test_encoding_substitution(self):
        # Here's the <meta> tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('<meta content="text/html; charset=x-sjis" '
                    'http-equiv="Content-type"/>')
        soup = self.soup(meta_tag)

        # Parse the document, and the charset apprears unchanged.
        assert soup.meta['content'] == 'text/html; charset=x-sjis'

        # Encode the document into some encoding, and the encoding is
        # substituted into the meta tag.
        utf_8 = soup.encode("utf-8")
        assert b"charset=utf-8" in utf_8

        euc_jp = soup.encode("euc_jp")
        assert b"charset=euc_jp" in euc_jp

        shift_jis = soup.encode("shift-jis")
        assert b"charset=shift-jis" in shift_jis

        utf_16_u = soup.encode("utf-16").decode("utf-16")
        assert "charset=utf-16" in utf_16_u

    def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self):
        markup = ('<head><meta content="text/html; charset=x-sjis" '
                    'http-equiv="Content-type"/></head><pre>foo</pre>')

        # Beautiful Soup used to try to rewrite the meta tag even if the
        # meta tag got filtered out by the strainer. This test makes
        # sure that doesn't happen.
        strainer = SoupStrainer('pre')
        soup = self.soup(markup, parse_only=strainer)
        assert soup.contents[0].name == 'pre'


class TestPersistence(SoupTest):
    "Testing features like pickle and deepcopy."

    def setup_method(self):
        self.page = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"
"http://www.w3.org/TR/REC-html40/transitional.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Beautiful Soup: We called him Tortoise because he taught us.</title>
<link rev="made" href="mailto:leonardr@segfault.org">
<meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping.">
<meta name="generator" content="Markov Approximation 1.4 (module: leonardr)">
<meta name="author" content="Leonard Richardson">
</head>
<body>
<a href="foo">foo</a>
<a href="foo"><b>bar</b></a>
</body>
</html>"""
        self.tree = self.soup(self.page)

    def test_pickle_and_unpickle_identity(self):
        # Pickling a tree, then unpickling it, yields a tree identical
        # to the original.
        dumped = pickle.dumps(self.tree, 2)
        loaded = pickle.loads(dumped)
        assert loaded.__class__ == BeautifulSoup
        assert loaded.decode() == self.tree.decode()
        
    def test_deepcopy_identity(self):
        # Making a deepcopy of a tree yields an identical tree.
        copied = copy.deepcopy(self.tree)
        assert copied.decode() == self.tree.decode()

    def test_copy_deeply_nested_document(self):
        # This test verifies that copy and deepcopy don't involve any
        # recursive function calls. If they did, this test would
        # overflow the Python interpreter stack.
        limit = sys.getrecursionlimit() + 1
        markup = "<span>" * limit

        soup = self.soup(markup)
        
        copied = copy.copy(soup)
        copied = copy.deepcopy(soup)

    def test_copy_preserves_encoding(self):
        soup = BeautifulSoup(b'<p>&nbsp;</p>', 'html.parser')
        encoding = soup.original_encoding
        copy = soup.__copy__()
        assert "<p> </p>" == str(copy)
        assert encoding == copy.original_encoding

    def test_copy_preserves_builder_information(self):

        tag = self.soup('<p></p>').p

        # Simulate a tag obtained from a source file.
        tag.sourceline = 10
        tag.sourcepos = 33
        
        copied = tag.__copy__()

        # The TreeBuilder object is no longer availble, but information
        # obtained from it gets copied over to the new Tag object.
        assert tag.sourceline == copied.sourceline
        assert tag.sourcepos == copied.sourcepos
        assert tag.can_be_empty_element == copied.can_be_empty_element
        assert tag.cdata_list_attributes == copied.cdata_list_attributes
        assert tag.preserve_whitespace_tags == copied.preserve_whitespace_tags
        assert tag.interesting_string_types == copied.interesting_string_types
        
    def test_unicode_pickle(self):
        # A tree containing Unicode characters can be pickled.
        html = "<b>\N{SNOWMAN}</b>"
        soup = self.soup(html)
        dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL)
        loaded = pickle.loads(dumped)
        assert loaded.decode() == soup.decode()

    def test_copy_navigablestring_is_not_attached_to_tree(self):
        html = "<b>Foo<a></a></b><b>Bar</b>"
        soup = self.soup(html)
        s1 = soup.find(string="Foo")
        s2 = copy.copy(s1)
        assert s1 == s2
        assert None == s2.parent
        assert None == s2.next_element
        assert None != s1.next_sibling
        assert None == s2.next_sibling
        assert None == s2.previous_element

    def test_copy_navigablestring_subclass_has_same_type(self):
        html = "<b><!--Foo--></b>"
        soup = self.soup(html)
        s1 = soup.string
        s2 = copy.copy(s1)
        assert s1 == s2
        assert isinstance(s2, Comment)

    def test_copy_entire_soup(self):
        html = "<div><b>Foo<a></a></b><b>Bar</b></div>end"
        soup = self.soup(html)
        soup_copy = copy.copy(soup)
        assert soup == soup_copy

    def test_copy_tag_copies_contents(self):
        html = "<div><b>Foo<a></a></b><b>Bar</b></div>end"
        soup = self.soup(html)
        div = soup.div
        div_copy = copy.copy(div)

        # The two tags look the same, and evaluate to equal.
        assert str(div) == str(div_copy)
        assert div == div_copy

        # But they're not the same object.
        assert div is not div_copy

        # And they don't have the same relation to the parse tree. The
        # copy is not associated with a parse tree at all.
        assert None == div_copy.parent
        assert None == div_copy.previous_element
        assert None == div_copy.find(string='Bar').next_element
        assert None != div.find(string='Bar').next_element


env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_pageelement.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
"""Tests of Beautiful Soup as a whole."""

from pdb import set_trace
import logging
import os
import pickle
import pytest
import sys
import tempfile

from bs4 import (
    BeautifulSoup,
    BeautifulStoneSoup,
    GuessedAtParserWarning,
    MarkupResemblesLocatorWarning,
    dammit,
)
from bs4.builder import (
    builder_registry,
    TreeBuilder,
    ParserRejectedMarkup,
)
from bs4.element import (
    Comment,
    SoupStrainer,
    PYTHON_SPECIFIC_ENCODINGS,
    Tag,
    NavigableString,
)

from . import (
    default_builder,
    LXML_PRESENT,
    SoupTest,
)
import warnings
    
class TestConstructor(SoupTest):

    def test_short_unicode_input(self):
        data = "<h1>éé</h1>"
        soup = self.soup(data)
        assert "éé" == soup.h1.string

    def test_embedded_null(self):
        data = "<h1>foo\0bar</h1>"
        soup = self.soup(data)
        assert "foo\0bar" == soup.h1.string

    def test_exclude_encodings(self):
        utf8_data = "Räksmörgås".encode("utf-8")
        soup = self.soup(utf8_data, exclude_encodings=["utf-8"])
        assert "windows-1252" == soup.original_encoding

    def test_custom_builder_class(self):
        # Verify that you can pass in a custom Builder class and
        # it'll be instantiated with the appropriate keyword arguments.
        class Mock(object):
            def __init__(self, **kwargs):
                self.called_with = kwargs
                self.is_xml = True
                self.store_line_numbers = False
                self.cdata_list_attributes = []
                self.preserve_whitespace_tags = []
                self.string_containers = {}
            def initialize_soup(self, soup):
                pass
            def feed(self, markup):
                self.fed = markup
            def reset(self):
                pass
            def ignore(self, ignore):
                pass
            set_up_substitutions = can_be_empty_element = ignore
            def prepare_markup(self, *args, **kwargs):
                yield "prepared markup", "original encoding", "declared encoding", "contains replacement characters"
                
        kwargs = dict(
            var="value",
            # This is a deprecated BS3-era keyword argument, which
            # will be stripped out.
            convertEntities=True,
        )
        with warnings.catch_warnings(record=True):
            soup = BeautifulSoup('', builder=Mock, **kwargs)
        assert isinstance(soup.builder, Mock)
        assert dict(var="value") == soup.builder.called_with
        assert "prepared markup" == soup.builder.fed
        
        # You can also instantiate the TreeBuilder yourself. In this
        # case, that specific object is used and any keyword arguments
        # to the BeautifulSoup constructor are ignored.
        builder = Mock(**kwargs)
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulSoup(
                '', builder=builder, ignored_value=True,
            )
        msg = str(w[0].message)
        assert msg.startswith("Keyword arguments to the BeautifulSoup constructor will be ignored.")
        assert builder == soup.builder
        assert kwargs == builder.called_with

    def test_parser_markup_rejection(self):
        # If markup is completely rejected by the parser, an
        # explanatory ParserRejectedMarkup exception is raised.
        class Mock(TreeBuilder):
            def feed(self, *args, **kwargs):
                raise ParserRejectedMarkup("Nope.")

        def prepare_markup(self, *args, **kwargs):
            # We're going to try two different ways of preparing this markup,
            # but feed() will reject both of them.
            yield markup, None, None, False
            yield markup, None, None, False
            

        import re
        with pytest.raises(ParserRejectedMarkup) as exc_info:
            BeautifulSoup('', builder=Mock)
        assert "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help." in str(exc_info.value)
        
    def test_cdata_list_attributes(self):
        # Most attribute values are represented as scalars, but the
        # HTML standard says that some attributes, like 'class' have
        # space-separated lists as values.
        markup = '<a id=" an id " class=" a class "></a>'
        soup = self.soup(markup)

        # Note that the spaces are stripped for 'class' but not for 'id'.
        a = soup.a
        assert " an id " == a['id']
        assert ["a", "class"] == a['class']

        # TreeBuilder takes an argument called 'multi_valued_attributes'  which lets
        # you customize or disable this. As always, you can customize the TreeBuilder
        # by passing in a keyword argument to the BeautifulSoup constructor.
        soup = self.soup(markup, builder=default_builder, multi_valued_attributes=None)
        assert " a class " == soup.a['class']

        # Here are two ways of saying that `id` is a multi-valued
        # attribute in this context, but 'class' is not.
        for switcheroo in ({'*': 'id'}, {'a': 'id'}):
            with warnings.catch_warnings(record=True) as w:
                # This will create a warning about not explicitly
                # specifying a parser, but we'll ignore it.
                soup = self.soup(markup, builder=None, multi_valued_attributes=switcheroo)
            a = soup.a
            assert ["an", "id"] == a['id']
            assert " a class " == a['class']

    def test_replacement_classes(self):
        # Test the ability to pass in replacements for element classes
        # which will be used when building the tree.
        class TagPlus(Tag):
            pass

        class StringPlus(NavigableString):
            pass

        class CommentPlus(Comment):
            pass
        
        soup = self.soup(
            "<a><b>foo</b>bar</a><!--whee-->",
            element_classes = {
                Tag: TagPlus,
                NavigableString: StringPlus,
                Comment: CommentPlus,
            }
        )

        # The tree was built with TagPlus, StringPlus, and CommentPlus objects,
        # rather than Tag, String, and Comment objects.
        assert all(
            isinstance(x, (TagPlus, StringPlus, CommentPlus))
            for x in soup.recursiveChildGenerator()
        )

    def test_alternate_string_containers(self):
        # Test the ability to customize the string containers for
        # different types of tags.
        class PString(NavigableString):
            pass

        class BString(NavigableString):
            pass

        soup = self.soup(
            "<div>Hello.<p>Here is <b>some <i>bolded</i></b> text",
            string_containers = {
                'b': BString,
                'p': PString,
            }
        )

        # The string before the <p> tag is a regular NavigableString.
        assert isinstance(soup.div.contents[0], NavigableString)
        
        # The string inside the <p> tag, but not inside the <i> tag,
        # is a PString.
        assert isinstance(soup.p.contents[0], PString)

        # Every string inside the <b> tag is a BString, even the one that
        # was also inside an <i> tag.
        for s in soup.b.strings:
            assert isinstance(s, BString)

        # Now that parsing was complete, the string_container_stack
        # (where this information was kept) has been cleared out.
        assert [] == soup.string_container_stack


class TestOutput(SoupTest):

    @pytest.mark.parametrize(
        "eventual_encoding,actual_encoding", [
            ("utf-8", "utf-8"),
            ("utf-16", "utf-16"),
        ]
    )
    def test_decode_xml_declaration(self, eventual_encoding, actual_encoding):
        # Most of the time, calling decode() on an XML document will
        # give you a document declaration that mentions the encoding
        # you intend to use when encoding the document as a
        # bytestring.
        soup = self.soup("<tag></tag>")
        soup.is_xml = True
        assert (f'<?xml version="1.0" encoding="{actual_encoding}"?>\n<tag></tag>'
                == soup.decode(eventual_encoding=eventual_encoding))

    @pytest.mark.parametrize(
        "eventual_encoding", [x for x in PYTHON_SPECIFIC_ENCODINGS] + [None]
    )
    def test_decode_xml_declaration_with_missing_or_python_internal_eventual_encoding(self, eventual_encoding):
        # But if you pass a Python internal encoding into decode(), or
        # omit the eventual_encoding altogether, the document
        # declaration won't mention any particular encoding.
        soup = BeautifulSoup("<tag></tag>", "html.parser")
        soup.is_xml = True
        assert (f'<?xml version="1.0"?>\n<tag></tag>'
                == soup.decode(eventual_encoding=eventual_encoding))

    def test(self):
        # BeautifulSoup subclasses Tag and extends the decode() method.
        # Make sure the other Tag methods which call decode() call
        # it correctly.
        soup = self.soup("<tag></tag>")
        assert b"<tag></tag>" == soup.encode(encoding="utf-8")
        assert b"<tag></tag>" == soup.encode_contents(encoding="utf-8")
        assert "<tag></tag>" == soup.decode_contents()
        assert "<tag>\n</tag>\n" == soup.prettify()

        
class TestWarnings(SoupTest):
    # Note that some of the tests in this class create BeautifulSoup
    # objects directly rather than using self.soup(). That's
    # because SoupTest.soup is defined in a different file,
    # which will throw off the assertion in _assert_warning
    # that the code that triggered the warning is in the same
    # file as the test.

    def _assert_warning(self, warnings, cls):
        for w in warnings:
            if isinstance(w.message, cls):
                assert w.filename == __file__
                return w
        raise Exception("%s warning not found in %r" % (cls, warnings))
    
    def _assert_no_parser_specified(self, w):
        warning = self._assert_warning(w, GuessedAtParserWarning)
        message = str(warning.message)
        assert message.startswith(BeautifulSoup.NO_PARSER_SPECIFIED_WARNING[:60])

    def test_warning_if_no_parser_specified(self):
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulSoup("<a><b></b></a>")
        self._assert_no_parser_specified(w)

    def test_warning_if_parser_specified_too_vague(self):
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulSoup("<a><b></b></a>", "html")
        self._assert_no_parser_specified(w)

    def test_no_warning_if_explicit_parser_specified(self):
        with warnings.catch_warnings(record=True) as w:
            soup = self.soup("<a><b></b></a>")
        assert [] == w

    def test_parseOnlyThese_renamed_to_parse_only(self):
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulSoup(
                "<a><b></b></a>", "html.parser",
                parseOnlyThese=SoupStrainer("b"),
            )
        warning = self._assert_warning(w, DeprecationWarning)
        msg = str(warning.message)
        assert "parseOnlyThese" in msg
        assert "parse_only" in msg
        assert b"<b></b>" == soup.encode()

    def test_fromEncoding_renamed_to_from_encoding(self):
        with warnings.catch_warnings(record=True) as w:
            utf8 = b"\xc3\xa9"
            soup = BeautifulSoup(
                utf8, "html.parser", fromEncoding="utf8"
            )
        warning = self._assert_warning(w, DeprecationWarning)
        msg = str(warning.message)
        assert "fromEncoding" in msg
        assert "from_encoding" in msg
        assert "utf8" == soup.original_encoding

    def test_unrecognized_keyword_argument(self):
        with pytest.raises(TypeError):
            self.soup("<a>", no_such_argument=True)

    @pytest.mark.parametrize(
        "extension",
        ['markup.html', 'markup.htm', 'markup.HTML', 'markup.txt',
         'markup.xhtml', 'markup.xml', "/home/user/file", "c:\\user\file"]
    )
    def test_resembles_filename_warning(self, extension):
        # A warning is issued if the "markup" looks like the name of
        # an HTML or text file, or a full path to a file on disk.
        with warnings.catch_warnings(record=True) as w:
            soup = BeautifulSoup("markup" + extension, "html.parser")
            warning = self._assert_warning(w, MarkupResemblesLocatorWarning)
            assert "looks more like a filename" in str(warning.message)

    @pytest.mark.parametrize(
        "extension",
        ['markuphtml', 'markup.com', '', 'markup.js']
    )
    def test_resembles_filename_no_warning(self, extension):
        # The 'looks more like a filename' warning is not issued if
        # the markup looks like a bare string, a domain name, or a
        # file that's not an HTML file.
        with warnings.catch_warnings(record=True) as w:
            soup = self.soup("markup" + extension)
        assert [] == w

    def test_url_warning_with_bytes_url(self):
        url = b"http://www.crummybytes.com/"
        with warnings.catch_warnings(record=True) as warning_list:
            soup = BeautifulSoup(url, "html.parser")
        warning = self._assert_warning(
            warning_list, MarkupResemblesLocatorWarning
        )
        assert "looks more like a URL" in str(warning.message)
        assert url not in str(warning.message).encode("utf8")
        
    def test_url_warning_with_unicode_url(self):
        url = "http://www.crummyunicode.com/"
        with warnings.catch_warnings(record=True) as warning_list:
            # note - this url must differ from the bytes one otherwise
            # python's warnings system swallows the second warning
            soup = BeautifulSoup(url, "html.parser")
        warning = self._assert_warning(
            warning_list, MarkupResemblesLocatorWarning
        )
        assert "looks more like a URL" in str(warning.message)
        assert url not in str(warning.message)

    def test_url_warning_with_bytes_and_space(self):
        # Here the markup contains something besides a URL, so no warning
        # is issued.
        with warnings.catch_warnings(record=True) as warning_list:
            soup = self.soup(b"http://www.crummybytes.com/ is great")
        assert not any("looks more like a URL" in str(w.message) 
                       for w in warning_list)

    def test_url_warning_with_unicode_and_space(self):
        with warnings.catch_warnings(record=True) as warning_list:
            soup = self.soup("http://www.crummyunicode.com/ is great")
        assert not any("looks more like a URL" in str(w.message) 
                       for w in warning_list)


class TestSelectiveParsing(SoupTest):

    def test_parse_with_soupstrainer(self):
        markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>"
        strainer = SoupStrainer("b")
        soup = self.soup(markup, parse_only=strainer)
        assert soup.encode() == b"<b>Yes</b><b>Yes <c>Yes</c></b>"

        
class TestNewTag(SoupTest):
    """Test the BeautifulSoup.new_tag() method."""
    def test_new_tag(self):
        soup = self.soup("")
        new_tag = soup.new_tag("foo", bar="baz", attrs={"name": "a name"})
        assert isinstance(new_tag, Tag)
        assert "foo" == new_tag.name
        assert dict(bar="baz", name="a name") == new_tag.attrs
        assert None == new_tag.parent

    @pytest.mark.skipif(
        not LXML_PRESENT,
        reason="lxml not installed, cannot parse XML document"
    )
    def test_xml_tag_inherits_self_closing_rules_from_builder(self):
        xml_soup = BeautifulSoup("", "xml")
        xml_br = xml_soup.new_tag("br")
        xml_p = xml_soup.new_tag("p")

        # Both the <br> and <p> tag are empty-element, just because
        # they have no contents.
        assert b"<br/>" == xml_br.encode()
        assert b"<p/>" == xml_p.encode()

    def test_tag_inherits_self_closing_rules_from_builder(self):
        html_soup = BeautifulSoup("", "html.parser")
        html_br = html_soup.new_tag("br")
        html_p = html_soup.new_tag("p")

        # The HTML builder users HTML's rules about which tags are
        # empty-element tags, and the new tags reflect these rules.
        assert b"<br/>" == html_br.encode()
        assert b"<p></p>" == html_p.encode()

class TestNewString(SoupTest):
    """Test the BeautifulSoup.new_string() method."""
    def test_new_string_creates_navigablestring(self):
        soup = self.soup("")
        s = soup.new_string("foo")
        assert "foo" == s
        assert isinstance(s, NavigableString)

    def test_new_string_can_create_navigablestring_subclass(self):
        soup = self.soup("")
        s = soup.new_string("foo", Comment)
        assert "foo" == s
        assert isinstance(s, Comment)


class TestPickle(SoupTest):
   # Test our ability to pickle the BeautifulSoup object itself.

    def test_normal_pickle(self):
        soup = self.soup("<a>some markup</a>")
        pickled = pickle.dumps(soup)
        unpickled = pickle.loads(pickled)
        assert "some markup" == unpickled.a.string
        
    def test_pickle_with_no_builder(self):
        # We had a bug that prevented pickling from working if
        # the builder wasn't set.
        soup = self.soup("some markup")
        soup.builder = None
        pickled = pickle.dumps(soup)
        unpickled = pickle.loads(pickled)
        assert "some markup" == unpickled.string

class TestEncodingConversion(SoupTest):
    # Test Beautiful Soup's ability to decode and encode from various
    # encodings.

    def setup_method(self):
        self.unicode_data = '<html><head><meta charset="utf-8"/></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>'
        self.utf8_data = self.unicode_data.encode("utf-8")
        # Just so you know what it looks like.
        assert self.utf8_data == b'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>'

    def test_ascii_in_unicode_out(self):
        # ASCII input is converted to Unicode. The original_encoding
        # attribute is set to 'utf-8', a superset of ASCII.
        chardet = dammit.chardet_dammit
        logging.disable(logging.WARNING)
        try:
            def noop(str):
                return None
            # Disable chardet, which will realize that the ASCII is ASCII.
            dammit.chardet_dammit = noop
            ascii = b"<foo>a</foo>"
            soup_from_ascii = self.soup(ascii)
            unicode_output = soup_from_ascii.decode()
            assert isinstance(unicode_output, str)
            assert unicode_output == self.document_for(ascii.decode())
            assert soup_from_ascii.original_encoding.lower() == "utf-8"
        finally:
            logging.disable(logging.NOTSET)
            dammit.chardet_dammit = chardet

    def test_unicode_in_unicode_out(self):
        # Unicode input is left alone. The original_encoding attribute
        # is not set.
        soup_from_unicode = self.soup(self.unicode_data)
        assert soup_from_unicode.decode() == self.unicode_data
        assert soup_from_unicode.foo.string == 'Sacr\xe9 bleu!'
        assert soup_from_unicode.original_encoding == None

    def test_utf8_in_unicode_out(self):
        # UTF-8 input is converted to Unicode. The original_encoding
        # attribute is set.
        soup_from_utf8 = self.soup(self.utf8_data)
        assert soup_from_utf8.decode() == self.unicode_data
        assert soup_from_utf8.foo.string == 'Sacr\xe9 bleu!'

    def test_utf8_out(self):
        # The internal data structures can be encoded as UTF-8.
        soup_from_unicode = self.soup(self.unicode_data)
        assert soup_from_unicode.encode('utf-8') == self.utf8_data

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_soup.py
NikolaBlagojevic/pyrecodes
import warnings
from bs4.element import (
    Comment,
    NavigableString,
)
from . import SoupTest

class TestTag(SoupTest):
    """Test various methods of Tag which aren't so complicated they
    need their own classes.
    """

    def test__should_pretty_print(self):
        # Test the rules about when a tag should be pretty-printed.
        tag = self.soup("").new_tag("a_tag")

        # No list of whitespace-preserving tags -> pretty-print
        tag._preserve_whitespace_tags = None
        assert True == tag._should_pretty_print(0)

        # List exists but tag is not on the list -> pretty-print
        tag.preserve_whitespace_tags = ["some_other_tag"]
        assert True == tag._should_pretty_print(1)

        # Indent level is None -> don't pretty-print
        assert False == tag._should_pretty_print(None)
        
        # Tag is on the whitespace-preserving list -> don't pretty-print
        tag.preserve_whitespace_tags = ["some_other_tag", "a_tag"]
        assert False == tag._should_pretty_print(1)
    
    def test_len(self):
        """The length of a Tag is its number of children."""
        soup = self.soup("<top>1<b>2</b>3</top>")

        # The BeautifulSoup object itself contains one element: the
        # <top> tag.
        assert len(soup.contents) == 1
        assert len(soup) == 1

        # The <top> tag contains three elements: the text node "1", the
        # <b> tag, and the text node "3".
        assert len(soup.top) == 3
        assert len(soup.top.contents) == 3

    def test_member_access_invokes_find(self):
        """Accessing a Python member .foo invokes find('foo')"""
        soup = self.soup('<b><i></i></b>')
        assert soup.b == soup.find('b')
        assert soup.b.i == soup.find('b').find('i')
        assert soup.a == None

    def test_deprecated_member_access(self):
        soup = self.soup('<b><i></i></b>')
        with warnings.catch_warnings(record=True) as w:
            tag = soup.bTag
        assert soup.b == tag
        assert '.bTag is deprecated, use .find("b") instead. If you really were looking for a tag called bTag, use .find("bTag")' == str(w[0].message)

    def test_has_attr(self):
        """has_attr() checks for the presence of an attribute.

        Please note note: has_attr() is different from
        __in__. has_attr() checks the tag's attributes and __in__
        checks the tag's chidlren.
        """
        soup = self.soup("<foo attr='bar'>")
        assert soup.foo.has_attr('attr')
        assert not soup.foo.has_attr('attr2')

    def test_attributes_come_out_in_alphabetical_order(self):
        markup = '<b a="1" z="5" m="3" f="2" y="4"></b>'
        self.assertSoupEquals(markup, '<b a="1" f="2" m="3" y="4" z="5"></b>')

    def test_string(self):
        # A Tag that contains only a text node makes that node
        # available as .string.
        soup = self.soup("<b>foo</b>")
        assert soup.b.string == 'foo'

    def test_empty_tag_has_no_string(self):
        # A Tag with no children has no .stirng.
        soup = self.soup("<b></b>")
        assert soup.b.string == None

    def test_tag_with_multiple_children_has_no_string(self):
        # A Tag with no children has no .string.
        soup = self.soup("<a>foo<b></b><b></b></b>")
        assert soup.b.string == None

        soup = self.soup("<a>foo<b></b>bar</b>")
        assert soup.b.string == None

        # Even if all the children are strings, due to trickery,
        # it won't work--but this would be a good optimization.
        soup = self.soup("<a>foo</b>")
        soup.a.insert(1, "bar")
        assert soup.a.string == None

    def test_tag_with_recursive_string_has_string(self):
        # A Tag with a single child which has a .string inherits that
        # .string.
        soup = self.soup("<a><b>foo</b></a>")
        assert soup.a.string == "foo"
        assert soup.string == "foo"

    def test_lack_of_string(self):
        """Only a Tag containing a single text node has a .string."""
        soup = self.soup("<b>f<i>e</i>o</b>")
        assert soup.b.string is None

        soup = self.soup("<b></b>")
        assert soup.b.string is None

    def test_all_text(self):
        """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated"""
        soup = self.soup("<a>a<b>r</b>   <r> t </r></a>")
        assert soup.a.text == "ar  t "
        assert soup.a.get_text(strip=True) == "art"
        assert soup.a.get_text(",") == "a,r, , t "
        assert soup.a.get_text(",", strip=True) == "a,r,t"

    def test_get_text_ignores_special_string_containers(self):
        soup = self.soup("foo<!--IGNORE-->bar")
        assert soup.get_text() == "foobar"

        assert soup.get_text(types=(NavigableString, Comment)) == "fooIGNOREbar"
        assert soup.get_text(types=None) == "fooIGNOREbar"

        soup = self.soup("foo<style>CSS</style><script>Javascript</script>bar")
        assert soup.get_text() == "foobar"
        
    def test_all_strings_ignores_special_string_containers(self):
        soup = self.soup("foo<!--IGNORE-->bar")
        assert ['foo', 'bar'] == list(soup.strings)

        soup = self.soup("foo<style>CSS</style><script>Javascript</script>bar")
        assert ['foo', 'bar'] == list(soup.strings)

    def test_string_methods_inside_special_string_container_tags(self):
        # Strings inside tags like <script> are generally ignored by
        # methods like get_text, because they're not what humans
        # consider 'text'. But if you call get_text on the <script>
        # tag itself, those strings _are_ considered to be 'text',
        # because there's nothing else you might be looking for.
        
        style = self.soup("<div>a<style>Some CSS</style></div>")
        template = self.soup("<div>a<template><p>Templated <b>text</b>.</p><!--With a comment.--></template></div>")
        script = self.soup("<div>a<script><!--a comment-->Some text</script></div>")
        
        assert style.div.get_text() == "a"
        assert list(style.div.strings) == ["a"]
        assert style.div.style.get_text() == "Some CSS"
        assert list(style.div.style.strings) == ['Some CSS']
        
        # The comment is not picked up here. That's because it was
        # parsed into a Comment object, which is not considered
        # interesting by template.strings.
        assert template.div.get_text() == "a"
        assert list(template.div.strings) == ["a"]
        assert template.div.template.get_text() == "Templated text."
        assert list(template.div.template.strings) == ["Templated ", "text", "."]

        # The comment is included here, because it didn't get parsed
        # into a Comment object--it's part of the Script string.
        assert script.div.get_text() == "a"
        assert list(script.div.strings) == ["a"]
        assert script.div.script.get_text() == "<!--a comment-->Some text"
        assert list(script.div.script.strings) == ['<!--a comment-->Some text']


class TestMultiValuedAttributes(SoupTest):
    """Test the behavior of multi-valued attributes like 'class'.

    The values of such attributes are always presented as lists.
    """

    def test_single_value_becomes_list(self):
        soup = self.soup("<a class='foo'>")
        assert ["foo"] ==soup.a['class']

    def test_multiple_values_becomes_list(self):
        soup = self.soup("<a class='foo bar'>")
        assert ["foo", "bar"] == soup.a['class']

    def test_multiple_values_separated_by_weird_whitespace(self):
        soup = self.soup("<a class='foo\tbar\nbaz'>")
        assert ["foo", "bar", "baz"] ==soup.a['class']

    def test_attributes_joined_into_string_on_output(self):
        soup = self.soup("<a class='foo\tbar'>")
        assert b'<a class="foo bar"></a>' == soup.a.encode()

    def test_get_attribute_list(self):
        soup = self.soup("<a id='abc def'>")
        assert ['abc def'] == soup.a.get_attribute_list('id')
        
    def test_accept_charset(self):
        soup = self.soup('<form accept-charset="ISO-8859-1 UTF-8">')
        assert ['ISO-8859-1', 'UTF-8'] == soup.form['accept-charset']

    def test_cdata_attribute_applying_only_to_one_tag(self):
        data = '<a accept-charset="ISO-8859-1 UTF-8"></a>'
        soup = self.soup(data)
        # We saw in another test that accept-charset is a cdata-list
        # attribute for the <form> tag. But it's not a cdata-list
        # attribute for any other tag.
        assert 'ISO-8859-1 UTF-8' == soup.a['accept-charset']

    def test_customization(self):
        # It's possible to change which attributes of which tags
        # are treated as multi-valued attributes.
        #
        # Here, 'id' is a multi-valued attribute and 'class' is not.
        #
        # TODO: This code is in the builder and should be tested there.
        soup = self.soup(
            '<a class="foo" id="bar">', multi_valued_attributes={ '*' : 'id' }
        )
        assert soup.a['class'] == 'foo'
        assert soup.a['id'] == ['bar']

env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_tag.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
"""Tests for Beautiful Soup's tree traversal methods.

The tree traversal methods are the main advantage of using Beautiful
Soup over just using a parser.

Different parsers will build different Beautiful Soup trees given the
same markup, but all Beautiful Soup trees can be traversed with the
methods tested here.
"""

from pdb import set_trace
import pytest
import re
import warnings
from bs4 import BeautifulSoup
from bs4.builder import (
    builder_registry,
    HTMLParserTreeBuilder,
)
from bs4.element import (
    CData,
    Comment,
    Declaration,
    Doctype,
    Formatter,
    NavigableString,
    Script,
    SoupStrainer,
    Stylesheet,
    Tag,
    TemplateString,
)
from . import (
    SoupTest,
)

class TestFind(SoupTest):
    """Basic tests of the find() method.

    find() just calls find_all() with limit=1, so it's not tested all
    that thouroughly here.
    """

    def test_find_tag(self):
        soup = self.soup("<a>1</a><b>2</b><a>3</a><b>4</b>")
        assert soup.find("b").string == "2"

    def test_unicode_text_find(self):
        soup = self.soup('<h1>Räksmörgås</h1>')
        assert soup.find(string='Räksmörgås') == 'Räksmörgås'

    def test_unicode_attribute_find(self):
        soup = self.soup('<h1 id="Räksmörgås">here it is</h1>')
        str(soup)
        assert "here it is" == soup.find(id='Räksmörgås').text


    def test_find_everything(self):
        """Test an optimization that finds all tags."""
        soup = self.soup("<a>foo</a><b>bar</b>")
        assert 2 == len(soup.find_all())

    def test_find_everything_with_name(self):
        """Test an optimization that finds all tags with a given name."""
        soup = self.soup("<a>foo</a><b>bar</b><a>baz</a>")
        assert 2 == len(soup.find_all('a'))

class TestFindAll(SoupTest):
    """Basic tests of the find_all() method."""

    def test_find_all_text_nodes(self):
        """You can search the tree for text nodes."""
        soup = self.soup("<html>Foo<b>bar</b>\xbb</html>")
        # Exact match.
        assert soup.find_all(string="bar") == ["bar"]

       
        # Match any of a number of strings.
        assert soup.find_all(string=["Foo", "bar"]) == ["Foo", "bar"]
        # Match a regular expression.
        assert soup.find_all(string=re.compile('.*')) == ["Foo", "bar", '\xbb']
        # Match anything.
        assert soup.find_all(string=True) == ["Foo", "bar", '\xbb']

    def test_find_all_limit(self):
        """You can limit the number of items returned by find_all."""
        soup = self.soup("<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>")
        self.assert_selects(soup.find_all('a', limit=3), ["1", "2", "3"])
        self.assert_selects(soup.find_all('a', limit=1), ["1"])
        self.assert_selects(
            soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"])

        # A limit of 0 means no limit.
        self.assert_selects(
            soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"])

    def test_calling_a_tag_is_calling_findall(self):
        soup = self.soup("<a>1</a><b>2<a id='foo'>3</a></b>")
        self.assert_selects(soup('a', limit=1), ["1"])
        self.assert_selects(soup.b(id="foo"), ["3"])

    def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self):
        soup = self.soup("<a></a>")
        # Create a self-referential list.
        l = []
        l.append(l)

        # Without special code in _normalize_search_value, this would cause infinite
        # recursion.
        assert [] == soup.find_all(l)

    def test_find_all_resultset(self):
        """All find_all calls return a ResultSet"""
        soup = self.soup("<a></a>")
        result = soup.find_all("a")
        assert hasattr(result, "source")

        result = soup.find_all(True)
        assert hasattr(result, "source")

        result = soup.find_all(string="foo")
        assert hasattr(result, "source")


class TestFindAllBasicNamespaces(SoupTest):

    def test_find_by_namespaced_name(self):
        soup = self.soup('<mathml:msqrt>4</mathml:msqrt><a svg:fill="red">')
        assert "4" == soup.find("mathml:msqrt").string
        assert "a" == soup.find(attrs= { "svg:fill" : "red" }).name


class TestFindAllByName(SoupTest):
    """Test ways of finding tags by tag name."""

    def setup_method(self):
        self.tree =  self.soup("""<a>First tag.</a>
                                  <b>Second tag.</b>
                                  <c>Third <a>Nested tag.</a> tag.</c>""")

    def test_find_all_by_tag_name(self):
        # Find all the <a> tags.
        self.assert_selects(
            self.tree.find_all('a'), ['First tag.', 'Nested tag.'])

    def test_find_all_by_name_and_text(self):
        self.assert_selects(
            self.tree.find_all('a', string='First tag.'), ['First tag.'])

        self.assert_selects(
            self.tree.find_all('a', string=True), ['First tag.', 'Nested tag.'])

        self.assert_selects(
            self.tree.find_all('a', string=re.compile("tag")),
            ['First tag.', 'Nested tag.'])


    def test_find_all_on_non_root_element(self):
        # You can call find_all on any node, not just the root.
        self.assert_selects(self.tree.c.find_all('a'), ['Nested tag.'])

    def test_calling_element_invokes_find_all(self):
        self.assert_selects(self.tree('a'), ['First tag.', 'Nested tag.'])

    def test_find_all_by_tag_strainer(self):
        self.assert_selects(
            self.tree.find_all(SoupStrainer('a')),
            ['First tag.', 'Nested tag.'])

    def test_find_all_by_tag_names(self):
        self.assert_selects(
            self.tree.find_all(['a', 'b']),
            ['First tag.', 'Second tag.', 'Nested tag.'])

    def test_find_all_by_tag_dict(self):
        self.assert_selects(
            self.tree.find_all({'a' : True, 'b' : True}),
            ['First tag.', 'Second tag.', 'Nested tag.'])

    def test_find_all_by_tag_re(self):
        self.assert_selects(
            self.tree.find_all(re.compile('^[ab]$')),
            ['First tag.', 'Second tag.', 'Nested tag.'])

    def test_find_all_with_tags_matching_method(self):
        # You can define an oracle method that determines whether
        # a tag matches the search.
        def id_matches_name(tag):
            return tag.name == tag.get('id')

        tree = self.soup("""<a id="a">Match 1.</a>
                            <a id="1">Does not match.</a>
                            <b id="b">Match 2.</a>""")

        self.assert_selects(
            tree.find_all(id_matches_name), ["Match 1.", "Match 2."])

    def test_find_with_multi_valued_attribute(self):
        soup = self.soup(
            "<div class='a b'>1</div><div class='a c'>2</div><div class='a d'>3</div>"
        )
        r1 = soup.find('div', 'a d');
        r2 = soup.find('div', re.compile(r'a d'));
        r3, r4 = soup.find_all('div', ['a b', 'a d']);
        assert '3' == r1.string
        assert '3' == r2.string
        assert '1' == r3.string
        assert '3' == r4.string

        
class TestFindAllByAttribute(SoupTest):

    def test_find_all_by_attribute_name(self):
        # You can pass in keyword arguments to find_all to search by
        # attribute.
        tree = self.soup("""
                         <a id="first">Matching a.</a>
                         <a id="second">
                          Non-matching <b id="first">Matching b.</b>a.
                         </a>""")
        self.assert_selects(tree.find_all(id='first'),
                           ["Matching a.", "Matching b."])

    def test_find_all_by_utf8_attribute_value(self):
        peace = "םולש".encode("utf8")
        data = '<a title="םולש"></a>'.encode("utf8")
        soup = self.soup(data)
        assert [soup.a] == soup.find_all(title=peace)
        assert [soup.a] == soup.find_all(title=peace.decode("utf8"))
        assert [soup.a], soup.find_all(title=[peace, "something else"])

    def test_find_all_by_attribute_dict(self):
        # You can pass in a dictionary as the argument 'attrs'. This
        # lets you search for attributes like 'name' (a fixed argument
        # to find_all) and 'class' (a reserved word in Python.)
        tree = self.soup("""
                         <a name="name1" class="class1">Name match.</a>
                         <a name="name2" class="class2">Class match.</a>
                         <a name="name3" class="class3">Non-match.</a>
                         <name1>A tag called 'name1'.</name1>
                         """)

        # This doesn't do what you want.
        self.assert_selects(tree.find_all(name='name1'),
                           ["A tag called 'name1'."])
        # This does what you want.
        self.assert_selects(tree.find_all(attrs={'name' : 'name1'}),
                           ["Name match."])

        self.assert_selects(tree.find_all(attrs={'class' : 'class2'}),
                           ["Class match."])

    def test_find_all_by_class(self):
        tree = self.soup("""
                         <a class="1">Class 1.</a>
                         <a class="2">Class 2.</a>
                         <b class="1">Class 1.</b>
                         <c class="3 4">Class 3 and 4.</c>
                         """)

        # Passing in the class_ keyword argument will search against
        # the 'class' attribute.
        self.assert_selects(tree.find_all('a', class_='1'), ['Class 1.'])
        self.assert_selects(tree.find_all('c', class_='3'), ['Class 3 and 4.'])
        self.assert_selects(tree.find_all('c', class_='4'), ['Class 3 and 4.'])

        # Passing in a string to 'attrs' will also search the CSS class.
        self.assert_selects(tree.find_all('a', '1'), ['Class 1.'])
        self.assert_selects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.'])
        self.assert_selects(tree.find_all('c', '3'), ['Class 3 and 4.'])
        self.assert_selects(tree.find_all('c', '4'), ['Class 3 and 4.'])

    def test_find_by_class_when_multiple_classes_present(self):
        tree = self.soup("<gar class='foo bar'>Found it</gar>")

        f = tree.find_all("gar", class_=re.compile("o"))
        self.assert_selects(f, ["Found it"])

        f = tree.find_all("gar", class_=re.compile("a"))
        self.assert_selects(f, ["Found it"])

        # If the search fails to match the individual strings "foo" and "bar",
        # it will be tried against the combined string "foo bar".
        f = tree.find_all("gar", class_=re.compile("o b"))
        self.assert_selects(f, ["Found it"])

    def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self):
        soup = self.soup("<a class='bar'>Found it</a>")

        self.assert_selects(soup.find_all("a", re.compile("ba")), ["Found it"])

        def big_attribute_value(value):
            return len(value) > 3

        self.assert_selects(soup.find_all("a", big_attribute_value), [])

        def small_attribute_value(value):
            return len(value) <= 3

        self.assert_selects(
            soup.find_all("a", small_attribute_value), ["Found it"])

    def test_find_all_with_string_for_attrs_finds_multiple_classes(self):
        soup = self.soup('<a class="foo bar"></a><a class="foo"></a>')
        a, a2 = soup.find_all("a")
        assert [a, a2], soup.find_all("a", "foo")
        assert [a], soup.find_all("a", "bar")

        # If you specify the class as a string that contains a
        # space, only that specific value will be found.
        assert [a] == soup.find_all("a", class_="foo bar")
        assert [a] == soup.find_all("a", "foo bar")
        assert [] == soup.find_all("a", "bar foo")

    def test_find_all_by_attribute_soupstrainer(self):
        tree = self.soup("""
                         <a id="first">Match.</a>
                         <a id="second">Non-match.</a>""")

        strainer = SoupStrainer(attrs={'id' : 'first'})
        self.assert_selects(tree.find_all(strainer), ['Match.'])

    def test_find_all_with_missing_attribute(self):
        # You can pass in None as the value of an attribute to find_all.
        # This will match tags that do not have that attribute set.
        tree = self.soup("""<a id="1">ID present.</a>
                            <a>No ID present.</a>
                            <a id="">ID is empty.</a>""")
        self.assert_selects(tree.find_all('a', id=None), ["No ID present."])

    def test_find_all_with_defined_attribute(self):
        # You can pass in None as the value of an attribute to find_all.
        # This will match tags that have that attribute set to any value.
        tree = self.soup("""<a id="1">ID present.</a>
                            <a>No ID present.</a>
                            <a id="">ID is empty.</a>""")
        self.assert_selects(
            tree.find_all(id=True), ["ID present.", "ID is empty."])

    def test_find_all_with_numeric_attribute(self):
        # If you search for a number, it's treated as a string.
        tree = self.soup("""<a id=1>Unquoted attribute.</a>
                            <a id="1">Quoted attribute.</a>""")

        expected = ["Unquoted attribute.", "Quoted attribute."]
        self.assert_selects(tree.find_all(id=1), expected)
        self.assert_selects(tree.find_all(id="1"), expected)

    def test_find_all_with_list_attribute_values(self):
        # You can pass a list of attribute values instead of just one,
        # and you'll get tags that match any of the values.
        tree = self.soup("""<a id="1">1</a>
                            <a id="2">2</a>
                            <a id="3">3</a>
                            <a>No ID.</a>""")
        self.assert_selects(tree.find_all(id=["1", "3", "4"]),
                           ["1", "3"])

    def test_find_all_with_regular_expression_attribute_value(self):
        # You can pass a regular expression as an attribute value, and
        # you'll get tags whose values for that attribute match the
        # regular expression.
        tree = self.soup("""<a id="a">One a.</a>
                            <a id="aa">Two as.</a>
                            <a id="ab">Mixed as and bs.</a>
                            <a id="b">One b.</a>
                            <a>No ID.</a>""")

        self.assert_selects(tree.find_all(id=re.compile("^a+$")),
                           ["One a.", "Two as."])

    def test_find_by_name_and_containing_string(self):
        soup = self.soup("<b>foo</b><b>bar</b><a>foo</a>")
        a = soup.a

        assert [a] == soup.find_all("a", string="foo")
        assert [] == soup.find_all("a", string="bar")

    def test_find_by_name_and_containing_string_when_string_is_buried(self):
        soup = self.soup("<a>foo</a><a><b><c>foo</c></b></a>")
        assert soup.find_all("a") == soup.find_all("a", string="foo")

    def test_find_by_attribute_and_containing_string(self):
        soup = self.soup('<b id="1">foo</b><a id="2">foo</a>')
        a = soup.a

        assert [a] == soup.find_all(id=2, string="foo")
        assert [] == soup.find_all(id=1, string="bar")


class TestSmooth(SoupTest):
    """Test Tag.smooth."""

    def test_smooth(self):
        soup = self.soup("<div>a</div>")
        div = soup.div
        div.append("b")
        div.append("c")
        div.append(Comment("Comment 1"))
        div.append(Comment("Comment 2"))
        div.append("d")
        builder = self.default_builder()
        span = Tag(soup, builder, 'span')
        span.append('1')
        span.append('2')
        div.append(span)

        # At this point the tree has a bunch of adjacent
        # NavigableStrings. This is normal, but it has no meaning in
        # terms of HTML, so we may want to smooth things out for
        # output.

        # Since the <span> tag has two children, its .string is None.
        assert None == div.span.string

        assert 7 == len(div.contents)
        div.smooth()
        assert 5 == len(div.contents)

        # The three strings at the beginning of div.contents have been
        # merged into on string.
        #
        assert 'abc' == div.contents[0]

        # The call is recursive -- the <span> tag was also smoothed.
        assert '12' == div.span.string

        # The two comments have _not_ been merged, even though
        # comments are strings. Merging comments would change the
        # meaning of the HTML.
        assert 'Comment 1' == div.contents[1]
        assert 'Comment 2' == div.contents[2]


class TestIndex(SoupTest):
    """Test Tag.index"""
    def test_index(self):
        tree = self.soup("""<div>
                            <a>Identical</a>
                            <b>Not identical</b>
                            <a>Identical</a>

                            <c><d>Identical with child</d></c>
                            <b>Also not identical</b>
                            <c><d>Identical with child</d></c>
                            </div>""")
        div = tree.div
        for i, element in enumerate(div.contents):
            assert i == div.index(element)
        with pytest.raises(ValueError):
            tree.index(1)


class TestParentOperations(SoupTest):
    """Test navigation and searching through an element's parents."""

    def setup_method(self):
        self.tree = self.soup('''<ul id="empty"></ul>
                                 <ul id="top">
                                  <ul id="middle">
                                   <ul id="bottom">
                                    <b>Start here</b>
                                   </ul>
                                  </ul>''')
        self.start = self.tree.b


    def test_parent(self):
        assert self.start.parent['id'] == 'bottom'
        assert self.start.parent.parent['id'] == 'middle'
        assert self.start.parent.parent.parent['id'] == 'top'

    def test_parent_of_top_tag_is_soup_object(self):
        top_tag = self.tree.contents[0]
        assert top_tag.parent == self.tree

    def test_soup_object_has_no_parent(self):
        assert None == self.tree.parent

    def test_find_parents(self):
        self.assert_selects_ids(
            self.start.find_parents('ul'), ['bottom', 'middle', 'top'])
        self.assert_selects_ids(
            self.start.find_parents('ul', id="middle"), ['middle'])

    def test_find_parent(self):
        assert self.start.find_parent('ul')['id'] == 'bottom'
        assert self.start.find_parent('ul', id='top')['id'] == 'top'

    def test_parent_of_text_element(self):
        text = self.tree.find(string="Start here")
        assert text.parent.name == 'b'

    def test_text_element_find_parent(self):
        text = self.tree.find(string="Start here")
        assert text.find_parent('ul')['id'] == 'bottom'

    def test_parent_generator(self):
        parents = [parent['id'] for parent in self.start.parents
                   if parent is not None and 'id' in parent.attrs]
        assert parents, ['bottom', 'middle' == 'top']


class ProximityTest(SoupTest):

    def setup_method(self):
        self.tree = self.soup(
            '<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>')


class TestNextOperations(ProximityTest):

    def setup_method(self):
        super(TestNextOperations, self).setup_method()
        self.start = self.tree.b

    def test_next(self):
        assert self.start.next_element == "One"
        assert self.start.next_element.next_element['id'] == "2"

    def test_next_of_last_item_is_none(self):
        last = self.tree.find(string="Three")
        assert last.next_element == None

    def test_next_of_root_is_none(self):
        # The document root is outside the next/previous chain.
        assert self.tree.next_element == None

    def test_find_all_next(self):
        self.assert_selects(self.start.find_all_next('b'), ["Two", "Three"])
        self.start.find_all_next(id=3)
        self.assert_selects(self.start.find_all_next(id=3), ["Three"])

    def test_find_next(self):
        assert self.start.find_next('b')['id'] == '2'
        assert self.start.find_next(string="Three") == "Three"

    def test_find_next_for_text_element(self):
        text = self.tree.find(string="One")
        assert text.find_next("b").string == "Two"
        self.assert_selects(text.find_all_next("b"), ["Two", "Three"])

    def test_next_generator(self):
        start = self.tree.find(string="Two")
        successors = [node for node in start.next_elements]
        # There are two successors: the final <b> tag and its text contents.
        tag, contents = successors
        assert tag['id'] == '3'
        assert contents == "Three"

class TestPreviousOperations(ProximityTest):

    def setup_method(self):
        super(TestPreviousOperations, self).setup_method()
        self.end = self.tree.find(string="Three")

    def test_previous(self):
        assert self.end.previous_element['id'] == "3"
        assert self.end.previous_element.previous_element == "Two"

    def test_previous_of_first_item_is_none(self):
        first = self.tree.find('html')
        assert first.previous_element == None

    def test_previous_of_root_is_none(self):
        # The document root is outside the next/previous chain.
        assert self.tree.previous_element == None

    def test_find_all_previous(self):
        # The <b> tag containing the "Three" node is the predecessor
        # of the "Three" node itself, which is why "Three" shows up
        # here.
        self.assert_selects(
            self.end.find_all_previous('b'), ["Three", "Two", "One"])
        self.assert_selects(self.end.find_all_previous(id=1), ["One"])

    def test_find_previous(self):
        assert self.end.find_previous('b')['id'] == '3'
        assert self.end.find_previous(string="One") == "One"

    def test_find_previous_for_text_element(self):
        text = self.tree.find(string="Three")
        assert text.find_previous("b").string == "Three"
        self.assert_selects(
            text.find_all_previous("b"), ["Three", "Two", "One"])

    def test_previous_generator(self):
        start = self.tree.find(string="One")
        predecessors = [node for node in start.previous_elements]

        # There are four predecessors: the <b> tag containing "One"
        # the <body> tag, the <head> tag, and the <html> tag.
        b, body, head, html = predecessors
        assert b['id'] == '1'
        assert body.name == "body"
        assert head.name == "head"
        assert html.name == "html"


class SiblingTest(SoupTest):

    def setup_method(self):
        markup = '''<html>
                    <span id="1">
                     <span id="1.1"></span>
                    </span>
                    <span id="2">
                     <span id="2.1"></span>
                    </span>
                    <span id="3">
                     <span id="3.1"></span>
                    </span>
                    <span id="4"></span>
                    </html>'''
        # All that whitespace looks good but makes the tests more
        # difficult. Get rid of it.
        markup = re.compile(r"\n\s*").sub("", markup)
        self.tree = self.soup(markup)


class TestNextSibling(SiblingTest):

    def setup_method(self):
        super(TestNextSibling, self).setup_method()
        self.start = self.tree.find(id="1")

    def test_next_sibling_of_root_is_none(self):
        assert self.tree.next_sibling == None

    def test_next_sibling(self):
        assert self.start.next_sibling['id'] == '2'
        assert self.start.next_sibling.next_sibling['id'] == '3'

        # Note the difference between next_sibling and next_element.
        assert self.start.next_element['id'] == '1.1'

    def test_next_sibling_may_not_exist(self):
        assert self.tree.html.next_sibling == None

        nested_span = self.tree.find(id="1.1")
        assert nested_span.next_sibling == None

        last_span = self.tree.find(id="4")
        assert last_span.next_sibling == None

    def test_find_next_sibling(self):
        assert self.start.find_next_sibling('span')['id'] == '2'

    def test_next_siblings(self):
        self.assert_selects_ids(self.start.find_next_siblings("span"),
                              ['2', '3', '4'])

        self.assert_selects_ids(self.start.find_next_siblings(id='3'), ['3'])

    def test_next_sibling_for_text_element(self):
        soup = self.soup("Foo<b>bar</b>baz")
        start = soup.find(string="Foo")
        assert start.next_sibling.name == 'b'
        assert start.next_sibling.next_sibling == 'baz'

        self.assert_selects(start.find_next_siblings('b'), ['bar'])
        assert start.find_next_sibling(string="baz") == "baz"
        assert start.find_next_sibling(string="nonesuch") == None


class TestPreviousSibling(SiblingTest):

    def setup_method(self):
        super(TestPreviousSibling, self).setup_method()
        self.end = self.tree.find(id="4")

    def test_previous_sibling_of_root_is_none(self):
        assert self.tree.previous_sibling == None

    def test_previous_sibling(self):
        assert self.end.previous_sibling['id'] == '3'
        assert self.end.previous_sibling.previous_sibling['id'] == '2'

        # Note the difference between previous_sibling and previous_element.
        assert self.end.previous_element['id'] == '3.1'

    def test_previous_sibling_may_not_exist(self):
        assert self.tree.html.previous_sibling == None

        nested_span = self.tree.find(id="1.1")
        assert nested_span.previous_sibling == None

        first_span = self.tree.find(id="1")
        assert first_span.previous_sibling == None

    def test_find_previous_sibling(self):
        assert self.end.find_previous_sibling('span')['id'] == '3'

    def test_previous_siblings(self):
        self.assert_selects_ids(self.end.find_previous_siblings("span"),
                              ['3', '2', '1'])

        self.assert_selects_ids(self.end.find_previous_siblings(id='1'), ['1'])

    def test_previous_sibling_for_text_element(self):
        soup = self.soup("Foo<b>bar</b>baz")
        start = soup.find(string="baz")
        assert start.previous_sibling.name == 'b'
        assert start.previous_sibling.previous_sibling == 'Foo'

        self.assert_selects(start.find_previous_siblings('b'), ['bar'])
        assert start.find_previous_sibling(string="Foo") == "Foo"
        assert start.find_previous_sibling(string="nonesuch") == None


class TestTreeModification(SoupTest):

    def test_attribute_modification(self):
        soup = self.soup('<a id="1"></a>')
        soup.a['id'] = 2
        assert soup.decode() == self.document_for('<a id="2"></a>')
        del(soup.a['id'])
        assert soup.decode() == self.document_for('<a></a>')
        soup.a['id2'] = 'foo'
        assert soup.decode() == self.document_for('<a id2="foo"></a>')

    def test_new_tag_creation(self):
        builder = builder_registry.lookup('html')()
        soup = self.soup("<body></body>", builder=builder)
        a = Tag(soup, builder, 'a')
        ol = Tag(soup, builder, 'ol')
        a['href'] = 'http://foo.com/'
        soup.body.insert(0, a)
        soup.body.insert(1, ol)
        assert soup.body.encode() == b'<body><a href="http://foo.com/"></a><ol></ol></body>'

    def test_append_to_contents_moves_tag(self):
        doc = """<p id="1">Don't leave me <b>here</b>.</p>
                <p id="2">Don\'t leave!</p>"""
        soup = self.soup(doc)
        second_para = soup.find(id='2')
        bold = soup.b

        # Move the <b> tag to the end of the second paragraph.
        soup.find(id='2').append(soup.b)

        # The <b> tag is now a child of the second paragraph.
        assert bold.parent == second_para

        assert soup.decode() == self.document_for(
                '<p id="1">Don\'t leave me .</p>\n'
                '<p id="2">Don\'t leave!<b>here</b></p>'
        )

    def test_replace_with_returns_thing_that_was_replaced(self):
        text = "<a></a><b><c></c></b>"
        soup = self.soup(text)
        a = soup.a
        new_a = a.replace_with(soup.c)
        assert a == new_a

    def test_unwrap_returns_thing_that_was_replaced(self):
        text = "<a><b></b><c></c></a>"
        soup = self.soup(text)
        a = soup.a
        new_a = a.unwrap()
        assert a == new_a

    def test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent(self):
        soup = self.soup("<a><b>Foo</b></a><c>Bar</c>")
        a = soup.a
        a.extract()
        assert None == a.parent
        with pytest.raises(ValueError):
            a.unwrap()
        with pytest.raises(ValueError):
            a.replace_with(soup.c)

    def test_replace_tag_with_itself(self):
        text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>"
        soup = self.soup(text)
        c = soup.c
        soup.c.replace_with(c)
        assert soup.decode() == self.document_for(text)

    def test_replace_tag_with_its_parent_raises_exception(self):
        text = "<a><b></b></a>"
        soup = self.soup(text)
        with pytest.raises(ValueError):
            soup.b.replace_with(soup.a)

    def test_insert_tag_into_itself_raises_exception(self):
        text = "<a><b></b></a>"
        soup = self.soup(text)
        with pytest.raises(ValueError):
            soup.a.insert(0, soup.a)

    def test_insert_beautifulsoup_object_inserts_children(self):
        """Inserting one BeautifulSoup object into another actually inserts all
        of its children -- you'll never combine BeautifulSoup objects.
        """
        soup = self.soup("<p>And now, a word:</p><p>And we're back.</p>")
        
        text = "<p>p2</p><p>p3</p>"
        to_insert = self.soup(text)
        soup.insert(1, to_insert)

        for i in soup.descendants:
            assert not isinstance(i, BeautifulSoup)
        
        p1, p2, p3, p4 = list(soup.children)
        assert "And now, a word:" == p1.string
        assert "p2" == p2.string
        assert "p3" == p3.string
        assert "And we're back." == p4.string
        
        
    def test_replace_with_maintains_next_element_throughout(self):
        soup = self.soup('<p><a>one</a><b>three</b></p>')
        a = soup.a
        b = a.contents[0]
        # Make it so the <a> tag has two text children.
        a.insert(1, "two")

        # Now replace each one with the empty string.
        left, right = a.contents
        left.replaceWith('')
        right.replaceWith('')

        # The <b> tag is still connected to the tree.
        assert "three" == soup.b.string

    def test_replace_final_node(self):
        soup = self.soup("<b>Argh!</b>")
        soup.find(string="Argh!").replace_with("Hooray!")
        new_text = soup.find(string="Hooray!")
        b = soup.b
        assert new_text.previous_element == b
        assert new_text.parent == b
        assert new_text.previous_element.next_element == new_text
        assert new_text.next_element == None

    def test_consecutive_text_nodes(self):
        # A builder should never create two consecutive text nodes,
        # but if you insert one next to another, Beautiful Soup will
        # handle it correctly.
        soup = self.soup("<a><b>Argh!</b><c></c></a>")
        soup.b.insert(1, "Hooray!")

        assert soup.decode() == self.document_for(
            "<a><b>Argh!Hooray!</b><c></c></a>"
        )

        new_text = soup.find(string="Hooray!")
        assert new_text.previous_element == "Argh!"
        assert new_text.previous_element.next_element == new_text

        assert new_text.previous_sibling == "Argh!"
        assert new_text.previous_sibling.next_sibling == new_text

        assert new_text.next_sibling == None
        assert new_text.next_element == soup.c

    def test_insert_string(self):
        soup = self.soup("<a></a>")
        soup.a.insert(0, "bar")
        soup.a.insert(0, "foo")
        # The string were added to the tag.
        assert ["foo", "bar"] == soup.a.contents
        # And they were converted to NavigableStrings.
        assert soup.a.contents[0].next_element == "bar"

    def test_insert_tag(self):
        builder = self.default_builder()
        soup = self.soup(
            "<a><b>Find</b><c>lady!</c><d></d></a>", builder=builder)
        magic_tag = Tag(soup, builder, 'magictag')
        magic_tag.insert(0, "the")
        soup.a.insert(1, magic_tag)

        assert soup.decode() == self.document_for(
                "<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>"
        )

        # Make sure all the relationships are hooked up correctly.
        b_tag = soup.b
        assert b_tag.next_sibling == magic_tag
        assert magic_tag.previous_sibling == b_tag

        find = b_tag.find(string="Find")
        assert find.next_element == magic_tag
        assert magic_tag.previous_element == find

        c_tag = soup.c
        assert magic_tag.next_sibling == c_tag
        assert c_tag.previous_sibling == magic_tag

        the = magic_tag.find(string="the")
        assert the.parent == magic_tag
        assert the.next_element == c_tag
        assert c_tag.previous_element == the

    def test_append_child_thats_already_at_the_end(self):
        data = "<a><b></b></a>"
        soup = self.soup(data)
        soup.a.append(soup.b)
        assert data == soup.decode()

    def test_extend(self):
        data = "<a><b><c><d><e><f><g></g></f></e></d></c></b></a>"
        soup = self.soup(data)
        l = [soup.g, soup.f, soup.e, soup.d, soup.c, soup.b]
        soup.a.extend(l)
        assert "<a><g></g><f></f><e></e><d></d><c></c><b></b></a>" == soup.decode()

    @pytest.mark.parametrize(
        "get_tags", [lambda tag: tag, lambda tag: tag.contents]
    )
    def test_extend_with_another_tags_contents(self, get_tags):
        data = '<body><div id="d1"><a>1</a><a>2</a><a>3</a><a>4</a></div><div id="d2"></div></body>'
        soup = self.soup(data)
        d1 = soup.find('div', id='d1')
        d2 = soup.find('div', id='d2')
        tags = get_tags(d1)
        d2.extend(tags)
        assert '<div id="d1"></div>' == d1.decode()
        assert '<div id="d2"><a>1</a><a>2</a><a>3</a><a>4</a></div>' == d2.decode()
        
    def test_move_tag_to_beginning_of_parent(self):
        data = "<a><b></b><c></c><d></d></a>"
        soup = self.soup(data)
        soup.a.insert(0, soup.d)
        assert "<a><d></d><b></b><c></c></a>" == soup.decode()

    def test_insert_works_on_empty_element_tag(self):
        # This is a little strange, since most HTML parsers don't allow
        # markup like this to come through. But in general, we don't
        # know what the parser would or wouldn't have allowed, so
        # I'm letting this succeed for now.
        soup = self.soup("<br/>")
        soup.br.insert(1, "Contents")
        assert str(soup.br) == "<br>Contents</br>"

    def test_insert_before(self):
        soup = self.soup("<a>foo</a><b>bar</b>")
        soup.b.insert_before("BAZ")
        soup.a.insert_before("QUUX")
        assert soup.decode() == self.document_for(
            "QUUX<a>foo</a>BAZ<b>bar</b>"
        )

        soup.a.insert_before(soup.b)
        assert soup.decode() == self.document_for("QUUX<b>bar</b><a>foo</a>BAZ")

        # Can't insert an element before itself.
        b = soup.b
        with pytest.raises(ValueError):
            b.insert_before(b)

        # Can't insert before if an element has no parent.
        b.extract()
        with pytest.raises(ValueError):
            b.insert_before("nope")

        # Can insert an identical element
        soup = self.soup("<a>")
        soup.a.insert_before(soup.new_tag("a"))

        # TODO: OK but what happens?
        
    def test_insert_multiple_before(self):
        soup = self.soup("<a>foo</a><b>bar</b>")
        soup.b.insert_before("BAZ", " ", "QUUX")
        soup.a.insert_before("QUUX", " ", "BAZ")
        assert soup.decode() == self.document_for(
            "QUUX BAZ<a>foo</a>BAZ QUUX<b>bar</b>"
        )

        soup.a.insert_before(soup.b, "FOO")
        assert soup.decode() == self.document_for(
            "QUUX BAZ<b>bar</b>FOO<a>foo</a>BAZ QUUX"
        )

    def test_insert_after(self):
        soup = self.soup("<a>foo</a><b>bar</b>")
        soup.b.insert_after("BAZ")
        soup.a.insert_after("QUUX")
        assert soup.decode() == self.document_for(
            "<a>foo</a>QUUX<b>bar</b>BAZ"
        )
        soup.b.insert_after(soup.a)
        assert soup.decode() == self.document_for("QUUX<b>bar</b><a>foo</a>BAZ")

        # Can't insert an element after itself.
        b = soup.b
        with pytest.raises(ValueError):
            b.insert_after(b)

        # Can't insert after if an element has no parent.
        b.extract()
        with pytest.raises(ValueError):
            b.insert_after("nope")

        # Can insert an identical element
        soup = self.soup("<a>")
        soup.a.insert_before(soup.new_tag("a"))

        # TODO: OK but what does it look like?
        
    def test_insert_multiple_after(self):
        soup = self.soup("<a>foo</a><b>bar</b>")
        soup.b.insert_after("BAZ", " ", "QUUX")
        soup.a.insert_after("QUUX", " ", "BAZ")
        assert soup.decode() == self.document_for(
            "<a>foo</a>QUUX BAZ<b>bar</b>BAZ QUUX"
        )
        soup.b.insert_after(soup.a, "FOO ")
        assert soup.decode() == self.document_for(
            "QUUX BAZ<b>bar</b><a>foo</a>FOO BAZ QUUX"
        )

    def test_insert_after_raises_exception_if_after_has_no_meaning(self):
        soup = self.soup("")
        tag = soup.new_tag("a")
        string = soup.new_string("")
        with pytest.raises(ValueError):
            string.insert_after(tag)
        with pytest.raises(NotImplementedError):
            soup.insert_after(tag)
        with pytest.raises(ValueError):
            tag.insert_after(tag)

    def test_insert_before_raises_notimplementederror_if_before_has_no_meaning(self):
        soup = self.soup("")
        tag = soup.new_tag("a")
        string = soup.new_string("")
        with pytest.raises(ValueError):
            string.insert_before(tag)
        with pytest.raises(NotImplementedError):
            soup.insert_before(tag)
        with pytest.raises(ValueError):
            tag.insert_before(tag)

    def test_replace_with(self):
        soup = self.soup(
                "<p>There's <b>no</b> business like <b>show</b> business</p>")
        no, show = soup.find_all('b')
        show.replace_with(no)
        assert soup.decode() == self.document_for(
            "<p>There's  business like <b>no</b> business</p>"
        )

        assert show.parent == None
        assert no.parent == soup.p
        assert no.next_element == "no"
        assert no.next_sibling == " business"

    def test_replace_with_errors(self):
        # Can't replace a tag that's not part of a tree.
        a_tag = Tag(name="a")
        with pytest.raises(ValueError):
            a_tag.replace_with("won't work")

        # Can't replace a tag with its parent.
        a_tag = self.soup("<a><b></b></a>").a
        with pytest.raises(ValueError):
            a_tag.b.replace_with(a_tag)

        # Or with a list that includes its parent.
        with pytest.raises(ValueError):
            a_tag.b.replace_with("string1", a_tag, "string2")
        
    def test_replace_with_multiple(self):
        data = "<a><b></b><c></c></a>"
        soup = self.soup(data)
        d_tag = soup.new_tag("d")
        d_tag.string = "Text In D Tag"
        e_tag = soup.new_tag("e")
        f_tag = soup.new_tag("f")
        a_string = "Random Text"
        soup.c.replace_with(d_tag, e_tag, a_string, f_tag)
        assert soup.decode() == "<a><b></b><d>Text In D Tag</d><e></e>Random Text<f></f></a>"
        assert soup.b.next_element == d_tag
        assert d_tag.string.next_element==e_tag
        assert e_tag.next_element.string == a_string
        assert e_tag.next_element.next_element == f_tag
        
    def test_replace_first_child(self):
        data = "<a><b></b><c></c></a>"
        soup = self.soup(data)
        soup.b.replace_with(soup.c)
        assert "<a><c></c></a>" == soup.decode()

    def test_replace_last_child(self):
        data = "<a><b></b><c></c></a>"
        soup = self.soup(data)
        soup.c.replace_with(soup.b)
        assert "<a><b></b></a>" == soup.decode()

    def test_nested_tag_replace_with(self):
        soup = self.soup(
            """<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>""")

        # Replace the entire <b> tag and its contents ("reserve the
        # right") with the <f> tag ("refuse").
        remove_tag = soup.b
        move_tag = soup.f
        remove_tag.replace_with(move_tag)

        assert soup.decode() == self.document_for(
            "<a>We<f>refuse</f></a><e>to<g>service</g></e>"
        )

        # The <b> tag is now an orphan.
        assert remove_tag.parent == None
        assert remove_tag.find(string="right").next_element == None
        assert remove_tag.previous_element == None
        assert remove_tag.next_sibling == None
        assert remove_tag.previous_sibling == None

        # The <f> tag is now connected to the <a> tag.
        assert move_tag.parent == soup.a
        assert move_tag.previous_element == "We"
        assert move_tag.next_element.next_element == soup.e
        assert move_tag.next_sibling == None

        # The gap where the <f> tag used to be has been mended, and
        # the word "to" is now connected to the <g> tag.
        to_text = soup.find(string="to")
        g_tag = soup.g
        assert to_text.next_element == g_tag
        assert to_text.next_sibling == g_tag
        assert g_tag.previous_element == to_text
        assert g_tag.previous_sibling == to_text

    def test_unwrap(self):
        tree = self.soup("""
            <p>Unneeded <em>formatting</em> is unneeded</p>
            """)
        tree.em.unwrap()
        assert tree.em == None
        assert tree.p.text == "Unneeded formatting is unneeded"

    def test_wrap(self):
        soup = self.soup("I wish I was bold.")
        value = soup.string.wrap(soup.new_tag("b"))
        assert value.decode() == "<b>I wish I was bold.</b>"
        assert soup.decode() == self.document_for("<b>I wish I was bold.</b>")

    def test_wrap_extracts_tag_from_elsewhere(self):
        soup = self.soup("<b></b>I wish I was bold.")
        soup.b.next_sibling.wrap(soup.b)
        assert soup.decode() == self.document_for("<b>I wish I was bold.</b>")

    def test_wrap_puts_new_contents_at_the_end(self):
        soup = self.soup("<b>I like being bold.</b>I wish I was bold.")
        soup.b.next_sibling.wrap(soup.b)
        assert 2 == len(soup.b.contents)
        assert soup.decode() == self.document_for(
            "<b>I like being bold.I wish I was bold.</b>"
        )

    def test_extract(self):
        soup = self.soup(
            '<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>')

        assert len(soup.body.contents) == 3
        extracted = soup.find(id="nav").extract()

        assert soup.decode() == "<html><body>Some content.  More content.</body></html>"
        assert extracted.decode() == '<div id="nav">Nav crap</div>'

        # The extracted tag is now an orphan.
        assert len(soup.body.contents) == 2
        assert extracted.parent == None
        assert extracted.previous_element == None
        assert extracted.next_element.next_element == None

        # The gap where the extracted tag used to be has been mended.
        content_1 = soup.find(string="Some content. ")
        content_2 = soup.find(string=" More content.")
        assert content_1.next_element == content_2
        assert content_1.next_sibling == content_2
        assert content_2.previous_element == content_1
        assert content_2.previous_sibling == content_1

    def test_extract_distinguishes_between_identical_strings(self):
        soup = self.soup("<a>foo</a><b>bar</b>")
        foo_1 = soup.a.string
        bar_1 = soup.b.string
        foo_2 = soup.new_string("foo")
        bar_2 = soup.new_string("bar")
        soup.a.append(foo_2)
        soup.b.append(bar_2)

        # Now there are two identical strings in the <a> tag, and two
        # in the <b> tag. Let's remove the first "foo" and the second
        # "bar".
        foo_1.extract()
        bar_2.extract()
        assert foo_2 == soup.a.string
        assert bar_2 == soup.b.string

    def test_extract_multiples_of_same_tag(self):
        soup = self.soup("""
<html>
<head>
<script>foo</script>
</head>
<body>
 <script>bar</script>
 <a></a>
</body>
<script>baz</script>
</html>""")
        [soup.script.extract() for i in soup.find_all("script")]
        assert "<body>\n\n<a></a>\n</body>" == str(soup.body)


    def test_extract_works_when_element_is_surrounded_by_identical_strings(self):
        soup = self.soup(
 '<html>\n'
 '<body>hi</body>\n'
 '</html>')
        soup.find('body').extract()
        assert None == soup.find('body')


    def test_clear(self):
        """Tag.clear()"""
        soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>")
        # clear using extract()
        a = soup.a
        soup.p.clear()
        assert len(soup.p.contents) == 0
        assert hasattr(a, "contents")

        # clear using decompose()
        em = a.em
        a.clear(decompose=True)
        assert 0 == len(em.contents)

       
    def test_decompose(self):
        # Test PageElement.decompose() and PageElement.decomposed
        soup = self.soup("<p><a>String <em>Italicized</em></a></p><p>Another para</p>")
        p1, p2 = soup.find_all('p')
        a = p1.a
        text = p1.em.string
        for i in [p1, p2, a, text]:
            assert False == i.decomposed

        # This sets p1 and everything beneath it to decomposed.
        p1.decompose()
        for i in [p1, a, text]:
            assert True == i.decomposed
        # p2 is unaffected.
        assert False == p2.decomposed
            
    def test_string_set(self):
        """Tag.string = 'string'"""
        soup = self.soup("<a></a> <b><c></c></b>")
        soup.a.string = "foo"
        assert soup.a.contents == ["foo"]
        soup.b.string = "bar"
        assert soup.b.contents == ["bar"]

    def test_string_set_does_not_affect_original_string(self):
        soup = self.soup("<a><b>foo</b><c>bar</c>")
        soup.b.string = soup.c.string
        assert soup.a.encode() == b"<a><b>bar</b><c>bar</c></a>"

    def test_set_string_preserves_class_of_string(self):
        soup = self.soup("<a></a>")
        cdata = CData("foo")
        soup.a.string = cdata
        assert isinstance(soup.a.string, CData)


class TestDeprecatedArguments(SoupTest):

    @pytest.mark.parametrize(
        "method_name", [
            "find", "find_all", "find_parent", "find_parents",
            "find_next", "find_all_next", "find_previous",
            "find_all_previous", "find_next_sibling", "find_next_siblings",
            "find_previous_sibling", "find_previous_siblings",
        ]
    )
    def test_find_type_method_string(self, method_name):
        soup = self.soup("<a>some</a><b>markup</b>")
        method = getattr(soup.b, method_name)
        with warnings.catch_warnings(record=True) as w:
            method(text='markup')
            [warning] = w
            assert warning.filename == __file__
            msg = str(warning.message)
            assert msg == "The 'text' argument to find()-type methods is deprecated. Use 'string' instead."

    def test_soupstrainer_constructor_string(self):
        with warnings.catch_warnings(record=True) as w:
            strainer = SoupStrainer(text="text")
            assert strainer.text == 'text'
            [warning] = w
            msg = str(warning.message)
            assert warning.filename == __file__
            assert msg == "The 'text' argument to the SoupStrainer constructor is deprecated. Use 'string' instead."


env_pyrecodes/lib/python3.9/site-packages/bs4/tests/test_tree.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0

"""CacheControl import Interface.

Make it easy to import from cachecontrol without long namespaces.
"""
__author__ = "Eric Larson"
__email__ = "eric@ionrock.org"
__version__ = "0.13.1"

from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.controller import CacheController
from cachecontrol.wrapper import CacheControl

__all__ = [
    "__author__",
    "__email__",
    "__version__",
    "CacheControlAdapter",
    "CacheController",
    "CacheControl",
]

import logging

logging.getLogger(__name__).addHandler(logging.NullHandler())

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/__init__.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

import logging
from argparse import ArgumentParser
from typing import TYPE_CHECKING

import requests

from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.cache import DictCache
from cachecontrol.controller import logger

if TYPE_CHECKING:
    from argparse import Namespace

    from cachecontrol.controller import CacheController


def setup_logging() -> None:
    logger.setLevel(logging.DEBUG)
    handler = logging.StreamHandler()
    logger.addHandler(handler)


def get_session() -> requests.Session:
    adapter = CacheControlAdapter(
        DictCache(), cache_etags=True, serializer=None, heuristic=None
    )
    sess = requests.Session()
    sess.mount("http://", adapter)
    sess.mount("https://", adapter)

    sess.cache_controller = adapter.controller  # type: ignore[attr-defined]
    return sess


def get_args() -> Namespace:
    parser = ArgumentParser()
    parser.add_argument("url", help="The URL to try and cache")
    return parser.parse_args()


def main() -> None:
    args = get_args()
    sess = get_session()

    # Make a request to get a response
    resp = sess.get(args.url)

    # Turn on logging
    setup_logging()

    # try setting the cache
    cache_controller: CacheController = (
        sess.cache_controller  # type: ignore[attr-defined]
    )
    cache_controller.cache_response(resp.request, resp.raw)

    # Now try to get it
    if cache_controller.cached_request(resp.request):
        print("Cached!")
    else:
        print("Not cached :(")


if __name__ == "__main__":
    main()

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/_cmd.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

import functools
import types
import zlib
from typing import TYPE_CHECKING, Any, Collection, Mapping

from requests.adapters import HTTPAdapter

from cachecontrol.cache import DictCache
from cachecontrol.controller import PERMANENT_REDIRECT_STATUSES, CacheController
from cachecontrol.filewrapper import CallbackFileWrapper

if TYPE_CHECKING:
    from requests import PreparedRequest, Response
    from urllib3 import HTTPResponse

    from cachecontrol.cache import BaseCache
    from cachecontrol.heuristics import BaseHeuristic
    from cachecontrol.serialize import Serializer


class CacheControlAdapter(HTTPAdapter):
    invalidating_methods = {"PUT", "PATCH", "DELETE"}

    def __init__(
        self,
        cache: BaseCache | None = None,
        cache_etags: bool = True,
        controller_class: type[CacheController] | None = None,
        serializer: Serializer | None = None,
        heuristic: BaseHeuristic | None = None,
        cacheable_methods: Collection[str] | None = None,
        *args: Any,
        **kw: Any,
    ) -> None:
        super().__init__(*args, **kw)
        self.cache = DictCache() if cache is None else cache
        self.heuristic = heuristic
        self.cacheable_methods = cacheable_methods or ("GET",)

        controller_factory = controller_class or CacheController
        self.controller = controller_factory(
            self.cache, cache_etags=cache_etags, serializer=serializer
        )

    def send(
        self,
        request: PreparedRequest,
        stream: bool = False,
        timeout: None | float | tuple[float, float] | tuple[float, None] = None,
        verify: bool | str = True,
        cert: (None | bytes | str | tuple[bytes | str, bytes | str]) = None,
        proxies: Mapping[str, str] | None = None,
        cacheable_methods: Collection[str] | None = None,
    ) -> Response:
        """
        Send a request. Use the request information to see if it
        exists in the cache and cache the response if we need to and can.
        """
        cacheable = cacheable_methods or self.cacheable_methods
        if request.method in cacheable:
            try:
                cached_response = self.controller.cached_request(request)
            except zlib.error:
                cached_response = None
            if cached_response:
                return self.build_response(request, cached_response, from_cache=True)

            # check for etags and add headers if appropriate
            request.headers.update(self.controller.conditional_headers(request))

        resp = super().send(request, stream, timeout, verify, cert, proxies)

        return resp

    def build_response(
        self,
        request: PreparedRequest,
        response: HTTPResponse,
        from_cache: bool = False,
        cacheable_methods: Collection[str] | None = None,
    ) -> Response:
        """
        Build a response by making a request or using the cache.

        This will end up calling send and returning a potentially
        cached response
        """
        cacheable = cacheable_methods or self.cacheable_methods
        if not from_cache and request.method in cacheable:
            # Check for any heuristics that might update headers
            # before trying to cache.
            if self.heuristic:
                response = self.heuristic.apply(response)

            # apply any expiration heuristics
            if response.status == 304:
                # We must have sent an ETag request. This could mean
                # that we've been expired already or that we simply
                # have an etag. In either case, we want to try and
                # update the cache if that is the case.
                cached_response = self.controller.update_cached_response(
                    request, response
                )

                if cached_response is not response:
                    from_cache = True

                # We are done with the server response, read a
                # possible response body (compliant servers will
                # not return one, but we cannot be 100% sure) and
                # release the connection back to the pool.
                response.read(decode_content=False)
                response.release_conn()

                response = cached_response

            # We always cache the 301 responses
            elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
                self.controller.cache_response(request, response)
            else:
                # Wrap the response file with a wrapper that will cache the
                #   response when the stream has been consumed.
                response._fp = CallbackFileWrapper(  # type: ignore[attr-defined]
                    response._fp,  # type: ignore[attr-defined]
                    functools.partial(
                        self.controller.cache_response, request, response
                    ),
                )
                if response.chunked:
                    super_update_chunk_length = response._update_chunk_length  # type: ignore[attr-defined]

                    def _update_chunk_length(self: HTTPResponse) -> None:
                        super_update_chunk_length()
                        if self.chunk_left == 0:
                            self._fp._close()  # type: ignore[attr-defined]

                    response._update_chunk_length = types.MethodType(  # type: ignore[attr-defined]
                        _update_chunk_length, response
                    )

        resp: Response = super().build_response(request, response)  # type: ignore[no-untyped-call]

        # See if we should invalidate the cache.
        if request.method in self.invalidating_methods and resp.ok:
            assert request.url is not None
            cache_url = self.controller.cache_url(request.url)
            self.cache.delete(cache_url)

        # Give the request a from_cache attr to let people use it
        resp.from_cache = from_cache  # type: ignore[attr-defined]

        return resp

    def close(self) -> None:
        self.cache.close()
        super().close()  # type: ignore[no-untyped-call]

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/adapter.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0

"""
The cache object API for implementing caches. The default is a thread
safe in-memory dictionary.
"""
from __future__ import annotations

from threading import Lock
from typing import IO, TYPE_CHECKING, MutableMapping

if TYPE_CHECKING:
    from datetime import datetime


class BaseCache:
    def get(self, key: str) -> bytes | None:
        raise NotImplementedError()

    def set(
        self, key: str, value: bytes, expires: int | datetime | None = None
    ) -> None:
        raise NotImplementedError()

    def delete(self, key: str) -> None:
        raise NotImplementedError()

    def close(self) -> None:
        pass


class DictCache(BaseCache):
    def __init__(self, init_dict: MutableMapping[str, bytes] | None = None) -> None:
        self.lock = Lock()
        self.data = init_dict or {}

    def get(self, key: str) -> bytes | None:
        return self.data.get(key, None)

    def set(
        self, key: str, value: bytes, expires: int | datetime | None = None
    ) -> None:
        with self.lock:
            self.data.update({key: value})

    def delete(self, key: str) -> None:
        with self.lock:
            if key in self.data:
                self.data.pop(key)


class SeparateBodyBaseCache(BaseCache):
    """
    In this variant, the body is not stored mixed in with the metadata, but is
    passed in (as a bytes-like object) in a separate call to ``set_body()``.

    That is, the expected interaction pattern is::

        cache.set(key, serialized_metadata)
        cache.set_body(key)

    Similarly, the body should be loaded separately via ``get_body()``.
    """

    def set_body(self, key: str, body: bytes) -> None:
        raise NotImplementedError()

    def get_body(self, key: str) -> IO[bytes] | None:
        """
        Return the body as file-like object.
        """
        raise NotImplementedError()

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/cache.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0

from cachecontrol.caches.file_cache import FileCache, SeparateBodyFileCache
from cachecontrol.caches.redis_cache import RedisCache

__all__ = ["FileCache", "SeparateBodyFileCache", "RedisCache"]

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/caches/__init__.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

import hashlib
import os
from textwrap import dedent
from typing import IO, TYPE_CHECKING

from cachecontrol.cache import BaseCache, SeparateBodyBaseCache
from cachecontrol.controller import CacheController

if TYPE_CHECKING:
    from datetime import datetime

    from filelock import BaseFileLock


def _secure_open_write(filename: str, fmode: int) -> IO[bytes]:
    # We only want to write to this file, so open it in write only mode
    flags = os.O_WRONLY

    # os.O_CREAT | os.O_EXCL will fail if the file already exists, so we only
    #  will open *new* files.
    # We specify this because we want to ensure that the mode we pass is the
    # mode of the file.
    flags |= os.O_CREAT | os.O_EXCL

    # Do not follow symlinks to prevent someone from making a symlink that
    # we follow and insecurely open a cache file.
    if hasattr(os, "O_NOFOLLOW"):
        flags |= os.O_NOFOLLOW

    # On Windows we'll mark this file as binary
    if hasattr(os, "O_BINARY"):
        flags |= os.O_BINARY

    # Before we open our file, we want to delete any existing file that is
    # there
    try:
        os.remove(filename)
    except OSError:
        # The file must not exist already, so we can just skip ahead to opening
        pass

    # Open our file, the use of os.O_CREAT | os.O_EXCL will ensure that if a
    # race condition happens between the os.remove and this line, that an
    # error will be raised. Because we utilize a lockfile this should only
    # happen if someone is attempting to attack us.
    fd = os.open(filename, flags, fmode)
    try:
        return os.fdopen(fd, "wb")

    except:
        # An error occurred wrapping our FD in a file object
        os.close(fd)
        raise


class _FileCacheMixin:
    """Shared implementation for both FileCache variants."""

    def __init__(
        self,
        directory: str,
        forever: bool = False,
        filemode: int = 0o0600,
        dirmode: int = 0o0700,
        lock_class: type[BaseFileLock] | None = None,
    ) -> None:
        try:
            if lock_class is None:
                from filelock import FileLock

                lock_class = FileLock
        except ImportError:
            notice = dedent(
                """
            NOTE: In order to use the FileCache you must have
            filelock installed. You can install it via pip:
              pip install filelock
            """
            )
            raise ImportError(notice)

        self.directory = directory
        self.forever = forever
        self.filemode = filemode
        self.dirmode = dirmode
        self.lock_class = lock_class

    @staticmethod
    def encode(x: str) -> str:
        return hashlib.sha224(x.encode()).hexdigest()

    def _fn(self, name: str) -> str:
        # NOTE: This method should not change as some may depend on it.
        #       See: https://github.com/ionrock/cachecontrol/issues/63
        hashed = self.encode(name)
        parts = list(hashed[:5]) + [hashed]
        return os.path.join(self.directory, *parts)

    def get(self, key: str) -> bytes | None:
        name = self._fn(key)
        try:
            with open(name, "rb") as fh:
                return fh.read()

        except FileNotFoundError:
            return None

    def set(
        self, key: str, value: bytes, expires: int | datetime | None = None
    ) -> None:
        name = self._fn(key)
        self._write(name, value)

    def _write(self, path: str, data: bytes) -> None:
        """
        Safely write the data to the given path.
        """
        # Make sure the directory exists
        try:
            os.makedirs(os.path.dirname(path), self.dirmode)
        except OSError:
            pass

        with self.lock_class(path + ".lock"):
            # Write our actual file
            with _secure_open_write(path, self.filemode) as fh:
                fh.write(data)

    def _delete(self, key: str, suffix: str) -> None:
        name = self._fn(key) + suffix
        if not self.forever:
            try:
                os.remove(name)
            except FileNotFoundError:
                pass


class FileCache(_FileCacheMixin, BaseCache):
    """
    Traditional FileCache: body is stored in memory, so not suitable for large
    downloads.
    """

    def delete(self, key: str) -> None:
        self._delete(key, "")


class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache):
    """
    Memory-efficient FileCache: body is stored in a separate file, reducing
    peak memory usage.
    """

    def get_body(self, key: str) -> IO[bytes] | None:
        name = self._fn(key) + ".body"
        try:
            return open(name, "rb")
        except FileNotFoundError:
            return None

    def set_body(self, key: str, body: bytes) -> None:
        name = self._fn(key) + ".body"
        self._write(name, body)

    def delete(self, key: str) -> None:
        self._delete(key, "")
        self._delete(key, ".body")


def url_to_file_path(url: str, filecache: FileCache) -> str:
    """Return the file cache path based on the URL.

    This does not ensure the file exists!
    """
    key = CacheController.cache_url(url)
    return filecache._fn(key)

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/caches/file_cache.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations


from datetime import datetime, timezone
from typing import TYPE_CHECKING

from cachecontrol.cache import BaseCache

if TYPE_CHECKING:
    from redis import Redis


class RedisCache(BaseCache):
    def __init__(self, conn: Redis[bytes]) -> None:
        self.conn = conn

    def get(self, key: str) -> bytes | None:
        return self.conn.get(key)

    def set(
        self, key: str, value: bytes, expires: int | datetime | None = None
    ) -> None:
        if not expires:
            self.conn.set(key, value)
        elif isinstance(expires, datetime):
            now_utc = datetime.now(timezone.utc)
            if expires.tzinfo is None:
                now_utc = now_utc.replace(tzinfo=None)
            delta = expires - now_utc
            self.conn.setex(key, int(delta.total_seconds()), value)
        else:
            self.conn.setex(key, expires, value)

    def delete(self, key: str) -> None:
        self.conn.delete(key)

    def clear(self) -> None:
        """Helper for clearing all the keys in a database. Use with
        caution!"""
        for key in self.conn.keys():
            self.conn.delete(key)

    def close(self) -> None:
        """Redis uses connection pooling, no need to close the connection."""
        pass

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/caches/redis_cache.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0

"""
The httplib2 algorithms ported for use with requests.
"""
from __future__ import annotations

import calendar
import logging
import re
import time
from email.utils import parsedate_tz
from typing import TYPE_CHECKING, Collection, Mapping

from requests.structures import CaseInsensitiveDict

from cachecontrol.cache import DictCache, SeparateBodyBaseCache
from cachecontrol.serialize import Serializer

if TYPE_CHECKING:
    from typing import Literal

    from requests import PreparedRequest
    from urllib3 import HTTPResponse

    from cachecontrol.cache import BaseCache

logger = logging.getLogger(__name__)

URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")

PERMANENT_REDIRECT_STATUSES = (301, 308)


def parse_uri(uri: str) -> tuple[str, str, str, str, str]:
    """Parses a URI using the regex given in Appendix B of RFC 3986.

    (scheme, authority, path, query, fragment) = parse_uri(uri)
    """
    match = URI.match(uri)
    assert match is not None
    groups = match.groups()
    return (groups[1], groups[3], groups[4], groups[6], groups[8])


class CacheController:
    """An interface to see if request should cached or not."""

    def __init__(
        self,
        cache: BaseCache | None = None,
        cache_etags: bool = True,
        serializer: Serializer | None = None,
        status_codes: Collection[int] | None = None,
    ):
        self.cache = DictCache() if cache is None else cache
        self.cache_etags = cache_etags
        self.serializer = serializer or Serializer()
        self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308)

    @classmethod
    def _urlnorm(cls, uri: str) -> str:
        """Normalize the URL to create a safe key for the cache"""
        (scheme, authority, path, query, fragment) = parse_uri(uri)
        if not scheme or not authority:
            raise Exception("Only absolute URIs are allowed. uri = %s" % uri)

        scheme = scheme.lower()
        authority = authority.lower()

        if not path:
            path = "/"

        # Could do syntax based normalization of the URI before
        # computing the digest. See Section 6.2.2 of Std 66.
        request_uri = query and "?".join([path, query]) or path
        defrag_uri = scheme + "://" + authority + request_uri

        return defrag_uri

    @classmethod
    def cache_url(cls, uri: str) -> str:
        return cls._urlnorm(uri)

    def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | None]:
        known_directives = {
            # https://tools.ietf.org/html/rfc7234#section-5.2
            "max-age": (int, True),
            "max-stale": (int, False),
            "min-fresh": (int, True),
            "no-cache": (None, False),
            "no-store": (None, False),
            "no-transform": (None, False),
            "only-if-cached": (None, False),
            "must-revalidate": (None, False),
            "public": (None, False),
            "private": (None, False),
            "proxy-revalidate": (None, False),
            "s-maxage": (int, True),
        }

        cc_headers = headers.get("cache-control", headers.get("Cache-Control", ""))

        retval: dict[str, int | None] = {}

        for cc_directive in cc_headers.split(","):
            if not cc_directive.strip():
                continue

            parts = cc_directive.split("=", 1)
            directive = parts[0].strip()

            try:
                typ, required = known_directives[directive]
            except KeyError:
                logger.debug("Ignoring unknown cache-control directive: %s", directive)
                continue

            if not typ or not required:
                retval[directive] = None
            if typ:
                try:
                    retval[directive] = typ(parts[1].strip())
                except IndexError:
                    if required:
                        logger.debug(
                            "Missing value for cache-control " "directive: %s",
                            directive,
                        )
                except ValueError:
                    logger.debug(
                        "Invalid value for cache-control directive " "%s, must be %s",
                        directive,
                        typ.__name__,
                    )

        return retval

    def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None:
        """
        Load a cached response, or return None if it's not available.
        """
        cache_url = request.url
        assert cache_url is not None
        cache_data = self.cache.get(cache_url)
        if cache_data is None:
            logger.debug("No cache entry available")
            return None

        if isinstance(self.cache, SeparateBodyBaseCache):
            body_file = self.cache.get_body(cache_url)
        else:
            body_file = None

        result = self.serializer.loads(request, cache_data, body_file)
        if result is None:
            logger.warning("Cache entry deserialization failed, entry ignored")
        return result

    def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[False]:
        """
        Return a cached response if it exists in the cache, otherwise
        return False.
        """
        assert request.url is not None
        cache_url = self.cache_url(request.url)
        logger.debug('Looking up "%s" in the cache', cache_url)
        cc = self.parse_cache_control(request.headers)

        # Bail out if the request insists on fresh data
        if "no-cache" in cc:
            logger.debug('Request header has "no-cache", cache bypassed')
            return False

        if "max-age" in cc and cc["max-age"] == 0:
            logger.debug('Request header has "max_age" as 0, cache bypassed')
            return False

        # Check whether we can load the response from the cache:
        resp = self._load_from_cache(request)
        if not resp:
            return False

        # If we have a cached permanent redirect, return it immediately. We
        # don't need to test our response for other headers b/c it is
        # intrinsically "cacheable" as it is Permanent.
        #
        # See:
        #   https://tools.ietf.org/html/rfc7231#section-6.4.2
        #
        # Client can try to refresh the value by repeating the request
        # with cache busting headers as usual (ie no-cache).
        if int(resp.status) in PERMANENT_REDIRECT_STATUSES:
            msg = (
                "Returning cached permanent redirect response "
                "(ignoring date and etag information)"
            )
            logger.debug(msg)
            return resp

        headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers)
        if not headers or "date" not in headers:
            if "etag" not in headers:
                # Without date or etag, the cached response can never be used
                # and should be deleted.
                logger.debug("Purging cached response: no date or etag")
                self.cache.delete(cache_url)
            logger.debug("Ignoring cached response: no date")
            return False

        now = time.time()
        time_tuple = parsedate_tz(headers["date"])
        assert time_tuple is not None
        date = calendar.timegm(time_tuple[:6])
        current_age = max(0, now - date)
        logger.debug("Current age based on date: %i", current_age)

        # TODO: There is an assumption that the result will be a
        #       urllib3 response object. This may not be best since we
        #       could probably avoid instantiating or constructing the
        #       response until we know we need it.
        resp_cc = self.parse_cache_control(headers)

        # determine freshness
        freshness_lifetime = 0

        # Check the max-age pragma in the cache control header
        max_age = resp_cc.get("max-age")
        if max_age is not None:
            freshness_lifetime = max_age
            logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime)

        # If there isn't a max-age, check for an expires header
        elif "expires" in headers:
            expires = parsedate_tz(headers["expires"])
            if expires is not None:
                expire_time = calendar.timegm(expires[:6]) - date
                freshness_lifetime = max(0, expire_time)
                logger.debug("Freshness lifetime from expires: %i", freshness_lifetime)

        # Determine if we are setting freshness limit in the
        # request. Note, this overrides what was in the response.
        max_age = cc.get("max-age")
        if max_age is not None:
            freshness_lifetime = max_age
            logger.debug(
                "Freshness lifetime from request max-age: %i", freshness_lifetime
            )

        min_fresh = cc.get("min-fresh")
        if min_fresh is not None:
            # adjust our current age by our min fresh
            current_age += min_fresh
            logger.debug("Adjusted current age from min-fresh: %i", current_age)

        # Return entry if it is fresh enough
        if freshness_lifetime > current_age:
            logger.debug('The response is "fresh", returning cached response')
            logger.debug("%i > %i", freshness_lifetime, current_age)
            return resp

        # we're not fresh. If we don't have an Etag, clear it out
        if "etag" not in headers:
            logger.debug('The cached response is "stale" with no etag, purging')
            self.cache.delete(cache_url)

        # return the original handler
        return False

    def conditional_headers(self, request: PreparedRequest) -> dict[str, str]:
        resp = self._load_from_cache(request)
        new_headers = {}

        if resp:
            headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(resp.headers)

            if "etag" in headers:
                new_headers["If-None-Match"] = headers["ETag"]

            if "last-modified" in headers:
                new_headers["If-Modified-Since"] = headers["Last-Modified"]

        return new_headers

    def _cache_set(
        self,
        cache_url: str,
        request: PreparedRequest,
        response: HTTPResponse,
        body: bytes | None = None,
        expires_time: int | None = None,
    ) -> None:
        """
        Store the data in the cache.
        """
        if isinstance(self.cache, SeparateBodyBaseCache):
            # We pass in the body separately; just put a placeholder empty
            # string in the metadata.
            self.cache.set(
                cache_url,
                self.serializer.dumps(request, response, b""),
                expires=expires_time,
            )
            # body is None can happen when, for example, we're only updating
            # headers, as is the case in update_cached_response().
            if body is not None:
                self.cache.set_body(cache_url, body)
        else:
            self.cache.set(
                cache_url,
                self.serializer.dumps(request, response, body),
                expires=expires_time,
            )

    def cache_response(
        self,
        request: PreparedRequest,
        response: HTTPResponse,
        body: bytes | None = None,
        status_codes: Collection[int] | None = None,
    ) -> None:
        """
        Algorithm for caching requests.

        This assumes a requests Response object.
        """
        # From httplib2: Don't cache 206's since we aren't going to
        #                handle byte range requests
        cacheable_status_codes = status_codes or self.cacheable_status_codes
        if response.status not in cacheable_status_codes:
            logger.debug(
                "Status code %s not in %s", response.status, cacheable_status_codes
            )
            return

        response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
            response.headers
        )

        if "date" in response_headers:
            time_tuple = parsedate_tz(response_headers["date"])
            assert time_tuple is not None
            date = calendar.timegm(time_tuple[:6])
        else:
            date = 0

        # If we've been given a body, our response has a Content-Length, that
        # Content-Length is valid then we can check to see if the body we've
        # been given matches the expected size, and if it doesn't we'll just
        # skip trying to cache it.
        if (
            body is not None
            and "content-length" in response_headers
            and response_headers["content-length"].isdigit()
            and int(response_headers["content-length"]) != len(body)
        ):
            return

        cc_req = self.parse_cache_control(request.headers)
        cc = self.parse_cache_control(response_headers)

        assert request.url is not None
        cache_url = self.cache_url(request.url)
        logger.debug('Updating cache with response from "%s"', cache_url)

        # Delete it from the cache if we happen to have it stored there
        no_store = False
        if "no-store" in cc:
            no_store = True
            logger.debug('Response header has "no-store"')
        if "no-store" in cc_req:
            no_store = True
            logger.debug('Request header has "no-store"')
        if no_store and self.cache.get(cache_url):
            logger.debug('Purging existing cache entry to honor "no-store"')
            self.cache.delete(cache_url)
        if no_store:
            return

        # https://tools.ietf.org/html/rfc7234#section-4.1:
        # A Vary header field-value of "*" always fails to match.
        # Storing such a response leads to a deserialization warning
        # during cache lookup and is not allowed to ever be served,
        # so storing it can be avoided.
        if "*" in response_headers.get("vary", ""):
            logger.debug('Response header has "Vary: *"')
            return

        # If we've been given an etag, then keep the response
        if self.cache_etags and "etag" in response_headers:
            expires_time = 0
            if response_headers.get("expires"):
                expires = parsedate_tz(response_headers["expires"])
                if expires is not None:
                    expires_time = calendar.timegm(expires[:6]) - date

            expires_time = max(expires_time, 14 * 86400)

            logger.debug(f"etag object cached for {expires_time} seconds")
            logger.debug("Caching due to etag")
            self._cache_set(cache_url, request, response, body, expires_time)

        # Add to the cache any permanent redirects. We do this before looking
        # that the Date headers.
        elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
            logger.debug("Caching permanent redirect")
            self._cache_set(cache_url, request, response, b"")

        # Add to the cache if the response headers demand it. If there
        # is no date header then we can't do anything about expiring
        # the cache.
        elif "date" in response_headers:
            time_tuple = parsedate_tz(response_headers["date"])
            assert time_tuple is not None
            date = calendar.timegm(time_tuple[:6])
            # cache when there is a max-age > 0
            max_age = cc.get("max-age")
            if max_age is not None and max_age > 0:
                logger.debug("Caching b/c date exists and max-age > 0")
                expires_time = max_age
                self._cache_set(
                    cache_url,
                    request,
                    response,
                    body,
                    expires_time,
                )

            # If the request can expire, it means we should cache it
            # in the meantime.
            elif "expires" in response_headers:
                if response_headers["expires"]:
                    expires = parsedate_tz(response_headers["expires"])
                    if expires is not None:
                        expires_time = calendar.timegm(expires[:6]) - date
                    else:
                        expires_time = None

                    logger.debug(
                        "Caching b/c of expires header. expires in {} seconds".format(
                            expires_time
                        )
                    )
                    self._cache_set(
                        cache_url,
                        request,
                        response,
                        body,
                        expires_time,
                    )

    def update_cached_response(
        self, request: PreparedRequest, response: HTTPResponse
    ) -> HTTPResponse:
        """On a 304 we will get a new set of headers that we want to
        update our cached value with, assuming we have one.

        This should only ever be called when we've sent an ETag and
        gotten a 304 as the response.
        """
        assert request.url is not None
        cache_url = self.cache_url(request.url)
        cached_response = self._load_from_cache(request)

        if not cached_response:
            # we didn't have a cached response
            return response

        # Lets update our headers with the headers from the new request:
        # http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
        #
        # The server isn't supposed to send headers that would make
        # the cached body invalid. But... just in case, we'll be sure
        # to strip out ones we know that might be problmatic due to
        # typical assumptions.
        excluded_headers = ["content-length"]

        cached_response.headers.update(
            {
                k: v
                for k, v in response.headers.items()  # type: ignore[no-untyped-call]
                if k.lower() not in excluded_headers
            }
        )

        # we want a 200 b/c we have content via the cache
        cached_response.status = 200

        # update our cache
        self._cache_set(cache_url, request, cached_response)

        return cached_response

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/controller.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

import mmap
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any, Callable

if TYPE_CHECKING:
    from http.client import HTTPResponse


class CallbackFileWrapper:
    """
    Small wrapper around a fp object which will tee everything read into a
    buffer, and when that file is closed it will execute a callback with the
    contents of that buffer.

    All attributes are proxied to the underlying file object.

    This class uses members with a double underscore (__) leading prefix so as
    not to accidentally shadow an attribute.

    The data is stored in a temporary file until it is all available.  As long
    as the temporary files directory is disk-based (sometimes it's a
    memory-backed-``tmpfs`` on Linux), data will be unloaded to disk if memory
    pressure is high.  For small files the disk usually won't be used at all,
    it'll all be in the filesystem memory cache, so there should be no
    performance impact.
    """

    def __init__(
        self, fp: HTTPResponse, callback: Callable[[bytes], None] | None
    ) -> None:
        self.__buf = NamedTemporaryFile("rb+", delete=True)
        self.__fp = fp
        self.__callback = callback

    def __getattr__(self, name: str) -> Any:
        # The vaguaries of garbage collection means that self.__fp is
        # not always set.  By using __getattribute__ and the private
        # name[0] allows looking up the attribute value and raising an
        # AttributeError when it doesn't exist. This stop thigns from
        # infinitely recursing calls to getattr in the case where
        # self.__fp hasn't been set.
        #
        # [0] https://docs.python.org/2/reference/expressions.html#atom-identifiers
        fp = self.__getattribute__("_CallbackFileWrapper__fp")
        return getattr(fp, name)

    def __is_fp_closed(self) -> bool:
        try:
            return self.__fp.fp is None

        except AttributeError:
            pass

        try:
            closed: bool = self.__fp.closed
            return closed

        except AttributeError:
            pass

        # We just don't cache it then.
        # TODO: Add some logging here...
        return False

    def _close(self) -> None:
        if self.__callback:
            if self.__buf.tell() == 0:
                # Empty file:
                result = b""
            else:
                # Return the data without actually loading it into memory,
                # relying on Python's buffer API and mmap(). mmap() just gives
                # a view directly into the filesystem's memory cache, so it
                # doesn't result in duplicate memory use.
                self.__buf.seek(0, 0)
                result = memoryview(
                    mmap.mmap(self.__buf.fileno(), 0, access=mmap.ACCESS_READ)
                )
            self.__callback(result)

        # We assign this to None here, because otherwise we can get into
        # really tricky problems where the CPython interpreter dead locks
        # because the callback is holding a reference to something which
        # has a __del__ method. Setting this to None breaks the cycle
        # and allows the garbage collector to do it's thing normally.
        self.__callback = None

        # Closing the temporary file releases memory and frees disk space.
        # Important when caching big files.
        self.__buf.close()

    def read(self, amt: int | None = None) -> bytes:
        data: bytes = self.__fp.read(amt)
        if data:
            # We may be dealing with b'', a sign that things are over:
            # it's passed e.g. after we've already closed self.__buf.
            self.__buf.write(data)
        if self.__is_fp_closed():
            self._close()

        return data

    def _safe_read(self, amt: int) -> bytes:
        data: bytes = self.__fp._safe_read(amt)  # type: ignore[attr-defined]
        if amt == 2 and data == b"\r\n":
            # urllib executes this read to toss the CRLF at the end
            # of the chunk.
            return data

        self.__buf.write(data)
        if self.__is_fp_closed():
            self._close()

        return data

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/filewrapper.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

import calendar
import time
from datetime import datetime, timedelta, timezone
from email.utils import formatdate, parsedate, parsedate_tz
from typing import TYPE_CHECKING, Any, Mapping

if TYPE_CHECKING:
    from urllib3 import HTTPResponse

TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"


def expire_after(delta: timedelta, date: datetime | None = None) -> datetime:
    date = date or datetime.now(timezone.utc)
    return date + delta


def datetime_to_header(dt: datetime) -> str:
    return formatdate(calendar.timegm(dt.timetuple()))


class BaseHeuristic:
    def warning(self, response: HTTPResponse) -> str | None:
        """
        Return a valid 1xx warning header value describing the cache
        adjustments.

        The response is provided too allow warnings like 113
        http://tools.ietf.org/html/rfc7234#section-5.5.4 where we need
        to explicitly say response is over 24 hours old.
        """
        return '110 - "Response is Stale"'

    def update_headers(self, response: HTTPResponse) -> dict[str, str]:
        """Update the response headers with any new headers.

        NOTE: This SHOULD always include some Warning header to
              signify that the response was cached by the client, not
              by way of the provided headers.
        """
        return {}

    def apply(self, response: HTTPResponse) -> HTTPResponse:
        updated_headers = self.update_headers(response)

        if updated_headers:
            response.headers.update(updated_headers)
            warning_header_value = self.warning(response)
            if warning_header_value is not None:
                response.headers.update({"Warning": warning_header_value})

        return response


class OneDayCache(BaseHeuristic):
    """
    Cache the response by providing an expires 1 day in the
    future.
    """

    def update_headers(self, response: HTTPResponse) -> dict[str, str]:
        headers = {}

        if "expires" not in response.headers:
            date = parsedate(response.headers["date"])
            expires = expire_after(timedelta(days=1), date=datetime(*date[:6], tzinfo=timezone.utc))  # type: ignore[misc]
            headers["expires"] = datetime_to_header(expires)
            headers["cache-control"] = "public"
        return headers


class ExpiresAfter(BaseHeuristic):
    """
    Cache **all** requests for a defined time period.
    """

    def __init__(self, **kw: Any) -> None:
        self.delta = timedelta(**kw)

    def update_headers(self, response: HTTPResponse) -> dict[str, str]:
        expires = expire_after(self.delta)
        return {"expires": datetime_to_header(expires), "cache-control": "public"}

    def warning(self, response: HTTPResponse) -> str | None:
        tmpl = "110 - Automatically cached for %s. Response might be stale"
        return tmpl % self.delta


class LastModified(BaseHeuristic):
    """
    If there is no Expires header already, fall back on Last-Modified
    using the heuristic from
    http://tools.ietf.org/html/rfc7234#section-4.2.2
    to calculate a reasonable value.

    Firefox also does something like this per
    https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching_FAQ
    http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397
    Unlike mozilla we limit this to 24-hr.
    """

    cacheable_by_default_statuses = {
        200,
        203,
        204,
        206,
        300,
        301,
        404,
        405,
        410,
        414,
        501,
    }

    def update_headers(self, resp: HTTPResponse) -> dict[str, str]:
        headers: Mapping[str, str] = resp.headers

        if "expires" in headers:
            return {}

        if "cache-control" in headers and headers["cache-control"] != "public":
            return {}

        if resp.status not in self.cacheable_by_default_statuses:
            return {}

        if "date" not in headers or "last-modified" not in headers:
            return {}

        time_tuple = parsedate_tz(headers["date"])
        assert time_tuple is not None
        date = calendar.timegm(time_tuple[:6])
        last_modified = parsedate(headers["last-modified"])
        if last_modified is None:
            return {}

        now = time.time()
        current_age = max(0, now - date)
        delta = date - calendar.timegm(last_modified)
        freshness_lifetime = max(0, min(delta / 10, 24 * 3600))
        if freshness_lifetime <= current_age:
            return {}

        expires = date + freshness_lifetime
        return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))}

    def warning(self, resp: HTTPResponse) -> str | None:
        return None

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/heuristics.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

import io
from typing import IO, TYPE_CHECKING, Any, Mapping, cast

import msgpack
from requests.structures import CaseInsensitiveDict
from urllib3 import HTTPResponse

if TYPE_CHECKING:
    from requests import PreparedRequest


class Serializer:
    serde_version = "4"

    def dumps(
        self,
        request: PreparedRequest,
        response: HTTPResponse,
        body: bytes | None = None,
    ) -> bytes:
        response_headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
            response.headers
        )

        if body is None:
            # When a body isn't passed in, we'll read the response. We
            # also update the response with a new file handler to be
            # sure it acts as though it was never read.
            body = response.read(decode_content=False)
            response._fp = io.BytesIO(body)  # type: ignore[attr-defined]
            response.length_remaining = len(body)

        data = {
            "response": {
                "body": body,  # Empty bytestring if body is stored separately
                "headers": {str(k): str(v) for k, v in response.headers.items()},  # type: ignore[no-untyped-call]
                "status": response.status,
                "version": response.version,
                "reason": str(response.reason),
                "decode_content": response.decode_content,
            }
        }

        # Construct our vary headers
        data["vary"] = {}
        if "vary" in response_headers:
            varied_headers = response_headers["vary"].split(",")
            for header in varied_headers:
                header = str(header).strip()
                header_value = request.headers.get(header, None)
                if header_value is not None:
                    header_value = str(header_value)
                data["vary"][header] = header_value

        return b",".join([f"cc={self.serde_version}".encode(), self.serialize(data)])

    def serialize(self, data: dict[str, Any]) -> bytes:
        return cast(bytes, msgpack.dumps(data, use_bin_type=True))

    def loads(
        self,
        request: PreparedRequest,
        data: bytes,
        body_file: IO[bytes] | None = None,
    ) -> HTTPResponse | None:
        # Short circuit if we've been given an empty set of data
        if not data:
            return None

        # Determine what version of the serializer the data was serialized
        # with
        try:
            ver, data = data.split(b",", 1)
        except ValueError:
            ver = b"cc=0"

        # Make sure that our "ver" is actually a version and isn't a false
        # positive from a , being in the data stream.
        if ver[:3] != b"cc=":
            data = ver + data
            ver = b"cc=0"

        # Get the version number out of the cc=N
        verstr = ver.split(b"=", 1)[-1].decode("ascii")

        # Dispatch to the actual load method for the given version
        try:
            return getattr(self, f"_loads_v{verstr}")(request, data, body_file)  # type: ignore[no-any-return]

        except AttributeError:
            # This is a version we don't have a loads function for, so we'll
            # just treat it as a miss and return None
            return None

    def prepare_response(
        self,
        request: PreparedRequest,
        cached: Mapping[str, Any],
        body_file: IO[bytes] | None = None,
    ) -> HTTPResponse | None:
        """Verify our vary headers match and construct a real urllib3
        HTTPResponse object.
        """
        # Special case the '*' Vary value as it means we cannot actually
        # determine if the cached response is suitable for this request.
        # This case is also handled in the controller code when creating
        # a cache entry, but is left here for backwards compatibility.
        if "*" in cached.get("vary", {}):
            return None

        # Ensure that the Vary headers for the cached response match our
        # request
        for header, value in cached.get("vary", {}).items():
            if request.headers.get(header, None) != value:
                return None

        body_raw = cached["response"].pop("body")

        headers: CaseInsensitiveDict[str] = CaseInsensitiveDict(
            data=cached["response"]["headers"]
        )
        if headers.get("transfer-encoding", "") == "chunked":
            headers.pop("transfer-encoding")

        cached["response"]["headers"] = headers

        try:
            body: IO[bytes]
            if body_file is None:
                body = io.BytesIO(body_raw)
            else:
                body = body_file
        except TypeError:
            # This can happen if cachecontrol serialized to v1 format (pickle)
            # using Python 2. A Python 2 str(byte string) will be unpickled as
            # a Python 3 str (unicode string), which will cause the above to
            # fail with:
            #
            #     TypeError: 'str' does not support the buffer interface
            body = io.BytesIO(body_raw.encode("utf8"))

        # Discard any `strict` parameter serialized by older version of cachecontrol.
        cached["response"].pop("strict", None)

        return HTTPResponse(body=body, preload_content=False, **cached["response"])

    def _loads_v0(
        self,
        request: PreparedRequest,
        data: bytes,
        body_file: IO[bytes] | None = None,
    ) -> None:
        # The original legacy cache data. This doesn't contain enough
        # information to construct everything we need, so we'll treat this as
        # a miss.
        return None

    def _loads_v1(
        self,
        request: PreparedRequest,
        data: bytes,
        body_file: IO[bytes] | None = None,
    ) -> HTTPResponse | None:
        # The "v1" pickled cache format. This is no longer supported
        # for security reasons, so we treat it as a miss.
        return None

    def _loads_v2(
        self,
        request: PreparedRequest,
        data: bytes,
        body_file: IO[bytes] | None = None,
    ) -> HTTPResponse | None:
        # The "v2" compressed base64 cache format.
        # This has been removed due to age and poor size/performance
        # characteristics, so we treat it as a miss.
        return None

    def _loads_v3(
        self,
        request: PreparedRequest,
        data: bytes,
        body_file: IO[bytes] | None = None,
    ) -> None:
        # Due to Python 2 encoding issues, it's impossible to know for sure
        # exactly how to load v3 entries, thus we'll treat these as a miss so
        # that they get rewritten out as v4 entries.
        return None

    def _loads_v4(
        self,
        request: PreparedRequest,
        data: bytes,
        body_file: IO[bytes] | None = None,
    ) -> HTTPResponse | None:
        try:
            cached = msgpack.loads(data, raw=False)
        except ValueError:
            return None

        return self.prepare_response(request, cached, body_file)

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/serialize.py
NikolaBlagojevic/pyrecodes
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

from typing import TYPE_CHECKING, Collection

from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.cache import DictCache

if TYPE_CHECKING:
    import requests

    from cachecontrol.cache import BaseCache
    from cachecontrol.controller import CacheController
    from cachecontrol.heuristics import BaseHeuristic
    from cachecontrol.serialize import Serializer


def CacheControl(
    sess: requests.Session,
    cache: BaseCache | None = None,
    cache_etags: bool = True,
    serializer: Serializer | None = None,
    heuristic: BaseHeuristic | None = None,
    controller_class: type[CacheController] | None = None,
    adapter_class: type[CacheControlAdapter] | None = None,
    cacheable_methods: Collection[str] | None = None,
) -> requests.Session:
    cache = DictCache() if cache is None else cache
    adapter_class = adapter_class or CacheControlAdapter
    adapter = adapter_class(
        cache,
        cache_etags=cache_etags,
        serializer=serializer,
        heuristic=heuristic,
        controller_class=controller_class,
        cacheable_methods=cacheable_methods,
    )
    sess.mount("http://", adapter)
    sess.mount("https://", adapter)

    return sess

env_pyrecodes/lib/python3.9/site-packages/cachecontrol/wrapper.py
NikolaBlagojevic/pyrecodes
import os, sys, io
from . import ffiplatform, model
from .error import VerificationError
from .cffi_opcode import *

VERSION_BASE = 0x2601
VERSION_EMBEDDED = 0x2701
VERSION_CHAR16CHAR32 = 0x2801

USE_LIMITED_API = (sys.platform != 'win32' or sys.version_info < (3, 0) or
                   sys.version_info >= (3, 5))


class GlobalExpr:
    def __init__(self, name, address, type_op, size=0, check_value=0):
        self.name = name
        self.address = address
        self.type_op = type_op
        self.size = size
        self.check_value = check_value

    def as_c_expr(self):
        return '  { "%s", (void *)%s, %s, (void *)%s },' % (
            self.name, self.address, self.type_op.as_c_expr(), self.size)

    def as_python_expr(self):
        return "b'%s%s',%d" % (self.type_op.as_python_bytes(), self.name,
                               self.check_value)

class FieldExpr:
    def __init__(self, name, field_offset, field_size, fbitsize, field_type_op):
        self.name = name
        self.field_offset = field_offset
        self.field_size = field_size
        self.fbitsize = fbitsize
        self.field_type_op = field_type_op

    def as_c_expr(self):
        spaces = " " * len(self.name)
        return ('  { "%s", %s,\n' % (self.name, self.field_offset) +
                '     %s   %s,\n' % (spaces, self.field_size) +
                '     %s   %s },' % (spaces, self.field_type_op.as_c_expr()))

    def as_python_expr(self):
        raise NotImplementedError

    def as_field_python_expr(self):
        if self.field_type_op.op == OP_NOOP:
            size_expr = ''
        elif self.field_type_op.op == OP_BITFIELD:
            size_expr = format_four_bytes(self.fbitsize)
        else:
            raise NotImplementedError
        return "b'%s%s%s'" % (self.field_type_op.as_python_bytes(),
                              size_expr,
                              self.name)

class StructUnionExpr:
    def __init__(self, name, type_index, flags, size, alignment, comment,
                 first_field_index, c_fields):
        self.name = name
        self.type_index = type_index
        self.flags = flags
        self.size = size
        self.alignment = alignment
        self.comment = comment
        self.first_field_index = first_field_index
        self.c_fields = c_fields

    def as_c_expr(self):
        return ('  { "%s", %d, %s,' % (self.name, self.type_index, self.flags)
                + '\n    %s, %s, ' % (self.size, self.alignment)
                + '%d, %d ' % (self.first_field_index, len(self.c_fields))
                + ('/* %s */ ' % self.comment if self.comment else '')
                + '},')

    def as_python_expr(self):
        flags = eval(self.flags, G_FLAGS)
        fields_expr = [c_field.as_field_python_expr()
                       for c_field in self.c_fields]
        return "(b'%s%s%s',%s)" % (
            format_four_bytes(self.type_index),
            format_four_bytes(flags),
            self.name,
            ','.join(fields_expr))

class EnumExpr:
    def __init__(self, name, type_index, size, signed, allenums):
        self.name = name
        self.type_index = type_index
        self.size = size
        self.signed = signed
        self.allenums = allenums

    def as_c_expr(self):
        return ('  { "%s", %d, _cffi_prim_int(%s, %s),\n'
                '    "%s" },' % (self.name, self.type_index,
                                 self.size, self.signed, self.allenums))

    def as_python_expr(self):
        prim_index = {
            (1, 0): PRIM_UINT8,  (1, 1):  PRIM_INT8,
            (2, 0): PRIM_UINT16, (2, 1):  PRIM_INT16,
            (4, 0): PRIM_UINT32, (4, 1):  PRIM_INT32,
            (8, 0): PRIM_UINT64, (8, 1):  PRIM_INT64,
            }[self.size, self.signed]
        return "b'%s%s%s\\x00%s'" % (format_four_bytes(self.type_index),
                                     format_four_bytes(prim_index),
                                     self.name, self.allenums)

class TypenameExpr:
    def __init__(self, name, type_index):
        self.name = name
        self.type_index = type_index

    def as_c_expr(self):
        return '  { "%s", %d },' % (self.name, self.type_index)

    def as_python_expr(self):
        return "b'%s%s'" % (format_four_bytes(self.type_index), self.name)


# ____________________________________________________________


class Recompiler:
    _num_externpy = 0

    def __init__(self, ffi, module_name, target_is_python=False):
        self.ffi = ffi
        self.module_name = module_name
        self.target_is_python = target_is_python
        self._version = VERSION_BASE

    def needs_version(self, ver):
        self._version = max(self._version, ver)

    def collect_type_table(self):
        self._typesdict = {}
        self._generate("collecttype")
        #
        all_decls = sorted(self._typesdict, key=str)
        #
        # prepare all FUNCTION bytecode sequences first
        self.cffi_types = []
        for tp in all_decls:
            if tp.is_raw_function:
                assert self._typesdict[tp] is None
                self._typesdict[tp] = len(self.cffi_types)
                self.cffi_types.append(tp)     # placeholder
                for tp1 in tp.args:
                    assert isinstance(tp1, (model.VoidType,
                                            model.BasePrimitiveType,
                                            model.PointerType,
                                            model.StructOrUnionOrEnum,
                                            model.FunctionPtrType))
                    if self._typesdict[tp1] is None:
                        self._typesdict[tp1] = len(self.cffi_types)
                    self.cffi_types.append(tp1)   # placeholder
                self.cffi_types.append('END')     # placeholder
        #
        # prepare all OTHER bytecode sequences
        for tp in all_decls:
            if not tp.is_raw_function and self._typesdict[tp] is None:
                self._typesdict[tp] = len(self.cffi_types)
                self.cffi_types.append(tp)        # placeholder
                if tp.is_array_type and tp.length is not None:
                    self.cffi_types.append('LEN') # placeholder
        assert None not in self._typesdict.values()
        #
        # collect all structs and unions and enums
        self._struct_unions = {}
        self._enums = {}
        for tp in all_decls:
            if isinstance(tp, model.StructOrUnion):
                self._struct_unions[tp] = None
            elif isinstance(tp, model.EnumType):
                self._enums[tp] = None
        for i, tp in enumerate(sorted(self._struct_unions,
                                      key=lambda tp: tp.name)):
            self._struct_unions[tp] = i
        for i, tp in enumerate(sorted(self._enums,
                                      key=lambda tp: tp.name)):
            self._enums[tp] = i
        #
        # emit all bytecode sequences now
        for tp in all_decls:
            method = getattr(self, '_emit_bytecode_' + tp.__class__.__name__)
            method(tp, self._typesdict[tp])
        #
        # consistency check
        for op in self.cffi_types:
            assert isinstance(op, CffiOp)
        self.cffi_types = tuple(self.cffi_types)    # don't change any more

    def _enum_fields(self, tp):
        # When producing C, expand all anonymous struct/union fields.
        # That's necessary to have C code checking the offsets of the
        # individual fields contained in them.  When producing Python,
        # don't do it and instead write it like it is, with the
        # corresponding fields having an empty name.  Empty names are
        # recognized at runtime when we import the generated Python
        # file.
        expand_anonymous_struct_union = not self.target_is_python
        return tp.enumfields(expand_anonymous_struct_union)

    def _do_collect_type(self, tp):
        if not isinstance(tp, model.BaseTypeByIdentity):
            if isinstance(tp, tuple):
                for x in tp:
                    self._do_collect_type(x)
            return
        if tp not in self._typesdict:
            self._typesdict[tp] = None
            if isinstance(tp, model.FunctionPtrType):
                self._do_collect_type(tp.as_raw_function())
            elif isinstance(tp, model.StructOrUnion):
                if tp.fldtypes is not None and (
                        tp not in self.ffi._parser._included_declarations):
                    for name1, tp1, _, _ in self._enum_fields(tp):
                        self._do_collect_type(self._field_type(tp, name1, tp1))
            else:
                for _, x in tp._get_items():
                    self._do_collect_type(x)

    def _generate(self, step_name):
        lst = self.ffi._parser._declarations.items()
        for name, (tp, quals) in sorted(lst):
            kind, realname = name.split(' ', 1)
            try:
                method = getattr(self, '_generate_cpy_%s_%s' % (kind,
                                                                step_name))
            except AttributeError:
                raise VerificationError(
                    "not implemented in recompile(): %r" % name)
            try:
                self._current_quals = quals
                method(tp, realname)
            except Exception as e:
                model.attach_exception_info(e, name)
                raise

    # ----------

    ALL_STEPS = ["global", "field", "struct_union", "enum", "typename"]

    def collect_step_tables(self):
        # collect the declarations for '_cffi_globals', '_cffi_typenames', etc.
        self._lsts = {}
        for step_name in self.ALL_STEPS:
            self._lsts[step_name] = []
        self._seen_struct_unions = set()
        self._generate("ctx")
        self._add_missing_struct_unions()
        #
        for step_name in self.ALL_STEPS:
            lst = self._lsts[step_name]
            if step_name != "field":
                lst.sort(key=lambda entry: entry.name)
            self._lsts[step_name] = tuple(lst)    # don't change any more
        #
        # check for a possible internal inconsistency: _cffi_struct_unions
        # should have been generated with exactly self._struct_unions
        lst = self._lsts["struct_union"]
        for tp, i in self._struct_unions.items():
            assert i < len(lst)
            assert lst[i].name == tp.name
        assert len(lst) == len(self._struct_unions)
        # same with enums
        lst = self._lsts["enum"]
        for tp, i in self._enums.items():
            assert i < len(lst)
            assert lst[i].name == tp.name
        assert len(lst) == len(self._enums)

    # ----------

    def _prnt(self, what=''):
        self._f.write(what + '\n')

    def write_source_to_f(self, f, preamble):
        if self.target_is_python:
            assert preamble is None
            self.write_py_source_to_f(f)
        else:
            assert preamble is not None
            self.write_c_source_to_f(f, preamble)

    def _rel_readlines(self, filename):
        g = open(os.path.join(os.path.dirname(__file__), filename), 'r')
        lines = g.readlines()
        g.close()
        return lines

    def write_c_source_to_f(self, f, preamble):
        self._f = f
        prnt = self._prnt
        if self.ffi._embedding is not None:
            prnt('#define _CFFI_USE_EMBEDDING')
        if not USE_LIMITED_API:
            prnt('#define _CFFI_NO_LIMITED_API')
        #
        # first the '#include' (actually done by inlining the file's content)
        lines = self._rel_readlines('_cffi_include.h')
        i = lines.index('#include "parse_c_type.h"\n')
        lines[i:i+1] = self._rel_readlines('parse_c_type.h')
        prnt(''.join(lines))
        #
        # if we have ffi._embedding != None, we give it here as a macro
        # and include an extra file
        base_module_name = self.module_name.split('.')[-1]
        if self.ffi._embedding is not None:
            prnt('#define _CFFI_MODULE_NAME  "%s"' % (self.module_name,))
            prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {')
            self._print_string_literal_in_array(self.ffi._embedding)
            prnt('0 };')
            prnt('#ifdef PYPY_VERSION')
            prnt('# define _CFFI_PYTHON_STARTUP_FUNC  _cffi_pypyinit_%s' % (
                base_module_name,))
            prnt('#elif PY_MAJOR_VERSION >= 3')
            prnt('# define _CFFI_PYTHON_STARTUP_FUNC  PyInit_%s' % (
                base_module_name,))
            prnt('#else')
            prnt('# define _CFFI_PYTHON_STARTUP_FUNC  init%s' % (
                base_module_name,))
            prnt('#endif')
            lines = self._rel_readlines('_embedding.h')
            i = lines.index('#include "_cffi_errors.h"\n')
            lines[i:i+1] = self._rel_readlines('_cffi_errors.h')
            prnt(''.join(lines))
            self.needs_version(VERSION_EMBEDDED)
        #
        # then paste the C source given by the user, verbatim.
        prnt('/************************************************************/')
        prnt()
        prnt(preamble)
        prnt()
        prnt('/************************************************************/')
        prnt()
        #
        # the declaration of '_cffi_types'
        prnt('static void *_cffi_types[] = {')
        typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()])
        for i, op in enumerate(self.cffi_types):
            comment = ''
            if i in typeindex2type:
                comment = ' // ' + typeindex2type[i]._get_c_name()
            prnt('/* %2d */ %s,%s' % (i, op.as_c_expr(), comment))
        if not self.cffi_types:
            prnt('  0')
        prnt('};')
        prnt()
        #
        # call generate_cpy_xxx_decl(), for every xxx found from
        # ffi._parser._declarations.  This generates all the functions.
        self._seen_constants = set()
        self._generate("decl")
        #
        # the declaration of '_cffi_globals' and '_cffi_typenames'
        nums = {}
        for step_name in self.ALL_STEPS:
            lst = self._lsts[step_name]
            nums[step_name] = len(lst)
            if nums[step_name] > 0:
                prnt('static const struct _cffi_%s_s _cffi_%ss[] = {' % (
                    step_name, step_name))
                for entry in lst:
                    prnt(entry.as_c_expr())
                prnt('};')
                prnt()
        #
        # the declaration of '_cffi_includes'
        if self.ffi._included_ffis:
            prnt('static const char * const _cffi_includes[] = {')
            for ffi_to_include in self.ffi._included_ffis:
                try:
                    included_module_name, included_source = (
                        ffi_to_include._assigned_source[:2])
                except AttributeError:
                    raise VerificationError(
                        "ffi object %r includes %r, but the latter has not "
                        "been prepared with set_source()" % (
                            self.ffi, ffi_to_include,))
                if included_source is None:
                    raise VerificationError(
                        "not implemented yet: ffi.include() of a Python-based "
                        "ffi inside a C-based ffi")
                prnt('  "%s",' % (included_module_name,))
            prnt('  NULL')
            prnt('};')
            prnt()
        #
        # the declaration of '_cffi_type_context'
        prnt('static const struct _cffi_type_context_s _cffi_type_context = {')
        prnt('  _cffi_types,')
        for step_name in self.ALL_STEPS:
            if nums[step_name] > 0:
                prnt('  _cffi_%ss,' % step_name)
            else:
                prnt('  NULL,  /* no %ss */' % step_name)
        for step_name in self.ALL_STEPS:
            if step_name != "field":
                prnt('  %d,  /* num_%ss */' % (nums[step_name], step_name))
        if self.ffi._included_ffis:
            prnt('  _cffi_includes,')
        else:
            prnt('  NULL,  /* no includes */')
        prnt('  %d,  /* num_types */' % (len(self.cffi_types),))
        flags = 0
        if self._num_externpy > 0 or self.ffi._embedding is not None:
            flags |= 1     # set to mean that we use extern "Python"
        prnt('  %d,  /* flags */' % flags)
        prnt('};')
        prnt()
        #
        # the init function
        prnt('#ifdef __GNUC__')
        prnt('#  pragma GCC visibility push(default)  /* for -fvisibility= */')
        prnt('#endif')
        prnt()
        prnt('#ifdef PYPY_VERSION')
        prnt('PyMODINIT_FUNC')
        prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
        prnt('{')
        if flags & 1:
            prnt('    if (((intptr_t)p[0]) >= 0x0A03) {')
            prnt('        _cffi_call_python_org = '
                 '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
            prnt('    }')
        prnt('    p[0] = (const void *)0x%x;' % self._version)
        prnt('    p[1] = &_cffi_type_context;')
        prnt('#if PY_MAJOR_VERSION >= 3')
        prnt('    return NULL;')
        prnt('#endif')
        prnt('}')
        # on Windows, distutils insists on putting init_cffi_xyz in
        # 'export_symbols', so instead of fighting it, just give up and
        # give it one
        prnt('#  ifdef _MSC_VER')
        prnt('     PyMODINIT_FUNC')
        prnt('#  if PY_MAJOR_VERSION >= 3')
        prnt('     PyInit_%s(void) { return NULL; }' % (base_module_name,))
        prnt('#  else')
        prnt('     init%s(void) { }' % (base_module_name,))
        prnt('#  endif')
        prnt('#  endif')
        prnt('#elif PY_MAJOR_VERSION >= 3')
        prnt('PyMODINIT_FUNC')
        prnt('PyInit_%s(void)' % (base_module_name,))
        prnt('{')
        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
            self.module_name, self._version))
        prnt('}')
        prnt('#else')
        prnt('PyMODINIT_FUNC')
        prnt('init%s(void)' % (base_module_name,))
        prnt('{')
        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
            self.module_name, self._version))
        prnt('}')
        prnt('#endif')
        prnt()
        prnt('#ifdef __GNUC__')
        prnt('#  pragma GCC visibility pop')
        prnt('#endif')
        self._version = None

    def _to_py(self, x):
        if isinstance(x, str):
            return "b'%s'" % (x,)
        if isinstance(x, (list, tuple)):
            rep = [self._to_py(item) for item in x]
            if len(rep) == 1:
                rep.append('')
            return "(%s)" % (','.join(rep),)
        return x.as_python_expr()  # Py2: unicode unexpected; Py3: bytes unexp.

    def write_py_source_to_f(self, f):
        self._f = f
        prnt = self._prnt
        #
        # header
        prnt("# auto-generated file")
        prnt("import _cffi_backend")
        #
        # the 'import' of the included ffis
        num_includes = len(self.ffi._included_ffis or ())
        for i in range(num_includes):
            ffi_to_include = self.ffi._included_ffis[i]
            try:
                included_module_name, included_source = (
                    ffi_to_include._assigned_source[:2])
            except AttributeError:
                raise VerificationError(
                    "ffi object %r includes %r, but the latter has not "
                    "been prepared with set_source()" % (
                        self.ffi, ffi_to_include,))
            if included_source is not None:
                raise VerificationError(
                    "not implemented yet: ffi.include() of a C-based "
                    "ffi inside a Python-based ffi")
            prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
        prnt()
        prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
        prnt("    _version = 0x%x," % (self._version,))
        self._version = None
        #
        # the '_types' keyword argument
        self.cffi_types = tuple(self.cffi_types)    # don't change any more
        types_lst = [op.as_python_bytes() for op in self.cffi_types]
        prnt('    _types = %s,' % (self._to_py(''.join(types_lst)),))
        typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()])
        #
        # the keyword arguments from ALL_STEPS
        for step_name in self.ALL_STEPS:
            lst = self._lsts[step_name]
            if len(lst) > 0 and step_name != "field":
                prnt('    _%ss = %s,' % (step_name, self._to_py(lst)))
        #
        # the '_includes' keyword argument
        if num_includes > 0:
            prnt('    _includes = (%s,),' % (
                ', '.join(['_ffi%d' % i for i in range(num_includes)]),))
        #
        # the footer
        prnt(')')

    # ----------

    def _gettypenum(self, type):
        # a KeyError here is a bug.  please report it! :-)
        return self._typesdict[type]

    def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
        extraarg = ''
        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
            if tp.is_integer_type() and tp.name != '_Bool':
                converter = '_cffi_to_c_int'
                extraarg = ', %s' % tp.name
            elif isinstance(tp, model.UnknownFloatType):
                # don't check with is_float_type(): it may be a 'long
                # double' here, and _cffi_to_c_double would loose precision
                converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
            else:
                cname = tp.get_c_name('')
                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                   tp.name.replace(' ', '_'))
                if cname in ('char16_t', 'char32_t'):
                    self.needs_version(VERSION_CHAR16CHAR32)
            errvalue = '-1'
        #
        elif isinstance(tp, model.PointerType):
            self._convert_funcarg_to_c_ptr_or_array(tp, fromvar,
                                                    tovar, errcode)
            return
        #
        elif (isinstance(tp, model.StructOrUnionOrEnum) or
              isinstance(tp, model.BasePrimitiveType)):
            # a struct (not a struct pointer) as a function argument;
            # or, a complex (the same code works)
            self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                      % (tovar, self._gettypenum(tp), fromvar))
            self._prnt('    %s;' % errcode)
            return
        #
        elif isinstance(tp, model.FunctionPtrType):
            converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('')
            extraarg = ', _cffi_type(%d)' % self._gettypenum(tp)
            errvalue = 'NULL'
        #
        else:
            raise NotImplementedError(tp)
        #
        self._prnt('  %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg))
        self._prnt('  if (%s == (%s)%s && PyErr_Occurred())' % (
            tovar, tp.get_c_name(''), errvalue))
        self._prnt('    %s;' % errcode)

    def _extra_local_variables(self, tp, localvars, freelines):
        if isinstance(tp, model.PointerType):
            localvars.add('Py_ssize_t datasize')
            localvars.add('struct _cffi_freeme_s *large_args_free = NULL')
            freelines.add('if (large_args_free != NULL)'
                          ' _cffi_free_array_arguments(large_args_free);')

    def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode):
        self._prnt('  datasize = _cffi_prepare_pointer_call_argument(')
        self._prnt('      _cffi_type(%d), %s, (char **)&%s);' % (
            self._gettypenum(tp), fromvar, tovar))
        self._prnt('  if (datasize != 0) {')
        self._prnt('    %s = ((size_t)datasize) <= 640 ? '
                   '(%s)alloca((size_t)datasize) : NULL;' % (
            tovar, tp.get_c_name('')))
        self._prnt('    if (_cffi_convert_array_argument(_cffi_type(%d), %s, '
                   '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar))
        self._prnt('            datasize, &large_args_free) < 0)')
        self._prnt('      %s;' % errcode)
        self._prnt('  }')

    def _convert_expr_from_c(self, tp, var, context):
        if isinstance(tp, model.BasePrimitiveType):
            if tp.is_integer_type() and tp.name != '_Bool':
                return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
            elif isinstance(tp, model.UnknownFloatType):
                return '_cffi_from_c_double(%s)' % (var,)
            elif tp.name != 'long double' and not tp.is_complex_type():
                cname = tp.name.replace(' ', '_')
                if cname in ('char16_t', 'char32_t'):
                    self.needs_version(VERSION_CHAR16CHAR32)
                return '_cffi_from_c_%s(%s)' % (cname, var)
            else:
                return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                    var, self._gettypenum(tp))
        elif isinstance(tp, (model.PointerType, model.FunctionPtrType)):
            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                var, self._gettypenum(tp))
        elif isinstance(tp, model.ArrayType):
            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                var, self._gettypenum(model.PointerType(tp.item)))
        elif isinstance(tp, model.StructOrUnion):
            if tp.fldnames is None:
                raise TypeError("'%s' is used as %s, but is opaque" % (
                    tp._get_c_name(), context))
            return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % (
                var, self._gettypenum(tp))
        elif isinstance(tp, model.EnumType):
            return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                var, self._gettypenum(tp))
        else:
            raise NotImplementedError(tp)

    # ----------
    # typedefs

    def _typedef_type(self, tp, name):
        return self._global_type(tp, "(*(%s *)0)" % (name,))

    def _generate_cpy_typedef_collecttype(self, tp, name):
        self._do_collect_type(self._typedef_type(tp, name))

    def _generate_cpy_typedef_decl(self, tp, name):
        pass

    def _typedef_ctx(self, tp, name):
        type_index = self._typesdict[tp]
        self._lsts["typename"].append(TypenameExpr(name, type_index))

    def _generate_cpy_typedef_ctx(self, tp, name):
        tp = self._typedef_type(tp, name)
        self._typedef_ctx(tp, name)
        if getattr(tp, "origin", None) == "unknown_type":
            self._struct_ctx(tp, tp.name, approxname=None)
        elif isinstance(tp, model.NamedPointerType):
            self._struct_ctx(tp.totype, tp.totype.name, approxname=tp.name,
                             named_ptr=tp)

    # ----------
    # function declarations

    def _generate_cpy_function_collecttype(self, tp, name):
        self._do_collect_type(tp.as_raw_function())
        if tp.ellipsis and not self.target_is_python:
            self._do_collect_type(tp)

    def _generate_cpy_function_decl(self, tp, name):
        assert not self.target_is_python
        assert isinstance(tp, model.FunctionPtrType)
        if tp.ellipsis:
            # cannot support vararg functions better than this: check for its
            # exact type (including the fixed arguments), and build it as a
            # constant function pointer (no CPython wrapper)
            self._generate_cpy_constant_decl(tp, name)
            return
        prnt = self._prnt
        numargs = len(tp.args)
        if numargs == 0:
            argname = 'noarg'
        elif numargs == 1:
            argname = 'arg0'
        else:
            argname = 'args'
        #
        # ------------------------------
        # the 'd' version of the function, only for addressof(lib, 'func')
        arguments = []
        call_arguments = []
        context = 'argument of %s' % name
        for i, type in enumerate(tp.args):
            arguments.append(type.get_c_name(' x%d' % i, context))
            call_arguments.append('x%d' % i)
        repr_arguments = ', '.join(arguments)
        repr_arguments = repr_arguments or 'void'
        if tp.abi:
            abi = tp.abi + ' '
        else:
            abi = ''
        name_and_arguments = '%s_cffi_d_%s(%s)' % (abi, name, repr_arguments)
        prnt('static %s' % (tp.result.get_c_name(name_and_arguments),))
        prnt('{')
        call_arguments = ', '.join(call_arguments)
        result_code = 'return '
        if isinstance(tp.result, model.VoidType):
            result_code = ''
        prnt('  %s%s(%s);' % (result_code, name, call_arguments))
        prnt('}')
        #
        prnt('#ifndef PYPY_VERSION')        # ------------------------------
        #
        prnt('static PyObject *')
        prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname))
        prnt('{')
        #
        context = 'argument of %s' % name
        for i, type in enumerate(tp.args):
            arg = type.get_c_name(' x%d' % i, context)
            prnt('  %s;' % arg)
        #
        localvars = set()
        freelines = set()
        for type in tp.args:
            self._extra_local_variables(type, localvars, freelines)
        for decl in sorted(localvars):
            prnt('  %s;' % (decl,))
        #
        if not isinstance(tp.result, model.VoidType):
            result_code = 'result = '
            context = 'result of %s' % name
            result_decl = '  %s;' % tp.result.get_c_name(' result', context)
            prnt(result_decl)
            prnt('  PyObject *pyresult;')
        else:
            result_decl = None
            result_code = ''
        #
        if len(tp.args) > 1:
            rng = range(len(tp.args))
            for i in rng:
                prnt('  PyObject *arg%d;' % i)
            prnt()
            prnt('  if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % (
                name, len(rng), len(rng),
                ', '.join(['&arg%d' % i for i in rng])))
            prnt('    return NULL;')
        prnt()
        #
        for i, type in enumerate(tp.args):
            self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i,
                                       'return NULL')
            prnt()
        #
        prnt('  Py_BEGIN_ALLOW_THREADS')
        prnt('  _cffi_restore_errno();')
        call_arguments = ['x%d' % i for i in range(len(tp.args))]
        call_arguments = ', '.join(call_arguments)
        prnt('  { %s%s(%s); }' % (result_code, name, call_arguments))
        prnt('  _cffi_save_errno();')
        prnt('  Py_END_ALLOW_THREADS')
        prnt()
        #
        prnt('  (void)self; /* unused */')
        if numargs == 0:
            prnt('  (void)noarg; /* unused */')
        if result_code:
            prnt('  pyresult = %s;' %
                 self._convert_expr_from_c(tp.result, 'result', 'result type'))
            for freeline in freelines:
                prnt('  ' + freeline)
            prnt('  return pyresult;')
        else:
            for freeline in freelines:
                prnt('  ' + freeline)
            prnt('  Py_INCREF(Py_None);')
            prnt('  return Py_None;')
        prnt('}')
        #
        prnt('#else')        # ------------------------------
        #
        # the PyPy version: need to replace struct/union arguments with
        # pointers, and if the result is a struct/union, insert a first
        # arg that is a pointer to the result.  We also do that for
        # complex args and return type.
        def need_indirection(type):
            return (isinstance(type, model.StructOrUnion) or
                    (isinstance(type, model.PrimitiveType) and
                     type.is_complex_type()))
        difference = False
        arguments = []
        call_arguments = []
        context = 'argument of %s' % name
        for i, type in enumerate(tp.args):
            indirection = ''
            if need_indirection(type):
                indirection = '*'
                difference = True
            arg = type.get_c_name(' %sx%d' % (indirection, i), context)
            arguments.append(arg)
            call_arguments.append('%sx%d' % (indirection, i))
        tp_result = tp.result
        if need_indirection(tp_result):
            context = 'result of %s' % name
            arg = tp_result.get_c_name(' *result', context)
            arguments.insert(0, arg)
            tp_result = model.void_type
            result_decl = None
            result_code = '*result = '
            difference = True
        if difference:
            repr_arguments = ', '.join(arguments)
            repr_arguments = repr_arguments or 'void'
            name_and_arguments = '%s_cffi_f_%s(%s)' % (abi, name,
                                                       repr_arguments)
            prnt('static %s' % (tp_result.get_c_name(name_and_arguments),))
            prnt('{')
            if result_decl:
                prnt(result_decl)
            call_arguments = ', '.join(call_arguments)
            prnt('  { %s%s(%s); }' % (result_code, name, call_arguments))
            if result_decl:
                prnt('  return result;')
            prnt('}')
        else:
            prnt('#  define _cffi_f_%s _cffi_d_%s' % (name, name))
        #
        prnt('#endif')        # ------------------------------
        prnt()

    def _generate_cpy_function_ctx(self, tp, name):
        if tp.ellipsis and not self.target_is_python:
            self._generate_cpy_constant_ctx(tp, name)
            return
        type_index = self._typesdict[tp.as_raw_function()]
        numargs = len(tp.args)
        if self.target_is_python:
            meth_kind = OP_DLOPEN_FUNC
        elif numargs == 0:
            meth_kind = OP_CPYTHON_BLTN_N   # 'METH_NOARGS'
        elif numargs == 1:
            meth_kind = OP_CPYTHON_BLTN_O   # 'METH_O'
        else:
            meth_kind = OP_CPYTHON_BLTN_V   # 'METH_VARARGS'
        self._lsts["global"].append(
            GlobalExpr(name, '_cffi_f_%s' % name,
                       CffiOp(meth_kind, type_index),
                       size='_cffi_d_%s' % name))

    # ----------
    # named structs or unions

    def _field_type(self, tp_struct, field_name, tp_field):
        if isinstance(tp_field, model.ArrayType):
            actual_length = tp_field.length
            if actual_length == '...':
                ptr_struct_name = tp_struct.get_c_name('*')
                actual_length = '_cffi_array_len(((%s)0)->%s)' % (
                    ptr_struct_name, field_name)
            tp_item = self._field_type(tp_struct, '%s[0]' % field_name,
                                       tp_field.item)
            tp_field = model.ArrayType(tp_item, actual_length)
        return tp_field

    def _struct_collecttype(self, tp):
        self._do_collect_type(tp)
        if self.target_is_python:
            # also requires nested anon struct/unions in ABI mode, recursively
            for fldtype in tp.anonymous_struct_fields():
                self._struct_collecttype(fldtype)

    def _struct_decl(self, tp, cname, approxname):
        if tp.fldtypes is None:
            return
        prnt = self._prnt
        checkfuncname = '_cffi_checkfld_%s' % (approxname,)
        prnt('_CFFI_UNUSED_FN')
        prnt('static void %s(%s *p)' % (checkfuncname, cname))
        prnt('{')
        prnt('  /* only to generate compile-time warnings or errors */')
        prnt('  (void)p;')
        for fname, ftype, fbitsize, fqual in self._enum_fields(tp):
            try:
                if ftype.is_integer_type() or fbitsize >= 0:
                    # accept all integers, but complain on float or double
                    if fname != '':
                        prnt("  (void)((p->%s) | 0);  /* check that '%s.%s' is "
                             "an integer */" % (fname, cname, fname))
                    continue
                # only accept exactly the type declared, except that '[]'
                # is interpreted as a '*' and so will match any array length.
                # (It would also match '*', but that's harder to detect...)
                while (isinstance(ftype, model.ArrayType)
                       and (ftype.length is None or ftype.length == '...')):
                    ftype = ftype.item
                    fname = fname + '[0]'
                prnt('  { %s = &p->%s; (void)tmp; }' % (
                    ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual),
                    fname))
            except VerificationError as e:
                prnt('  /* %s */' % str(e))   # cannot verify it, ignore
        prnt('}')
        prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname))
        prnt()

    def _struct_ctx(self, tp, cname, approxname, named_ptr=None):
        type_index = self._typesdict[tp]
        reason_for_not_expanding = None
        flags = []
        if isinstance(tp, model.UnionType):
            flags.append("_CFFI_F_UNION")
        if tp.fldtypes is None:
            flags.append("_CFFI_F_OPAQUE")
            reason_for_not_expanding = "opaque"
        if (tp not in self.ffi._parser._included_declarations and
                (named_ptr is None or
                 named_ptr not in self.ffi._parser._included_declarations)):
            if tp.fldtypes is None:
                pass    # opaque
            elif tp.partial or any(tp.anonymous_struct_fields()):
                pass    # field layout obtained silently from the C compiler
            else:
                flags.append("_CFFI_F_CHECK_FIELDS")
            if tp.packed:
                if tp.packed > 1:
                    raise NotImplementedError(
                        "%r is declared with 'pack=%r'; only 0 or 1 are "
                        "supported in API mode (try to use \"...;\", which "
                        "does not require a 'pack' declaration)" %
                        (tp, tp.packed))
                flags.append("_CFFI_F_PACKED")
        else:
            flags.append("_CFFI_F_EXTERNAL")
            reason_for_not_expanding = "external"
        flags = '|'.join(flags) or '0'
        c_fields = []
        if reason_for_not_expanding is None:
            enumfields = list(self._enum_fields(tp))
            for fldname, fldtype, fbitsize, fqual in enumfields:
                fldtype = self._field_type(tp, fldname, fldtype)
                self._check_not_opaque(fldtype,
                                       "field '%s.%s'" % (tp.name, fldname))
                # cname is None for _add_missing_struct_unions() only
                op = OP_NOOP
                if fbitsize >= 0:
                    op = OP_BITFIELD
                    size = '%d /* bits */' % fbitsize
                elif cname is None or (
                        isinstance(fldtype, model.ArrayType) and
                        fldtype.length is None):
                    size = '(size_t)-1'
                else:
                    size = 'sizeof(((%s)0)->%s)' % (
                        tp.get_c_name('*') if named_ptr is None
                                           else named_ptr.name,
                        fldname)
                if cname is None or fbitsize >= 0:
                    offset = '(size_t)-1'
                elif named_ptr is not None:
                    offset = '((char *)&((%s)0)->%s) - (char *)0' % (
                        named_ptr.name, fldname)
                else:
                    offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname)
                c_fields.append(
                    FieldExpr(fldname, offset, size, fbitsize,
                              CffiOp(op, self._typesdict[fldtype])))
            first_field_index = len(self._lsts["field"])
            self._lsts["field"].extend(c_fields)
            #
            if cname is None:  # unknown name, for _add_missing_struct_unions
                size = '(size_t)-2'
                align = -2
                comment = "unnamed"
            else:
                if named_ptr is not None:
                    size = 'sizeof(*(%s)0)' % (named_ptr.name,)
                    align = '-1 /* unknown alignment */'
                else:
                    size = 'sizeof(%s)' % (cname,)
                    align = 'offsetof(struct _cffi_align_%s, y)' % (approxname,)
                comment = None
        else:
            size = '(size_t)-1'
            align = -1
            first_field_index = -1
            comment = reason_for_not_expanding
        self._lsts["struct_union"].append(
            StructUnionExpr(tp.name, type_index, flags, size, align, comment,
                            first_field_index, c_fields))
        self._seen_struct_unions.add(tp)

    def _check_not_opaque(self, tp, location):
        while isinstance(tp, model.ArrayType):
            tp = tp.item
        if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None:
            raise TypeError(
                "%s is of an opaque type (not declared in cdef())" % location)

    def _add_missing_struct_unions(self):
        # not very nice, but some struct declarations might be missing
        # because they don't have any known C name.  Check that they are
        # not partial (we can't complete or verify them!) and emit them
        # anonymously.
        lst = list(self._struct_unions.items())
        lst.sort(key=lambda tp_order: tp_order[1])
        for tp, order in lst:
            if tp not in self._seen_struct_unions:
                if tp.partial:
                    raise NotImplementedError("internal inconsistency: %r is "
                                              "partial but was not seen at "
                                              "this point" % (tp,))
                if tp.name.startswith('$') and tp.name[1:].isdigit():
                    approxname = tp.name[1:]
                elif tp.name == '_IO_FILE' and tp.forcename == 'FILE':
                    approxname = 'FILE'
                    self._typedef_ctx(tp, 'FILE')
                else:
                    raise NotImplementedError("internal inconsistency: %r" %
                                              (tp,))
                self._struct_ctx(tp, None, approxname)

    def _generate_cpy_struct_collecttype(self, tp, name):
        self._struct_collecttype(tp)
    _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype

    def _struct_names(self, tp):
        cname = tp.get_c_name('')
        if ' ' in cname:
            return cname, cname.replace(' ', '_')
        else:
            return cname, '_' + cname

    def _generate_cpy_struct_decl(self, tp, name):
        self._struct_decl(tp, *self._struct_names(tp))
    _generate_cpy_union_decl = _generate_cpy_struct_decl

    def _generate_cpy_struct_ctx(self, tp, name):
        self._struct_ctx(tp, *self._struct_names(tp))
    _generate_cpy_union_ctx = _generate_cpy_struct_ctx

    # ----------
    # 'anonymous' declarations.  These are produced for anonymous structs
    # or unions; the 'name' is obtained by a typedef.

    def _generate_cpy_anonymous_collecttype(self, tp, name):
        if isinstance(tp, model.EnumType):
            self._generate_cpy_enum_collecttype(tp, name)
        else:
            self._struct_collecttype(tp)

    def _generate_cpy_anonymous_decl(self, tp, name):
        if isinstance(tp, model.EnumType):
            self._generate_cpy_enum_decl(tp)
        else:
            self._struct_decl(tp, name, 'typedef_' + name)

    def _generate_cpy_anonymous_ctx(self, tp, name):
        if isinstance(tp, model.EnumType):
            self._enum_ctx(tp, name)
        else:
            self._struct_ctx(tp, name, 'typedef_' + name)

    # ----------
    # constants, declared with "static const ..."

    def _generate_cpy_const(self, is_int, name, tp=None, category='const',
                            check_value=None):
        if (category, name) in self._seen_constants:
            raise VerificationError(
                "duplicate declaration of %s '%s'" % (category, name))
        self._seen_constants.add((category, name))
        #
        prnt = self._prnt
        funcname = '_cffi_%s_%s' % (category, name)
        if is_int:
            prnt('static int %s(unsigned long long *o)' % funcname)
            prnt('{')
            prnt('  int n = (%s) <= 0;' % (name,))
            prnt('  *o = (unsigned long long)((%s) | 0);'
                 '  /* check that %s is an integer */' % (name, name))
            if check_value is not None:
                if check_value > 0:
                    check_value = '%dU' % (check_value,)
                prnt('  if (!_cffi_check_int(*o, n, %s))' % (check_value,))
                prnt('    n |= 2;')
            prnt('  return n;')
            prnt('}')
        else:
            assert check_value is None
            prnt('static void %s(char *o)' % funcname)
            prnt('{')
            prnt('  *(%s)o = %s;' % (tp.get_c_name('*'), name))
            prnt('}')
        prnt()

    def _generate_cpy_constant_collecttype(self, tp, name):
        is_int = tp.is_integer_type()
        if not is_int or self.target_is_python:
            self._do_collect_type(tp)

    def _generate_cpy_constant_decl(self, tp, name):
        is_int = tp.is_integer_type()
        self._generate_cpy_const(is_int, name, tp)

    def _generate_cpy_constant_ctx(self, tp, name):
        if not self.target_is_python and tp.is_integer_type():
            type_op = CffiOp(OP_CONSTANT_INT, -1)
        else:
            if self.target_is_python:
                const_kind = OP_DLOPEN_CONST
            else:
                const_kind = OP_CONSTANT
            type_index = self._typesdict[tp]
            type_op = CffiOp(const_kind, type_index)
        self._lsts["global"].append(
            GlobalExpr(name, '_cffi_const_%s' % name, type_op))

    # ----------
    # enums

    def _generate_cpy_enum_collecttype(self, tp, name):
        self._do_collect_type(tp)

    def _generate_cpy_enum_decl(self, tp, name=None):
        for enumerator in tp.enumerators:
            self._generate_cpy_const(True, enumerator)

    def _enum_ctx(self, tp, cname):
        type_index = self._typesdict[tp]
        type_op = CffiOp(OP_ENUM, -1)
        if self.target_is_python:
            tp.check_not_partial()
        for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
            self._lsts["global"].append(
                GlobalExpr(enumerator, '_cffi_const_%s' % enumerator, type_op,
                           check_value=enumvalue))
        #
        if cname is not None and '$' not in cname and not self.target_is_python:
            size = "sizeof(%s)" % cname
            signed = "((%s)-1) <= 0" % cname
        else:
            basetp = tp.build_baseinttype(self.ffi, [])
            size = self.ffi.sizeof(basetp)
            signed = int(int(self.ffi.cast(basetp, -1)) < 0)
        allenums = ",".join(tp.enumerators)
        self._lsts["enum"].append(
            EnumExpr(tp.name, type_index, size, signed, allenums))

    def _generate_cpy_enum_ctx(self, tp, name):
        self._enum_ctx(tp, tp._get_c_name())

    # ----------
    # macros: for now only for integers

    def _generate_cpy_macro_collecttype(self, tp, name):
        pass

    def _generate_cpy_macro_decl(self, tp, name):
        if tp == '...':
            check_value = None
        else:
            check_value = tp     # an integer
        self._generate_cpy_const(True, name, check_value=check_value)

    def _generate_cpy_macro_ctx(self, tp, name):
        if tp == '...':
            if self.target_is_python:
                raise VerificationError(
                    "cannot use the syntax '...' in '#define %s ...' when "
                    "using the ABI mode" % (name,))
            check_value = None
        else:
            check_value = tp     # an integer
        type_op = CffiOp(OP_CONSTANT_INT, -1)
        self._lsts["global"].append(
            GlobalExpr(name, '_cffi_const_%s' % name, type_op,
                       check_value=check_value))

    # ----------
    # global variables

    def _global_type(self, tp, global_name):
        if isinstance(tp, model.ArrayType):
            actual_length = tp.length
            if actual_length == '...':
                actual_length = '_cffi_array_len(%s)' % (global_name,)
            tp_item = self._global_type(tp.item, '%s[0]' % global_name)
            tp = model.ArrayType(tp_item, actual_length)
        return tp

    def _generate_cpy_variable_collecttype(self, tp, name):
        self._do_collect_type(self._global_type(tp, name))

    def _generate_cpy_variable_decl(self, tp, name):
        prnt = self._prnt
        tp = self._global_type(tp, name)
        if isinstance(tp, model.ArrayType) and tp.length is None:
            tp = tp.item
            ampersand = ''
        else:
            ampersand = '&'
        # This code assumes that casts from "tp *" to "void *" is a
        # no-op, i.e. a function that returns a "tp *" can be called
        # as if it returned a "void *".  This should be generally true
        # on any modern machine.  The only exception to that rule (on
        # uncommon architectures, and as far as I can tell) might be
        # if 'tp' were a function type, but that is not possible here.
        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
        # **" to "void *" are again no-ops, as far as I can tell.)
        decl = '*_cffi_var_%s(void)' % (name,)
        prnt('static ' + tp.get_c_name(decl, quals=self._current_quals))
        prnt('{')
        prnt('  return %s(%s);' % (ampersand, name))
        prnt('}')
        prnt()

    def _generate_cpy_variable_ctx(self, tp, name):
        tp = self._global_type(tp, name)
        type_index = self._typesdict[tp]
        if self.target_is_python:
            op = OP_GLOBAL_VAR
        else:
            op = OP_GLOBAL_VAR_F
        self._lsts["global"].append(
            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))

    # ----------
    # extern "Python"

    def _generate_cpy_extern_python_collecttype(self, tp, name):
        assert isinstance(tp, model.FunctionPtrType)
        self._do_collect_type(tp)
    _generate_cpy_dllexport_python_collecttype = \
      _generate_cpy_extern_python_plus_c_collecttype = \
      _generate_cpy_extern_python_collecttype

    def _extern_python_decl(self, tp, name, tag_and_space):
        prnt = self._prnt
        if isinstance(tp.result, model.VoidType):
            size_of_result = '0'
        else:
            context = 'result of %s' % name
            size_of_result = '(int)sizeof(%s)' % (
                tp.result.get_c_name('', context),)
        prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
        prnt('  { "%s.%s", %s, 0, 0 };' % (
            self.module_name, name, size_of_result))
        prnt()
        #
        arguments = []
        context = 'argument of %s' % name
        for i, type in enumerate(tp.args):
            arg = type.get_c_name(' a%d' % i, context)
            arguments.append(arg)
        #
        repr_arguments = ', '.join(arguments)
        repr_arguments = repr_arguments or 'void'
        name_and_arguments = '%s(%s)' % (name, repr_arguments)
        if tp.abi == "__stdcall":
            name_and_arguments = '_cffi_stdcall ' + name_and_arguments
        #
        def may_need_128_bits(tp):
            return (isinstance(tp, model.PrimitiveType) and
                    tp.name == 'long double')
        #
        size_of_a = max(len(tp.args)*8, 8)
        if may_need_128_bits(tp.result):
            size_of_a = max(size_of_a, 16)
        if isinstance(tp.result, model.StructOrUnion):
            size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % (
                tp.result.get_c_name(''), size_of_a,
                tp.result.get_c_name(''), size_of_a)
        prnt('%s%s' % (tag_and_space, tp.result.get_c_name(name_and_arguments)))
        prnt('{')
        prnt('  char a[%s];' % size_of_a)
        prnt('  char *p = a;')
        for i, type in enumerate(tp.args):
            arg = 'a%d' % i
            if (isinstance(type, model.StructOrUnion) or
                    may_need_128_bits(type)):
                arg = '&' + arg
                type = model.PointerType(type)
            prnt('  *(%s)(p + %d) = %s;' % (type.get_c_name('*'), i*8, arg))
        prnt('  _cffi_call_python(&_cffi_externpy__%s, p);' % name)
        if not isinstance(tp.result, model.VoidType):
            prnt('  return *(%s)p;' % (tp.result.get_c_name('*'),))
        prnt('}')
        prnt()
        self._num_externpy += 1

    def _generate_cpy_extern_python_decl(self, tp, name):
        self._extern_python_decl(tp, name, 'static ')

    def _generate_cpy_dllexport_python_decl(self, tp, name):
        self._extern_python_decl(tp, name, 'CFFI_DLLEXPORT ')

    def _generate_cpy_extern_python_plus_c_decl(self, tp, name):
        self._extern_python_decl(tp, name, '')

    def _generate_cpy_extern_python_ctx(self, tp, name):
        if self.target_is_python:
            raise VerificationError(
                "cannot use 'extern \"Python\"' in the ABI mode")
        if tp.ellipsis:
            raise NotImplementedError("a vararg function is extern \"Python\"")
        type_index = self._typesdict[tp]
        type_op = CffiOp(OP_EXTERN_PYTHON, type_index)
        self._lsts["global"].append(
            GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name))

    _generate_cpy_dllexport_python_ctx = \
      _generate_cpy_extern_python_plus_c_ctx = \
      _generate_cpy_extern_python_ctx

    def _print_string_literal_in_array(self, s):
        prnt = self._prnt
        prnt('// # NB. this is not a string because of a size limit in MSVC')
        if not isinstance(s, bytes):    # unicode
            s = s.encode('utf-8')       # -> bytes
        else:
            s.decode('utf-8')           # got bytes, check for valid utf-8
        try:
            s.decode('ascii')
        except UnicodeDecodeError:
            s = b'# -*- encoding: utf8 -*-\n' + s
        for line in s.splitlines(True):
            comment = line
            if type('//') is bytes:     # python2
                line = map(ord, line)   #     make a list of integers
            else:                       # python3
                # type(line) is bytes, which enumerates like a list of integers
                comment = ascii(comment)[1:-1]
            prnt(('// ' + comment).rstrip())
            printed_line = ''
            for c in line:
                if len(printed_line) >= 76:
                    prnt(printed_line)
                    printed_line = ''
                printed_line += '%d,' % (c,)
            prnt(printed_line)

    # ----------
    # emitting the opcodes for individual types

    def _emit_bytecode_VoidType(self, tp, index):
        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, PRIM_VOID)

    def _emit_bytecode_PrimitiveType(self, tp, index):
        prim_index = PRIMITIVE_TO_INDEX[tp.name]
        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index)

    def _emit_bytecode_UnknownIntegerType(self, tp, index):
        s = ('_cffi_prim_int(sizeof(%s), (\n'
             '           ((%s)-1) | 0 /* check that %s is an integer type */\n'
             '         ) <= 0)' % (tp.name, tp.name, tp.name))
        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)

    def _emit_bytecode_UnknownFloatType(self, tp, index):
        s = ('_cffi_prim_float(sizeof(%s) *\n'
             '           (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n'
             '         )' % (tp.name, tp.name))
        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)

    def _emit_bytecode_RawFunctionType(self, tp, index):
        self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
        index += 1
        for tp1 in tp.args:
            realindex = self._typesdict[tp1]
            if index != realindex:
                if isinstance(tp1, model.PrimitiveType):
                    self._emit_bytecode_PrimitiveType(tp1, index)
                else:
                    self.cffi_types[index] = CffiOp(OP_NOOP, realindex)
            index += 1
        flags = int(tp.ellipsis)
        if tp.abi is not None:
            if tp.abi == '__stdcall':
                flags |= 2
            else:
                raise NotImplementedError("abi=%r" % (tp.abi,))
        self.cffi_types[index] = CffiOp(OP_FUNCTION_END, flags)

    def _emit_bytecode_PointerType(self, tp, index):
        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype])

    _emit_bytecode_ConstPointerType = _emit_bytecode_PointerType
    _emit_bytecode_NamedPointerType = _emit_bytecode_PointerType

    def _emit_bytecode_FunctionPtrType(self, tp, index):
        raw = tp.as_raw_function()
        self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw])

    def _emit_bytecode_ArrayType(self, tp, index):
        item_index = self._typesdict[tp.item]
        if tp.length is None:
            self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index)
        elif tp.length == '...':
            raise VerificationError(
                "type %s badly placed: the '...' array length can only be "
                "used on global arrays or on fields of structures" % (
                    str(tp).replace('/*...*/', '...'),))
        else:
            assert self.cffi_types[index + 1] == 'LEN'
            self.cffi_types[index] = CffiOp(OP_ARRAY, item_index)
            self.cffi_types[index + 1] = CffiOp(None, str(tp.length))

    def _emit_bytecode_StructType(self, tp, index):
        struct_index = self._struct_unions[tp]
        self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index)
    _emit_bytecode_UnionType = _emit_bytecode_StructType

    def _emit_bytecode_EnumType(self, tp, index):
        enum_index = self._enums[tp]
        self.cffi_types[index] = CffiOp(OP_ENUM, enum_index)


if sys.version_info >= (3,):
    NativeIO = io.StringIO
else:
    class NativeIO(io.BytesIO):
        def write(self, s):
            if isinstance(s, unicode):
                s = s.encode('ascii')
            super(NativeIO, self).write(s)

def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose):
    if verbose:
        print("generating %s" % (target_file,))
    recompiler = Recompiler(ffi, module_name,
                            target_is_python=(preamble is None))
    recompiler.collect_type_table()
    recompiler.collect_step_tables()
    f = NativeIO()
    recompiler.write_source_to_f(f, preamble)
    output = f.getvalue()
    try:
        with open(target_file, 'r') as f1:
            if f1.read(len(output) + 1) != output:
                raise IOError
        if verbose:
            print("(already up-to-date)")
        return False     # already up-to-date
    except IOError:
        tmp_file = '%s.~%d' % (target_file, os.getpid())
        with open(tmp_file, 'w') as f1:
            f1.write(output)
        try:
            os.rename(tmp_file, target_file)
        except OSError:
            os.unlink(target_file)
            os.rename(tmp_file, target_file)
        return True

def make_c_source(ffi, module_name, preamble, target_c_file, verbose=False):
    assert preamble is not None
    return _make_c_or_py_source(ffi, module_name, preamble, target_c_file,
                                verbose)

def make_py_source(ffi, module_name, target_py_file, verbose=False):
    return _make_c_or_py_source(ffi, module_name, None, target_py_file,
                                verbose)

def _modname_to_file(outputdir, modname, extension):
    parts = modname.split('.')
    try:
        os.makedirs(os.path.join(outputdir, *parts[:-1]))
    except OSError:
        pass
    parts[-1] += extension
    return os.path.join(outputdir, *parts), parts


# Aaargh.  Distutils is not tested at all for the purpose of compiling
# DLLs that are not extension modules.  Here are some hacks to work
# around that, in the _patch_for_*() functions...

def _patch_meth(patchlist, cls, name, new_meth):
    old = getattr(cls, name)
    patchlist.append((cls, name, old))
    setattr(cls, name, new_meth)
    return old

def _unpatch_meths(patchlist):
    for cls, name, old_meth in reversed(patchlist):
        setattr(cls, name, old_meth)

def _patch_for_embedding(patchlist):
    if sys.platform == 'win32':
        # we must not remove the manifest when building for embedding!
        from distutils.msvc9compiler import MSVCCompiler
        _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref',
                    lambda self, manifest_file: manifest_file)

    if sys.platform == 'darwin':
        # we must not make a '-bundle', but a '-dynamiclib' instead
        from distutils.ccompiler import CCompiler
        def my_link_shared_object(self, *args, **kwds):
            if '-bundle' in self.linker_so:
                self.linker_so = list(self.linker_so)
                i = self.linker_so.index('-bundle')
                self.linker_so[i] = '-dynamiclib'
            return old_link_shared_object(self, *args, **kwds)
        old_link_shared_object = _patch_meth(patchlist, CCompiler,
                                             'link_shared_object',
                                             my_link_shared_object)

def _patch_for_target(patchlist, target):
    from distutils.command.build_ext import build_ext
    # if 'target' is different from '*', we need to patch some internal
    # method to just return this 'target' value, instead of having it
    # built from module_name
    if target.endswith('.*'):
        target = target[:-2]
        if sys.platform == 'win32':
            target += '.dll'
        elif sys.platform == 'darwin':
            target += '.dylib'
        else:
            target += '.so'
    _patch_meth(patchlist, build_ext, 'get_ext_filename',
                lambda self, ext_name: target)


def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
              c_file=None, source_extension='.c', extradir=None,
              compiler_verbose=1, target=None, debug=None, **kwds):
    if not isinstance(module_name, str):
        module_name = module_name.encode('ascii')
    if ffi._windows_unicode:
        ffi._apply_windows_unicode(kwds)
    if preamble is not None:
        embedding = (ffi._embedding is not None)
        if embedding:
            ffi._apply_embedding_fix(kwds)
        if c_file is None:
            c_file, parts = _modname_to_file(tmpdir, module_name,
                                             source_extension)
            if extradir:
                parts = [extradir] + parts
            ext_c_file = os.path.join(*parts)
        else:
            ext_c_file = c_file
        #
        if target is None:
            if embedding:
                target = '%s.*' % module_name
            else:
                target = '*'
        #
        ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds)
        updated = make_c_source(ffi, module_name, preamble, c_file,
                                verbose=compiler_verbose)
        if call_c_compiler:
            patchlist = []
            cwd = os.getcwd()
            try:
                if embedding:
                    _patch_for_embedding(patchlist)
                if target != '*':
                    _patch_for_target(patchlist, target)
                if compiler_verbose:
                    if tmpdir == '.':
                        msg = 'the current directory is'
                    else:
                        msg = 'setting the current directory to'
                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                os.chdir(tmpdir)
                outputfilename = ffiplatform.compile('.', ext,
                                                     compiler_verbose, debug)
            finally:
                os.chdir(cwd)
                _unpatch_meths(patchlist)
            return outputfilename
        else:
            return ext, updated
    else:
        if c_file is None:
            c_file, _ = _modname_to_file(tmpdir, module_name, '.py')
        updated = make_py_source(ffi, module_name, c_file,
                                 verbose=compiler_verbose)
        if call_c_compiler:
            return c_file
        else:
            return None, updated


env_pyrecodes/lib/python3.9/site-packages/cffi/recompiler.py
NikolaBlagojevic/pyrecodes
"""
Built from something like this:

    print [
        (
            row[2].text_content().strip(),
            eval(row[4].text_content().strip())
        )
        for row in lxml.html.parse('http://www.w3.org/TR/css3-color/')
            .xpath("//*[@class='colortable']//tr[position()>1]")
    ]

by Simon Sapin
"""

COLORS = {
    'transparent': (0, 0, 0, 0.0),
    'black': (0, 0, 0, 1.0),
    'silver': (192, 192, 192, 1.0),
    'gray': (128, 128, 128, 1.0),
    'white': (255, 255, 255, 1.0),
    'maroon': (128, 0, 0, 1.0),
    'red': (255, 0, 0, 1.0),
    'purple': (128, 0, 128, 1.0),
    'fuchsia': (255, 0, 255, 1.0),
    'green': (0, 128, 0, 1.0),
    'lime': (0, 255, 0, 1.0),
    'olive': (128, 128, 0, 1.0),
    'yellow': (255, 255, 0, 1.0),
    'navy': (0, 0, 128, 1.0),
    'blue': (0, 0, 255, 1.0),
    'teal': (0, 128, 128, 1.0),
    'aqua': (0, 255, 255, 1.0),
    'aliceblue': (240, 248, 255, 1.0),
    'antiquewhite': (250, 235, 215, 1.0),
    'aquamarine': (127, 255, 212, 1.0),
    'azure': (240, 255, 255, 1.0),
    'beige': (245, 245, 220, 1.0),
    'bisque': (255, 228, 196, 1.0),
    'blanchedalmond': (255, 235, 205, 1.0),
    'blueviolet': (138, 43, 226, 1.0),
    'brown': (165, 42, 42, 1.0),
    'burlywood': (222, 184, 135, 1.0),
    'cadetblue': (95, 158, 160, 1.0),
    'chartreuse': (127, 255, 0, 1.0),
    'chocolate': (210, 105, 30, 1.0),
    'coral': (255, 127, 80, 1.0),
    'cornflowerblue': (100, 149, 237, 1.0),
    'cornsilk': (255, 248, 220, 1.0),
    'crimson': (220, 20, 60, 1.0),
    'cyan': (0, 255, 255, 1.0),
    'darkblue': (0, 0, 139, 1.0),
    'darkcyan': (0, 139, 139, 1.0),
    'darkgoldenrod': (184, 134, 11, 1.0),
    'darkgray': (169, 169, 169, 1.0),
    'darkgreen': (0, 100, 0, 1.0),
    'darkgrey': (169, 169, 169, 1.0),
    'darkkhaki': (189, 183, 107, 1.0),
    'darkmagenta': (139, 0, 139, 1.0),
    'darkolivegreen': (85, 107, 47, 1.0),
    'darkorange': (255, 140, 0, 1.0),
    'darkorchid': (153, 50, 204, 1.0),
    'darkred': (139, 0, 0, 1.0),
    'darksalmon': (233, 150, 122, 1.0),
    'darkseagreen': (143, 188, 143, 1.0),
    'darkslateblue': (72, 61, 139, 1.0),
    'darkslategray': (47, 79, 79, 1.0),
    'darkslategrey': (47, 79, 79, 1.0),
    'darkturquoise': (0, 206, 209, 1.0),
    'darkviolet': (148, 0, 211, 1.0),
    'deeppink': (255, 20, 147, 1.0),
    'deepskyblue': (0, 191, 255, 1.0),
    'dimgray': (105, 105, 105, 1.0),
    'dimgrey': (105, 105, 105, 1.0),
    'dodgerblue': (30, 144, 255, 1.0),
    'firebrick': (178, 34, 34, 1.0),
    'floralwhite': (255, 250, 240, 1.0),
    'forestgreen': (34, 139, 34, 1.0),
    'gainsboro': (220, 220, 220, 1.0),
    'ghostwhite': (248, 248, 255, 1.0),
    'gold': (255, 215, 0, 1.0),
    'goldenrod': (218, 165, 32, 1.0),
    'greenyellow': (173, 255, 47, 1.0),
    'grey': (128, 128, 128, 1.0),
    'honeydew': (240, 255, 240, 1.0),
    'hotpink': (255, 105, 180, 1.0),
    'indianred': (205, 92, 92, 1.0),
    'indigo': (75, 0, 130, 1.0),
    'ivory': (255, 255, 240, 1.0),
    'khaki': (240, 230, 140, 1.0),
    'lavender': (230, 230, 250, 1.0),
    'lavenderblush': (255, 240, 245, 1.0),
    'lawngreen': (124, 252, 0, 1.0),
    'lemonchiffon': (255, 250, 205, 1.0),
    'lightblue': (173, 216, 230, 1.0),
    'lightcoral': (240, 128, 128, 1.0),
    'lightcyan': (224, 255, 255, 1.0),
    'lightgoldenrodyellow': (250, 250, 210, 1.0),
    'lightgray': (211, 211, 211, 1.0),
    'lightgreen': (144, 238, 144, 1.0),
    'lightgrey': (211, 211, 211, 1.0),
    'lightpink': (255, 182, 193, 1.0),
    'lightsalmon': (255, 160, 122, 1.0),
    'lightseagreen': (32, 178, 170, 1.0),
    'lightskyblue': (135, 206, 250, 1.0),
    'lightslategray': (119, 136, 153, 1.0),
    'lightslategrey': (119, 136, 153, 1.0),
    'lightsteelblue': (176, 196, 222, 1.0),
    'lightyellow': (255, 255, 224, 1.0),
    'limegreen': (50, 205, 50, 1.0),
    'linen': (250, 240, 230, 1.0),
    'magenta': (255, 0, 255, 1.0),
    'mediumaquamarine': (102, 205, 170, 1.0),
    'mediumblue': (0, 0, 205, 1.0),
    'mediumorchid': (186, 85, 211, 1.0),
    'mediumpurple': (147, 112, 219, 1.0),
    'mediumseagreen': (60, 179, 113, 1.0),
    'mediumslateblue': (123, 104, 238, 1.0),
    'mediumspringgreen': (0, 250, 154, 1.0),
    'mediumturquoise': (72, 209, 204, 1.0),
    'mediumvioletred': (199, 21, 133, 1.0),
    'midnightblue': (25, 25, 112, 1.0),
    'mintcream': (245, 255, 250, 1.0),
    'mistyrose': (255, 228, 225, 1.0),
    'moccasin': (255, 228, 181, 1.0),
    'navajowhite': (255, 222, 173, 1.0),
    'oldlace': (253, 245, 230, 1.0),
    'olivedrab': (107, 142, 35, 1.0),
    'orange': (255, 165, 0, 1.0),
    'orangered': (255, 69, 0, 1.0),
    'orchid': (218, 112, 214, 1.0),
    'palegoldenrod': (238, 232, 170, 1.0),
    'palegreen': (152, 251, 152, 1.0),
    'paleturquoise': (175, 238, 238, 1.0),
    'palevioletred': (219, 112, 147, 1.0),
    'papayawhip': (255, 239, 213, 1.0),
    'peachpuff': (255, 218, 185, 1.0),
    'peru': (205, 133, 63, 1.0),
    'pink': (255, 192, 203, 1.0),
    'plum': (221, 160, 221, 1.0),
    'powderblue': (176, 224, 230, 1.0),
    'rosybrown': (188, 143, 143, 1.0),
    'royalblue': (65, 105, 225, 1.0),
    'saddlebrown': (139, 69, 19, 1.0),
    'salmon': (250, 128, 114, 1.0),
    'sandybrown': (244, 164, 96, 1.0),
    'seagreen': (46, 139, 87, 1.0),
    'seashell': (255, 245, 238, 1.0),
    'sienna': (160, 82, 45, 1.0),
    'skyblue': (135, 206, 235, 1.0),
    'slateblue': (106, 90, 205, 1.0),
    'slategray': (112, 128, 144, 1.0),
    'slategrey': (112, 128, 144, 1.0),
    'snow': (255, 250, 250, 1.0),
    'springgreen': (0, 255, 127, 1.0),
    'steelblue': (70, 130, 180, 1.0),
    'tan': (210, 180, 140, 1.0),
    'thistle': (216, 191, 216, 1.0),
    'tomato': (255, 99, 71, 1.0),
    'turquoise': (64, 224, 208, 1.0),
    'violet': (238, 130, 238, 1.0),
    'wheat': (245, 222, 179, 1.0),
    'whitesmoke': (245, 245, 245, 1.0),
    'yellowgreen': (154, 205, 50, 1.0),
}

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/colors.py
NikolaBlagojevic/pyrecodes
"""CSSImportRule implements DOM Level 2 CSS CSSImportRule plus the
``name`` property from http://www.w3.org/TR/css3-cascade/#cascading."""
__all__ = ['CSSImportRule']

from . import cssrule
import cssutils
import os
import urllib.parse
import xml.dom


class CSSImportRule(cssrule.CSSRule):
    """
    Represents an @import rule within a CSS style sheet.  The @import rule
    is used to import style rules from other style sheets.

    Format::

        import
          : IMPORT_SYM S*
          [STRING|URI] S* [ medium [ COMMA S* medium]* ]? S* STRING? S* ';' S*
          ;
    """

    def __init__(
        self,
        href=None,
        mediaText=None,
        name=None,
        parentRule=None,
        parentStyleSheet=None,
        readonly=False,
    ):
        """
        If readonly allows setting of properties in constructor only

        :param href:
            location of the style sheet to be imported.
        :param mediaText:
            A list of media types for which this style sheet may be used
            as a string
        :param name:
            Additional name of imported style sheet
        """
        super().__init__(parentRule=parentRule, parentStyleSheet=parentStyleSheet)
        self._atkeyword = '@import'
        self._styleSheet = None

        # string or uri used for reserialization
        self.hreftype = None

        # prepare seq
        seq = self._tempSeq()
        seq.append(None, 'href')
        # seq.append(None, 'media')
        seq.append(None, 'name')
        self._setSeq(seq)

        # 1. media
        if mediaText:
            self.media = mediaText
        else:
            # must be all for @import
            self.media = cssutils.stylesheets.MediaList(mediaText='all')
        # 2. name
        self.name = name
        # 3. href and styleSheet
        self.href = href

        self._readonly = readonly

    def __repr__(self):
        if self._usemedia:
            mediaText = self.media.mediaText
        else:
            mediaText = None
        return "cssutils.css.{}(href={!r}, mediaText={!r}, name={!r})".format(
            self.__class__.__name__,
            self.href,
            mediaText,
            self.name,
        )

    def __str__(self):
        if self._usemedia:
            mediaText = self.media.mediaText
        else:
            mediaText = None
        return "<cssutils.css.{} object href={!r} mediaText={!r} name={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.href,
            mediaText,
            self.name,
            id(self),
        )

    _usemedia = property(
        lambda self: self.media.mediaText not in ('', 'all'),
        doc="if self.media is used (or simply empty)",
    )

    def _getCssText(self):
        """Return serialized property cssText."""
        return cssutils.ser.do_CSSImportRule(self)

    def _setCssText(self, cssText):  # noqa: C901
        """
        :exceptions:
            - :exc:`~xml.dom.HierarchyRequestErr`:
              Raised if the rule cannot be inserted at this point in the
              style sheet.
            - :exc:`~xml.dom.InvalidModificationErr`:
              Raised if the specified CSS string value represents a different
              type of rule than the current one.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if the rule is readonly.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.
        """
        super()._setCssText(cssText)
        tokenizer = self._tokenize2(cssText)
        attoken = self._nexttoken(tokenizer, None)
        if self._type(attoken) != self._prods.IMPORT_SYM:
            self._log.error(
                'CSSImportRule: No CSSImportRule found: %s' % self._valuestr(cssText),
                error=xml.dom.InvalidModificationErr,
            )
        else:
            # for closures: must be a mutable
            new = {
                'keyword': self._tokenvalue(attoken),
                'href': None,
                'hreftype': None,
                'media': None,
                'name': None,
                'wellformed': True,
            }

            def __doname(seq, token):
                # called by _string or _ident
                new['name'] = self._stringtokenvalue(token)
                seq.append(new['name'], 'name')
                return ';'

            def _string(expected, seq, token, tokenizer=None):
                if 'href' == expected:
                    # href
                    new['href'] = self._stringtokenvalue(token)
                    new['hreftype'] = 'string'
                    seq.append(new['href'], 'href')
                    return 'media name ;'
                elif 'name' in expected:
                    # name
                    return __doname(seq, token)
                else:
                    new['wellformed'] = False
                    self._log.error('CSSImportRule: Unexpected string.', token)
                    return expected

            def _uri(expected, seq, token, tokenizer=None):
                # href
                if 'href' == expected:
                    uri = self._uritokenvalue(token)
                    new['hreftype'] = 'uri'
                    new['href'] = uri
                    seq.append(new['href'], 'href')
                    return 'media name ;'
                else:
                    new['wellformed'] = False
                    self._log.error('CSSImportRule: Unexpected URI.', token)
                    return expected

            def _ident(expected, seq, token, tokenizer=None):
                # medialist ending with ; which is checked upon too
                if expected.startswith('media'):
                    mediatokens = self._tokensupto2(
                        tokenizer, importmediaqueryendonly=True
                    )
                    mediatokens.insert(0, token)  # push found token

                    last = mediatokens.pop()  # retrieve ;
                    lastval, lasttyp = self._tokenvalue(last), self._type(last)
                    if lastval != ';' and lasttyp not in ('EOF', self._prods.STRING):
                        new['wellformed'] = False
                        self._log.error(
                            'CSSImportRule: No ";" found: %s' % self._valuestr(cssText),
                            token=token,
                        )

                    newMedia = cssutils.stylesheets.MediaList(parentRule=self)
                    newMedia.mediaText = mediatokens
                    if newMedia.wellformed:
                        new['media'] = newMedia
                        seq.append(newMedia, 'media')
                    else:
                        new['wellformed'] = False
                        self._log.error(
                            'CSSImportRule: Invalid MediaList: %s'
                            % self._valuestr(cssText),
                            token=token,
                        )

                    if lasttyp == self._prods.STRING:
                        # name
                        return __doname(seq, last)
                    else:
                        return 'EOF'  # ';' is token "last"
                else:
                    new['wellformed'] = False
                    self._log.error('CSSImportRule: Unexpected ident.', token)
                    return expected

            def _char(expected, seq, token, tokenizer=None):
                # final ;
                val = self._tokenvalue(token)
                if expected.endswith(';') and ';' == val:
                    return 'EOF'
                else:
                    new['wellformed'] = False
                    self._log.error('CSSImportRule: Unexpected char.', token)
                    return expected

            # import : IMPORT_SYM S* [STRING|URI]
            #            S* [ medium [ ',' S* medium]* ]? ';' S*
            #         STRING? # see http://www.w3.org/TR/css3-cascade/#cascading
            #        ;
            newseq = self._tempSeq()
            wellformed, expected = self._parse(
                expected='href',
                seq=newseq,
                tokenizer=tokenizer,
                productions={
                    'STRING': _string,
                    'URI': _uri,
                    'IDENT': _ident,
                    'CHAR': _char,
                },
                new=new,
            )

            # wellformed set by parse
            ok = wellformed and new['wellformed']

            # post conditions
            if not new['href']:
                ok = False
                self._log.error(
                    'CSSImportRule: No href found: %s' % self._valuestr(cssText)
                )

            if expected != 'EOF':
                ok = False
                self._log.error(
                    'CSSImportRule: No ";" found: %s' % self._valuestr(cssText)
                )

            # set all
            if ok:
                self._setSeq(newseq)

                self.atkeyword = new['keyword']
                self.hreftype = new['hreftype']
                self.name = new['name']

                if new['media']:
                    self.media = new['media']
                else:
                    # must be all for @import
                    self.media = cssutils.stylesheets.MediaList(mediaText='all')

                # needs new self.media
                self.href = new['href']

    cssText = property(
        fget=_getCssText,
        fset=_setCssText,
        doc="(DOM) The parsable textual representation of this rule.",
    )

    def _setHref(self, href):
        # set new href
        self._href = href
        # update seq
        for i, item in enumerate(self.seq):
            type_ = item.type
            if 'href' == type_:
                self._seq[i] = (href, type_, item.line, item.col)
                break

        importedSheet = cssutils.css.CSSStyleSheet(
            media=self.media, ownerRule=self, title=self.name
        )
        self.hrefFound = False
        # set styleSheet
        if href and self.parentStyleSheet:
            # loading errors are all catched!

            # relative href
            parentHref = self.parentStyleSheet.href
            if parentHref is None:
                # use cwd instead
                parentHref = cssutils.helper.path2url(os.getcwd()) + '/'

            fullhref = urllib.parse.urljoin(parentHref, self.href)

            # all possible exceptions are ignored
            try:
                usedEncoding, enctype, cssText = self.parentStyleSheet._resolveImport(
                    fullhref
                )

                if cssText is None:
                    # catched in next except below!
                    raise OSError('Cannot read Stylesheet.')

                # contentEncoding with parentStyleSheet.overrideEncoding,
                # HTTP or parent
                encodingOverride, encoding = None, None

                if enctype == 0:
                    encodingOverride = usedEncoding
                elif 0 < enctype < 5:
                    encoding = usedEncoding

                # inherit fetcher for @imports in styleSheet
                importedSheet._href = fullhref
                importedSheet._setFetcher(self.parentStyleSheet._fetcher)
                importedSheet._setCssTextWithEncodingOverride(
                    cssText, encodingOverride=encodingOverride, encoding=encoding
                )

            except (OSError, ValueError) as e:
                self._log.warn(
                    'CSSImportRule: While processing imported '
                    'style sheet href=%s: %r' % (self.href, e),
                    neverraise=True,
                )

            else:
                # used by resolveImports if to keep unprocessed href
                self.hrefFound = True

        self._styleSheet = importedSheet

    _href = None  # needs to be set
    href = property(
        lambda self: self._href,
        _setHref,
        doc="Location of the style sheet to be imported.",
    )

    def _setMedia(self, media):
        """
        :param media:
            a :class:`~cssutils.stylesheets.MediaList` or string
        """
        self._checkReadonly()
        if isinstance(media, str):
            self._media = cssutils.stylesheets.MediaList(
                mediaText=media, parentRule=self
            )
        else:
            media._parentRule = self
            self._media = media

        # update seq
        ihref = 0
        for i, item in enumerate(self.seq):
            if item.type == 'href':
                ihref = i
            elif item.type == 'media':
                self.seq[i] = (self._media, 'media', None, None)
                break
        else:
            # if no media until now add after href
            self.seq.insert(ihref + 1, self._media, 'media', None, None)

    media = property(
        lambda self: self._media,
        _setMedia,
        doc="(DOM) A list of media types for this rule "
        "of type :class:`~cssutils.stylesheets.MediaList`.",
    )

    def _setName(self, name=''):
        """Raises xml.dom.SyntaxErr if name is not a string."""
        if name is None or isinstance(name, str):
            # "" or '' handled as None
            if not name:
                name = None

            # save name
            self._name = name

            # update seq
            for i, item in enumerate(self.seq):
                typ = item.type
                if 'name' == typ:
                    self._seq[i] = (name, typ, item.line, item.col)
                    break

            # set title of imported sheet
            if self.styleSheet:
                self.styleSheet.title = name

        else:
            self._log.error('CSSImportRule: Not a valid name: %s' % name)

    def _getName(self):
        return self._name

    name = property(
        _getName,
        _setName,
        doc="An optional name for the imported sheet.",
    )

    styleSheet = property(
        lambda self: self._styleSheet,
        doc="(readonly) The style sheet referred to by this " "rule.",
    )

    type = property(
        lambda self: self.IMPORT_RULE,
        doc="The type of this rule, as defined by a CSSRule " "type constant.",
    )

    def _getWellformed(self):
        "Depending on if media is used at all."
        if self._usemedia:
            return bool(self.href and self.media.wellformed)
        else:
            return bool(self.href)

    wellformed = property(_getWellformed)

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssimportrule.py
NikolaBlagojevic/pyrecodes
"""CSSMediaRule implements DOM Level 2 CSS CSSMediaRule."""
__all__ = ['CSSMediaRule']

from . import cssrule
import cssutils
import xml.dom


class CSSMediaRule(cssrule.CSSRuleRules):
    """
    Objects implementing the CSSMediaRule interface can be identified by the
    MEDIA_RULE constant. On these objects the type attribute must return the
    value of that constant.

    Format::

      : MEDIA_SYM S* medium [ COMMA S* medium ]*

          STRING? # the name

      LBRACE S* ruleset* '}' S*;

    ``cssRules``
        All Rules in this media rule, a :class:`~cssutils.css.CSSRuleList`.
    """

    def __init__(
        self,
        mediaText='all',
        name=None,
        parentRule=None,
        parentStyleSheet=None,
        readonly=False,
    ):
        """constructor"""
        super().__init__(parentRule=parentRule, parentStyleSheet=parentStyleSheet)
        self._atkeyword = '@media'

        # 1. media
        if mediaText:
            self.media = mediaText
        else:
            self.media = cssutils.stylesheets.MediaList()

        self.name = name
        self._readonly = readonly

    def __repr__(self):
        return "cssutils.css.{}(mediaText={!r})".format(
            self.__class__.__name__,
            self.media.mediaText,
        )

    def __str__(self):
        return "<cssutils.css.{} object mediaText={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.media.mediaText,
            id(self),
        )

    def _getCssText(self):
        """Return serialized property cssText."""
        return cssutils.ser.do_CSSMediaRule(self)

    def _setCssText(self, cssText):  # noqa: C901
        """
        :param cssText:
            a parseable string or a tuple of (cssText, dict-of-namespaces)
        :Exceptions:
            - :exc:`~xml.dom.NamespaceErr`:
              Raised if a specified selector uses an unknown namespace
              prefix.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.
            - :exc:`~xml.dom.InvalidModificationErr`:
              Raised if the specified CSS string value represents a different
              type of rule than the current one.
            - :exc:`~xml.dom.HierarchyRequestErr`:
              Raised if the rule cannot be inserted at this point in the
              style sheet.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if the rule is readonly.
        """
        # media "name"? { cssRules }
        super()._setCssText(cssText)

        # might be (cssText, namespaces)
        cssText, namespaces = self._splitNamespacesOff(cssText)

        tokenizer = self._tokenize2(cssText)
        attoken = self._nexttoken(tokenizer, None)
        if self._type(attoken) != self._prods.MEDIA_SYM:
            self._log.error(
                'CSSMediaRule: No CSSMediaRule found: %s' % self._valuestr(cssText),
                error=xml.dom.InvalidModificationErr,
            )

        else:
            # save if parse goes wrong
            oldMedia = self._media
            oldCssRules = self._cssRules

            ok = True

            # media
            mediatokens, end = self._tokensupto2(
                tokenizer, mediaqueryendonly=True, separateEnd=True
            )
            if '{' == self._tokenvalue(end) or self._prods.STRING == self._type(end):
                self.media = cssutils.stylesheets.MediaList(parentRule=self)
                # TODO: remove special case
                self.media.mediaText = mediatokens
                ok = ok and self.media.wellformed
            else:
                ok = False

            # name (optional)
            name = None
            nameseq = self._tempSeq()
            if self._prods.STRING == self._type(end):
                name = self._stringtokenvalue(end)
                # TODO: for now comments are lost after name
                nametokens, end = self._tokensupto2(
                    tokenizer, blockstartonly=True, separateEnd=True
                )
                wellformed, expected = self._parse(None, nameseq, nametokens, {})
                if not wellformed:
                    ok = False
                    self._log.error(
                        'CSSMediaRule: Syntax Error: %s' % self._valuestr(cssText)
                    )

            # check for {
            if '{' != self._tokenvalue(end):
                self._log.error(
                    'CSSMediaRule: No "{" found: %s' % self._valuestr(cssText)
                )
                return

            # cssRules
            cssrulestokens, braceOrEOF = self._tokensupto2(
                tokenizer, mediaendonly=True, separateEnd=True
            )
            nonetoken = self._nexttoken(tokenizer, None)
            if 'EOF' == self._type(braceOrEOF):
                # HACK!!!
                # TODO: Not complete, add EOF to rule and } to @media
                cssrulestokens.append(braceOrEOF)
                braceOrEOF = ('CHAR', '}', 0, 0)
                self._log.debug(
                    'CSSMediaRule: Incomplete, adding "}".',
                    token=braceOrEOF,
                    neverraise=True,
                )

            if '}' != self._tokenvalue(braceOrEOF):
                self._log.error('CSSMediaRule: No "}" found.', token=braceOrEOF)
            elif nonetoken:
                self._log.error(
                    'CSSMediaRule: Trailing content found.', token=nonetoken
                )
            else:
                # for closures: must be a mutable
                new = {'wellformed': True}

                def COMMENT(expected, seq, token, tokenizer=None):
                    self.insertRule(
                        cssutils.css.CSSComment(
                            [token],
                            parentRule=self,
                            parentStyleSheet=self.parentStyleSheet,
                        )
                    )
                    return expected

                def ruleset(expected, seq, token, tokenizer):
                    rule = cssutils.css.CSSStyleRule(
                        parentRule=self, parentStyleSheet=self.parentStyleSheet
                    )
                    rule.cssText = self._tokensupto2(tokenizer, token)
                    if rule.wellformed:
                        self.insertRule(rule)
                    return expected

                def atrule(expected, seq, token, tokenizer):
                    # TODO: get complete rule!
                    tokens = self._tokensupto2(tokenizer, token)
                    atval = self._tokenvalue(token)
                    factories = {
                        '@page': cssutils.css.CSSPageRule,
                        '@media': CSSMediaRule,
                    }
                    if atval in (
                        '@charset ',
                        '@font-face',
                        '@import',
                        '@namespace',
                        '@variables',
                    ):
                        self._log.error(
                            'CSSMediaRule: This rule is not '
                            'allowed in CSSMediaRule - ignored: '
                            '%s.' % self._valuestr(tokens),
                            token=token,
                            error=xml.dom.HierarchyRequestErr,
                        )
                    elif atval in factories:
                        rule = factories[atval](
                            parentRule=self, parentStyleSheet=self.parentStyleSheet
                        )
                        rule.cssText = tokens
                        if rule.wellformed:
                            self.insertRule(rule)
                    else:
                        rule = cssutils.css.CSSUnknownRule(
                            tokens,
                            parentRule=self,
                            parentStyleSheet=self.parentStyleSheet,
                        )
                        if rule.wellformed:
                            self.insertRule(rule)
                    return expected

                # save for possible reset
                oldCssRules = self.cssRules

                self.cssRules = cssutils.css.CSSRuleList()
                seq = []  # not used really

                tokenizer = iter(cssrulestokens)
                wellformed, expected = self._parse(
                    braceOrEOF,
                    seq,
                    tokenizer,
                    {
                        'COMMENT': COMMENT,
                        'CHARSET_SYM': atrule,
                        'FONT_FACE_SYM': atrule,
                        'IMPORT_SYM': atrule,
                        'NAMESPACE_SYM': atrule,
                        'PAGE_SYM': atrule,
                        'MEDIA_SYM': atrule,
                        'ATKEYWORD': atrule,
                    },
                    default=ruleset,
                    new=new,
                )
                ok = ok and wellformed

            if ok:
                self.name = name
                self._setSeq(nameseq)
            else:
                self._media = oldMedia
                self._cssRules = oldCssRules

    cssText = property(
        _getCssText,
        _setCssText,
        doc="(DOM) The parsable textual representation of this " "rule.",
    )

    @property
    def name(self):
        """An optional name for this media rule."""
        return self._name

    @name.setter
    def name(self, name):
        if isinstance(name, str) or name is None:
            # "" or ''
            if not name:
                name = None

            self._name = name
        else:
            self._log.error('CSSImportRule: Not a valid name: %s' % name)

    def _setName(self, name):
        self.name = name

    def _setMedia(self, media):
        """
        :param media:
            a :class:`~cssutils.stylesheets.MediaList` or string
        """
        self._checkReadonly()
        if isinstance(media, str):
            self._media = cssutils.stylesheets.MediaList(
                mediaText=media, parentRule=self
            )
        else:
            media._parentRule = self
            self._media = media

        # NOT IN @media seq at all?!

    #        # update seq
    #        for i, item in enumerate(self.seq):
    #            if item.type == 'media':
    #                self._seq[i] = (self._media, 'media', None, None)
    #                break
    #        else:
    #            # insert after @media if not in seq at all
    #            self.seq.insert(0,
    #                             self._media, 'media', None, None)

    media = property(
        lambda self: self._media,
        _setMedia,
        doc="(DOM) A list of media types for this rule "
        "of type :class:`~cssutils.stylesheets.MediaList`.",
    )

    def insertRule(self, rule, index=None):
        """Implements base ``insertRule``."""
        rule, index = self._prepareInsertRule(rule, index)

        if rule is False or rule is True:
            # done or error
            return

        # check hierarchy
        if (
            isinstance(rule, cssutils.css.CSSCharsetRule)
            or isinstance(rule, cssutils.css.CSSFontFaceRule)
            or isinstance(rule, cssutils.css.CSSImportRule)
            or isinstance(rule, cssutils.css.CSSNamespaceRule)
            or isinstance(rule, cssutils.css.MarginRule)
        ):
            self._log.error(
                '%s: This type of rule is not allowed here: %s'
                % (self.__class__.__name__, rule.cssText),
                error=xml.dom.HierarchyRequestErr,
            )
            return

        return self._finishInsertRule(rule, index)

    type = property(
        lambda self: self.MEDIA_RULE,
        doc="The type of this rule, as defined by a CSSRule " "type constant.",
    )

    wellformed = property(lambda self: self.media.wellformed)

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssmediarule.py
NikolaBlagojevic/pyrecodes
"""CSSRule implements DOM Level 2 CSS CSSRule."""
__all__ = ['CSSRule']

import cssutils
import xml.dom


class CSSRule(cssutils.util.Base2):
    """Abstract base interface for any type of CSS statement. This includes
    both rule sets and at-rules. An implementation is expected to preserve
    all rules specified in a CSS style sheet, even if the rule is not
    recognized by the parser. Unrecognized rules are represented using the
    ``CSSUnknownRule`` interface.
    """

    """
    CSSRule type constants.
    An integer indicating which type of rule this is.
    """
    UNKNOWN_RULE = 0
    "``cssutils.css.CSSUnknownRule`` (not used in CSSOM anymore)"
    STYLE_RULE = 1
    "``cssutils.css.CSSStyleRule``"
    CHARSET_RULE = 2
    "``cssutils.css.CSSCharsetRule`` (not used in CSSOM anymore)"
    IMPORT_RULE = 3
    "``cssutils.css.CSSImportRule``"
    MEDIA_RULE = 4
    "``cssutils.css.CSSMediaRule``"
    FONT_FACE_RULE = 5
    "``cssutils.css.CSSFontFaceRule``"
    PAGE_RULE = 6
    "``cssutils.css.CSSPageRule``"
    NAMESPACE_RULE = 10
    """``cssutils.css.CSSNamespaceRule``,
    Value has changed in 0.9.7a3 due to a change in the CSSOM spec."""
    COMMENT = 1001  # was -1, cssutils only
    """``cssutils.css.CSSComment`` - not in the offical spec,
    Value has changed in 0.9.7a3"""
    VARIABLES_RULE = 1008
    """``cssutils.css.CSSVariablesRule`` - experimental rule
    not in the offical spec"""

    MARGIN_RULE = 1006
    """``cssutils.css.MarginRule`` - experimental rule
    not in the offical spec"""

    _typestrings = {
        UNKNOWN_RULE: 'UNKNOWN_RULE',
        STYLE_RULE: 'STYLE_RULE',
        CHARSET_RULE: 'CHARSET_RULE',
        IMPORT_RULE: 'IMPORT_RULE',
        MEDIA_RULE: 'MEDIA_RULE',
        FONT_FACE_RULE: 'FONT_FACE_RULE',
        PAGE_RULE: 'PAGE_RULE',
        NAMESPACE_RULE: 'NAMESPACE_RULE',
        COMMENT: 'COMMENT',
        VARIABLES_RULE: 'VARIABLES_RULE',
        MARGIN_RULE: 'MARGIN_RULE',
    }

    def __init__(self, parentRule=None, parentStyleSheet=None, readonly=False):
        """Set common attributes for all rules."""
        super().__init__()
        self._parent = parentRule
        self._parentRule = parentRule
        self._parentStyleSheet = parentStyleSheet
        self._setSeq(self._tempSeq())
        # self._atkeyword = None
        # must be set after initialization of #inheriting rule is done
        self._readonly = False

    def _setAtkeyword(self, keyword):
        """Check if new keyword fits the rule it is used for."""
        atkeyword = self._normalize(keyword)
        if not self.atkeyword or (self.atkeyword == atkeyword):
            self._atkeyword = atkeyword
            self._keyword = keyword
        else:
            self._log.error(
                '{}: Invalid atkeyword for this rule: {!r}'.format(
                    self.atkeyword, keyword
                ),
                error=xml.dom.InvalidModificationErr,
            )

    atkeyword = property(
        lambda self: self._atkeyword,
        _setAtkeyword,
        doc="Normalized  keyword of an @rule (e.g. ``@import``).",
    )

    def _setCssText(self, cssText):
        """
        :param cssText:
            A parsable DOMString.
        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.
            - :exc:`~xml.dom.InvalidModificationErr`:
              Raised if the specified CSS string value represents a different
              type of rule than the current one.
            - :exc:`~xml.dom.HierarchyRequestErr`:
              Raised if the rule cannot be inserted at this point in the
              style sheet.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if the rule is readonly.
        """
        self._checkReadonly()

    cssText = property(
        lambda self: '',
        _setCssText,
        doc="(DOM) The parsable textual representation of the "
        "rule. This reflects the current state of the rule "
        "and not its initial value.",
    )

    @property
    def parent(self):
        """The Parent Node of this CSSRule or None."""
        return self._parent

    parentRule = property(
        lambda self: self._parentRule,
        doc="If this rule is contained inside another rule "
        "(e.g. a style rule inside an @media block), this "
        "is the containing rule. If this rule is not nested "
        "inside any other rules, this returns None.",
    )

    def _getParentStyleSheet(self):
        # rules contained in other rules (@media) use that rules parent
        if self.parentRule:
            return self.parentRule._parentStyleSheet
        else:
            return self._parentStyleSheet

    parentStyleSheet = property(
        _getParentStyleSheet, doc="The style sheet that contains this rule."
    )

    type = property(
        lambda self: self.UNKNOWN_RULE,
        doc="The type of this rule, as defined by a CSSRule " "type constant.",
    )

    typeString = property(
        lambda self: CSSRule._typestrings[self.type],
        doc="Descriptive name of this rule's type.",
    )

    wellformed = property(lambda self: False, doc="If the rule is wellformed.")


class CSSRuleRules(CSSRule):
    """Abstract base interface for rules that contain other rules
    like @media or @page. Methods may be overwritten if a rule
    has specific stuff to do like checking the order of insertion like
    @media does.
    """

    def __init__(self, parentRule=None, parentStyleSheet=None):
        super().__init__(parentRule=parentRule, parentStyleSheet=parentStyleSheet)

        self.cssRules = cssutils.css.CSSRuleList()

    def __iter__(self):
        """Generator iterating over these rule's cssRules."""
        yield from self._cssRules

    def _setCssRules(self, cssRules):
        "Set new cssRules and update contained rules refs."
        cssRules.append = self.insertRule
        cssRules.extend = self.insertRule
        cssRules.__delitem__ == self.deleteRule

        for rule in cssRules:
            rule._parentRule = self
            rule._parentStyleSheet = None

        self._cssRules = cssRules

    cssRules = property(
        lambda self: self._cssRules,
        _setCssRules,
        "All Rules in this style sheet, a " ":class:`~cssutils.css.CSSRuleList`.",
    )

    def deleteRule(self, index):
        """
        Delete the rule at `index` from rules ``cssRules``.

        :param index:
            The `index` of the rule to be removed from the rules cssRules
            list. For an `index` < 0 **no** :exc:`~xml.dom.IndexSizeErr` is
            raised but rules for normal Python lists are used. E.g.
            ``deleteRule(-1)`` removes the last rule in cssRules.

            `index` may also be a CSSRule object which will then be removed.

        :Exceptions:
            - :exc:`~xml.dom.IndexSizeErr`:
              Raised if the specified index does not correspond to a rule in
              the media rule list.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this media rule is readonly.
        """
        self._checkReadonly()

        if isinstance(index, CSSRule):
            for i, r in enumerate(self.cssRules):
                if index == r:
                    index = i
                    break
            else:
                raise xml.dom.IndexSizeErr(
                    "%s: Not a rule in "
                    "this rule'a cssRules list: %s" % (self.__class__.__name__, index)
                )

        try:
            # detach
            self._cssRules[index]._parentRule = None
            del self._cssRules[index]

        except IndexError:
            raise xml.dom.IndexSizeErr(
                '%s: %s is not a valid index '
                'in the rulelist of length %i'
                % (self.__class__.__name__, index, self._cssRules.length)
            )

    def _prepareInsertRule(self, rule, index=None):
        "return checked `index` and optional parsed `rule`"
        self._checkReadonly()

        # check index
        if index is None:
            index = len(self._cssRules)

        elif index < 0 or index > self._cssRules.length:
            raise xml.dom.IndexSizeErr(
                '%s: Invalid index %s for '
                'CSSRuleList with a length of %s.'
                % (self.__class__.__name__, index, self._cssRules.length)
            )

        # check and optionally parse rule
        if isinstance(rule, str):
            tempsheet = cssutils.css.CSSStyleSheet()
            tempsheet.cssText = rule
            if len(tempsheet.cssRules) != 1 or (
                tempsheet.cssRules
                and not isinstance(tempsheet.cssRules[0], cssutils.css.CSSRule)
            ):
                self._log.error(f'{self.__class__.__name__}: Invalid Rule: {rule}')
                return False, False
            rule = tempsheet.cssRules[0]

        elif isinstance(rule, cssutils.css.CSSRuleList):
            # insert all rules
            for i, r in enumerate(rule):
                self.insertRule(r, index + i)
            return True, True

        elif not isinstance(rule, cssutils.css.CSSRule):
            self._log.error(f'{rule}: Not a CSSRule: {self.__class__.__name__}')
            return False, False

        return rule, index

    def _finishInsertRule(self, rule, index):
        "add `rule` at `index`"
        rule._parentRule = self
        rule._parentStyleSheet = None
        self._cssRules.insert(index, rule)
        return index

    def add(self, rule):
        """Add `rule` to page rule. Same as ``insertRule(rule)``."""
        return self.insertRule(rule)

    def insertRule(self, rule, index=None):
        """
        Insert `rule` into the rules ``cssRules``.

        :param rule:
            the parsable text representing the `rule` to be inserted. For rule
            sets this contains both the selector and the style declaration.
            For at-rules, this specifies both the at-identifier and the rule
            content.

            cssutils also allows rule to be a valid
            :class:`~cssutils.css.CSSRule` object.

        :param index:
            before the `index` the specified `rule` will be inserted.
            If the specified `index` is equal to the length of the rules
            rule collection, the rule will be added to the end of the rule.
            If index is not given or None rule will be appended to rule
            list.

        :returns:
            the index of the newly inserted rule.

        :exceptions:
            - :exc:`~xml.dom.HierarchyRequestErr`:
              Raised if the `rule` cannot be inserted at the specified `index`,
              e.g., if an @import rule is inserted after a standard rule set
              or other at-rule.
            - :exc:`~xml.dom.IndexSizeErr`:
              Raised if the specified `index` is not a valid insertion point.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this rule is readonly.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified `rule` has a syntax error and is
              unparsable.
        """
        return self._prepareInsertRule(rule, index)

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssrule.py
NikolaBlagojevic/pyrecodes
"""CSSRuleList implements DOM Level 2 CSS CSSRuleList.
Partly also http://dev.w3.org/csswg/cssom/#the-cssrulelist."""
__all__ = ['CSSRuleList']


class CSSRuleList(list):
    """The CSSRuleList object represents an (ordered) list of statements.

    The items in the CSSRuleList are accessible via an integral index,
    starting from 0.

    Subclasses a standard Python list so theoretically all standard list
    methods are available. Setting methods like ``__init__``, ``append``,
    ``extend`` or ``__setslice__`` are added later on instances of this
    class if so desired.
    E.g. CSSStyleSheet adds ``append`` which is not available in a simple
    instance of this class!
    """

    def __init__(self, *ignored):
        "Nothing is set as this must also be defined later."
        pass

    def __notimplemented(self, *ignored):
        "Implemented in class using a CSSRuleList only."
        raise NotImplementedError(
            'Must be implemented by class using an instance of this class.'
        )

    append = extend = __setitem__ = __setslice__ = __notimplemented

    def item(self, index):
        """(DOM) Retrieve a CSS rule by ordinal `index`. The order in this
        collection represents the order of the rules in the CSS style
        sheet. If index is greater than or equal to the number of rules in
        the list, this returns None.

        Returns CSSRule, the style rule at the index position in the
        CSSRuleList, or None if that is not a valid index.
        """
        try:
            return self[index]
        except IndexError:
            return None

    @property
    def length(self):
        """(DOM) The number of CSSRules in the list."""
        return len(self)

    def rulesOfType(self, type):
        """Yield the rules which have the given `type` only, one of the
        constants defined in :class:`cssutils.css.CSSRule`."""
        for r in self:
            if r.type == type:
                yield r

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssrulelist.py
NikolaBlagojevic/pyrecodes
"""CSSStyleDeclaration implements DOM Level 2 CSS CSSStyleDeclaration and
extends CSS2Properties

see
    http://www.w3.org/TR/1998/REC-CSS2-19980512/syndata.html#parsing-errors

Unknown properties
------------------
User agents must ignore a declaration with an unknown property.
For example, if the style sheet is::

    H1 { color: red; rotation: 70minutes }

the user agent will treat this as if the style sheet had been::

    H1 { color: red }

Cssutils gives a message about any unknown properties but
keeps any property (if syntactically correct).

Illegal values
--------------
User agents must ignore a declaration with an illegal value. For example::

    IMG { float: left }       /* correct CSS2 */
    IMG { float: left here }  /* "here" is not a value of 'float' */
    IMG { background: "red" } /* keywords cannot be quoted in CSS2 */
    IMG { border-width: 3 }   /* a unit must be specified for length values */

A CSS2 parser would honor the first rule and ignore the rest, as if the
style sheet had been::

    IMG { float: left }
    IMG { }
    IMG { }
    IMG { }

Cssutils again will issue a message (WARNING in this case) about invalid
CSS2 property values.

TODO:
    This interface is also used to provide a read-only access to the
    computed values of an element. See also the ViewCSS interface.

    - return computed values and not literal values
    - simplify unit pairs/triples/quadruples
      2px 2px 2px 2px -> 2px for border/padding...
    - normalize compound properties like:
      background: no-repeat left url()  #fff
      -> background: #fff url() no-repeat left
"""
__all__ = ['CSSStyleDeclaration', 'Property']

from .cssproperties import CSS2Properties
from .property import Property
import cssutils


class CSSStyleDeclaration(CSS2Properties, cssutils.util.Base2):
    """The CSSStyleDeclaration class represents a single CSS declaration
    block. This class may be used to determine the style properties
    currently set in a block or to set style properties explicitly
    within the block.

    While an implementation may not recognize all CSS properties within
    a CSS declaration block, it is expected to provide access to all
    specified properties in the style sheet through the
    CSSStyleDeclaration interface.
    Furthermore, implementations that support a specific level of CSS
    should correctly handle CSS shorthand properties for that level. For
    a further discussion of shorthand properties, see the CSS2Properties
    interface.

    Additionally the CSS2Properties interface is implemented.

    $css2propertyname
        All properties defined in the CSS2Properties class are available
        as direct properties of CSSStyleDeclaration with their respective
        DOM name, so e.g. ``fontStyle`` for property 'font-style'.

        These may be used as::

            >>> style = CSSStyleDeclaration(cssText='color: red')
            >>> style.color = 'green'
            >>> print(style.color)
            green
            >>> del style.color
            >>> print(style.color)
            <BLANKLINE>

    Format::

        [Property: Value Priority?;]* [Property: Value Priority?]?
    """

    def __init__(self, cssText='', parentRule=None, readonly=False, validating=None):
        """
        :param cssText:
            Shortcut, sets CSSStyleDeclaration.cssText
        :param parentRule:
            The CSS rule that contains this declaration block or
            None if this CSSStyleDeclaration is not attached to a CSSRule.
        :param readonly:
            defaults to False
        :param validating:
            a flag defining if this sheet should be validated on change.
            Defaults to None, which means defer to the parent stylesheet.
        """
        super().__init__()
        self._parentRule = parentRule
        self.validating = validating
        self.cssText = cssText
        self._readonly = readonly

    def __contains__(self, nameOrProperty):
        """Check if a property (or a property with given name) is in style.

        :param name:
            a string or Property, uses normalized name and not literalname
        """
        if isinstance(nameOrProperty, Property):
            name = nameOrProperty.name
        else:
            name = self._normalize(nameOrProperty)
        return name in self.__nnames()

    def __iter__(self):
        """Iterator of set Property objects with different normalized names."""

        def properties():
            for name in self.__nnames():
                yield self.getProperty(name)

        return properties()

    def keys(self):
        """Analoguous to standard dict returns property names which are set in
        this declaration."""
        return list(self.__nnames())

    def __getitem__(self, CSSName):
        """Retrieve the value of property ``CSSName`` from this declaration.

        ``CSSName`` will be always normalized.
        """
        return self.getPropertyValue(CSSName)

    def __setitem__(self, CSSName, value):
        """Set value of property ``CSSName``. ``value`` may also be a tuple of
        (value, priority), e.g. style['color'] = ('red', 'important')

        ``CSSName`` will be always normalized.
        """
        priority = None
        if isinstance(value, tuple):
            value, priority = value

        return self.setProperty(CSSName, value, priority)

    def __delitem__(self, CSSName):
        """Delete property ``CSSName`` from this declaration.
        If property is not in this declaration return u'' just like
        removeProperty.

        ``CSSName`` will be always normalized.
        """
        return self.removeProperty(CSSName)

    def __setattr__(self, n, v):
        """Prevent setting of unknown properties on CSSStyleDeclaration
        which would not work anyway. For these
        ``CSSStyleDeclaration.setProperty`` MUST be called explicitly!

        TODO:
            implementation of known is not really nice, any alternative?
        """
        known = [
            '_tokenizer',
            '_log',
            '_ttypes',
            '_seq',
            'seq',
            'parentRule',
            '_parentRule',
            'cssText',
            'valid',
            'wellformed',
            'validating',
            '_readonly',
            '_profiles',
            '_validating',
        ]
        known.extend(CSS2Properties._properties)
        if n in known:
            super().__setattr__(n, v)
        else:
            raise AttributeError(
                'Unknown CSS Property, '
                '``CSSStyleDeclaration.setProperty("%s", '
                '...)`` MUST be used.' % n
            )

    def __repr__(self):
        return "cssutils.css.{}(cssText={!r})".format(
            self.__class__.__name__,
            self.getCssText(separator=' '),
        )

    def __str__(self):
        return "<cssutils.css.{} object length={!r} (all: {!r}) at 0x{:x}>".format(
            self.__class__.__name__,
            self.length,
            len(self.getProperties(all=True)),
            id(self),
        )

    def __nnames(self):
        """Return iterator for all different names in order as set
        if names are set twice the last one is used (double reverse!)
        """
        names = []
        for item in reversed(self.seq):
            val = item.value
            if isinstance(val, Property) and val.name not in names:
                names.append(val.name)
        return reversed(names)

    # overwritten accessor functions for CSS2Properties' properties
    def _getP(self, CSSName):
        """(DOM CSS2Properties) Overwritten here and effectively the same as
        ``self.getPropertyValue(CSSname)``.

        Parameter is in CSSname format ('font-style'), see CSS2Properties.

        Example::

            >>> style = CSSStyleDeclaration(cssText='font-style:italic;')
            >>> print(style.fontStyle)
            italic
        """
        return self.getPropertyValue(CSSName)

    def _setP(self, CSSName, value):
        """(DOM CSS2Properties) Overwritten here and effectively the same as
        ``self.setProperty(CSSname, value)``.

        Only known CSS2Properties may be set this way, otherwise an
        AttributeError is raised.
        For these unknown properties ``setPropertyValue(CSSname, value)``
        has to be called explicitly.
        Also setting the priority of properties needs to be done with a
        call like ``setPropertyValue(CSSname, value, priority)``.

        Example::

            >>> style = CSSStyleDeclaration()
            >>> style.fontStyle = 'italic'
            >>> # or
            >>> style.setProperty('font-style', 'italic', '!important')

        """
        self.setProperty(CSSName, value)
        # TODO: Shorthand ones

    def _delP(self, CSSName):
        """(cssutils only) Overwritten here and effectively the same as
        ``self.removeProperty(CSSname)``.

        Example::

            >>> style = CSSStyleDeclaration(cssText='font-style:italic;')
            >>> del style.fontStyle
            >>> print(style.fontStyle)
            <BLANKLINE>

        """
        self.removeProperty(CSSName)

    def children(self):
        """Generator yielding any known child in this declaration including
        *all* properties, comments or CSSUnknownrules.
        """
        for item in self._seq:
            yield item.value

    def _getCssText(self):
        """Return serialized property cssText."""
        return cssutils.ser.do_css_CSSStyleDeclaration(self)

    def _setCssText(self, cssText):
        """Setting this attribute will result in the parsing of the new value
        and resetting of all the properties in the declaration block
        including the removal or addition of properties.

        :exceptions:
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this declaration is readonly or a property is readonly.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.
        """
        self._checkReadonly()
        tokenizer = self._tokenize2(cssText)

        def ident(expected, seq, token, tokenizer=None):
            # a property

            tokens = self._tokensupto2(tokenizer, starttoken=token, semicolon=True)
            if self._tokenvalue(tokens[-1]) == ';':
                tokens.pop()
            property = Property(parent=self)
            property.cssText = tokens
            if property.wellformed:
                seq.append(property, 'Property')
            else:
                self._log.error(
                    'CSSStyleDeclaration: Syntax Error in '
                    'Property: %s' % self._valuestr(tokens)
                )
            # does not matter in this case
            return expected

        def unexpected(expected, seq, token, tokenizer=None):
            # error, find next ; or } to omit upto next property
            ignored = self._tokenvalue(token) + self._valuestr(
                self._tokensupto2(tokenizer, propertyvalueendonly=True)
            )
            self._log.error(
                'CSSStyleDeclaration: Unexpected token, ignoring ' 'upto %r.' % ignored,
                token,
            )
            # does not matter in this case
            return expected

        def char(expected, seq, token, tokenizer=None):
            # a standalone ; or error...
            if self._tokenvalue(token) == ';':
                self._log.info(
                    'CSSStyleDeclaration: Stripped standalone semicolon'
                    ': %s' % self._valuestr([token]),
                    neverraise=True,
                )
                return expected
            else:
                return unexpected(expected, seq, token, tokenizer)

        # [Property: Value;]* Property: Value?
        newseq = self._tempSeq()
        wellformed, expected = self._parse(
            expected=None,
            seq=newseq,
            tokenizer=tokenizer,
            productions={'IDENT': ident, 'CHAR': char},
            default=unexpected,
        )
        # wellformed set by parse

        for item in newseq:
            item.value._parent = self

        # do not check wellformed as invalid things are removed anyway
        self._setSeq(newseq)

    cssText = property(
        _getCssText,
        _setCssText,
        doc="(DOM) A parsable textual representation of the "
        "declaration block excluding the surrounding curly "
        "braces.",
    )

    def getCssText(self, separator=None):
        """
        :returns:
            serialized property cssText, each property separated by
            given `separator` which may e.g. be ``u''`` to be able to use
            cssText directly in an HTML style attribute. ``;`` is part of
            each property (except the last one) and **cannot** be set with
            separator!
        """
        return cssutils.ser.do_css_CSSStyleDeclaration(self, separator)

    def _setParentRule(self, parentRule):
        self._parentRule = parentRule

    #        for x in self.children():
    #            x.parent = self

    parentRule = property(
        lambda self: self._parentRule,
        _setParentRule,
        doc="(DOM) The CSS rule that contains this declaration block or "
        "None if this CSSStyleDeclaration is not attached to a CSSRule.",
    )

    def getProperties(self, name=None, all=False):
        """
        :param name:
            optional `name` of properties which are requested.
            Only properties with this **always normalized** `name` are returned.
            If `name` is ``None`` all properties are returned (at least one for
            each set name depending on parameter `all`).
        :param all:
            if ``False`` (DEFAULT) only the effective properties are returned.
            If name is given a list with only one property is returned.

            if ``True`` all properties including properties set multiple times
            with different values or priorities for different UAs are returned.
            The order of the properties is fully kept as in the original
            stylesheet.
        :returns:
            a list of :class:`~cssutils.css.Property` objects set in
            this declaration.
        """
        if name and not all:
            # single prop but list
            p = self.getProperty(name)
            if p:
                return [p]
            else:
                return []
        elif not all:
            # effective Properties in name order
            return [self.getProperty(name) for name in self.__nnames()]
        else:
            # all properties or all with this name
            nname = self._normalize(name)
            properties = []
            for item in self.seq:
                val = item.value
                if isinstance(val, Property) and ((not nname) or (val.name == nname)):
                    properties.append(val)
            return properties

    def getProperty(self, name, normalize=True):
        r"""
        :param name:
            of the CSS property, always lowercase (even if not normalized)
        :param normalize:
            if ``True`` (DEFAULT) name will be normalized (lowercase, no simple
            escapes) so "color", "COLOR" or "C\olor" will all be equivalent

            If ``False`` may return **NOT** the effective value but the
            effective for the unnormalized name.
        :returns:
            the effective :class:`~cssutils.css.Property` object.
        """
        nname = self._normalize(name)
        found = None
        for item in reversed(self.seq):
            val = item.value
            if isinstance(val, Property):
                if (normalize and nname == val.name) or name == val.literalname:
                    if val.priority:
                        return val
                    elif not found:
                        found = val
        return found

    def getPropertyCSSValue(self, name, normalize=True):
        r"""
        :param name:
            of the CSS property, always lowercase (even if not normalized)
        :param normalize:
            if ``True`` (DEFAULT) name will be normalized (lowercase, no simple
            escapes) so "color", "COLOR" or "C\olor" will all be equivalent

            If ``False`` may return **NOT** the effective value but the
            effective for the unnormalized name.
        :returns:
            ``~cssutils.css.CSSValue``, the value of the effective
            property if it has been explicitly set for this declaration block.

        (DOM)
        Used to retrieve the object representation of the value of a CSS
        property if it has been explicitly set within this declaration
        block. Returns None if the property has not been set.

        (This method returns None if the property is a shorthand
        property. Shorthand property values can only be accessed and
        modified as strings, using the getPropertyValue and setProperty
        methods.)

        **cssutils currently always returns a CSSValue if the property is
        set.**

        for more on shorthand properties see
            http://www.dustindiaz.com/css-shorthand/
        """
        nname = self._normalize(name)
        if nname in self._SHORTHANDPROPERTIES:
            self._log.info(
                'CSSValue for shorthand property "%s" should be '
                'None, this may be implemented later.' % nname,
                neverraise=True,
            )

        p = self.getProperty(name, normalize)
        if p:
            return p.propertyValue
        else:
            return None

    def getPropertyValue(self, name, normalize=True):
        r"""
        :param name:
            of the CSS property, always lowercase (even if not normalized)
        :param normalize:
            if ``True`` (DEFAULT) name will be normalized (lowercase, no simple
            escapes) so "color", "COLOR" or "C\olor" will all be equivalent

            If ``False`` may return **NOT** the effective value but the
            effective for the unnormalized name.
        :returns:
            the value of the effective property if it has been explicitly set
            for this declaration block. Returns the empty string if the
            property has not been set.
        """
        p = self.getProperty(name, normalize)
        if p:
            return p.value
        else:
            return ''

    def getPropertyPriority(self, name, normalize=True):
        r"""
        :param name:
            of the CSS property, always lowercase (even if not normalized)
        :param normalize:
            if ``True`` (DEFAULT) name will be normalized (lowercase, no simple
            escapes) so "color", "COLOR" or "C\olor" will all be equivalent

            If ``False`` may return **NOT** the effective value but the
            effective for the unnormalized name.
        :returns:
            the priority of the effective CSS property (e.g. the
            "important" qualifier) if the property has been explicitly set in
            this declaration block. The empty string if none exists.
        """
        p = self.getProperty(name, normalize)
        if p:
            return p.priority
        else:
            return ''

    def removeProperty(self, name, normalize=True):
        r"""
        (DOM)
        Used to remove a CSS property if it has been explicitly set within
        this declaration block.

        :param name:
            of the CSS property
        :param normalize:
            if ``True`` (DEFAULT) name will be normalized (lowercase, no simple
            escapes) so "color", "COLOR" or "C\olor" will all be equivalent.
            The effective Property value is returned and *all* Properties
            with ``Property.name == name`` are removed.

            If ``False`` may return **NOT** the effective value but the
            effective for the unnormalized `name` only. Also only the
            Properties with the literal name `name` are removed.
        :returns:
            the value of the property if it has been explicitly set for
            this declaration block. Returns the empty string if the property
            has not been set or the property name does not correspond to a
            known CSS property


        :exceptions:
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this declaration is readonly or the property is
              readonly.
        """
        self._checkReadonly()
        r = self.getPropertyValue(name, normalize=normalize)
        newseq = self._tempSeq()
        if normalize:
            # remove all properties with name == nname
            nname = self._normalize(name)
            for item in self.seq:
                if not (isinstance(item.value, Property) and item.value.name == nname):
                    newseq.appendItem(item)
        else:
            # remove all properties with literalname == name
            for item in self.seq:
                if not (
                    isinstance(item.value, Property) and item.value.literalname == name
                ):
                    newseq.appendItem(item)
        self._setSeq(newseq)
        return r

    def setProperty(self, name, value=None, priority='', normalize=True, replace=True):
        r"""(DOM) Set a property value and priority within this declaration
        block.

        :param name:
            of the CSS property to set (in W3C DOM the parameter is called
            "propertyName"), always lowercase (even if not normalized)

            If a property with this `name` is present it will be reset.

            cssutils also allowed `name` to be a
            :class:`~cssutils.css.Property` object, all other
            parameter are ignored in this case

        :param value:
            the new value of the property, ignored if `name` is a Property.
        :param priority:
            the optional priority of the property (e.g. "important"),
            ignored if `name` is a Property.
        :param normalize:
            if True (DEFAULT) `name` will be normalized (lowercase, no simple
            escapes) so "color", "COLOR" or "C\olor" will all be equivalent
        :param replace:
            if True (DEFAULT) the given property will replace a present
            property. If False a new property will be added always.
            The difference to `normalize` is that two or more properties with
            the same name may be set, useful for e.g. stuff like::

                background: red;
                background: rgba(255, 0, 0, 0.5);

            which defines the same property but only capable UAs use the last
            property value, older ones use the first value.

        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified value has a syntax error and is
              unparsable.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this declaration is readonly or the property is
              readonly.
        """
        self._checkReadonly()

        if isinstance(name, Property):
            newp = name
            name = newp.literalname
        elif not value:
            # empty string or None effectively removed property
            return self.removeProperty(name)
        else:
            newp = Property(name, value, priority, parent=self)

        if newp.wellformed:
            if replace:
                # check if update
                nname = self._normalize(name)
                properties = self.getProperties(name, all=(not normalize))
                for property in reversed(properties):
                    if normalize and property.name == nname:
                        property.propertyValue = newp.propertyValue.cssText
                        property.priority = newp.priority
                        return
                    elif property.literalname == name:
                        property.propertyValue = newp.propertyValue.cssText
                        property.priority = newp.priority
                        return

            # not yet set or forced omit replace
            newp.parent = self
            self.seq._readonly = False
            self.seq.append(newp, 'Property')
            self.seq._readonly = True

        else:
            self._log.warn(f'Invalid Property: {name}: {value} {priority}')

    def item(self, index):
        """(DOM) Retrieve the properties that have been explicitly set in
        this declaration block. The order of the properties retrieved using
        this method does not have to be the order in which they were set.
        This method can be used to iterate over all properties in this
        declaration block.

        :param index:
            of the property to retrieve, negative values behave like
            negative indexes on Python lists, so -1 is the last element

        :returns:
            the name of the property at this ordinal position. The
            empty string if no property exists at this position.

        **ATTENTION:**
        Only properties with different names are counted. If two
        properties with the same name are present in this declaration
        only the effective one is included.

        :meth:`item` and :attr:`length` work on the same set here.
        """
        names = list(self.__nnames())
        try:
            return names[index]
        except IndexError:
            return ''

    length = property(
        lambda self: len(list(self.__nnames())),
        doc="(DOM) The number of distinct properties that have "
        "been explicitly in this declaration block. The "
        "range of valid indices is 0 to length-1 inclusive. "
        "These are properties with a different ``name`` "
        "only. :meth:`item` and :attr:`length` work on the "
        "same set here.",
    )

    def _getValidating(self):
        try:
            # CSSParser.parseX() sets validating of stylesheet
            return self.parentRule.parentStyleSheet.validating
        except AttributeError:
            # CSSParser.parseStyle() sets validating of declaration
            if self._validating is not None:
                return self._validating
        # default
        return True

    def _setValidating(self, validating):
        self._validating = validating

    validating = property(
        _getValidating,
        _setValidating,
        doc="If ``True`` this declaration validates "
        "contained properties. The parent StyleSheet "
        "validation setting does *always* win though so "
        "even if validating is True it may not validate "
        "if the StyleSheet defines else!",
    )

    def _getValid(self):
        """Check each contained property for validity."""
        return all(prop.valid for prop in self.getProperties())

    valid = property(_getValid, doc='``True`` if each property is valid.')

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssstyledeclaration.py
NikolaBlagojevic/pyrecodes
"""CSSStyleSheet implements DOM Level 2 CSS CSSStyleSheet.

Partly also:
    - http://dev.w3.org/csswg/cssom/#the-cssstylesheet
    - http://www.w3.org/TR/2006/WD-css3-namespace-20060828/

TODO:
    - ownerRule and ownerNode
"""
__all__ = ['CSSStyleSheet']

from cssutils.helper import Deprecated
from cssutils.util import _Namespaces, _readUrl
from .cssrule import CSSRule
from .cssvariablesdeclaration import CSSVariablesDeclaration
import cssutils.stylesheets
import xml.dom


class CSSStyleSheet(cssutils.stylesheets.StyleSheet):
    """CSSStyleSheet represents a CSS style sheet.

    Format::

        stylesheet
          : [ CHARSET_SYM S* STRING S* ';' ]?
            [S|CDO|CDC]* [ import [S|CDO|CDC]* ]*
            [ namespace [S|CDO|CDC]* ]* # according to @namespace WD
            [ [ ruleset | media | page ] [S|CDO|CDC]* ]*

    ``cssRules``
        All Rules in this style sheet, a :class:`~cssutils.css.CSSRuleList`.
    """

    def __init__(
        self,
        href=None,
        media=None,
        title='',
        disabled=None,
        ownerNode=None,
        parentStyleSheet=None,
        readonly=False,
        ownerRule=None,
        validating=True,
    ):
        """
        For parameters see :class:`~cssutils.stylesheets.StyleSheet`
        """
        super().__init__(
            'text/css',
            href,
            media,
            title,
            disabled,
            ownerNode,
            parentStyleSheet,
            validating=validating,
        )

        self._ownerRule = ownerRule
        self.cssRules = cssutils.css.CSSRuleList()
        self._namespaces = _Namespaces(parentStyleSheet=self, log=self._log)
        self._variables = CSSVariablesDeclaration()
        self._readonly = readonly

        # used only during setting cssText by parse*()
        self.__encodingOverride = None
        self._fetcher = None

    def __iter__(self):
        "Generator which iterates over cssRules."
        yield from self._cssRules

    def __repr__(self):
        if self.media:
            mediaText = self.media.mediaText
        else:
            mediaText = None
        return "cssutils.css.{}(href={!r}, media={!r}, title={!r})".format(
            self.__class__.__name__,
            self.href,
            mediaText,
            self.title,
        )

    def __str__(self):
        if self.media:
            mediaText = self.media.mediaText
        else:
            mediaText = None
        return (
            "<cssutils.css.%s object encoding=%r href=%r "
            "media=%r title=%r namespaces=%r at 0x%x>"
            % (
                self.__class__.__name__,
                self.encoding,
                self.href,
                mediaText,
                self.title,
                self.namespaces.namespaces,
                id(self),
            )
        )

    def _cleanNamespaces(self):
        "Remove all namespace rules with same namespaceURI but last."
        rules = self.cssRules
        namespaceitems = list(self.namespaces.items())
        i = 0
        while i < len(rules):
            rule = rules[i]
            if (
                rule.type == rule.NAMESPACE_RULE
                and (rule.prefix, rule.namespaceURI) not in namespaceitems
            ):
                self.deleteRule(i)
            else:
                i += 1

    def _getUsedURIs(self):
        "Return set of URIs used in the sheet."
        useduris = set()
        for r1 in self:
            if r1.STYLE_RULE == r1.type:
                useduris.update(r1.selectorList._getUsedUris())
            elif r1.MEDIA_RULE == r1.type:
                for r2 in r1:
                    if r2.type == r2.STYLE_RULE:
                        useduris.update(r2.selectorList._getUsedUris())
        return useduris

    @property
    def cssRules(self):
        "All Rules in this style sheet, a :class:`~cssutils.css.CSSRuleList`."
        return self._cssRules

    @cssRules.setter
    def cssRules(self, cssRules):
        "Set new cssRules and update contained rules refs."
        cssRules.append = self.insertRule
        cssRules.extend = self.insertRule
        cssRules.__delitem__ = self.deleteRule

        for rule in cssRules:
            rule._parentStyleSheet = self

        self._cssRules = cssRules

    def _getCssText(self):
        "Textual representation of the stylesheet (a byte string)."
        return cssutils.ser.do_CSSStyleSheet(self)

    def _setCssText(self, cssText):  # noqa: C901
        """Parse `cssText` and overwrites the whole stylesheet.

        :param cssText:
            a parseable string or a tuple of (cssText, dict-of-namespaces)
        :exceptions:
            - :exc:`~xml.dom.NamespaceErr`:
              If a namespace prefix is found which is not declared.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if the rule is readonly.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.
        """
        self._checkReadonly()

        cssText, namespaces = self._splitNamespacesOff(cssText)
        tokenizer = self._tokenize2(cssText)

        def S(expected, seq, token, tokenizer=None):
            # @charset must be at absolute beginning of style sheet
            # or 0 for py3
            return max(1, expected or 0)

        def COMMENT(expected, seq, token, tokenizer=None):
            "special: sets parent*"
            self.insertRule(cssutils.css.CSSComment([token], parentStyleSheet=self))
            # or 0 for py3
            return max(1, expected or 0)

        def charsetrule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSCharsetRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)

            if expected > 0:
                self._log.error(
                    'CSSStylesheet: CSSCharsetRule only allowed '
                    'at beginning of stylesheet.',
                    token,
                    xml.dom.HierarchyRequestErr,
                )
                return expected
            elif rule.wellformed:
                self.insertRule(rule)

            return 1

        def importrule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSImportRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)

            if expected > 1:
                self._log.error(
                    'CSSStylesheet: CSSImportRule not allowed ' 'here.',
                    token,
                    xml.dom.HierarchyRequestErr,
                )
                return expected
            elif rule.wellformed:
                self.insertRule(rule)

            return 1

        def namespacerule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSNamespaceRule(
                cssText=self._tokensupto2(tokenizer, token), parentStyleSheet=self
            )

            if expected > 2:
                self._log.error(
                    'CSSStylesheet: CSSNamespaceRule not allowed ' 'here.',
                    token,
                    xml.dom.HierarchyRequestErr,
                )
                return expected
            elif rule.wellformed:
                if rule.prefix not in self.namespaces:
                    # add new if not same prefix
                    self.insertRule(rule, _clean=False)
                else:
                    # same prefix => replace namespaceURI
                    for r in self.cssRules.rulesOfType(rule.NAMESPACE_RULE):
                        if r.prefix == rule.prefix:
                            r._replaceNamespaceURI(rule.namespaceURI)

                self._namespaces[rule.prefix] = rule.namespaceURI

            return 2

        def variablesrule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSVariablesRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)

            if expected > 2:
                self._log.error(
                    'CSSStylesheet: CSSVariablesRule not allowed ' 'here.',
                    token,
                    xml.dom.HierarchyRequestErr,
                )
                return expected
            elif rule.wellformed:
                self.insertRule(rule)
                self._updateVariables()

            return 2

        def fontfacerule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSFontFaceRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)
            if rule.wellformed:
                self.insertRule(rule)
            return 3

        def mediarule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSMediaRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)
            if rule.wellformed:
                self.insertRule(rule)
            return 3

        def pagerule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSPageRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)
            if rule.wellformed:
                self.insertRule(rule)
            return 3

        def unknownrule(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            if token[1] in cssutils.css.MarginRule.margins:
                self._log.error(
                    'CSSStylesheet: MarginRule out CSSPageRule.', token, neverraise=True
                )
                rule = cssutils.css.MarginRule(parentStyleSheet=self)
                rule.cssText = self._tokensupto2(tokenizer, token)
            else:
                self._log.warn(
                    'CSSStylesheet: Unknown @rule found.', token, neverraise=True
                )
                rule = cssutils.css.CSSUnknownRule(parentStyleSheet=self)
                rule.cssText = self._tokensupto2(tokenizer, token)

            if rule.wellformed:
                self.insertRule(rule)

            # or 0 for py3
            return max(1, expected or 0)

        def ruleset(expected, seq, token, tokenizer):
            # parse and consume tokens in any case
            rule = cssutils.css.CSSStyleRule(parentStyleSheet=self)
            rule.cssText = self._tokensupto2(tokenizer, token)
            if rule.wellformed:
                self.insertRule(rule)
            return 3

        # save for possible reset
        oldCssRules = self.cssRules
        oldNamespaces = self._namespaces

        self.cssRules = cssutils.css.CSSRuleList()
        # simple during parse
        self._namespaces = namespaces
        self._variables = CSSVariablesDeclaration()

        # not used?!
        newseq = []

        # ['CHARSET', 'IMPORT', ('VAR', NAMESPACE'), ('PAGE', 'MEDIA', ruleset)]
        wellformed, expected = self._parse(
            0,
            newseq,
            tokenizer,
            {
                'S': S,
                'COMMENT': COMMENT,
                'CDO': lambda *ignored: None,
                'CDC': lambda *ignored: None,
                'CHARSET_SYM': charsetrule,
                'FONT_FACE_SYM': fontfacerule,
                'IMPORT_SYM': importrule,
                'NAMESPACE_SYM': namespacerule,
                'PAGE_SYM': pagerule,
                'MEDIA_SYM': mediarule,
                'VARIABLES_SYM': variablesrule,
                'ATKEYWORD': unknownrule,
            },
            default=ruleset,
        )

        if wellformed:
            # use proper namespace object
            self._namespaces = _Namespaces(parentStyleSheet=self, log=self._log)
            self._cleanNamespaces()

        else:
            # reset
            self._cssRules = oldCssRules
            self._namespaces = oldNamespaces
            self._updateVariables()
            self._cleanNamespaces()

    cssText = property(
        _getCssText,
        _setCssText,
        "Textual representation of the stylesheet (a byte string)",
    )

    def _resolveImport(self, url):
        """Read (encoding, enctype, decodedContent) from `url` for @import
        sheets."""
        try:
            # only available during parsing of a complete sheet
            parentEncoding = self.__newEncoding

        except AttributeError:
            try:
                # explicit @charset
                parentEncoding = self._cssRules[0].encoding
            except (IndexError, AttributeError):
                # default not UTF-8 but None!
                parentEncoding = None

        return _readUrl(
            url,
            fetcher=self._fetcher,
            overrideEncoding=self.__encodingOverride,
            parentEncoding=parentEncoding,
        )

    def _setCssTextWithEncodingOverride(
        self, cssText, encodingOverride=None, encoding=None
    ):
        """Set `cssText` but use `encodingOverride` to overwrite detected
        encoding. This is used by parse and @import during setting of cssText.

        If `encoding` is given use this but do not save as `encodingOverride`.
        """
        if encodingOverride:
            # encoding during resolving of @import
            self.__encodingOverride = encodingOverride

        if encoding:
            # save for nested @import
            self.__newEncoding = encoding

        self.cssText = cssText

        if encodingOverride:
            # set encodingOverride explicit again!
            self.encoding = self.__encodingOverride
            # del?
            self.__encodingOverride = None
        elif encoding:
            # may e.g. be httpEncoding
            self.encoding = encoding
            try:
                del self.__newEncoding
            except AttributeError:
                pass

    def _setFetcher(self, fetcher=None):
        """Set @import URL loader, if None the default is used."""
        self._fetcher = fetcher

    def _getEncoding(self):
        """Encoding set in :class:`~cssutils.css.CSSCharsetRule` or if ``None``
        resulting in default ``utf-8`` encoding being used."""
        try:
            return self._cssRules[0].encoding
        except (IndexError, AttributeError):
            return 'utf-8'

    def _setEncoding(self, encoding):
        """Set `encoding` of charset rule if present in sheet or insert a new
        :class:`~cssutils.css.CSSCharsetRule` with given `encoding`.
        If `encoding` is None removes charsetrule if present resulting in
        default encoding of utf-8.
        """
        try:
            rule = self._cssRules[0]
        except IndexError:
            rule = None
        if rule and rule.CHARSET_RULE == rule.type:
            if encoding:
                rule.encoding = encoding
            else:
                self.deleteRule(0)
        elif encoding:
            self.insertRule(cssutils.css.CSSCharsetRule(encoding=encoding), 0)

    encoding = property(
        _getEncoding,
        _setEncoding,
        "(cssutils) Reflect encoding of an @charset rule or 'utf-8' "
        "(default) if set to ``None``",
    )

    @property
    def namespaces(self):
        """All Namespaces used in this CSSStyleSheet."""
        return self._namespaces

    def _updateVariables(self):
        """Updates self._variables, called when @import or @variables rules
        is added to sheet.
        """
        for r in self.cssRules.rulesOfType(CSSRule.IMPORT_RULE):
            s = r.styleSheet
            if s:
                for var in s.variables:
                    self._variables.setVariable(var, s.variables[var])
        # for r in self.cssRules.rulesOfType(CSSRule.IMPORT_RULE):
        #     for vr in r.styleSheet.cssRules.rulesOfType(CSSRule.VARIABLES_RULE):
        #         for var in vr.variables:
        #             self._variables.setVariable(var, vr.variables[var])
        for vr in self.cssRules.rulesOfType(CSSRule.VARIABLES_RULE):
            for var in vr.variables:
                self._variables.setVariable(var, vr.variables[var])

    variables = property(
        lambda self: self._variables,
        doc="A :class:`cssutils.css.CSSVariablesDeclaration` "
        "containing all available variables in this "
        "CSSStyleSheet including the ones defined in "
        "imported sheets.",
    )

    def add(self, rule):
        """Add `rule` to style sheet at appropriate position.
        Same as ``insertRule(rule, inOrder=True)``.
        """
        return self.insertRule(rule, index=None, inOrder=True)

    def deleteRule(self, index):
        """Delete rule at `index` from the style sheet.

        :param index:
            The `index` of the rule to be removed from the StyleSheet's rule
            list. For an `index` < 0 **no** :exc:`~xml.dom.IndexSizeErr` is
            raised but rules for normal Python lists are used. E.g.
            ``deleteRule(-1)`` removes the last rule in cssRules.

            `index` may also be a CSSRule object which will then be removed
            from the StyleSheet.

        :exceptions:
            - :exc:`~xml.dom.IndexSizeErr`:
              Raised if the specified index does not correspond to a rule in
              the style sheet's rule list.
            - :exc:`~xml.dom.NamespaceErr`:
              Raised if removing this rule would result in an invalid StyleSheet
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this style sheet is readonly.
        """
        self._checkReadonly()

        if isinstance(index, CSSRule):
            for i, r in enumerate(self.cssRules):
                if index == r:
                    index = i
                    break
            else:
                raise xml.dom.IndexSizeErr(
                    "CSSStyleSheet: Not a rule in"
                    " this sheets'a cssRules list: %s" % index
                )

        try:
            rule = self._cssRules[index]
        except IndexError:
            raise xml.dom.IndexSizeErr(
                'CSSStyleSheet: %s is not a valid index in the rulelist of '
                'length %i' % (index, self._cssRules.length)
            )
        else:
            if rule.type == rule.NAMESPACE_RULE:
                # check all namespacerules if used
                uris = [r.namespaceURI for r in self if r.type == r.NAMESPACE_RULE]
                useduris = self._getUsedURIs()
                if rule.namespaceURI in useduris and uris.count(rule.namespaceURI) == 1:
                    raise xml.dom.NoModificationAllowedErr(
                        'CSSStyleSheet: NamespaceURI defined in this rule is '
                        'used, cannot remove.'
                    )
                    return

            rule._parentStyleSheet = None  # detach
            del self._cssRules[index]  # delete from StyleSheet

    def insertRule(self, rule, index=None, inOrder=False, _clean=True):  # noqa: C901
        """
        Used to insert a new rule into the style sheet. The new rule now
        becomes part of the cascade.

        :param rule:
            a parsable DOMString, in cssutils also a
            :class:`~cssutils.css.CSSRule` or :class:`~cssutils.css.CSSRuleList`
        :param index:
            of the rule before the new rule will be inserted.
            If the specified `index` is equal to the length of the
            StyleSheet's rule collection, the rule will be added to the end
            of the style sheet.
            If `index` is not given or ``None`` rule will be appended to rule
            list.
        :param inOrder:
            if ``True`` the rule will be put to a proper location while
            ignoring `index` and without raising
            :exc:`~xml.dom.HierarchyRequestErr`.
            The resulting index is returned nevertheless.
        :returns: The index within the style sheet's rule collection
        :Exceptions:
            - :exc:`~xml.dom.HierarchyRequestErr`:
              Raised if the rule cannot be inserted at the specified `index`
              e.g. if an @import rule is inserted after a standard rule set
              or other at-rule.
            - :exc:`~xml.dom.IndexSizeErr`:
              Raised if the specified `index` is not a valid insertion point.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this style sheet is readonly.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified rule has a syntax error and is
              unparsable.
        """
        self._checkReadonly()

        # check position
        if index is None:
            index = len(self._cssRules)
        elif index < 0 or index > self._cssRules.length:
            raise xml.dom.IndexSizeErr(
                'CSSStyleSheet: Invalid index %s for CSSRuleList with a '
                'length of %s.' % (index, self._cssRules.length)
            )
            return

        if isinstance(rule, str):
            # init a temp sheet which has the same properties as self
            tempsheet = CSSStyleSheet(
                href=self.href,
                media=self.media,
                title=self.title,
                parentStyleSheet=self.parentStyleSheet,
                ownerRule=self.ownerRule,
            )
            tempsheet._ownerNode = self.ownerNode
            tempsheet._fetcher = self._fetcher
            # prepend encoding if in this sheet to be able to use it in
            # @import rules encoding resolution
            # do not add if new rule startswith "@charset" (which is exact!)
            if not rule.startswith('@charset') and (
                self._cssRules
                and self._cssRules[0].type == self._cssRules[0].CHARSET_RULE
            ):
                # rule 0 is @charset!
                newrulescount, newruleindex = 2, 1
                rule = self._cssRules[0].cssText + rule
            else:
                newrulescount, newruleindex = 1, 0

            # parse the new rule(s)
            tempsheet.cssText = (rule, self._namespaces)

            if len(tempsheet.cssRules) != newrulescount or (
                not isinstance(tempsheet.cssRules[newruleindex], cssutils.css.CSSRule)
            ):
                self._log.error('CSSStyleSheet: Not a CSSRule: %s' % rule)
                return
            rule = tempsheet.cssRules[newruleindex]
            rule._parentStyleSheet = None  # done later?

            # TODO:
            # tempsheet._namespaces = self._namespaces
            # variables?

        elif isinstance(rule, cssutils.css.CSSRuleList):
            # insert all rules
            for i, r in enumerate(rule):
                self.insertRule(r, index + i)
            return index

        if not rule.wellformed:
            self._log.error('CSSStyleSheet: Invalid rules cannot be added.')
            return

        # CHECK HIERARCHY
        # @charset
        if rule.type == rule.CHARSET_RULE:
            if inOrder:
                index = 0
                # always first and only
                if self._cssRules and self._cssRules[0].type == rule.CHARSET_RULE:
                    self._cssRules[0].encoding = rule.encoding
                else:
                    self._cssRules.insert(0, rule)
            elif index != 0 or (
                self._cssRules and self._cssRules[0].type == rule.CHARSET_RULE
            ):
                self._log.error(
                    'CSSStylesheet: @charset only allowed once at the'
                    ' beginning of a stylesheet.',
                    error=xml.dom.HierarchyRequestErr,
                )
                return
            else:
                self._cssRules.insert(index, rule)

        # @unknown or comment
        elif rule.type in (rule.UNKNOWN_RULE, rule.COMMENT) and not inOrder:
            if (
                index == 0
                and self._cssRules
                and self._cssRules[0].type == rule.CHARSET_RULE
            ):
                self._log.error(
                    'CSSStylesheet: @charset must be the first rule.',
                    error=xml.dom.HierarchyRequestErr,
                )
                return
            else:
                self._cssRules.insert(index, rule)

        # @import
        elif rule.type == rule.IMPORT_RULE:
            if inOrder:
                # automatic order
                if rule.type in (r.type for r in self):
                    # find last of this type
                    for i, r in enumerate(reversed(self._cssRules)):
                        if r.type == rule.type:
                            index = len(self._cssRules) - i
                            break
                else:
                    # find first point to insert
                    if self._cssRules and self._cssRules[0].type in (
                        rule.CHARSET_RULE,
                        rule.COMMENT,
                    ):
                        index = 1
                    else:
                        index = 0
            else:
                # after @charset
                if (
                    index == 0
                    and self._cssRules
                    and self._cssRules[0].type == rule.CHARSET_RULE
                ):
                    self._log.error(
                        'CSSStylesheet: Found @charset at index 0.',
                        error=xml.dom.HierarchyRequestErr,
                    )
                    return
                # before @namespace @variables @page @font-face @media stylerule
                for r in self._cssRules[:index]:
                    if r.type in (
                        r.NAMESPACE_RULE,
                        r.VARIABLES_RULE,
                        r.MEDIA_RULE,
                        r.PAGE_RULE,
                        r.STYLE_RULE,
                        r.FONT_FACE_RULE,
                    ):
                        self._log.error(
                            'CSSStylesheet: Cannot insert @import here,'
                            ' found @namespace, @variables, @media, @page or'
                            ' CSSStyleRule before index %s.' % index,
                            error=xml.dom.HierarchyRequestErr,
                        )
                        return
            self._cssRules.insert(index, rule)
            self._updateVariables()

        # @namespace
        elif rule.type == rule.NAMESPACE_RULE:
            if inOrder:
                if rule.type in (r.type for r in self):
                    # find last of this type
                    for i, r in enumerate(reversed(self._cssRules)):
                        if r.type == rule.type:
                            index = len(self._cssRules) - i
                            break
                else:
                    # find first point to insert
                    for i, r in enumerate(self._cssRules):
                        if r.type in (
                            r.VARIABLES_RULE,
                            r.MEDIA_RULE,
                            r.PAGE_RULE,
                            r.STYLE_RULE,
                            r.FONT_FACE_RULE,
                            r.UNKNOWN_RULE,
                            r.COMMENT,
                        ):
                            index = i  # before these
                            break
            else:
                # after @charset and @import
                for r in self._cssRules[index:]:
                    if r.type in (r.CHARSET_RULE, r.IMPORT_RULE):
                        self._log.error(
                            'CSSStylesheet: Cannot insert @namespace here,'
                            ' found @charset or @import after index %s.' % index,
                            error=xml.dom.HierarchyRequestErr,
                        )
                        return
                # before @variables @media @page @font-face and stylerule
                for r in self._cssRules[:index]:
                    if r.type in (
                        r.VARIABLES_RULE,
                        r.MEDIA_RULE,
                        r.PAGE_RULE,
                        r.STYLE_RULE,
                        r.FONT_FACE_RULE,
                    ):
                        self._log.error(
                            'CSSStylesheet: Cannot insert @namespace here,'
                            ' found @variables, @media, @page or CSSStyleRule'
                            ' before index %s.' % index,
                            error=xml.dom.HierarchyRequestErr,
                        )
                        return

            if not (
                rule.prefix in self.namespaces
                and self.namespaces[rule.prefix] == rule.namespaceURI
            ):
                # no doublettes
                self._cssRules.insert(index, rule)
                if _clean:
                    self._cleanNamespaces()

        # @variables
        elif rule.type == rule.VARIABLES_RULE:
            if inOrder:
                if rule.type in (r.type for r in self):
                    # find last of this type
                    for i, r in enumerate(reversed(self._cssRules)):
                        if r.type == rule.type:
                            index = len(self._cssRules) - i
                            break
                else:
                    # find first point to insert
                    for i, r in enumerate(self._cssRules):
                        if r.type in (
                            r.MEDIA_RULE,
                            r.PAGE_RULE,
                            r.STYLE_RULE,
                            r.FONT_FACE_RULE,
                            r.UNKNOWN_RULE,
                            r.COMMENT,
                        ):
                            index = i  # before these
                            break
            else:
                # after @charset @import @namespace
                for r in self._cssRules[index:]:
                    if r.type in (r.CHARSET_RULE, r.IMPORT_RULE, r.NAMESPACE_RULE):
                        self._log.error(
                            'CSSStylesheet: Cannot insert @variables here,'
                            ' found @charset, @import or @namespace after'
                            ' index %s.' % index,
                            error=xml.dom.HierarchyRequestErr,
                        )
                        return
                # before @media @page @font-face and stylerule
                for r in self._cssRules[:index]:
                    if r.type in (
                        r.MEDIA_RULE,
                        r.PAGE_RULE,
                        r.STYLE_RULE,
                        r.FONT_FACE_RULE,
                    ):
                        self._log.error(
                            'CSSStylesheet: Cannot insert @variables here,'
                            ' found @media, @page or CSSStyleRule'
                            ' before index %s.' % index,
                            error=xml.dom.HierarchyRequestErr,
                        )
                        return

            self._cssRules.insert(index, rule)
            self._updateVariables()

        # all other where order is not important
        else:
            if inOrder:
                # simply add to end as no specific order
                self._cssRules.append(rule)
                index = len(self._cssRules) - 1
            else:
                for r in self._cssRules[index:]:
                    if r.type in (r.CHARSET_RULE, r.IMPORT_RULE, r.NAMESPACE_RULE):
                        self._log.error(
                            'CSSStylesheet: Cannot insert rule here, found '
                            '@charset, @import or @namespace before index %s.' % index,
                            error=xml.dom.HierarchyRequestErr,
                        )
                        return
                self._cssRules.insert(index, rule)

        # post settings
        rule._parentStyleSheet = self

        if rule.IMPORT_RULE == rule.type and not rule.hrefFound:
            # try loading the imported sheet which has new relative href now
            rule.href = rule.href

        return index

    ownerRule = property(
        lambda self: self._ownerRule,
        doc='A ref to an @import rule if it is imported, ' 'else ``None``.',
    )

    def _getValid(self):
        """Check if each contained rule is valid."""
        for rule in self.cssRules:
            # Not all rules can be checked for validity
            if hasattr(rule, 'valid') and not rule.valid:
                return False
        return True

    valid = property(_getValid, doc='``True`` if all contained rules are valid')

    @Deprecated('Use ``cssutils.setSerializer(serializer)`` instead.')
    def setSerializer(self, cssserializer):
        """Set the cssutils global Serializer used for all output."""
        if isinstance(cssserializer, cssutils.CSSSerializer):
            cssutils.ser = cssserializer
        else:
            raise ValueError(
                'Serializer must be an instance of ' 'cssutils.CSSSerializer.'
            )

    @Deprecated('Set pref in ``cssutils.ser.prefs`` instead.')
    def setSerializerPref(self, pref, value):
        """Set a Preference of CSSSerializer used for output.
        See :class:`cssutils.serialize.Preferences` for possible
        preferences to be set.
        """
        cssutils.ser.prefs.__setattr__(pref, value)

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssstylesheet.py
NikolaBlagojevic/pyrecodes
"""CSSValue related classes

- CSSValue implements DOM Level 2 CSS CSSValue
- CSSPrimitiveValue implements DOM Level 2 CSS CSSPrimitiveValue
- CSSValueList implements DOM Level 2 CSS CSSValueList

"""
__all__ = ['CSSValue', 'CSSPrimitiveValue', 'CSSValueList', 'RGBColor', 'CSSVariable']

from cssutils.prodparser import Choice, PreDef, Prod, ProdParser, Sequence
import cssutils
import cssutils.helper
import math
import re
import xml.dom


class CSSValue(cssutils.util._NewBase):
    """The CSSValue interface represents a simple or a complex value.
    A CSSValue object only occurs in a context of a CSS property.
    """

    # The value is inherited and the cssText contains "inherit".
    CSS_INHERIT = 0
    # The value is a CSSPrimitiveValue.
    CSS_PRIMITIVE_VALUE = 1
    # The value is a CSSValueList.
    CSS_VALUE_LIST = 2
    # The value is a custom value.
    CSS_CUSTOM = 3
    # The value is a CSSVariable.
    CSS_VARIABLE = 4

    _typestrings = {
        0: 'CSS_INHERIT',
        1: 'CSS_PRIMITIVE_VALUE',
        2: 'CSS_VALUE_LIST',
        3: 'CSS_CUSTOM',
        4: 'CSS_VARIABLE',
    }

    def __init__(self, cssText=None, parent=None, readonly=False):
        """
        :param cssText:
            the parsable cssText of the value
        :param readonly:
            defaults to False
        """
        super().__init__()

        self._cssValueType = None
        self.wellformed = False
        self.parent = parent
        if cssText is not None:  # may be 0
            if isinstance(cssText, int):
                cssText = str(cssText)  # if it is an integer
            elif isinstance(cssText, float):
                cssText = '%f' % cssText  # if it is a floating point number

            self.cssText = cssText

        self._readonly = readonly

    def __repr__(self):
        return f"cssutils.css.{self.__class__.__name__}({self.cssText!r})"

    def __str__(self):
        return (
            "<cssutils.css.%s object cssValueTypeString=%r cssText=%r at "
            "0x%x>"
            % (self.__class__.__name__, self.cssValueTypeString, self.cssText, id(self))
        )

    def _setCssText(self, cssText):  # noqa: C901
        """
        Format::

            unary_operator
              : '-' | '+'
              ;
            operator
              : '/' S* | ',' S* | /* empty */
              ;
            expr
              : term [ operator term ]*
              ;
            term
              : unary_operator?
                [ NUMBER S* | PERCENTAGE S* | LENGTH S* | EMS S* | EXS S* |
                  ANGLE S* | TIME S* | FREQ S* ]
              | STRING S* | IDENT S* | URI S* | hexcolor | function
              | UNICODE-RANGE S*
              ;
            function
              : FUNCTION S* expr ')' S*
              ;
            /*
             * There is a constraint on the color that it must
             * have either 3 or 6 hex-digits (i.e., [0-9a-fA-F])
             * after the "#"; e.g., "#000" is OK, but "#abcd" is not.
             */
            hexcolor
              : HASH S*
              ;

        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error
              (according to the attached property) or is unparsable.
            - :exc:`~xml.dom.InvalidModificationErr`:
              TODO: Raised if the specified CSS string value represents a
              different type of values than the values allowed by the CSS
              property.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this value is readonly.
        """
        self._checkReadonly()

        # used as operator is , / or S
        nextSor = ',/'

        term = Choice(
            Sequence(
                PreDef.unary(),
                Choice(
                    PreDef.number(nextSor=nextSor),
                    PreDef.percentage(nextSor=nextSor),
                    PreDef.dimension(nextSor=nextSor),
                ),
            ),
            PreDef.string(nextSor=nextSor),
            PreDef.ident(nextSor=nextSor),
            PreDef.uri(nextSor=nextSor),
            PreDef.hexcolor(nextSor=nextSor),
            PreDef.unicode_range(nextSor=nextSor),
            # special case IE only expression
            Prod(
                name='expression',
                match=lambda t, v: t == self._prods.FUNCTION
                and (
                    cssutils.helper.normalize(v)
                    in (
                        'expression(',
                        'alpha(',
                        'blur(',
                        'chroma(',
                        'dropshadow(',
                        'fliph(',
                        'flipv(',
                        'glow(',
                        'gray(',
                        'invert(',
                        'mask(',
                        'shadow(',
                        'wave(',
                        'xray(',
                    )
                    or v.startswith('progid:DXImageTransform.Microsoft.')
                ),
                nextSor=nextSor,
                toSeq=lambda t, tokens: (
                    ExpressionValue._functionName,
                    ExpressionValue(cssutils.helper.pushtoken(t, tokens), parent=self),
                ),
            ),
            # CSS Variable var(
            PreDef.variable(
                nextSor=nextSor,
                toSeq=lambda t, tokens: (
                    'CSSVariable',
                    CSSVariable(cssutils.helper.pushtoken(t, tokens), parent=self),
                ),
            ),
            # calc(
            PreDef.calc(
                nextSor=nextSor,
                toSeq=lambda t, tokens: (
                    CalcValue._functionName,
                    CalcValue(cssutils.helper.pushtoken(t, tokens), parent=self),
                ),
            ),
            # TODO:
            # # rgb/rgba(
            # Prod(name='RGBColor',
            #      match=lambda t, v: t == self._prods.FUNCTION and (
            #         cssutils.helper.normalize(v) in (u'rgb(',
            #                                          u'rgba('
            #                                          )
            #      ),
            #      nextSor=nextSor,
            #             toSeq=lambda t, tokens: (RGBColor._functionName,
            #                                      RGBColor(
            #                   cssutils.helper.pushtoken(t, tokens),
            #                   parent=self)
            #                                      )
            # ),
            # other functions like rgb( etc
            PreDef.function(
                nextSor=nextSor,
                toSeq=lambda t, tokens: (
                    'FUNCTION',
                    CSSFunction(cssutils.helper.pushtoken(t, tokens), parent=self),
                ),
            ),
        )
        operator = Choice(
            PreDef.S(),
            PreDef.char('comma', ',', toSeq=lambda t, tokens: ('operator', t[1])),
            PreDef.char('slash', '/', toSeq=lambda t, tokens: ('operator', t[1])),
            optional=True,
        )
        # CSSValue PRODUCTIONS
        valueprods = Sequence(
            term,
            Sequence(
                operator,  # mayEnd this Sequence if whitespace
                # TODO: only when setting via other class
                # used by variabledeclaration currently
                PreDef.char('END', ';', stopAndKeep=True, optional=True),
                term,
                minmax=lambda: (0, None),
            ),
        )
        # parse
        wellformed, seq, store, notused = ProdParser().parse(
            cssText, 'CSSValue', valueprods, keepS=True
        )
        if wellformed:
            # - count actual values and set firstvalue which is used later on
            # - combine comma separated list, e.g. font-family to a single item
            # - remove S which should be an operator but is no needed
            count, firstvalue = 0, ()
            newseq = self._tempSeq()
            i, end = 0, len(seq)
            while i < end:
                item = seq[i]
                if item.type == self._prods.S:
                    pass

                elif (item.value, item.type) == (',', 'operator'):
                    # , separared counts as a single STRING for now
                    # URI or STRING value might be a single CHAR too!
                    newseq.appendItem(item)
                    count -= 1
                    if firstvalue:
                        # list of IDENTs is handled as STRING!
                        if firstvalue[1] == self._prods.IDENT:
                            firstvalue = firstvalue[0], 'STRING'

                elif item.value == '/':
                    # / separated items count as one
                    newseq.appendItem(item)

                elif item.value == '-' or item.value == '+':
                    # combine +- and following number or other
                    i += 1
                    try:
                        next = seq[i]
                    except IndexError:
                        firstvalue = ()  # raised later
                        break

                    newval = item.value + next.value
                    newseq.append(newval, next.type, item.line, item.col)
                    if not firstvalue:
                        firstvalue = (newval, next.type)
                    count += 1

                elif item.type != cssutils.css.CSSComment:
                    newseq.appendItem(item)
                    if not firstvalue:
                        firstvalue = (item.value, item.type)
                    count += 1

                else:
                    newseq.appendItem(item)

                i += 1

            if not firstvalue:
                self._log.error(
                    'CSSValue: Unknown syntax or no value: %r.'
                    % self._valuestr(cssText)
                )
            else:
                # ok and set
                self._setSeq(newseq)
                self.wellformed = wellformed

                if hasattr(self, '_value'):
                    # only in case of CSSPrimitiveValue, else remove!
                    del self._value

                if count == 1:
                    # inherit, primitive or variable
                    if isinstance(
                        firstvalue[0], str
                    ) and 'inherit' == cssutils.helper.normalize(firstvalue[0]):
                        self.__class__ = CSSValue
                        self._cssValueType = CSSValue.CSS_INHERIT
                    elif 'CSSVariable' == firstvalue[1]:
                        self.__class__ = CSSVariable
                        self._value = firstvalue
                        # TODO: remove major hack!
                        self._name = firstvalue[0]._name
                    else:
                        self.__class__ = CSSPrimitiveValue
                        self._value = firstvalue

                elif count > 1:
                    # valuelist
                    self.__class__ = CSSValueList

                    # change items in list to specific type (primitive etc)
                    newseq = self._tempSeq()
                    commalist = []
                    nexttocommalist = False

                    def itemValue(item):
                        "Reserialized simple item.value"
                        if self._prods.STRING == item.type:
                            return cssutils.helper.string(item.value)
                        elif self._prods.URI == item.type:
                            return cssutils.helper.uri(item.value)
                        elif (
                            self._prods.FUNCTION == item.type
                            or 'CSSVariable' == item.type
                        ):
                            return item.value.cssText
                        else:
                            return item.value

                    def saveifcommalist(commalist, newseq):
                        """
                        saves items in commalist to seq and items
                        if anything in there
                        """
                        if commalist:
                            newseq.replace(
                                -1,
                                CSSPrimitiveValue(cssText=''.join(commalist)),
                                CSSPrimitiveValue,
                                newseq[-1].line,
                                newseq[-1].col,
                            )
                            del commalist[:]

                    for i, item in enumerate(self._seq):
                        if issubclass(type(item.value), CSSValue):
                            # set parent of CSSValueList items to the lists
                            # parent
                            item.value.parent = self.parent

                        if item.type in (
                            self._prods.DIMENSION,
                            self._prods.FUNCTION,
                            self._prods.HASH,
                            self._prods.IDENT,
                            self._prods.NUMBER,
                            self._prods.PERCENTAGE,
                            self._prods.STRING,
                            self._prods.URI,
                            self._prods.UNICODE_RANGE,
                            'CSSVariable',
                        ):
                            if nexttocommalist:
                                # wait until complete
                                commalist.append(itemValue(item))
                            else:
                                saveifcommalist(commalist, newseq)
                                # append new item
                                if hasattr(item.value, 'cssText'):
                                    newseq.append(
                                        item.value,
                                        item.value.__class__,
                                        item.line,
                                        item.col,
                                    )

                                else:
                                    newseq.append(
                                        CSSPrimitiveValue(itemValue(item)),
                                        CSSPrimitiveValue,
                                        item.line,
                                        item.col,
                                    )

                            nexttocommalist = False

                        elif ',' == item.value:
                            if not commalist:
                                # save last item to commalist
                                commalist.append(itemValue(self._seq[i - 1]))
                            commalist.append(',')
                            nexttocommalist = True

                        else:
                            if nexttocommalist:
                                commalist.append(item.value.cssText)
                            else:
                                newseq.appendItem(item)

                    saveifcommalist(commalist, newseq)
                    self._setSeq(newseq)

                else:
                    # should not happen...
                    self.__class__ = CSSValue
                    self._cssValueType = CSSValue.CSS_CUSTOM

    cssText = property(
        lambda self: cssutils.ser.do_css_CSSValue(self),
        _setCssText,
        doc="A string representation of the current value.",
    )

    cssValueType = property(
        lambda self: self._cssValueType,
        doc="A (readonly) code defining the type of the value.",
    )

    cssValueTypeString = property(
        lambda self: CSSValue._typestrings.get(self.cssValueType, None),
        doc="(readonly) Name of cssValueType.",
    )


class CSSPrimitiveValue(CSSValue):
    """Represents a single CSS Value.  May be used to determine the value of a
    specific style property currently set in a block or to set a specific
    style property explicitly within the block. Might be obtained from the
    getPropertyCSSValue method of CSSStyleDeclaration.

    Conversions are allowed between absolute values (from millimeters to
    centimeters, from degrees to radians, and so on) but not between
    relative values. (For example, a pixel value cannot be converted to a
    centimeter value.) Percentage values can't be converted since they are
    relative to the parent value (or another property value). There is one
    exception for color percentage values: since a color percentage value
    is relative to the range 0-255, a color percentage value can be
    converted to a number; (see also the RGBColor interface).
    """

    # constant: type of this CSSValue class
    cssValueType = CSSValue.CSS_PRIMITIVE_VALUE

    __types = cssutils.cssproductions.CSSProductions

    # An integer indicating which type of unit applies to the value.
    CSS_UNKNOWN = 0  # only obtainable via cssText
    CSS_NUMBER = 1
    CSS_PERCENTAGE = 2
    CSS_EMS = 3
    CSS_EXS = 4
    CSS_PX = 5
    CSS_CM = 6
    CSS_MM = 7
    CSS_IN = 8
    CSS_PT = 9
    CSS_PC = 10
    CSS_DEG = 11
    CSS_RAD = 12
    CSS_GRAD = 13
    CSS_MS = 14
    CSS_S = 15
    CSS_HZ = 16
    CSS_KHZ = 17
    CSS_DIMENSION = 18
    CSS_STRING = 19
    CSS_URI = 20
    CSS_IDENT = 21
    CSS_ATTR = 22
    CSS_COUNTER = 23
    CSS_RECT = 24
    CSS_RGBCOLOR = 25
    # NOT OFFICIAL:
    CSS_RGBACOLOR = 26
    CSS_UNICODE_RANGE = 27

    _floattypes = (
        CSS_NUMBER,
        CSS_PERCENTAGE,
        CSS_EMS,
        CSS_EXS,
        CSS_PX,
        CSS_CM,
        CSS_MM,
        CSS_IN,
        CSS_PT,
        CSS_PC,
        CSS_DEG,
        CSS_RAD,
        CSS_GRAD,
        CSS_MS,
        CSS_S,
        CSS_HZ,
        CSS_KHZ,
        CSS_DIMENSION,
    )
    _stringtypes = (CSS_ATTR, CSS_IDENT, CSS_STRING, CSS_URI)
    _countertypes = (CSS_COUNTER,)
    _recttypes = (CSS_RECT,)
    _rbgtypes = (CSS_RGBCOLOR, CSS_RGBACOLOR)
    _lengthtypes = (
        CSS_NUMBER,
        CSS_EMS,
        CSS_EXS,
        CSS_PX,
        CSS_CM,
        CSS_MM,
        CSS_IN,
        CSS_PT,
        CSS_PC,
    )

    # oldtype: newType: converterfunc
    _converter = {
        # cm <-> mm <-> in, 1 inch is equal to 2.54 centimeters.
        # pt <-> pc, the points used by CSS 2.1 are equal to 1/72nd of an inch.
        # pc: picas - 1 pica is equal to 12 points
        (CSS_CM, CSS_MM): lambda x: x * 10,
        (CSS_MM, CSS_CM): lambda x: x / 10,
        (CSS_PT, CSS_PC): lambda x: x * 12,
        (CSS_PC, CSS_PT): lambda x: x / 12,
        (CSS_CM, CSS_IN): lambda x: x / 2.54,
        (CSS_IN, CSS_CM): lambda x: x * 2.54,
        (CSS_MM, CSS_IN): lambda x: x / 25.4,
        (CSS_IN, CSS_MM): lambda x: x * 25.4,
        (CSS_IN, CSS_PT): lambda x: x / 72,
        (CSS_PT, CSS_IN): lambda x: x * 72,
        (CSS_CM, CSS_PT): lambda x: x / 2.54 / 72,
        (CSS_PT, CSS_CM): lambda x: x * 72 * 2.54,
        (CSS_MM, CSS_PT): lambda x: x / 25.4 / 72,
        (CSS_PT, CSS_MM): lambda x: x * 72 * 25.4,
        (CSS_IN, CSS_PC): lambda x: x / 72 / 12,
        (CSS_PC, CSS_IN): lambda x: x * 12 * 72,
        (CSS_CM, CSS_PC): lambda x: x / 2.54 / 72 / 12,
        (CSS_PC, CSS_CM): lambda x: x * 12 * 72 * 2.54,
        (CSS_MM, CSS_PC): lambda x: x / 25.4 / 72 / 12,
        (CSS_PC, CSS_MM): lambda x: x * 12 * 72 * 25.4,
        # hz <-> khz
        (CSS_KHZ, CSS_HZ): lambda x: x * 1000,
        (CSS_HZ, CSS_KHZ): lambda x: x / 1000,
        # s <-> ms
        (CSS_S, CSS_MS): lambda x: x * 1000,
        (CSS_MS, CSS_S): lambda x: x / 1000,
        (CSS_RAD, CSS_DEG): lambda x: math.degrees(x),
        (CSS_DEG, CSS_RAD): lambda x: math.radians(x),
        # TODO: convert grad <-> deg or rad
        # (CSS_RAD, CSS_GRAD): lambda x: math.degrees(x),
        # (CSS_DEG, CSS_GRAD): lambda x: math.radians(x),
        # (CSS_GRAD, CSS_RAD): lambda x: math.radians(x),
        # (CSS_GRAD, CSS_DEG): lambda x: math.radians(x)
    }

    def __init__(self, cssText=None, parent=None, readonly=False):
        """See CSSPrimitiveValue.__init__()"""
        super().__init__(cssText=cssText, parent=parent, readonly=readonly)

    def __str__(self):
        return (
            "<cssutils.css.{} object primitiveType={} cssText={!r} at 0x{:x}>".format(
                self.__class__.__name__,
                self.primitiveTypeString,
                self.cssText,
                id(self),
            )
        )

    _unitnames = [
        'CSS_UNKNOWN',
        'CSS_NUMBER',
        'CSS_PERCENTAGE',
        'CSS_EMS',
        'CSS_EXS',
        'CSS_PX',
        'CSS_CM',
        'CSS_MM',
        'CSS_IN',
        'CSS_PT',
        'CSS_PC',
        'CSS_DEG',
        'CSS_RAD',
        'CSS_GRAD',
        'CSS_MS',
        'CSS_S',
        'CSS_HZ',
        'CSS_KHZ',
        'CSS_DIMENSION',
        'CSS_STRING',
        'CSS_URI',
        'CSS_IDENT',
        'CSS_ATTR',
        'CSS_COUNTER',
        'CSS_RECT',
        'CSS_RGBCOLOR',
        'CSS_RGBACOLOR',
        'CSS_UNICODE_RANGE',
    ]

    _reNumDim = re.compile(r'([+-]?\d*\.\d+|[+-]?\d+)(.*)$', re.I | re.U | re.X)

    def _unitDIMENSION(value):
        """Check val for dimension name."""
        units = {
            'em': 'CSS_EMS',
            'ex': 'CSS_EXS',
            'px': 'CSS_PX',
            'cm': 'CSS_CM',
            'mm': 'CSS_MM',
            'in': 'CSS_IN',
            'pt': 'CSS_PT',
            'pc': 'CSS_PC',
            'deg': 'CSS_DEG',
            'rad': 'CSS_RAD',
            'grad': 'CSS_GRAD',
            'ms': 'CSS_MS',
            's': 'CSS_S',
            'hz': 'CSS_HZ',
            'khz': 'CSS_KHZ',
        }
        val, dim = CSSPrimitiveValue._reNumDim.findall(
            cssutils.helper.normalize(value)
        )[0]
        return units.get(dim, 'CSS_DIMENSION')

    def _unitFUNCTION(value):
        """Check val for function name."""
        units = {
            'attr(': 'CSS_ATTR',
            'counter(': 'CSS_COUNTER',
            'rect(': 'CSS_RECT',
            'rgb(': 'CSS_RGBCOLOR',
            'rgba(': 'CSS_RGBACOLOR',
        }
        return units.get(
            re.findall(r'^(.*?\()', cssutils.helper.normalize(value.cssText), re.U)[0],
            'CSS_UNKNOWN',
        )

    __unitbytype = {
        __types.NUMBER: 'CSS_NUMBER',
        __types.PERCENTAGE: 'CSS_PERCENTAGE',
        __types.STRING: 'CSS_STRING',
        __types.UNICODE_RANGE: 'CSS_UNICODE_RANGE',
        __types.URI: 'CSS_URI',
        __types.IDENT: 'CSS_IDENT',
        __types.HASH: 'CSS_RGBCOLOR',
        __types.DIMENSION: _unitDIMENSION,
        __types.FUNCTION: _unitFUNCTION,
    }

    def __set_primitiveType(self):
        """primitiveType is readonly but is set lazy if accessed"""
        # TODO: check unary and font-family STRING a, b, "c"
        val, type_ = self._value
        # try get by type_
        pt = self.__unitbytype.get(type_, 'CSS_UNKNOWN')
        if callable(pt):
            # multiple options, check value too
            pt = pt(val)
        self._primitiveType = getattr(self, pt)

    def _getPrimitiveType(self):
        if not hasattr(self, '_primitivetype'):
            self.__set_primitiveType()
        return self._primitiveType

    primitiveType = property(
        _getPrimitiveType,
        doc="(readonly) The type of the value as defined "
        "by the constants in this class.",
    )

    def _getPrimitiveTypeString(self):
        return self._unitnames[self.primitiveType]

    primitiveTypeString = property(
        _getPrimitiveTypeString, doc="Name of primitive type of this value."
    )

    def _getCSSPrimitiveTypeString(self, type):
        "get TypeString by given type which may be unknown, used by setters"
        try:
            return self._unitnames[type]
        except (IndexError, TypeError):
            return '%r (UNKNOWN TYPE)' % type

    def _getNumDim(self, value=None):
        "Split self._value in numerical and dimension part."
        if value is None:
            value = cssutils.helper.normalize(self._value[0])

        try:
            val, dim = CSSPrimitiveValue._reNumDim.findall(value)[0]
        except IndexError:
            val, dim = value, ''
        try:
            val = float(val)
            if val == int(val):
                val = int(val)
        except ValueError:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue: No float value %r' % self._value[0]
            )

        return val, dim

    def getFloatValue(self, unitType=None):
        """(DOM) This method is used to get a float value in a
        specified unit. If this CSS value doesn't contain a float value
        or can't be converted into the specified unit, a DOMException
        is raised.

        :param unitType:
            to get the float value. The unit code can only be a float unit type
            (i.e. CSS_NUMBER, CSS_PERCENTAGE, CSS_EMS, CSS_EXS, CSS_PX, CSS_CM,
            CSS_MM, CSS_IN, CSS_PT, CSS_PC, CSS_DEG, CSS_RAD, CSS_GRAD, CSS_MS,
            CSS_S, CSS_HZ, CSS_KHZ, CSS_DIMENSION) or None in which case
            the current dimension is used.

        :returns:
            not necessarily a float but some cases just an integer
            e.g. if the value is ``1px`` it return ``1`` and **not** ``1.0``

            Conversions might return strange values like 1.000000000001
        """
        if unitType is not None and unitType not in self._floattypes:
            raise xml.dom.InvalidAccessErr('unitType Parameter is not a float type')

        val, dim = self._getNumDim()

        if unitType is not None and self.primitiveType != unitType:
            # convert if needed
            try:
                val = self._converter[self.primitiveType, unitType](val)
            except KeyError:
                raise xml.dom.InvalidAccessErr(
                    'CSSPrimitiveValue: Cannot coerce primitiveType %r to %r'
                    % (
                        self.primitiveTypeString,
                        self._getCSSPrimitiveTypeString(unitType),
                    )
                )

        if val == int(val):
            val = int(val)

        return val

    def setFloatValue(self, unitType, floatValue):
        """(DOM) A method to set the float value with a specified unit.
        If the property attached with this value can not accept the
        specified unit or the float value, the value will be unchanged and
        a DOMException will be raised.

        :param unitType:
            a unit code as defined above. The unit code can only be a float
            unit type
        :param floatValue:
            the new float value which does not have to be a float value but
            may simple be an int e.g. if setting::

                setFloatValue(CSS_PX, 1)

        :exceptions:
            - :exc:`~xml.dom.InvalidAccessErr`:
              Raised if the attached property doesn't
              support the float value or the unit type.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this property is readonly.
        """
        self._checkReadonly()
        if unitType not in self._floattypes:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue: unitType %r is not a float type'
                % self._getCSSPrimitiveTypeString(unitType)
            )
        try:
            val = float(floatValue)
        except ValueError:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue: floatValue %r is not a float' % floatValue
            )

        oldval, dim = self._getNumDim()
        if self.primitiveType != unitType:
            # convert if possible
            try:
                val = self._converter[unitType, self.primitiveType](val)
            except KeyError:
                raise xml.dom.InvalidAccessErr(
                    'CSSPrimitiveValue: Cannot coerce primitiveType %r to %r'
                    % (
                        self.primitiveTypeString,
                        self._getCSSPrimitiveTypeString(unitType),
                    )
                )

        if val == int(val):
            val = int(val)

        self.cssText = f'{val}{dim}'

    def getStringValue(self):
        """(DOM) This method is used to get the string value. If the
        CSS value doesn't contain a string value, a DOMException is raised.

        Some properties (like 'font-family' or 'voice-family')
        convert a whitespace separated list of idents to a string.

        Only the actual value is returned so e.g. all the following return the
        actual value ``a``: url(a), attr(a), "a", 'a'
        """
        if self.primitiveType not in self._stringtypes:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue %r is not a string type' % self.primitiveTypeString
            )

        if CSSPrimitiveValue.CSS_ATTR == self.primitiveType:
            return self._value[0].cssText[5:-1]
        else:
            return self._value[0]

    def setStringValue(self, stringType, stringValue):
        """(DOM) A method to set the string value with the specified
        unit. If the property attached to this value can't accept the
        specified unit or the string value, the value will be unchanged and
        a DOMException will be raised.

        :param stringType:
            a string code as defined above. The string code can only be a
            string unit type (i.e. CSS_STRING, CSS_URI, CSS_IDENT, and
            CSS_ATTR).
        :param stringValue:
            the new string value
            Only the actual value is expected so for (CSS_URI, "a") the
            new value will be ``url(a)``. For (CSS_STRING, "'a'")
            the new value will be ``"\\'a\\'"`` as the surrounding ``'`` are
            not part of the string value

        :exceptions:
            - :exc:`~xml.dom.InvalidAccessErr`:
              Raised if the CSS value doesn't contain a
              string value or if the string value can't be converted into
              the specified unit.

            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this property is readonly.
        """
        self._checkReadonly()
        # self not stringType
        if self.primitiveType not in self._stringtypes:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue %r is not a string type' % self.primitiveTypeString
            )
        # given stringType is no StringType
        if stringType not in self._stringtypes:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue: stringType %s is not a string type'
                % self._getCSSPrimitiveTypeString(stringType)
            )

        if self._primitiveType != stringType:
            raise xml.dom.InvalidAccessErr(
                'CSSPrimitiveValue: Cannot coerce primitiveType %r to %r'
                % (
                    self.primitiveTypeString,
                    self._getCSSPrimitiveTypeString(stringType),
                )
            )

        if CSSPrimitiveValue.CSS_STRING == self._primitiveType:
            self.cssText = cssutils.helper.string(stringValue)
        elif CSSPrimitiveValue.CSS_URI == self._primitiveType:
            self.cssText = cssutils.helper.uri(stringValue)
        elif CSSPrimitiveValue.CSS_ATTR == self._primitiveType:
            self.cssText = 'attr(%s)' % stringValue
        else:
            self.cssText = stringValue
        self._primitiveType = stringType

    def getCounterValue(self):
        """(DOM) This method is used to get the Counter value. If
        this CSS value doesn't contain a counter value, a DOMException
        is raised. Modification to the corresponding style property
        can be achieved using the Counter interface.

        **Not implemented.**
        """
        if not self.CSS_COUNTER == self.primitiveType:
            raise xml.dom.InvalidAccessErr('Value is not a counter type')
        # TODO: use Counter class
        raise NotImplementedError()

    def getRGBColorValue(self):
        """(DOM) This method is used to get the RGB color. If this
        CSS value doesn't contain a RGB color value, a DOMException
        is raised. Modification to the corresponding style property
        can be achieved using the RGBColor interface.
        """
        if self.primitiveType not in self._rbgtypes:
            raise xml.dom.InvalidAccessErr('Value is not a RGBColor value')
        return RGBColor(self._value[0])

    def getRectValue(self):
        """(DOM) This method is used to get the Rect value. If this CSS
        value doesn't contain a rect value, a DOMException is raised.
        Modification to the corresponding style property can be achieved
        using the Rect interface.

        **Not implemented.**
        """
        if self.primitiveType not in self._recttypes:
            raise xml.dom.InvalidAccessErr('value is not a Rect value')
        # TODO: use Rect class
        raise NotImplementedError()

    def _getCssText(self):
        """Overwrites CSSValue."""
        return cssutils.ser.do_css_CSSPrimitiveValue(self)

    def _setCssText(self, cssText):
        """Use CSSValue."""
        return super()._setCssText(cssText)

    cssText = property(
        _getCssText, _setCssText, doc="A string representation of the current value."
    )


class CSSValueList(CSSValue):
    """The CSSValueList interface provides the abstraction of an ordered
    collection of CSS values.

    Some properties allow an empty list into their syntax. In that case,
    these properties take the none identifier. So, an empty list means
    that the property has the value none.

    The items in the CSSValueList are accessible via an integral index,
    starting from 0.
    """

    cssValueType = CSSValue.CSS_VALUE_LIST

    def __init__(self, cssText=None, parent=None, readonly=False):
        """Init a new CSSValueList"""
        super().__init__(cssText=cssText, parent=parent, readonly=readonly)
        self._items = []

    def __iter__(self):
        "CSSValueList is iterable."
        for item in self.__items():
            yield item.value

    def __str__(self):
        return (
            "<cssutils.css.%s object cssValueType=%r cssText=%r length=%r "
            "at 0x%x>"
            % (
                self.__class__.__name__,
                self.cssValueTypeString,
                self.cssText,
                self.length,
                id(self),
            )
        )

    def __items(self):
        return [item for item in self._seq if isinstance(item.value, CSSValue)]

    def item(self, index):
        """(DOM) Retrieve a CSSValue by ordinal `index`. The
        order in this collection represents the order of the values in the
        CSS style property. If `index` is greater than or equal to the number
        of values in the list, this returns ``None``.
        """
        try:
            return self.__items()[index].value
        except IndexError:
            return None

    length = property(
        lambda self: len(self.__items()),
        doc="(DOM attribute) The number of CSSValues in the " "list.",
    )


class CSSFunction(CSSPrimitiveValue):
    """A CSS function value like rect() etc."""

    _functionName = 'CSSFunction'
    primitiveType = CSSPrimitiveValue.CSS_UNKNOWN

    def __init__(self, cssText=None, parent=None, readonly=False):
        """
        Init a new CSSFunction

        :param cssText:
            the parsable cssText of the value
        :param readonly:
            defaults to False
        """
        super().__init__(parent=parent)
        self._funcType = None
        self.valid = False
        self.wellformed = False
        if cssText is not None:
            self.cssText = cssText
        self._readonly = readonly

    def _productiondefinition(self):
        """Return definition used for parsing."""
        types = self._prods  # rename!

        value = Sequence(
            PreDef.unary(),
            Prod(
                name='PrimitiveValue',
                match=lambda t, v: t
                in (
                    types.DIMENSION,
                    types.HASH,
                    types.IDENT,
                    types.NUMBER,
                    types.PERCENTAGE,
                    types.STRING,
                ),
                toSeq=lambda t, tokens: (t[0], CSSPrimitiveValue(t[1])),
            ),
        )
        valueOrFunc = Choice(
            value,
            # FUNC is actually not in spec but used in e.g. Prince
            PreDef.function(
                toSeq=lambda t, tokens: (
                    'FUNCTION',
                    CSSFunction(cssutils.helper.pushtoken(t, tokens)),
                )
            ),
        )
        funcProds = Sequence(
            Prod(
                name='FUNC',
                match=lambda t, v: t == types.FUNCTION,
                toSeq=lambda t, tokens: (t[0], cssutils.helper.normalize(t[1])),
            ),
            Choice(
                Sequence(
                    valueOrFunc,
                    # more values starting with Comma
                    # should use store where colorType is saved to
                    # define min and may, closure?
                    Sequence(PreDef.comma(), valueOrFunc, minmax=lambda: (0, None)),
                    PreDef.funcEnd(stop=True),
                ),
                PreDef.funcEnd(stop=True),
            ),
        )
        return funcProds

    def _setCssText(self, cssText):
        self._checkReadonly()
        # store: colorType, parts
        wellformed, seq, store, unusedtokens = ProdParser().parse(
            cssText, self._functionName, self._productiondefinition(), keepS=True
        )
        if wellformed:
            # combine +/- and following CSSPrimitiveValue, remove S
            newseq = self._tempSeq()
            i, end = 0, len(seq)
            while i < end:
                item = seq[i]
                if item.type == self._prods.S:
                    pass
                elif item.value == '+' or item.value == '-':
                    i += 1
                    next = seq[i]
                    newval = next.value
                    if isinstance(newval, CSSPrimitiveValue):
                        newval.setFloatValue(
                            newval.primitiveType,
                            float(item.value + str(newval.getFloatValue())),
                        )
                        newseq.append(newval, next.type, item.line, item.col)
                    else:
                        # expressions only?
                        newseq.appendItem(item)
                        newseq.appendItem(next)
                else:
                    newseq.appendItem(item)

                i += 1

            self.wellformed = True
            self._setSeq(newseq)
            self._funcType = newseq[0].value

    cssText = property(
        lambda self: cssutils.ser.do_css_FunctionValue(self), _setCssText
    )

    funcType = property(lambda self: self._funcType)


class RGBColor(CSSFunction):
    """A CSS color like RGB, RGBA or a simple value like `#000` or `red`."""

    _functionName = 'Function rgb()'

    def __init__(self, cssText=None, parent=None, readonly=False):
        """
        Init a new RGBColor

        :param cssText:
            the parsable cssText of the value
        :param readonly:
            defaults to False
        """
        super(CSSFunction, self).__init__(parent=parent)
        self._colorType = None
        self.valid = False
        self.wellformed = False
        if cssText is not None:
            try:
                # if it is a Function object
                cssText = cssText.cssText
            except AttributeError:
                pass
            self.cssText = cssText

        self._readonly = readonly

    def __repr__(self):
        return f"cssutils.css.{self.__class__.__name__}({self.cssText!r})"

    def __str__(self):
        return "<cssutils.css.{} object colorType={!r} cssText={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.colorType,
            self.cssText,
            id(self),
        )

    def _setCssText(self, cssText):
        self._checkReadonly()
        types = self._prods  # rename!
        valueProd = Prod(
            name='value',
            match=lambda t, v: t in (types.NUMBER, types.PERCENTAGE),
            toSeq=lambda t, v: (CSSPrimitiveValue, CSSPrimitiveValue(v)),
            toStore='parts',
        )
        # COLOR PRODUCTION
        funccolor = Sequence(
            Prod(
                name='FUNC',
                match=lambda t, v: t == types.FUNCTION
                and cssutils.helper.normalize(v) in ('rgb(', 'rgba(', 'hsl(', 'hsla('),
                toSeq=lambda t, v: (t, v),  # cssutils.helper.normalize(v)),
                toStore='colorType',
            ),
            PreDef.unary(),
            valueProd,
            # 2 or 3 more values starting with Comma
            Sequence(PreDef.comma(), PreDef.unary(), valueProd, minmax=lambda: (2, 3)),
            PreDef.funcEnd(),
        )
        colorprods = Choice(
            funccolor,
            PreDef.hexcolor('colorType'),
            Prod(
                name='named color',
                match=lambda t, v: t == types.IDENT,
                toStore='colorType',
            ),
        )
        # store: colorType, parts
        wellformed, seq, store, unusedtokens = ProdParser().parse(
            cssText, 'RGBColor', colorprods, keepS=True, store={'parts': []}
        )

        if wellformed:
            self.wellformed = True
            if store['colorType'].type == self._prods.HASH:
                self._colorType = 'HEX'
            elif store['colorType'].type == self._prods.IDENT:
                self._colorType = 'Named Color'
            else:
                self._colorType = store['colorType'].value[:-1]
                # self._colorType = \
                # cssutils.helper.normalize(store['colorType'].value)[:-1]

            self._setSeq(seq)

    cssText = property(lambda self: cssutils.ser.do_css_RGBColor(self), _setCssText)

    colorType = property(lambda self: self._colorType)


class CalcValue(CSSFunction):
    """Calc Function"""

    _functionName = 'Function calc()'

    def _productiondefinition(self):
        """Return defintion used for parsing."""
        types = self._prods  # rename!

        def toSeq(t, tokens):
            "Do not normalize function name!"
            return t[0], t[1]

        funcProds = Sequence(
            Prod(name='calc', match=lambda t, v: t == types.FUNCTION, toSeq=toSeq),
            Sequence(
                Choice(
                    Prod(
                        name='nested function',
                        match=lambda t, v: t == self._prods.FUNCTION,
                        toSeq=lambda t, tokens: (
                            CSSFunction._functionName,
                            CSSFunction(cssutils.helper.pushtoken(t, tokens)),
                        ),
                    ),
                    Prod(
                        name='part',
                        match=lambda t, v: v != ')',
                        toSeq=lambda t, tokens: (t[0], t[1]),
                    ),
                ),
                minmax=lambda: (0, None),
            ),
            PreDef.funcEnd(stop=True),
        )
        return funcProds

    def _getCssText(self):
        return cssutils.ser.do_css_CalcValue(self)

    def _setCssText(self, cssText):
        return super()._setCssText(cssText)

    cssText = property(
        _getCssText, _setCssText, doc="A string representation of the current value."
    )


class ExpressionValue(CSSFunction):
    """Special IE only CSSFunction which may contain *anything*.
    Used for expressions and ``alpha(opacity=100)`` currently."""

    _functionName = 'Expression (IE only)'

    def _productiondefinition(self):
        """Return defintion used for parsing."""
        types = self._prods  # rename!

        def toSeq(t, tokens):
            "Do not normalize function name!"
            return t[0], t[1]

        funcProds = Sequence(
            Prod(
                name='expression', match=lambda t, v: t == types.FUNCTION, toSeq=toSeq
            ),
            Sequence(
                Choice(
                    Prod(
                        name='nested function',
                        match=lambda t, v: t == self._prods.FUNCTION,
                        toSeq=lambda t, tokens: (
                            ExpressionValue._functionName,
                            ExpressionValue(cssutils.helper.pushtoken(t, tokens)),
                        ),
                    ),
                    Prod(
                        name='part',
                        match=lambda t, v: v != ')',
                        toSeq=lambda t, tokens: (t[0], t[1]),
                    ),
                ),
                minmax=lambda: (0, None),
            ),
            PreDef.funcEnd(stop=True),
        )
        return funcProds

    def _getCssText(self):
        return cssutils.ser.do_css_ExpressionValue(self)

    def _setCssText(self, cssText):
        # self._log.warn(u'CSSValue: Unoffial and probably invalid MS value used!')
        return super()._setCssText(cssText)

    cssText = property(
        _getCssText, _setCssText, doc="A string representation of the current value."
    )


class CSSVariable(CSSValue):
    """The CSSVariable represents a call to CSS Variable."""

    def __init__(self, cssText=None, parent=None, readonly=False):
        """Init a new CSSVariable.

        :param cssText:
            the parsable cssText of the value, e.g. ``var(x)``
        :param readonly:
            defaults to False
        """
        self._name = None
        super().__init__(cssText=cssText, parent=parent, readonly=readonly)

    def __repr__(self):
        return f"cssutils.css.{self.__class__.__name__}({self.cssText!r})"

    def __str__(self):
        return "<cssutils.css.{} object name={!r} value={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.name,
            self.value,
            id(self),
        )

    def _setCssText(self, cssText):
        self._checkReadonly()

        types = self._prods  # rename!

        funcProds = Sequence(
            Prod(name='var', match=lambda t, v: t == types.FUNCTION),
            PreDef.ident(toStore='ident'),
            PreDef.funcEnd(stop=True),
        )

        # store: name of variable
        store = {'ident': None}
        wellformed, seq, store, unusedtokens = ProdParser().parse(
            cssText, 'CSSVariable', funcProds, keepS=True
        )
        if wellformed:
            self._name = store['ident'].value
            self._setSeq(seq)
            self.wellformed = True

    cssText = property(
        lambda self: cssutils.ser.do_css_CSSVariable(self),
        _setCssText,
        doc="A string representation of the current variable.",
    )

    cssValueType = CSSValue.CSS_VARIABLE

    # TODO: writable? check if var (value) available?
    name = property(lambda self: self._name)

    def _getValue(self):
        "Find contained sheet and @variables there"
        try:
            variables = self.parent.parent.parentRule.parentStyleSheet.variables
        except AttributeError:
            return None
        else:
            try:
                return variables[self.name]
            except KeyError:
                return None

    value = property(_getValue)

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssvalue.py
NikolaBlagojevic/pyrecodes
"""CSSVariablesDeclaration
http://disruptive-innovations.com/zoo/cssvariables/#mozTocId496530
"""
__all__ = ['CSSVariablesDeclaration']

from cssutils.prodparser import PreDef, Prod, ProdParser, Sequence
from cssutils.helper import normalize
from .value import PropertyValue
import cssutils
import itertools


class CSSVariablesDeclaration(cssutils.util._NewBase):
    """The CSSVariablesDeclaration interface represents a single block of
    variable declarations.
    """

    def __init__(self, cssText='', parentRule=None, readonly=False):
        """
        :param cssText:
            Shortcut, sets CSSVariablesDeclaration.cssText
        :param parentRule:
            The CSS rule that contains this declaration block or
            None if this CSSVariablesDeclaration is not attached to a CSSRule.
        :param readonly:
            defaults to False

        Format::

            variableset
                : vardeclaration [ ';' S* vardeclaration ]* S*
                ;

            vardeclaration
                : varname ':' S* term
                ;

            varname
                : IDENT S*
                ;
        """
        super().__init__()
        self._parentRule = parentRule
        self._vars = {}
        if cssText:
            self.cssText = cssText

        self._readonly = readonly

    def __repr__(self):
        return "cssutils.css.{}(cssText={!r})".format(
            self.__class__.__name__, self.cssText
        )

    def __str__(self):
        return "<cssutils.css.{} object length={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.length,
            id(self),
        )

    def __contains__(self, variableName):
        """Check if a variable is in variable declaration block.

        :param variableName:
            a string
        """
        return normalize(variableName) in list(self.keys())

    def __getitem__(self, variableName):
        """Retrieve the value of variable ``variableName`` from this
        declaration.
        """
        return self.getVariableValue(variableName)

    def __setitem__(self, variableName, value):
        self.setVariable(variableName, value)

    def __delitem__(self, variableName):
        return self.removeVariable(variableName)

    def __iter__(self):
        """Iterator of names of set variables."""
        yield from list(self.keys())

    def keys(self):
        """Analoguous to standard dict returns variable names which are set in
        this declaration."""
        return list(self._vars.keys())

    def _getCssText(self):
        """Return serialized property cssText."""
        return cssutils.ser.do_css_CSSVariablesDeclaration(self)

    def _setCssText(self, cssText):
        """Setting this attribute will result in the parsing of the new value
        and resetting of all the properties in the declaration block
        including the removal or addition of properties.

        :exceptions:
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this declaration is readonly or a property is readonly.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.

        Format::

            variableset
            : vardeclaration [ ';' S* vardeclaration ]*
            ;

            vardeclaration
            : varname ':' S* term
            ;

            varname
            : IDENT S*
            ;

            expr
            : [ VARCALL | term ] [ operator [ VARCALL | term ] ]*
            ;

        """
        self._checkReadonly()

        vardeclaration = Sequence(
            PreDef.ident(),
            PreDef.char(':', ':', toSeq=False, optional=True),
            # PreDef.S(toSeq=False, optional=True),
            Prod(
                name='term',
                match=lambda t, v: True,
                toSeq=lambda t, tokens: (
                    'value',
                    PropertyValue(itertools.chain([t], tokens), parent=self),
                ),
            ),
        )
        prods = Sequence(
            vardeclaration,
            Sequence(
                PreDef.S(optional=True),
                PreDef.char(';', ';', toSeq=False, optional=True),
                PreDef.S(optional=True),
                vardeclaration,
                minmax=lambda: (0, None),
            ),
            PreDef.S(optional=True),
            PreDef.char(';', ';', toSeq=False, optional=True),
        )
        # parse
        wellformed, seq, store, notused = ProdParser().parse(
            cssText, 'CSSVariableDeclaration', prods, emptyOk=True
        )
        if wellformed:
            newseq = self._tempSeq()
            newvars = {}

            # seq contains only name: value pairs plus comments etc
            nameitem = None
            for item in seq:
                if 'IDENT' == item.type:
                    nameitem = item
                elif 'value' == item.type:
                    nname = normalize(nameitem.value)
                    if nname in newvars:
                        # replace var with same name
                        for i, it in enumerate(newseq):
                            if normalize(it.value[0]) == nname:
                                newseq.replace(
                                    i,
                                    (nameitem.value, item.value),
                                    'var',
                                    nameitem.line,
                                    nameitem.col,
                                )
                    else:
                        # saved non normalized name for reserialization
                        newseq.append(
                            (nameitem.value, item.value),
                            'var',
                            nameitem.line,
                            nameitem.col,
                        )

                    #                    newseq.append((nameitem.value, item.value),
                    #                                  'var',
                    #                                  nameitem.line, nameitem.col)

                    newvars[nname] = item.value

                else:
                    newseq.appendItem(item)

            self._setSeq(newseq)
            self._vars = newvars
            self.wellformed = True

    cssText = property(
        _getCssText,
        _setCssText,
        doc="(DOM) A parsable textual representation of the declaration "
        "block excluding the surrounding curly braces.",
    )

    def _setParentRule(self, parentRule):
        self._parentRule = parentRule

    parentRule = property(
        lambda self: self._parentRule,
        _setParentRule,
        doc="(DOM) The CSS rule that contains this"
        " declaration block or None if this block"
        " is not attached to a CSSRule.",
    )

    def getVariableValue(self, variableName):
        """Used to retrieve the value of a variable if it has been explicitly
        set within this variable declaration block.

        :param variableName:
            The name of the variable.
        :returns:
            the value of the variable if it has been explicitly set in this
            variable declaration block. Returns the empty string if the
            variable has not been set.
        """
        try:
            return self._vars[normalize(variableName)].cssText
        except KeyError:
            return ''

    def removeVariable(self, variableName):
        """Used to remove a variable if it has been explicitly set within this
        variable declaration block.

        :param variableName:
            The name of the variable.
        :returns:
            the value of the variable if it has been explicitly set for this
            variable declaration block. Returns the empty string if the
            variable has not been set.

        :exceptions:
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this declaration is readonly is readonly.
        """
        normalname = variableName
        try:
            r = self._vars[normalname]
        except KeyError:
            return ''
        else:
            self.seq._readonly = False
            if normalname in self._vars:
                for i, x in enumerate(self.seq):
                    if x.value[0] == variableName:
                        del self.seq[i]
            self.seq._readonly = True
            del self._vars[normalname]

        return r.cssText

    def setVariable(self, variableName, value):
        """Used to set a variable value within this variable declaration block.

        :param variableName:
            The name of the CSS variable.
        :param value:
            The new value of the variable, may also be a PropertyValue object.

        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified value has a syntax error and is
              unparsable.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this declaration is readonly or the property is
              readonly.
        """
        self._checkReadonly()

        # check name
        wellformed, seq, store, unused = ProdParser().parse(
            normalize(variableName), 'variableName', Sequence(PreDef.ident())
        )
        if not wellformed:
            self._log.error(f'Invalid variableName: {variableName!r}: {value!r}')
        else:
            # check value
            if isinstance(value, PropertyValue):
                v = value
            else:
                v = PropertyValue(cssText=value, parent=self)

            if not v.wellformed:
                self._log.error(f'Invalid variable value: {variableName!r}: {value!r}')
            else:
                # update seq
                self.seq._readonly = False

                variableName = normalize(variableName)

                if variableName in self._vars:
                    for i, x in enumerate(self.seq):
                        if x.value[0] == variableName:
                            self.seq.replace(
                                i, [variableName, v], x.type, x.line, x.col
                            )
                            break
                else:
                    self.seq.append([variableName, v], 'var')
                self.seq._readonly = True
                self._vars[variableName] = v

    def item(self, index):
        """Used to retrieve the variables that have been explicitly set in
        this variable declaration block. The order of the variables
        retrieved using this method does not have to be the order in which
        they were set. This method can be used to iterate over all variables
        in this variable declaration block.

        :param index:
            of the variable name to retrieve, negative values behave like
            negative indexes on Python lists, so -1 is the last element

        :returns:
            The name of the variable at this ordinal position. The empty
            string if no variable exists at this position.
        """
        try:
            return list(self.keys())[index]
        except IndexError:
            return ''

    length = property(
        lambda self: len(self._vars),
        doc="The number of variables that have been explicitly set in this"
        " variable declaration block. The range of valid indices is 0"
        " to length-1 inclusive.",
    )

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/cssvariablesdeclaration.py
NikolaBlagojevic/pyrecodes
"""Property is a single CSS property in a CSSStyleDeclaration."""
__all__ = ['Property']

from cssutils.helper import Deprecated
from .value import PropertyValue
import cssutils


class Property(cssutils.util.Base):
    """A CSS property in a StyleDeclaration of a CSSStyleRule (cssutils).

    Format::

        property = name
          : IDENT S*
          ;

        expr = value
          : term [ operator term ]*
          ;
        term
          : unary_operator?
            [ NUMBER S* | PERCENTAGE S* | LENGTH S* | EMS S* | EXS S* |
              ANGLE S* | TIME S* | FREQ S* | function ]
          | STRING S* | IDENT S* | URI S* | hexcolor
          ;
        function
          : FUNCTION S* expr ')' S*
          ;
        /*
         * There is a constraint on the color that it must
         * have either 3 or 6 hex-digits (i.e., [0-9a-fA-F])
         * after the "#"; e.g., "#000" is OK, but "#abcd" is not.
         */
        hexcolor
          : HASH S*
          ;

        prio
          : IMPORTANT_SYM S*
          ;

    """

    def __init__(
        self, name=None, value=None, priority='', _mediaQuery=False, parent=None
    ):
        """
        :param name:
            a property name string (will be normalized)
        :param value:
            a property value string
        :param priority:
            an optional priority string which currently must be u'',
            u'!important' or u'important'
        :param _mediaQuery:
            if ``True`` value is optional (used by MediaQuery)
        :param parent:
            the parent object, normally a
            :class:`cssutils.css.CSSStyleDeclaration`
        """
        super().__init__()
        self.seqs = [[], None, []]
        self.wellformed = False
        self._mediaQuery = _mediaQuery
        self.parent = parent

        self.__nametoken = None
        self._name = ''
        self._literalname = ''
        self.seqs[1] = PropertyValue(parent=self)
        if name:
            self.name = name
            self.propertyValue = value

        self._priority = ''
        self._literalpriority = ''
        if priority:
            self.priority = priority

    def __repr__(self):
        return "cssutils.css.{}(name={!r}, value={!r}, priority={!r})".format(
            self.__class__.__name__,
            self.literalname,
            self.propertyValue.cssText,
            self.priority,
        )

    def __str__(self):
        return "<{}.{} object name={!r} value={!r} priority={!r} valid={!r} at 0x{:x}>".format(
            self.__class__.__module__,
            self.__class__.__name__,
            self.name,
            self.propertyValue.cssText,
            self.priority,
            self.valid,
            id(self),
        )

    def _isValidating(self):
        """Return True if validation is enabled."""
        try:
            return self.parent.validating
        except AttributeError:
            # default (no parent)
            return True

    def _getCssText(self):
        """Return serialized property cssText."""
        return cssutils.ser.do_Property(self)

    def _setCssText(self, cssText):
        """
        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error and
              is unparsable.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if the rule is readonly.
        """
        # check and prepare tokenlists for setting
        tokenizer = self._tokenize2(cssText)
        nametokens = self._tokensupto2(tokenizer, propertynameendonly=True)
        if nametokens:
            wellformed = True

            valuetokens = self._tokensupto2(tokenizer, propertyvalueendonly=True)
            prioritytokens = self._tokensupto2(tokenizer, propertypriorityendonly=True)

            if self._mediaQuery and not valuetokens:
                # MediaQuery may consist of name only
                self.name = nametokens
                self.propertyValue = None
                self.priority = None
                return

            # remove colon from nametokens
            colontoken = nametokens.pop()
            if self._tokenvalue(colontoken) != ':':
                wellformed = False
                self._log.error(
                    'Property: No ":" after name found: %s' % self._valuestr(cssText),
                    colontoken,
                )
            elif not nametokens:
                wellformed = False
                self._log.error(
                    'Property: No property name found: %s' % self._valuestr(cssText),
                    colontoken,
                )

            if valuetokens:
                if self._tokenvalue(valuetokens[-1]) == '!':
                    # priority given, move "!" to prioritytokens
                    prioritytokens.insert(0, valuetokens.pop(-1))
            else:
                wellformed = False
                self._log.error(
                    'Property: No property value found: %s' % self._valuestr(cssText),
                    colontoken,
                )

            if wellformed:
                self.wellformed = True
                self.name = nametokens
                self.propertyValue = valuetokens
                self.priority = prioritytokens

                # also invalid values are set!

                if self._isValidating():
                    self.validate()

        else:
            self._log.error(
                'Property: No property name found: %s' % self._valuestr(cssText)
            )

    cssText = property(
        fget=_getCssText, fset=_setCssText, doc="A parsable textual representation."
    )

    def _setName(self, name):
        """
        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified name has a syntax error and is
              unparsable.
        """
        # for closures: must be a mutable
        new = {'literalname': None, 'wellformed': True}

        def _ident(expected, seq, token, tokenizer=None):
            # name
            if 'name' == expected:
                new['literalname'] = self._tokenvalue(token).lower()
                seq.append(new['literalname'])
                return 'EOF'
            else:
                new['wellformed'] = False
                self._log.error('Property: Unexpected ident.', token)
                return expected

        newseq = []
        wellformed, expected = self._parse(
            expected='name',
            seq=newseq,
            tokenizer=self._tokenize2(name),
            productions={'IDENT': _ident},
        )
        wellformed = wellformed and new['wellformed']

        # post conditions
        # define a token for error logging
        if isinstance(name, list):
            token = name[0]
            self.__nametoken = token
        else:
            token = None

        if not new['literalname']:
            wellformed = False
            self._log.error(
                'Property: No name found: %s' % self._valuestr(name), token=token
            )

        if wellformed:
            self.wellformed = True
            self._literalname = new['literalname']
            self._name = self._normalize(self._literalname)
            self.seqs[0] = newseq

            # validate
            if self._isValidating() and self._name not in cssutils.profile.knownNames:
                # self.valid = False
                self._log.warn(
                    'Property: Unknown Property name.', token=token, neverraise=True
                )
            else:
                pass
        #                self.valid = True
        #                if self.propertyValue:
        #                    self.propertyValue._propertyName = self._name
        #                    #self.valid = self.propertyValue.valid
        else:
            self.wellformed = False

    name = property(lambda self: self._name, _setName, doc="Name of this property.")

    literalname = property(
        lambda self: self._literalname,
        doc="Readonly literal (not normalized) name " "of this property",
    )

    def _setPropertyValue(self, cssText):
        """
        See css.PropertyValue

        :exceptions:
        - :exc:`~xml.dom.SyntaxErr`:
          Raised if the specified CSS string value has a syntax error
          (according to the attached property) or is unparsable.
        - :exc:`~xml.dom.InvalidModificationErr`:
          TODO: Raised if the specified CSS string value represents a different
          type of values than the values allowed by the CSS property.
        """
        if self._mediaQuery and not cssText:
            self.seqs[1] = PropertyValue(parent=self)
        else:
            self.seqs[1].cssText = cssText
            self.wellformed = self.wellformed and self.seqs[1].wellformed

    propertyValue = property(
        lambda self: self.seqs[1],
        _setPropertyValue,
        doc="(cssutils) PropertyValue object of property",
    )

    def _getValue(self):
        if self.propertyValue:
            # value without comments
            return self.propertyValue.value
        else:
            return ''

    def _setValue(self, value):
        self._setPropertyValue(value)

    value = property(
        _getValue, _setValue, doc="The textual value of this Properties propertyValue."
    )

    def _setPriority(self, priority):
        self.priority = priority

    @property
    def priority(self):
        """Priority of this property."""
        return self._priority

    @priority.setter
    def priority(self, priority):  # noqa: C901
        """
        priority
            a string, currently either u'', u'!important' or u'important'

        Format::

            prio
              : IMPORTANT_SYM S*
              ;

            "!"{w}"important"   {return IMPORTANT_SYM;}

        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified priority has a syntax error and is
              unparsable.
              In this case a priority not equal to None, "" or "!{w}important".
              As CSSOM defines CSSStyleDeclaration.getPropertyPriority resulting
              in u'important' this value is also allowed to set a Properties
              priority
        """
        if self._mediaQuery:
            self._priority = ''
            self._literalpriority = ''
            if priority:
                self._log.error('Property: No priority in a MediaQuery - ' 'ignored.')
            return

        if isinstance(priority, str) and 'important' == self._normalize(priority):
            priority = '!%s' % priority

        # for closures: must be a mutable
        new = {'literalpriority': '', 'wellformed': True}

        def _char(expected, seq, token, tokenizer=None):
            # "!"
            val = self._tokenvalue(token)
            if '!' == expected == val:
                seq.append(val)
                return 'important'
            else:
                new['wellformed'] = False
                self._log.error('Property: Unexpected char.', token)
                return expected

        def _ident(expected, seq, token, tokenizer=None):
            # "important"
            val = self._tokenvalue(token)
            if 'important' == expected:
                new['literalpriority'] = val
                seq.append(val)
                return 'EOF'
            else:
                new['wellformed'] = False
                self._log.error('Property: Unexpected ident.', token)
                return expected

        newseq = []
        wellformed, expected = self._parse(
            expected='!',
            seq=newseq,
            tokenizer=self._tokenize2(priority),
            productions={'CHAR': _char, 'IDENT': _ident},
        )
        wellformed = wellformed and new['wellformed']

        # post conditions
        if priority and not new['literalpriority']:
            wellformed = False
            self._log.info('Property: Invalid priority: %s' % self._valuestr(priority))

        if wellformed:
            self.wellformed = self.wellformed and wellformed
            self._literalpriority = new['literalpriority']
            self._priority = self._normalize(self.literalpriority)
            self.seqs[2] = newseq
            # validate priority
            if self._priority not in ('', 'important'):
                self._log.error('Property: No CSS priority value: %s' % self._priority)

    literalpriority = property(
        lambda self: self._literalpriority,
        doc="Readonly literal (not normalized) priority of this property",
    )

    @property
    def parent(self):
        """The Parent Node (normally a CSSStyledeclaration) of this Property"""
        return self._parent

    @parent.setter
    def parent(self, parent):
        self._parent = parent

    def validate(self):  # noqa: C901
        """Validate value against `profiles` which are checked dynamically.
        properties in e.g. @font-face rules are checked against
        ``cssutils.profile.CSS3_FONT_FACE`` only.

        For each of the following cases a message is reported:

        - INVALID (so the property is known but not valid)
            ``ERROR    Property: Invalid value for "{PROFILE-1[/PROFILE-2...]"
            property: ...``

        - VALID but not in given profiles or defaultProfiles
            ``WARNING    Property: Not valid for profile "{PROFILE-X}" but valid
            "{PROFILE-Y}" property: ...``

        - VALID in current profile
            ``DEBUG    Found valid "{PROFILE-1[/PROFILE-2...]" property...``

        - UNKNOWN property
            ``WARNING    Unknown Property name...`` is issued

        so for example::

            cssutils.log.setLevel(logging.DEBUG)
            parser = cssutils.CSSParser()
            s = parser.parseString('''body {
                unknown-property: x;
                color: 4;
                color: rgba(1,2,3,4);
                color: red
            }''')

            # Log output:

            WARNING Property: Unknown Property name. [2:9: unknown-property]
            ERROR   Property: Invalid value for \
                "CSS Color Module Level 3/CSS Level 2.1" property: 4 [3:9: color]
            DEBUG   Property: Found valid \
                "CSS Color Module Level 3" value: rgba(1, 2, 3, 4) [4:9: color]
            DEBUG   Property: Found valid "CSS Level 2.1" value: red [5:9: color]


        and when setting an explicit default profile::

            cssutils.profile.defaultProfiles = cssutils.profile.CSS_LEVEL_2
            s = parser.parseString('''body {
                unknown-property: x;
                color: 4;
                color: rgba(1,2,3,4);
                color: red
            }''')

            # Log output:

            WARNING Property: Unknown Property name. [2:9: unknown-property]
            ERROR   Property: Invalid value for \
                "CSS Color Module Level 3/CSS Level 2.1" property: 4 [3:9: color]
            WARNING Property: Not valid for profile \
                "CSS Level 2.1" but valid "CSS Color Module Level 3" \
                value: rgba(1, 2, 3, 4)  [4:9: color]
            DEBUG   Property: Found valid "CSS Level 2.1" value: red [5:9: color]
        """
        valid = False

        profiles = None
        try:
            # if @font-face use that profile
            rule = self.parent.parentRule
        except AttributeError:
            pass
        else:
            if rule is not None:
                if rule.type == rule.FONT_FACE_RULE:
                    profiles = [cssutils.profile.CSS3_FONT_FACE]
                # TODO: same for @page

        if self.name and self.value:
            # TODO
            # cv = self.propertyValue
            # if cv.cssValueType == cv.CSS_VARIABLE and not cv.value:
            #     # TODO: false alarms too!
            #     cssutils.log.warn(u'No value for variable "%s" found, keeping '
            #                       u'variable.' % cv.name, neverraise=True)

            if self.name in cssutils.profile.knownNames:
                # add valid, matching, validprofiles...
                valid, matching, validprofiles = cssutils.profile.validateWithProfile(
                    self.name, self.value, profiles
                )

                if not valid:
                    self._log.error(
                        'Property: Invalid value for '
                        '"%s" property: %s' % ('/'.join(validprofiles), self.value),
                        token=self.__nametoken,
                        neverraise=True,
                    )

                # TODO: remove logic to profiles!
                elif (
                    valid and not matching
                ):  # (profiles and profiles not in validprofiles):
                    if not profiles:
                        notvalidprofiles = '/'.join(cssutils.profile.defaultProfiles)
                    else:
                        notvalidprofiles = profiles
                    self._log.warn(
                        'Property: Not valid for profile "%s" '
                        'but valid "%s" value: %s '
                        % (notvalidprofiles, '/'.join(validprofiles), self.value),
                        token=self.__nametoken,
                        neverraise=True,
                    )
                    valid = False

                elif valid:
                    self._log.debug(
                        'Property: Found valid "%s" value: %s'
                        % ('/'.join(validprofiles), self.value),
                        token=self.__nametoken,
                        neverraise=True,
                    )

        if self._priority not in ('', 'important'):
            valid = False

        return valid

    valid = property(
        validate,
        doc="Check if value of this property is valid " "in the properties context.",
    )

    @Deprecated('Use ``property.propertyValue`` instead.')
    def _getCSSValue(self):
        return self.propertyValue

    @Deprecated('Use ``property.propertyValue`` instead.')
    def _setCSSValue(self, cssText):
        self._setPropertyValue(cssText)

    cssValue = property(
        _getCSSValue,
        _setCSSValue,
        doc="(DEPRECATED) Use ``property.propertyValue`` instead.",
    )

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/property.py
NikolaBlagojevic/pyrecodes
"""Selector is a single Selector of a CSSStyleRule SelectorList.
Partly implements http://www.w3.org/TR/css3-selectors/.

TODO
    - .contains(selector)
    - .isSubselector(selector)
"""
__all__ = ['Selector']

from cssutils.helper import Deprecated
from cssutils.util import _SimpleNamespaces
import cssutils
import xml.dom


class Selector(cssutils.util.Base2):
    """
    (cssutils) a single selector in a :class:`~cssutils.css.SelectorList`
    of a :class:`~cssutils.css.CSSStyleRule`.

    Format::

        # implemented in SelectorList
        selectors_group
          : selector [ COMMA S* selector ]*
          ;

        selector
          : simple_selector_sequence [ combinator simple_selector_sequence ]*
          ;

        combinator
          /* combinators can be surrounded by white space */
          : PLUS S* | GREATER S* | TILDE S* | S+
          ;

        simple_selector_sequence
          : [ type_selector | universal ]
            [ HASH | class | attrib | pseudo | negation ]*
          | [ HASH | class | attrib | pseudo | negation ]+
          ;

        type_selector
          : [ namespace_prefix ]? element_name
          ;

        namespace_prefix
          : [ IDENT | '*' ]? '|'
          ;

        element_name
          : IDENT
          ;

        universal
          : [ namespace_prefix ]? '*'
          ;

        class
          : '.' IDENT
          ;

        attrib
          : '[' S* [ namespace_prefix ]? IDENT S*
                [ [ PREFIXMATCH |
                    SUFFIXMATCH |
                    SUBSTRINGMATCH |
                    '=' |
                    INCLUDES |
                    DASHMATCH ] S* [ IDENT | STRING ] S*
                ]? ']'
          ;

        pseudo
          /* '::' starts a pseudo-element, ':' a pseudo-class */
          /* Exceptions: :first-line, :first-letter, :before and :after. */
          /* Note that pseudo-elements are restricted to one per selector and */
          /* occur only in the last simple_selector_sequence. */
          : ':' ':'? [ IDENT | functional_pseudo ]
          ;

        functional_pseudo
          : FUNCTION S* expression ')'
          ;

        expression
          /* In CSS3, the expressions are identifiers, strings, */
          /* or of the form "an+b" */
          : [ [ PLUS | '-' | DIMENSION | NUMBER | STRING | IDENT ] S* ]+
          ;

        negation
          : NOT S* negation_arg S* ')'
          ;

        negation_arg
          : type_selector | universal | HASH | class | attrib | pseudo
          ;

    """

    def __init__(self, selectorText=None, parent=None, readonly=False):
        """
        :Parameters:
            selectorText
                initial value of this selector
            parent
                a SelectorList
            readonly
                default to False
        """
        super().__init__()

        self.__namespaces = _SimpleNamespaces(log=self._log)
        self._element = None
        self._parent = parent
        self._specificity = (0, 0, 0, 0)

        if selectorText:
            self.selectorText = selectorText

        self._readonly = readonly

    def __repr__(self):
        if self.__getNamespaces():
            st = (self.selectorText, self._getUsedNamespaces())
        else:
            st = self.selectorText
        return f"cssutils.css.{self.__class__.__name__}(selectorText={st!r})"

    def __str__(self):
        return (
            "<cssutils.css.%s object selectorText=%r specificity=%r"
            " _namespaces=%r at 0x%x>"
            % (
                self.__class__.__name__,
                self.selectorText,
                self.specificity,
                self._getUsedNamespaces(),
                id(self),
            )
        )

    def _getUsedUris(self):
        "Return list of actually used URIs in this Selector."
        uris = set()
        for item in self.seq:
            type_, val = item.type, item.value
            if (
                type_.endswith('-selector')
                or type_ == 'universal'
                and isinstance(val, tuple)
                and val[0] not in (None, '*')
            ):
                uris.add(val[0])
        return uris

    def _getUsedNamespaces(self):
        "Return actually used namespaces only."
        useduris = self._getUsedUris()
        namespaces = _SimpleNamespaces(log=self._log)
        for p, uri in list(self._namespaces.items()):
            if uri in useduris:
                namespaces[p] = uri
        return namespaces

    def __getNamespaces(self):
        "Use own namespaces if not attached to a sheet, else the sheet's ones."
        try:
            return self._parent.parentRule.parentStyleSheet.namespaces
        except AttributeError:
            return self.__namespaces

    _namespaces = property(
        __getNamespaces,
        doc="If this Selector is attached to a "
        "CSSStyleSheet the namespaces of that sheet "
        "are mirrored here. While the Selector (or "
        "parent SelectorList or parentRule(s) of that "
        "are not attached a own dict of {prefix: "
        "namespaceURI} is used.",
    )

    @property
    def element(self):
        """Effective element target of this selector."""
        return self._element

    parent = property(
        lambda self: self._parent,
        doc="(DOM) The SelectorList that contains this Selector "
        "or None if this Selector is not attached to a "
        "SelectorList.",
    )

    def _getSelectorText(self):
        """Return serialized format."""
        return cssutils.ser.do_css_Selector(self)

    def _setSelectorText(self, selectorText):  # noqa: C901
        """
        :param selectorText:
            parsable string or a tuple of (selectorText, dict-of-namespaces).
            Given namespaces are ignored if this object is attached to a
            CSSStyleSheet!

        :exceptions:
            - :exc:`~xml.dom.NamespaceErr`:
              Raised if the specified selector uses an unknown namespace
              prefix.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error
              and is unparsable.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this rule is readonly.
        """
        self._checkReadonly()

        # might be (selectorText, namespaces)
        selectorText, namespaces = self._splitNamespacesOff(selectorText)

        try:
            # uses parent stylesheets namespaces if available,
            # otherwise given ones
            namespaces = self.parent.parentRule.parentStyleSheet.namespaces
        except AttributeError:
            pass
        tokenizer = self._tokenize2(selectorText)
        if not tokenizer:
            self._log.error('Selector: No selectorText given.')
        else:
            # prepare tokenlist:
            #     "*" -> type "universal"
            #     "*"|IDENT + "|" -> combined to "namespace_prefix"
            #     "|" -> type "namespace_prefix"
            #     "." + IDENT -> combined to "class"
            #     ":" + IDENT, ":" + FUNCTION -> pseudo-class
            #     FUNCTION "not(" -> negation
            #     "::" + IDENT, "::" + FUNCTION -> pseudo-element
            tokens = []
            for t in tokenizer:
                typ, val, lin, col = t
                if val == ':' and tokens and self._tokenvalue(tokens[-1]) == ':':
                    # combine ":" and ":"
                    tokens[-1] = (typ, '::', lin, col)

                elif typ == 'IDENT' and tokens and self._tokenvalue(tokens[-1]) == '.':
                    # class: combine to .IDENT
                    tokens[-1] = ('class', '.' + val, lin, col)
                elif (
                    typ == 'IDENT'
                    and tokens
                    and self._tokenvalue(tokens[-1]).startswith(':')
                    and not self._tokenvalue(tokens[-1]).endswith('(')
                ):
                    # pseudo-X: combine to :IDENT or ::IDENT but not ":a(" + "b"
                    if self._tokenvalue(tokens[-1]).startswith('::'):
                        t = 'pseudo-element'
                    else:
                        t = 'pseudo-class'
                    tokens[-1] = (t, self._tokenvalue(tokens[-1]) + val, lin, col)

                elif (
                    typ == 'FUNCTION'
                    and val == 'not('
                    and tokens
                    and ':' == self._tokenvalue(tokens[-1])
                ):
                    tokens[-1] = ('negation', ':' + val, lin, tokens[-1][3])
                elif (
                    typ == 'FUNCTION'
                    and tokens
                    and self._tokenvalue(tokens[-1]).startswith(':')
                ):
                    # pseudo-X: combine to :FUNCTION( or ::FUNCTION(
                    if self._tokenvalue(tokens[-1]).startswith('::'):
                        t = 'pseudo-element'
                    else:
                        t = 'pseudo-class'
                    tokens[-1] = (t, self._tokenvalue(tokens[-1]) + val, lin, col)

                elif (
                    val == '*'
                    and tokens
                    and self._type(tokens[-1]) == 'namespace_prefix'
                    and self._tokenvalue(tokens[-1]).endswith('|')
                ):
                    # combine prefix|*
                    tokens[-1] = (
                        'universal',
                        self._tokenvalue(tokens[-1]) + val,
                        lin,
                        col,
                    )
                elif val == '*':
                    # universal: "*"
                    tokens.append(('universal', val, lin, col))

                elif (
                    val == '|'
                    and tokens
                    and self._type(tokens[-1]) in (self._prods.IDENT, 'universal')
                    and self._tokenvalue(tokens[-1]).find('|') == -1
                ):
                    # namespace_prefix: "IDENT|" or "*|"
                    tokens[-1] = (
                        'namespace_prefix',
                        self._tokenvalue(tokens[-1]) + '|',
                        lin,
                        col,
                    )
                elif val == '|':
                    # namespace_prefix: "|"
                    tokens.append(('namespace_prefix', val, lin, col))

                else:
                    tokens.append(t)

            tokenizer = iter(tokens)

            # for closures: must be a mutable
            new = {
                'context': [''],  # stack of: 'attrib', 'negation', 'pseudo'
                'element': None,
                '_PREFIX': None,
                'specificity': [0, 0, 0, 0],  # mutable, finally a tuple!
                'wellformed': True,
            }
            # used for equality checks and setting of a space combinator
            S = ' '

            def append(seq, val, typ=None, token=None):
                """
                appends to seq

                namespace_prefix, IDENT will be combined to a tuple
                (prefix, name) where prefix might be None, the empty string
                or a prefix.

                Saved are also:
                    - specificity definition: style, id, class/att, type
                    - element: the element this Selector is for
                """
                context = new['context'][-1]
                if token:
                    line, col = token[2], token[3]
                else:
                    line, col = None, None

                if typ == '_PREFIX':
                    # SPECIAL TYPE: save prefix for combination with next
                    new['_PREFIX'] = val[:-1]
                    # handle next time
                    return

                if new['_PREFIX'] is not None:
                    # as saved from before and reset to None
                    prefix, new['_PREFIX'] = new['_PREFIX'], None
                elif typ == 'universal' and '|' in val:
                    # val == *|* or prefix|*
                    prefix, val = val.split('|')
                else:
                    prefix = None

                # namespace
                if (typ.endswith('-selector') or typ == 'universal') and not (
                    'attribute-selector' == typ and not prefix
                ):
                    # att **IS NOT** in default ns
                    if prefix == '*':
                        # *|name: in ANY_NS
                        namespaceURI = cssutils._ANYNS
                    elif prefix is None:
                        # e or *: default namespace with prefix u''
                        # or local-name()
                        namespaceURI = namespaces.get('', None)
                    elif prefix == '':
                        # |name or |*: in no (or the empty) namespace
                        namespaceURI = ''
                    else:
                        # explicit namespace prefix
                        # does not raise KeyError, see _SimpleNamespaces
                        namespaceURI = namespaces[prefix]

                        if namespaceURI is None:
                            new['wellformed'] = False
                            self._log.error(
                                'Selector: No namespaceURI found '
                                'for prefix %r' % prefix,
                                token=token,
                                error=xml.dom.NamespaceErr,
                            )
                            return

                    # val is now (namespaceprefix, name) tuple
                    val = (namespaceURI, val)

                # specificity
                if not context or context == 'negation':
                    if 'id' == typ:
                        new['specificity'][1] += 1
                    elif 'class' == typ or '[' == val:
                        new['specificity'][2] += 1
                    elif typ in (
                        'type-selector',
                        'negation-type-selector',
                        'pseudo-element',
                    ):
                        new['specificity'][3] += 1
                if not context and typ in ('type-selector', 'universal'):
                    # define element
                    new['element'] = val

                seq.append(val, typ, line=line, col=col)

            # expected constants
            simple_selector_sequence = (
                'type_selector universal HASH class ' 'attrib pseudo negation '
            )
            simple_selector_sequence2 = 'HASH class attrib pseudo negation '

            element_name = 'element_name'

            negation_arg = 'type_selector universal HASH class attrib pseudo'
            negationend = ')'

            attname = 'prefix attribute'
            attname2 = 'attribute'
            attcombinator = 'combinator ]'  # optional
            attvalue = 'value'  # optional
            attend = ']'

            expressionstart = 'PLUS - DIMENSION NUMBER STRING IDENT'
            expression = expressionstart + ' )'

            combinator = ' combinator'

            def _COMMENT(expected, seq, token, tokenizer=None):
                "special implementation for comment token"
                append(seq, cssutils.css.CSSComment([token]), 'COMMENT', token=token)
                return expected

            def _S(expected, seq, token, tokenizer=None):
                # S
                context = new['context'][-1]
                if context.startswith('pseudo-'):
                    if seq and seq[-1].value not in '+-':
                        # e.g. x:func(a + b)
                        append(seq, S, 'S', token=token)
                    return expected

                elif context != 'attrib' and 'combinator' in expected:
                    append(seq, S, 'descendant', token=token)
                    return simple_selector_sequence + combinator

                else:
                    return expected

            def _universal(expected, seq, token, tokenizer=None):
                # *|* or prefix|*
                context = new['context'][-1]
                val = self._tokenvalue(token)
                if 'universal' in expected:
                    append(seq, val, 'universal', token=token)

                    if 'negation' == context:
                        return negationend
                    else:
                        return simple_selector_sequence2 + combinator

                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected universal.', token=token)
                    return expected

            def _namespace_prefix(expected, seq, token, tokenizer=None):
                # prefix| => element_name
                # or prefix| => attribute_name if attrib
                context = new['context'][-1]
                val = self._tokenvalue(token)
                if 'attrib' == context and 'prefix' in expected:
                    # [PREFIX|att]
                    append(seq, val, '_PREFIX', token=token)
                    return attname2
                elif 'type_selector' in expected:
                    # PREFIX|*
                    append(seq, val, '_PREFIX', token=token)
                    return element_name
                else:
                    new['wellformed'] = False
                    self._log.error(
                        'Selector: Unexpected namespace prefix.', token=token
                    )
                    return expected

            def _pseudo(expected, seq, token, tokenizer=None):
                # pseudo-class or pseudo-element :a ::a :a( ::a(
                """
                /* '::' starts a pseudo-element, ':' a pseudo-class */
                /* Exceptions: :first-line, :first-letter, :before and
                :after. */
                /* Note that pseudo-elements are restricted to one per selector
                and */
                /* occur only in the last simple_selector_sequence. */
                """
                context = new['context'][-1]
                val, typ = self._tokenvalue(token, normalize=True), self._type(token)
                if 'pseudo' in expected:
                    if val in (':first-line', ':first-letter', ':before', ':after'):
                        # always pseudo-element ???
                        typ = 'pseudo-element'
                    append(seq, val, typ, token=token)

                    if val.endswith('('):
                        # function
                        # "pseudo-" "class" or "element"
                        new['context'].append(typ)
                        return expressionstart
                    elif 'negation' == context:
                        return negationend
                    elif 'pseudo-element' == typ:
                        # only one per element, check at ) also!
                        return combinator
                    else:
                        return simple_selector_sequence2 + combinator

                else:
                    new['wellformed'] = False
                    self._log.error(
                        'Selector: Unexpected start of pseudo.', token=token
                    )
                    return expected

            def _expression(expected, seq, token, tokenizer=None):
                # [ [ PLUS | '-' | DIMENSION | NUMBER | STRING | IDENT ] S* ]+
                context = new['context'][-1]
                val, typ = self._tokenvalue(token), self._type(token)
                if context.startswith('pseudo-'):
                    append(seq, val, typ, token=token)
                    return expression
                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected %s.' % typ, token=token)
                    return expected

            def _attcombinator(expected, seq, token, tokenizer=None):
                # context: attrib
                # PREFIXMATCH | SUFFIXMATCH | SUBSTRINGMATCH | INCLUDES |
                # DASHMATCH
                context = new['context'][-1]
                val, typ = self._tokenvalue(token), self._type(token)
                if 'attrib' == context and 'combinator' in expected:
                    # combinator in attrib
                    append(seq, val, typ.lower(), token=token)
                    return attvalue
                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected %s.' % typ, token=token)
                    return expected

            def _string(expected, seq, token, tokenizer=None):
                # identifier
                context = new['context'][-1]
                typ, val = self._type(token), self._stringtokenvalue(token)

                # context: attrib
                if 'attrib' == context and 'value' in expected:
                    # attrib: [...=VALUE]
                    append(seq, val, typ, token=token)
                    return attend

                # context: pseudo
                elif context.startswith('pseudo-'):
                    # :func(...)
                    append(seq, val, typ, token=token)
                    return expression

                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected STRING.', token=token)
                    return expected

            def _ident(expected, seq, token, tokenizer=None):
                # identifier
                context = new['context'][-1]
                val, typ = self._tokenvalue(token), self._type(token)

                # context: attrib
                if 'attrib' == context and 'attribute' in expected:
                    # attrib: [...|ATT...]
                    append(seq, val, 'attribute-selector', token=token)
                    return attcombinator

                elif 'attrib' == context and 'value' in expected:
                    # attrib: [...=VALUE]
                    append(seq, val, 'attribute-value', token=token)
                    return attend

                # context: negation
                elif 'negation' == context:
                    # negation: (prefix|IDENT)
                    append(seq, val, 'negation-type-selector', token=token)
                    return negationend

                # context: pseudo
                elif context.startswith('pseudo-'):
                    # :func(...)
                    append(seq, val, typ, token=token)
                    return expression

                elif 'type_selector' in expected or element_name == expected:
                    # element name after ns or complete type_selector
                    append(seq, val, 'type-selector', token=token)
                    return simple_selector_sequence2 + combinator

                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected IDENT.', token=token)
                    return expected

            def _class(expected, seq, token, tokenizer=None):
                # .IDENT
                context = new['context'][-1]
                val = self._tokenvalue(token)
                if 'class' in expected:
                    append(seq, val, 'class', token=token)

                    if 'negation' == context:
                        return negationend
                    else:
                        return simple_selector_sequence2 + combinator

                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected class.', token=token)
                    return expected

            def _hash(expected, seq, token, tokenizer=None):
                # #IDENT
                context = new['context'][-1]
                val = self._tokenvalue(token)
                if 'HASH' in expected:
                    append(seq, val, 'id', token=token)

                    if 'negation' == context:
                        return negationend
                    else:
                        return simple_selector_sequence2 + combinator

                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected HASH.', token=token)
                    return expected

            def _char(expected, seq, token, tokenizer=None):
                # + > ~ ) [ ] + -
                context = new['context'][-1]
                val = self._tokenvalue(token)

                # context: attrib
                if ']' == val and 'attrib' == context and ']' in expected:
                    # end of attrib
                    append(seq, val, 'attribute-end', token=token)
                    context = new['context'].pop()  # attrib is done
                    context = new['context'][-1]
                    if 'negation' == context:
                        return negationend
                    else:
                        return simple_selector_sequence2 + combinator

                elif '=' == val and 'attrib' == context and 'combinator' in expected:
                    # combinator in attrib
                    append(seq, val, 'equals', token=token)
                    return attvalue

                # context: negation
                elif ')' == val and 'negation' == context and ')' in expected:
                    # not(negation_arg)"
                    append(seq, val, 'negation-end', token=token)
                    new['context'].pop()  # negation is done
                    context = new['context'][-1]
                    return simple_selector_sequence + combinator

                # context: pseudo (at least one expression)
                elif val in '+-' and context.startswith('pseudo-'):
                    # :func(+ -)"
                    _names = {'+': 'plus', '-': 'minus'}
                    if val == '+' and seq and seq[-1].value == S:
                        seq.replace(-1, val, _names[val])
                    else:
                        append(seq, val, _names[val], token=token)
                    return expression

                elif (
                    ')' == val
                    and context.startswith('pseudo-')
                    and expression == expected
                ):
                    # :func(expression)"
                    append(seq, val, 'function-end', token=token)
                    new['context'].pop()  # pseudo is done
                    if 'pseudo-element' == context:
                        return combinator
                    else:
                        return simple_selector_sequence + combinator

                # context: ROOT
                elif '[' == val and 'attrib' in expected:
                    # start of [attrib]
                    append(seq, val, 'attribute-start', token=token)
                    new['context'].append('attrib')
                    return attname

                elif val in '+>~' and 'combinator' in expected:
                    # no other combinator except S may be following
                    _names = {
                        '>': 'child',
                        '+': 'adjacent-sibling',
                        '~': 'following-sibling',
                    }
                    if seq and seq[-1].value == S:
                        seq.replace(-1, val, _names[val])
                    else:
                        append(seq, val, _names[val], token=token)
                    return simple_selector_sequence

                elif ',' == val:
                    # not a selectorlist
                    new['wellformed'] = False
                    self._log.error(
                        'Selector: Single selector only.',
                        error=xml.dom.InvalidModificationErr,
                        token=token,
                    )
                    return expected

                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected CHAR.', token=token)
                    return expected

            def _negation(expected, seq, token, tokenizer=None):
                # not(
                val = self._tokenvalue(token, normalize=True)
                if 'negation' in expected:
                    new['context'].append('negation')
                    append(seq, val, 'negation-start', token=token)
                    return negation_arg
                else:
                    new['wellformed'] = False
                    self._log.error('Selector: Unexpected negation.', token=token)
                    return expected

            def _atkeyword(expected, seq, token, tokenizer=None):
                "invalidates selector"
                new['wellformed'] = False
                self._log.error('Selector: Unexpected ATKEYWORD.', token=token)
                return expected

            # expected: only|not or mediatype, mediatype, feature, and
            newseq = self._tempSeq()

            wellformed, expected = self._parse(
                expected=simple_selector_sequence,
                seq=newseq,
                tokenizer=tokenizer,
                productions={
                    'CHAR': _char,
                    'class': _class,
                    'HASH': _hash,
                    'STRING': _string,
                    'IDENT': _ident,
                    'namespace_prefix': _namespace_prefix,
                    'negation': _negation,
                    'pseudo-class': _pseudo,
                    'pseudo-element': _pseudo,
                    'universal': _universal,
                    # pseudo
                    'NUMBER': _expression,
                    'DIMENSION': _expression,
                    # attribute
                    'PREFIXMATCH': _attcombinator,
                    'SUFFIXMATCH': _attcombinator,
                    'SUBSTRINGMATCH': _attcombinator,
                    'DASHMATCH': _attcombinator,
                    'INCLUDES': _attcombinator,
                    'S': _S,
                    'COMMENT': _COMMENT,
                    'ATKEYWORD': _atkeyword,
                },
            )
            wellformed = wellformed and new['wellformed']

            # post condition
            if len(new['context']) > 1 or not newseq:
                wellformed = False
                self._log.error(
                    'Selector: Invalid or incomplete selector: %s'
                    % self._valuestr(selectorText)
                )

            if expected == 'element_name':
                wellformed = False
                self._log.error(
                    'Selector: No element name found: %s' % self._valuestr(selectorText)
                )

            if expected == simple_selector_sequence and newseq:
                wellformed = False
                self._log.error(
                    'Selector: Cannot end with combinator: %s'
                    % self._valuestr(selectorText)
                )

            if (
                newseq
                and hasattr(newseq[-1].value, 'strip')
                and newseq[-1].value.strip() == ''
            ):
                del newseq[-1]

            # set
            if wellformed:
                self.__namespaces = namespaces
                self._element = new['element']
                self._specificity = tuple(new['specificity'])
                self._setSeq(newseq)
                # filter that only used ones are kept
                self.__namespaces = self._getUsedNamespaces()

    selectorText = property(
        _getSelectorText,
        _setSelectorText,
        doc="(DOM) The parsable textual representation of " "the selector.",
    )

    specificity = property(
        lambda self: self._specificity,
        doc="""Specificity of this selector (READONLY).
                Tuple of (a, b, c, d) where:

                a
                    presence of style in document, always 0 if not used on a
                    document
                b
                    number of ID selectors
                c
                    number of .class selectors
                d
                    number of Element (type) selectors""",
    )

    wellformed = property(lambda self: bool(len(self.seq)))

    @Deprecated('Use property parent instead')
    def _getParentList(self):
        return self.parent

    parentList = property(_getParentList, doc="DEPRECATED, see property parent instead")

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/selector.py
NikolaBlagojevic/pyrecodes
"""SelectorList is a list of CSS Selector objects.

TODO
    - remove duplicate Selectors. -> CSSOM canonicalize

    - ??? CSS2 gives a special meaning to the comma (,) in selectors.
        However, since it is not known if the comma may acquire other
        meanings in future versions of CSS, the whole statement should be
        ignored if there is an error anywhere in the selector, even though
        the rest of the selector may look reasonable in CSS2.

        Illegal example(s):

        For example, since the "&" is not a valid token in a CSS2 selector,
        a CSS2 user agent must ignore the whole second line, and not set
        the color of H3 to red:
"""
__all__ = ['SelectorList']

from .selector import Selector
import cssutils


class SelectorList(cssutils.util.Base, cssutils.util.ListSeq):
    """A list of :class:`~cssutils.css.Selector` objects
    of a :class:`~cssutils.css.CSSStyleRule`."""

    def __init__(self, selectorText=None, parentRule=None, readonly=False):
        """
        :Parameters:
            selectorText
                parsable list of Selectors
            parentRule
                the parent CSSRule if available
        """
        super().__init__()

        self._parentRule = parentRule

        if selectorText:
            self.selectorText = selectorText

        self._readonly = readonly

    def __repr__(self):
        if self._namespaces:
            st = (self.selectorText, self._namespaces)
        else:
            st = self.selectorText
        return f"cssutils.css.{self.__class__.__name__}(selectorText={st!r})"

    def __str__(self):
        return "<cssutils.css.%s object selectorText=%r _namespaces=%r at " "0x%x>" % (
            self.__class__.__name__,
            self.selectorText,
            self._namespaces,
            id(self),
        )

    def __setitem__(self, index, newSelector):
        """Overwrite ListSeq.__setitem__

        Any duplicate Selectors are **not** removed.
        """
        newSelector = self.__prepareset(newSelector)
        if newSelector:
            self.seq[index] = newSelector

    def __prepareset(self, newSelector, namespaces=None):
        "Used by appendSelector and __setitem__"
        if not namespaces:
            namespaces = {}
        self._checkReadonly()
        if not isinstance(newSelector, Selector):
            newSelector = Selector((newSelector, namespaces), parent=self)
        if newSelector.wellformed:
            newSelector._parent = self  # maybe set twice but must be!
            return newSelector

    def __getNamespaces(self):
        """Use children namespaces if not attached to a sheet, else the sheet's
        ones.
        """
        try:
            return self.parentRule.parentStyleSheet.namespaces
        except AttributeError:
            namespaces = {}
            for selector in self.seq:
                namespaces.update(selector._namespaces)
            return namespaces

    def _getUsedUris(self):
        "Used by CSSStyleSheet to check if @namespace rules are needed"
        uris = set()
        for s in self:
            uris.update(s._getUsedUris())
        return uris

    _namespaces = property(
        __getNamespaces,
        doc="""If this SelectorList is
        attached to a CSSStyleSheet the namespaces of that sheet are mirrored
        here. While the SelectorList (or parentRule(s) are
        not attached the namespaces of all children Selectors are used.""",
    )

    def append(self, newSelector):
        "Same as :meth:`appendSelector`."
        self.appendSelector(newSelector)

    def appendSelector(self, newSelector):
        """
        Append `newSelector` to this list (a string will be converted to a
        :class:`~cssutils.css.Selector`).

        :param newSelector:
            comma-separated list of selectors (as a single string) or a tuple of
            `(newSelector, dict-of-namespaces)`
        :returns: New :class:`~cssutils.css.Selector` or ``None`` if
            `newSelector` is not wellformed.
        :exceptions:
            - :exc:`~xml.dom.NamespaceErr`:
              Raised if the specified selector uses an unknown namespace
              prefix.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error
              and is unparsable.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this rule is readonly.
        """
        self._checkReadonly()

        # might be (selectorText, namespaces)
        newSelector, namespaces = self._splitNamespacesOff(newSelector)
        try:
            # use parent's only if available
            namespaces = self.parentRule.parentStyleSheet.namespaces
        except AttributeError:
            # use already present namespaces plus new given ones
            _namespaces = self._namespaces
            _namespaces.update(namespaces)
            namespaces = _namespaces

        newSelector = self.__prepareset(newSelector, namespaces)
        if newSelector:
            seq = self.seq[:]
            del self.seq[:]
            for s in seq:
                if s.selectorText != newSelector.selectorText:
                    self.seq.append(s)
            self.seq.append(newSelector)
            return newSelector

    def _getSelectorText(self):
        "Return serialized format."
        return cssutils.ser.do_css_SelectorList(self)

    def _setSelectorText(self, selectorText):
        """
        :param selectorText:
            comma-separated list of selectors or a tuple of
            (selectorText, dict-of-namespaces)
        :exceptions:
            - :exc:`~xml.dom.NamespaceErr`:
              Raised if the specified selector uses an unknown namespace
              prefix.
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error
              and is unparsable.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this rule is readonly.
        """
        self._checkReadonly()

        # might be (selectorText, namespaces)
        selectorText, namespaces = self._splitNamespacesOff(selectorText)
        try:
            # use parent's only if available
            namespaces = self.parentRule.parentStyleSheet.namespaces
        except AttributeError:
            pass

        wellformed = True
        tokenizer = self._tokenize2(selectorText)
        newseq = []

        expected = True
        while True:
            # find all upto and including next ",", EOF or nothing
            selectortokens = self._tokensupto2(tokenizer, listseponly=True)
            if selectortokens:
                if self._tokenvalue(selectortokens[-1]) == ',':
                    expected = selectortokens.pop()
                else:
                    expected = None

                selector = Selector((selectortokens, namespaces), parent=self)
                if selector.wellformed:
                    newseq.append(selector)
                else:
                    wellformed = False
                    self._log.error(
                        'SelectorList: Invalid Selector: %s'
                        % self._valuestr(selectortokens)
                    )
            else:
                break

        # post condition
        if ',' == expected:
            wellformed = False
            self._log.error(
                'SelectorList: Cannot end with ",": %r' % self._valuestr(selectorText)
            )
        elif expected:
            wellformed = False
            self._log.error(
                'SelectorList: Unknown Syntax: %r' % self._valuestr(selectorText)
            )
        if wellformed:
            self.seq = newseq

    selectorText = property(
        _getSelectorText,
        _setSelectorText,
        doc="(cssutils) The textual representation of the " "selector for a rule set.",
    )

    length = property(
        lambda self: len(self),
        doc="The number of :class:`~cssutils.css.Selector` " "objects in the list.",
    )

    parentRule = property(
        lambda self: self._parentRule,
        doc="(DOM) The CSS rule that contains this "
        "SelectorList or ``None`` if this SelectorList "
        "is not attached to a CSSRule.",
    )

    wellformed = property(lambda self: bool(len(self.seq)))

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/selectorlist.py
NikolaBlagojevic/pyrecodes
"""Value related classes.

DOM Level 2 CSS CSSValue, CSSPrimitiveValue and CSSValueList are **no longer**
supported and are replaced by these new classes.
"""
__all__ = [
    'PropertyValue',
    'Value',
    'ColorValue',
    'DimensionValue',
    'URIValue',
    'CSSFunction',
    'CSSCalc',
    'CSSVariable',
    'MSValue',
]

from cssutils.prodparser import Choice, PreDef, Prod, ProdParser, Sequence
import cssutils
from cssutils.helper import normalize, pushtoken
import colorsys
import re
import urllib.parse


class PropertyValue(cssutils.util._NewBase):
    """
    An unstructured list like holder for all values defined for a
    :class:`~cssutils.css.Property`. Contains :class:`~cssutils.css.Value`
    or subclass objects. Currently there is no access to the combinators of
    the defined values which might simply be space or comma or slash.

    You may:

    - iterate over all contained Value objects (not the separators like ``,``,
      ``/`` or `` `` though!)
    - get a Value item by index or use ``PropertyValue[index]``
    - find out the number of values defined (unstructured)
    """

    def __init__(self, cssText=None, parent=None, readonly=False):
        """
        :param cssText:
            the parsable cssText of the value
        :param readonly:
            defaults to False
        """
        super().__init__()

        self.parent = parent
        self.wellformed = False

        if cssText is not None:  # may be 0
            if isinstance(cssText, (int, float)):
                cssText = str(cssText)  # if it is a number
            self.cssText = cssText

        self._readonly = readonly

    def __len__(self):
        return len(list(self.__items()))

    def __getitem__(self, index):
        try:
            return list(self.__items())[index]
        except IndexError:
            return None

    def __iter__(self):
        "Generator which iterates over values."
        yield from self.__items()

    def __repr__(self):
        return f"cssutils.css.{self.__class__.__name__}({self.cssText!r})"

    def __str__(self):
        return "<cssutils.css.%s object length=%r cssText=%r at " "0x%x>" % (
            self.__class__.__name__,
            self.length,
            self.cssText,
            id(self),
        )

    def __items(self, seq=None):
        "a generator of Value obects only, no , / or ' '"
        if seq is None:
            seq = self.seq
        return (x.value for x in seq if isinstance(x.value, Value))

    def _setCssText(self, cssText):
        if isinstance(cssText, (int, float)):
            cssText = str(cssText)  # if it is a number
        """
        Format::

            unary_operator
              : '-' | '+'
              ;
            operator
              : '/' S* | ',' S* | /* empty */
              ;
            expr
              : term [ operator term ]*
              ;
            term
              : unary_operator?
                [ NUMBER S* | PERCENTAGE S* | LENGTH S* | EMS S* | EXS S* |
                  ANGLE S* | TIME S* | FREQ S* ]
              | STRING S* | IDENT S* | URI S* | hexcolor | function
              | UNICODE-RANGE S*
              ;
            function
              : FUNCTION S* expr ')' S*
              ;
            /*
             * There is a constraint on the color that it must
             * have either 3 or 6 hex-digits (i.e., [0-9a-fA-F])
             * after the "#"; e.g., "#000" is OK, but "#abcd" is not.
             */
            hexcolor
              : HASH S*
              ;

        :exceptions:
            - :exc:`~xml.dom.SyntaxErr`:
              Raised if the specified CSS string value has a syntax error
              (according to the attached property) or is unparsable.
            - :exc:`~xml.dom.InvalidModificationErr`:
              TODO: Raised if the specified CSS string value represents a
              different type of values than the values allowed by the CSS
              property.
            - :exc:`~xml.dom.NoModificationAllowedErr`:
              Raised if this value is readonly.
        """
        self._checkReadonly()

        # used as operator is , / or S
        nextSor = ',/'
        term = Choice(
            _ColorProd(self, nextSor),
            _DimensionProd(self, nextSor),
            _URIProd(self, nextSor),
            _ValueProd(self, nextSor),
            #                      _Rect(self, nextSor),
            # all other functions
            _CSSVariableProd(self, nextSor),
            _MSValueProd(self, nextSor),
            _CalcValueProd(self, nextSor),
            _CSSFunctionProd(self, nextSor),
        )
        operator = Choice(
            PreDef.S(toSeq=False),
            PreDef.char(
                'comma', ',', toSeq=lambda t, tokens: ('operator', t[1]), optional=True
            ),
            PreDef.char(
                'slash', '/', toSeq=lambda t, tokens: ('operator', t[1]), optional=True
            ),
            optional=True,
        )
        prods = Sequence(
            term,
            Sequence(  # mayEnd this Sequence if whitespace
                operator,
                # TODO: only when setting via other class
                # used by variabledeclaration currently
                PreDef.char('END', ';', stopAndKeep=True, optional=True),
                # TODO: } and !important ends too!
                term,
                minmax=lambda: (0, None),
            ),
        )
        # parse
        ok, seq, store, unused = ProdParser().parse(cssText, 'PropertyValue', prods)
        # must be at least one value!
        ok = ok and len(list(self.__items(seq))) > 0

        for item in seq:
            if hasattr(item.value, 'wellformed') and not item.value.wellformed:
                ok = False
                break

        self.wellformed = ok
        if ok:
            self._setSeq(seq)
        else:
            self._log.error(
                'PropertyValue: Unknown syntax or no value: %s'
                % self._valuestr(cssText)
            )

    cssText = property(
        lambda self: cssutils.ser.do_css_PropertyValue(self),
        _setCssText,
        doc="A string representation of the current value.",
    )

    def item(self, index):
        """
        The value at position `index`. Alternatively simple use
        ``PropertyValue[index]``.

        :param index:
            the parsable cssText of the value
        :exceptions:
            - :exc:`~IndexError`:
              Raised if index if out of bounds
        """
        return self[index]

    length = property(lambda self: len(self), doc="Number of values set.")

    value = property(
        lambda self: cssutils.ser.do_css_PropertyValue(self, valuesOnly=True),
        doc="A string representation of the current value "
        "without any comments used for validation.",
    )


class Value(cssutils.util._NewBase):
    """
    Represents a single CSS value. For now simple values of
    IDENT, STRING, or UNICODE-RANGE values are represented directly
    as Value objects. Other values like e.g. FUNCTIONs are represented by
    subclasses with an extended API.
    """

    IDENT = 'IDENT'
    STRING = 'STRING'
    UNICODE_RANGE = 'UNICODE-RANGE'
    URI = 'URI'

    DIMENSION = 'DIMENSION'
    NUMBER = 'NUMBER'
    PERCENTAGE = 'PERCENTAGE'

    COLOR_VALUE = 'COLOR_VALUE'
    HASH = 'HASH'

    FUNCTION = 'FUNCTION'
    CALC = 'CALC'
    VARIABLE = 'VARIABLE'

    _type = None
    _value = ''

    def __init__(self, cssText=None, parent=None, readonly=False):
        super().__init__()

        self.parent = parent
        self.wellformed = False

        if cssText:
            self.cssText = cssText

    def __repr__(self):
        return f"cssutils.css.{self.__class__.__name__}({self.cssText!r})"

    def __str__(self):
        return (
            "<cssutils.css.{} object type={} value={!r} cssText={!r} at 0x{:x}>".format(
                self.__class__.__name__,
                self.type,
                self.value,
                self.cssText,
                id(self),
            )
        )

    def _setCssText(self, cssText):
        self._checkReadonly()

        prods = Choice(
            PreDef.hexcolor(stop=True),
            PreDef.ident(stop=True),
            PreDef.string(stop=True),
            PreDef.unicode_range(stop=True),
        )
        ok, seq, store, unused = ProdParser().parse(cssText, 'Value', prods)
        self.wellformed = ok
        if ok:
            # only 1 value anyway!
            self._type = seq[0].type
            self._value = seq[0].value

            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_Value(self),
        _setCssText,
        doc='String value of this value.',
    )

    @property
    def type(self):
        """
        Type of this value, for now the production type
        like e.g. `DIMENSION` or `STRING`. All types are
        defined as constants in :class:`~cssutils.css.Value`.
        """
        return self._type

    def _setValue(self, value):
        # TODO: check!
        self._value = value

    value = property(
        lambda self: self._value,
        _setValue,
        doc="Actual value if possible: An int or float or else " " a string",
    )


class ColorValue(Value):
    """
    A color value like rgb(), rgba(), hsl(), hsla() or #rgb, #rrggbb

    TODO: Color Keywords
    """

    from .colors import COLORS

    type = Value.COLOR_VALUE
    # hexcolor, FUNCTION?
    _colorType = None
    _red = 0
    _green = 0
    _blue = 0
    _alpha = 0

    def __str__(self):
        return (
            "<cssutils.css.%s object type=%s value=%r colorType=%r "
            "red=%s blue=%s green=%s alpha=%s at 0x%x>"
            % (
                self.__class__.__name__,
                self.type,
                self.value,
                self.colorType,
                self.red,
                self.green,
                self.blue,
                self.alpha,
                id(self),
            )
        )

    def _setCssText(self, cssText):  # noqa: C901
        self._checkReadonly()
        types = self._prods  # rename!

        component = Choice(
            PreDef.unary(
                toSeq=lambda t, tokens: (
                    t[0],
                    DimensionValue(pushtoken(t, tokens), parent=self),
                )
            ),
            PreDef.number(
                toSeq=lambda t, tokens: (
                    t[0],
                    DimensionValue(pushtoken(t, tokens), parent=self),
                )
            ),
            PreDef.percentage(
                toSeq=lambda t, tokens: (
                    t[0],
                    DimensionValue(pushtoken(t, tokens), parent=self),
                )
            ),
        )
        noalp = Sequence(
            Prod(
                name='FUNCTION',
                match=lambda t, v: t == types.FUNCTION and v in ('rgb(', 'hsl('),
                toSeq=lambda t, tokens: (t[0], normalize(t[1])),
            ),
            component,
            Sequence(PreDef.comma(optional=True), component, minmax=lambda: (2, 2)),
            PreDef.funcEnd(stop=True),
        )
        witha = Sequence(
            Prod(
                name='FUNCTION',
                match=lambda t, v: t == types.FUNCTION and v in ('rgba(', 'hsla('),
                toSeq=lambda t, tokens: (t[0], normalize(t[1])),
            ),
            component,
            Sequence(PreDef.comma(optional=True), component, minmax=lambda: (3, 3)),
            PreDef.funcEnd(stop=True),
        )
        namedcolor = Prod(
            name='Named Color',
            match=lambda t, v: t == 'IDENT'
            and (normalize(v) in list(self.COLORS.keys())),
            stop=True,
        )

        prods = Choice(PreDef.hexcolor(stop=True), namedcolor, noalp, witha)

        ok, seq, store, unused = ProdParser().parse(cssText, self.type, prods)
        self.wellformed = ok
        if ok:
            t, v = seq[0].type, seq[0].value
            if 'IDENT' == t:
                rgba = self.COLORS[normalize(v)]
            if 'HASH' == t:
                if len(v) == 4:
                    # HASH #rgb
                    rgba = (
                        int(2 * v[1], 16),
                        int(2 * v[2], 16),
                        int(2 * v[3], 16),
                        1.0,
                    )
                else:
                    # HASH #rrggbb
                    rgba = (int(v[1:3], 16), int(v[3:5], 16), int(v[5:7], 16), 1.0)

            elif 'FUNCTION' == t:
                functiontype, raw, check = None, [], ''
                HSL = False

                for item in seq:
                    try:
                        type_ = item.value.type
                    except AttributeError:
                        # type of function, e.g. rgb(
                        if item.type == 'FUNCTION':
                            functiontype = item.value
                            HSL = functiontype in ('hsl(', 'hsla(')
                        continue

                    # save components
                    if type_ == Value.NUMBER:
                        raw.append(item.value.value)
                        check += 'N'
                    elif type_ == Value.PERCENTAGE:
                        if HSL:
                            # save as percentage fraction
                            raw.append(item.value.value / 100.0)
                        else:
                            # save as real value of percentage of 255
                            raw.append(int(255 * item.value.value / 100))
                        check += 'P'

                if HSL:
                    # convert to rgb
                    # h is 360 based (circle)
                    h, s, l_ = raw[0] / 360.0, raw[1], raw[2]
                    # ORDER h l s !!!
                    r, g, b = colorsys.hls_to_rgb(h, l_, s)
                    # back to 255 based
                    rgba = [
                        int(round(r * 255)),
                        int(round(g * 255)),
                        int(round(b * 255)),
                    ]

                    if len(raw) > 3:
                        rgba.append(raw[3])

                else:
                    # rgb, rgba
                    rgba = raw

                if len(rgba) < 4:
                    rgba.append(1.0)

                # validate
                checks = {
                    'rgb(': ('NNN', 'PPP'),
                    'rgba(': ('NNNN', 'PPPN'),
                    'hsl(': ('NPP',),
                    'hsla(': ('NPPN',),
                }
                if check not in checks[functiontype]:
                    self._log.error(
                        'ColorValue has invalid %s) parameters: '
                        '%s (N=Number, P=Percentage)' % (functiontype, check)
                    )

            self._colorType = t
            self._red, self._green, self._blue, self._alpha = tuple(rgba)
            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_ColorValue(self),
        _setCssText,
        doc="String value of this value.",
    )

    value = property(
        lambda self: cssutils.ser.do_css_CSSFunction(self, True),
        doc='Same as cssText but without comments.',
    )

    @property
    def type(self):
        """Type is fixed to Value.COLOR_VALUE."""
        return Value.COLOR_VALUE

    def _getName(self):
        for n, v in list(self.COLORS.items()):
            if v == (self.red, self.green, self.blue, self.alpha):
                return n

    colorType = property(
        lambda self: self._colorType,
        doc="IDENT (red), HASH (#f00) or FUNCTION (rgb(255, 0, 0).",
    )

    name = property(
        _getName, doc='Name of the color if known (in ColorValue.COLORS) ' 'else None'
    )

    red = property(lambda self: self._red, doc='red part as integer between 0 and 255')

    @property
    def green(self):
        """green part as integer between 0 and 255"""
        return self._green

    @property
    def blue(self):
        """blue part as integer between 0 and 255"""
        return self._blue

    @property
    def alpha(self):
        """alpha part as float between 0.0 and 1.0"""
        return self._alpha


class DimensionValue(Value):
    """
    A numerical value with an optional dimension like e.g. "px" or "%".

    Covers DIMENSION, PERCENTAGE or NUMBER values.
    """

    __reUnNumDim = re.compile(r'^([+-]?)(\d*\.\d+|\d+)(.*)$', re.I | re.U | re.X)
    _dimension = None
    _sign = None

    def __str__(self):
        return (
            "<cssutils.css.%s object type=%s value=%r dimension=%r cssText=%r at 0x%x>"
            % (
                self.__class__.__name__,
                self.type,
                self.value,
                self.dimension,
                self.cssText,
                id(self),
            )
        )

    def _setCssText(self, cssText):
        self._checkReadonly()

        prods = Sequence(  # PreDef.unary(),
            Choice(
                PreDef.dimension(stop=True),
                PreDef.number(stop=True),
                PreDef.percentage(stop=True),
            )
        )
        ok, seq, store, unused = ProdParser().parse(cssText, 'DimensionValue', prods)
        self.wellformed = ok
        if ok:
            item = seq[0]

            sign, v, d = self.__reUnNumDim.findall(normalize(item.value))[0]
            if '.' in v:
                val = float(sign + v)
            else:
                val = int(sign + v)

            dim = None
            if d:
                dim = d

            self._sign = sign
            self._value = val
            self._dimension = dim
            self._type = item.type

            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_Value(self),
        _setCssText,
        doc="String value of this value including dimension.",
    )

    dimension = property(
        lambda self: self._dimension,  # _setValue,
        doc="Dimension if a DIMENSION or PERCENTAGE value, " "else None",
    )


class URIValue(Value):
    """
    An URI value like ``url(example.png)``.
    """

    _type = Value.URI
    _uri = Value._value

    def __str__(self):
        return "<cssutils.css.{} object type={} value={!r} uri={!r} cssText={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.type,
            self.value,
            self.uri,
            self.cssText,
            id(self),
        )

    def _setCssText(self, cssText):
        self._checkReadonly()

        prods = Sequence(PreDef.uri(stop=True))

        ok, seq, store, unused = ProdParser().parse(cssText, 'URIValue', prods)
        self.wellformed = ok
        if ok:
            # only 1 value only anyway
            self._type = seq[0].type
            self._value = seq[0].value

            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_Value(self),
        _setCssText,
        doc='String value of this value.',
    )

    def _setUri(self, uri):
        # TODO: check?
        self._value = uri

    uri = property(
        lambda self: self._value,
        _setUri,
        doc="Actual URL without delimiters or the empty string",
    )

    def absoluteUri(self):
        """Actual URL, made absolute if possible, else same as `uri`."""
        # Ancestry: PropertyValue, Property, CSSStyleDeclaration, CSSStyleRule,
        # CSSStyleSheet
        try:
            # TODO: better way?
            styleSheet = self.parent.parent.parent.parentRule.parentStyleSheet
        except AttributeError:
            return self.uri
        else:
            return urllib.parse.urljoin(styleSheet.href, self.uri)

    absoluteUri = property(absoluteUri, doc=absoluteUri.__doc__)


class CSSFunction(Value):
    """
    A function value.
    """

    _functionName = 'Function'

    def _productions(self):
        """Return definition used for parsing."""
        types = self._prods  # rename!

        itemProd = Choice(
            _ColorProd(self),
            _DimensionProd(self),
            _URIProd(self),
            _ValueProd(self),
            _CalcValueProd(self),
            _CSSVariableProd(self),
            _CSSFunctionProd(self),
        )
        funcProds = Sequence(
            Prod(
                name='FUNCTION',
                match=lambda t, v: t == types.FUNCTION,
                toSeq=lambda t, tokens: (t[0], normalize(t[1])),
            ),
            Choice(
                Sequence(
                    itemProd,
                    Sequence(
                        PreDef.comma(optional=True), itemProd, minmax=lambda: (0, None)
                    ),
                    PreDef.funcEnd(stop=True),
                ),
                PreDef.funcEnd(stop=True),
            ),
        )
        return funcProds

    def _setCssText(self, cssText):
        self._checkReadonly()
        ok, seq, store, unused = ProdParser().parse(
            cssText, self.type, self._productions()
        )
        self.wellformed = ok
        if ok:
            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_CSSFunction(self),
        _setCssText,
        doc="String value of this value.",
    )

    value = property(
        lambda self: cssutils.ser.do_css_CSSFunction(self, True),
        doc='Same as cssText but without comments.',
    )

    type = property(lambda self: Value.FUNCTION, doc="Type is fixed to Value.FUNCTION.")


class MSValue(CSSFunction):
    """An IE specific Microsoft only function value which is much looser
    in what is syntactically allowed."""

    _functionName = 'MSValue'

    def _productions(self):
        """Return definition used for parsing."""
        types = self._prods  # rename!

        func = Prod(
            name='MSValue-Sub',
            match=lambda t, v: t == self._prods.FUNCTION,
            toSeq=lambda t, tokens: (
                MSValue._functionName,
                MSValue(pushtoken(t, tokens), parent=self),
            ),
        )

        funcProds = Sequence(
            Prod(
                name='FUNCTION',
                match=lambda t, v: t == types.FUNCTION,
                toSeq=lambda t, tokens: (t[0], t[1]),
            ),
            Sequence(
                Choice(
                    _ColorProd(self),
                    _DimensionProd(self),
                    _URIProd(self),
                    _ValueProd(self),
                    _MSValueProd(self),
                    # _CalcValueProd(self),
                    _CSSVariableProd(self),
                    func,
                    # _CSSFunctionProd(self),
                    Prod(
                        name='MSValuePart',
                        match=lambda t, v: v != ')',
                        toSeq=lambda t, tokens: (t[0], t[1]),
                    ),
                ),
                minmax=lambda: (0, None),
            ),
            PreDef.funcEnd(stop=True),
        )
        return funcProds

    def _setCssText(self, cssText):
        super()._setCssText(cssText)

    cssText = property(
        lambda self: cssutils.ser.do_css_MSValue(self),
        _setCssText,
        doc="String value of this value.",
    )


class CSSCalc(CSSFunction):
    """The CSSCalc function represents a CSS calc() function.

    No further API is provided. For multiplication and division no check
    if one operand is a NUMBER is made.
    """

    _functionName = 'CSSCalc'

    def __str__(self):
        return "<cssutils.css.{} object at 0x{:x}>".format(
            self.__class__.__name__, id(self)
        )

    def _setCssText(self, cssText):
        self._checkReadonly()

        types = self._prods  # rename!

        _operator = Choice(
            Prod(
                name='Operator */',
                match=lambda t, v: v in '*/',
                toSeq=lambda t, tokens: (t[0], t[1]),
            ),
            Sequence(
                PreDef.S(),
                Choice(
                    Sequence(
                        Prod(
                            name='Operator */',
                            match=lambda t, v: v in '*/',
                            toSeq=lambda t, tokens: (t[0], t[1]),
                        ),
                        PreDef.S(optional=True),
                    ),
                    Sequence(
                        Prod(
                            name='Operator +-',
                            match=lambda t, v: v in '+-',
                            toSeq=lambda t, tokens: (t[0], t[1]),
                        ),
                        PreDef.S(),
                    ),
                    PreDef.funcEnd(stop=True, mayEnd=True),
                ),
            ),
        )

        _operant = lambda: Choice(  # noqa:E731
            _DimensionProd(self), _CalcValueProd(self), _CSSVariableProd(self)
        )

        prods = Sequence(
            Prod(
                name='CALC',
                match=lambda t, v: t == types.FUNCTION and normalize(v) == 'calc(',
            ),
            PreDef.S(optional=True),
            _operant(),
            Sequence(_operator, _operant(), minmax=lambda: (0, None)),
            PreDef.funcEnd(stop=True),
        )

        # store: name of variable
        ok, seq, store, unused = ProdParser().parse(
            cssText, 'CSSCalc', prods, checkS=True
        )
        self.wellformed = ok
        if ok:
            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_CSSCalc(self),
        _setCssText,
        doc="String representation of calc function.",
    )

    type = property(lambda self: Value.CALC, doc="Type is fixed to Value.CALC.")


class CSSVariable(CSSFunction):
    """The CSSVariable represents a CSS variables like ``var(varname)``.

    A variable has a (nonnormalized!) `name` and a `value` which is
    tried to be resolved from any available CSSVariablesRule definition.
    """

    _functionName = 'CSSVariable'
    _name = None
    _fallback = None

    def __str__(self):
        return "<cssutils.css.{} object name={!r} value={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.name,
            self.value,
            id(self),
        )

    def _setCssText(self, cssText):
        self._checkReadonly()

        types = self._prods  # rename!
        prods = Sequence(
            Prod(
                name='var',
                match=lambda t, v: t == types.FUNCTION and normalize(v) == 'var(',
            ),
            PreDef.ident(toStore='ident'),
            Sequence(
                PreDef.comma(),
                Choice(
                    _ColorProd(self, toStore='fallback'),
                    _DimensionProd(self, toStore='fallback'),
                    _URIProd(self, toStore='fallback'),
                    _ValueProd(self, toStore='fallback'),
                    _CalcValueProd(self, toStore='fallback'),
                    _CSSVariableProd(self, toStore='fallback'),
                    _CSSFunctionProd(self, toStore='fallback'),
                ),
                minmax=lambda: (0, 1),
            ),
            PreDef.funcEnd(stop=True),
        )

        # store: name of variable
        store = {'ident': None, 'fallback': None}
        ok, seq, store, unused = ProdParser().parse(cssText, 'CSSVariable', prods)
        self.wellformed = ok

        if ok:
            self._name = store['ident'].value
            try:
                self._fallback = store['fallback'].value
            except KeyError:
                self._fallback = None

            self._setSeq(seq)

    cssText = property(
        lambda self: cssutils.ser.do_css_CSSVariable(self),
        _setCssText,
        doc="String representation of variable.",
    )

    # TODO: writable? check if var (value) available?
    name = property(
        lambda self: self._name,
        doc="The name identifier of this variable referring to "
        "a value in a "
        ":class:`cssutils.css.CSSVariablesDeclaration`.",
    )

    @property
    def fallback(self):
        """The fallback Value of this variable"""
        return self._fallback

    type = property(lambda self: Value.VARIABLE, doc="Type is fixed to Value.VARIABLE.")

    def _getValue(self):
        "Find contained sheet and @variables there"
        rel = self
        while True:
            # find node which has parentRule to get to StyleSheet
            if hasattr(rel, 'parent'):
                rel = rel.parent
            else:
                break
        try:
            variables = rel.parentRule.parentStyleSheet.variables
        except AttributeError:
            return None
        else:
            try:
                return variables[self.name]
            except KeyError:
                return None

    value = property(_getValue, doc='The resolved actual value or None.')


# helper for productions
def _ValueProd(parent, nextSor=False, toStore=None):
    return Prod(
        name='Value',
        match=lambda t, v: t in ('IDENT', 'STRING', 'UNICODE-RANGE'),
        nextSor=nextSor,
        toStore=toStore,
        toSeq=lambda t, tokens: ('Value', Value(pushtoken(t, tokens), parent=parent)),
    )


def _DimensionProd(parent, nextSor=False, toStore=None):
    return Prod(
        name='Dimension',
        match=lambda t, v: t in ('DIMENSION', 'NUMBER', 'PERCENTAGE'),
        nextSor=nextSor,
        toStore=toStore,
        toSeq=lambda t, tokens: (
            'DIMENSION',
            DimensionValue(pushtoken(t, tokens), parent=parent),
        ),
    )


def _URIProd(parent, nextSor=False, toStore=None):
    return Prod(
        name='URIValue',
        match=lambda t, v: t == 'URI',
        toStore=toStore,
        nextSor=nextSor,
        toSeq=lambda t, tokens: (
            'URIValue',
            URIValue(pushtoken(t, tokens), parent=parent),
        ),
    )


reHexcolor = re.compile(r'^\#(?:[0-9abcdefABCDEF]{3}|[0-9abcdefABCDEF]{6})$')


def _ColorProd(parent, nextSor=False, toStore=None):
    return Prod(
        name='ColorValue',
        match=lambda t, v: (t == 'HASH' and reHexcolor.match(v))
        or (t == 'FUNCTION' and normalize(v) in ('rgb(', 'rgba(', 'hsl(', 'hsla('))
        or (t == 'IDENT' and normalize(v) in list(ColorValue.COLORS.keys())),
        nextSor=nextSor,
        toStore=toStore,
        toSeq=lambda t, tokens: (
            'ColorValue',
            ColorValue(pushtoken(t, tokens), parent=parent),
        ),
    )


def _CSSFunctionProd(parent, nextSor=False, toStore=None):
    return PreDef.function(
        nextSor=nextSor,
        toStore=toStore,
        toSeq=lambda t, tokens: (
            CSSFunction._functionName,
            CSSFunction(pushtoken(t, tokens), parent=parent),
        ),
    )


def _CalcValueProd(parent, nextSor=False, toStore=None):
    return Prod(
        name=CSSCalc._functionName,
        match=lambda t, v: t == PreDef.types.FUNCTION and normalize(v) == 'calc(',
        toStore=toStore,
        toSeq=lambda t, tokens: (
            CSSCalc._functionName,
            CSSCalc(pushtoken(t, tokens), parent=parent),
        ),
        nextSor=nextSor,
    )


def _CSSVariableProd(parent, nextSor=False, toStore=None):
    return PreDef.variable(
        nextSor=nextSor,
        toStore=toStore,
        toSeq=lambda t, tokens: (
            CSSVariable._functionName,
            CSSVariable(pushtoken(t, tokens), parent=parent),
        ),
    )


def _MSValueProd(parent, nextSor=False):
    return Prod(
        name=MSValue._functionName,
        match=lambda t, v: (  # t == self._prods.FUNCTION and (
            normalize(v)
            in (
                'expression(',
                'alpha(',
                'blur(',
                'chroma(',
                'dropshadow(',
                'fliph(',
                'flipv(',
                'glow(',
                'gray(',
                'invert(',
                'mask(',
                'shadow(',
                'wave(',
                'xray(',
            )
            or v.startswith('progid:DXImageTransform.Microsoft.')
        ),
        nextSor=nextSor,
        toSeq=lambda t, tokens: (
            MSValue._functionName,
            MSValue(pushtoken(t, tokens), parent=parent),
        ),
    )


def MediaQueryValueProd(parent):
    return Choice(
        _ColorProd(parent),
        _DimensionProd(parent),
        _ValueProd(parent),
    )

env_pyrecodes/lib/python3.9/site-packages/cssutils/css/value.py
NikolaBlagojevic/pyrecodes
"""CSS profiles.

Profiles is based on code by Kevin D. Smith, orginally used as cssvalues,
thanks!
"""

# too many long lines
# flake8: noqa

__all__ = ['Profiles']

from cssutils import util
import re


class NoSuchProfileException(Exception):
    """Raised if no profile with given name is found"""

    pass


class Profiles:
    """
    All profiles used for validation. ``cssutils.profile`` is a
    preset object of this class and used by all properties for validation.

    Predefined profiles are (use
    :meth:`~cssutils.profiles.Profiles.propertiesByProfile` to
    get a list of defined properties):

    ``~cssutils.profiles.Profiles.CSS_LEVEL_2``
        Properties defined by CSS2.1
    ``~cssutils.profiles.Profiles.CSS3_BASIC_USER_INTERFACE``
        Currently resize and outline properties only
    ``~cssutils.profiles.Profiles.CSS3_BOX``
        Currently overflow related properties only
    ``~cssutils.profiles.Profiles.CSS3_COLOR``
        CSS 3 color properties
    ``~cssutils.profiles.Profiles.CSS3_PAGED_MEDIA``
        As defined at http://www.w3.org/TR/css3-page/ (at 090307)

    Predefined macros are:

    ``~cssutils.profiles.Profiles._TOKEN_MACROS``
        Macros containing the token values as defined to CSS2
    ``~cssutils.profiles.Profiles._MACROS``
        Additional general macros.

    If you want to redefine any of these macros do this in your custom
    macros.
    """

    CSS_LEVEL_2 = 'CSS Level 2.1'
    CSS3_BACKGROUNDS_AND_BORDERS = 'CSS Backgrounds and Borders Module Level 3'
    CSS3_BASIC_USER_INTERFACE = 'CSS3 Basic User Interface Module'
    CSS3_BOX = CSS_BOX_LEVEL_3 = 'CSS Box Module Level 3'
    CSS3_COLOR = CSS_COLOR_LEVEL_3 = 'CSS Color Module Level 3'
    CSS3_FONTS = 'CSS Fonts Module Level 3'
    CSS3_FONT_FACE = 'CSS Fonts Module Level 3 @font-face properties'
    CSS3_PAGED_MEDIA = 'CSS3 Paged Media Module'
    CSS3_TEXT = 'CSS Text Level 3'

    _TOKEN_MACROS = {
        'ident': r'[-]?{nmstart}{nmchar}*',
        'name': r'{nmchar}+',
        'nmstart': r'[_a-z]|{nonascii}|{escape}',
        'nonascii': r'[^\0-\177]',
        'unicode': r'\\[0-9a-f]{1,6}(\r\n|[ \n\r\t\f])?',
        'escape': r'{unicode}|\\[ -~\u0080-\u01ff]',
        #   'escape': r'{unicode}|\\[ -~\200-\4177777]',
        'int': r'[-]?\d+',
        'nmchar': r'[\w-]|{nonascii}|{escape}',
        'num': r'[-]?\d+|[-]?\d*\.\d+',
        'positivenum': r'\d+|\d*\.\d+',
        'number': r'{num}',
        'string': r'{string1}|{string2}',
        'string1': r'"(\\\"|[^\"])*"',
        'uri': r'url\({w}({string}|(\\\)|[^\)])+){w}\)',
        'string2': r"'(\\\'|[^\'])*'",
        'nl': r'\n|\r\n|\r|\f',
        'w': r'\s*',
    }
    _MACROS = {
        'hexcolor': r'#[0-9a-f]{3}|#[0-9a-f]{6}',
        'rgbcolor': r'rgb\({w}{int}{w}\,{w}{int}{w}\,{w}{int}{w}\)|rgb\({w}{num}%{w}\,{w}{num}%{w}\,{w}{num}%{w}\)',
        'namedcolor': r'(transparent|orange|maroon|red|orange|yellow|olive|purple|fuchsia|white|lime|green|navy|blue|aqua|teal|black|silver|gray)',
        'uicolor': r'(ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)',
        'color': r'{namedcolor}|{hexcolor}|{rgbcolor}|{uicolor}',
        # 'color': r'(maroon|red|orange|yellow|olive|purple|fuchsia|white|lime|green|navy|blue|aqua|teal|black|silver|gray|ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)|#[0-9a-f]{3}|#[0-9a-f]{6}|rgb\({w}{int}{w},{w}{int}{w},{w}{int}{w}\)|rgb\({w}{num}%{w},{w}{num}%{w},{w}{num}%{w}\)',
        'integer': r'{int}',
        'length': r'0|{num}(em|ex|px|in|cm|mm|pt|pc)',
        'positivelength': r'0|{positivenum}(em|ex|px|in|cm|mm|pt|pc)',
        'angle': r'0|{num}(deg|grad|rad)',
        'time': r'0|{num}m?s',
        'frequency': r'0|{num}k?Hz',
        'percentage': r'{num}%',
        'shadow': '(inset)?{w}{length}{w}{length}{w}{length}?{w}{length}?{w}{color}?',
    }

    def __init__(self, log=None):
        """A few profiles are predefined."""
        self._log = log

        # macro cache
        self._usedMacros = Profiles._TOKEN_MACROS.copy()
        self._usedMacros.update(Profiles._MACROS.copy())

        # to keep order, REFACTOR!
        self._profileNames = []
        # for reset if macro changes
        self._rawProfiles = {}
        # already compiled profiles: {profile: {property: checkfunc, ...}, ...}
        self._profilesProperties = {}

        self._defaultProfiles = None

        self.addProfiles(
            [
                (
                    self.CSS_LEVEL_2,
                    properties[self.CSS_LEVEL_2],
                    macros[self.CSS_LEVEL_2],
                ),
                (
                    self.CSS3_BACKGROUNDS_AND_BORDERS,
                    properties[self.CSS3_BACKGROUNDS_AND_BORDERS],
                    macros[self.CSS3_BACKGROUNDS_AND_BORDERS],
                ),
                (
                    self.CSS3_BASIC_USER_INTERFACE,
                    properties[self.CSS3_BASIC_USER_INTERFACE],
                    macros[self.CSS3_BASIC_USER_INTERFACE],
                ),
                (self.CSS3_BOX, properties[self.CSS3_BOX], macros[self.CSS3_BOX]),
                (self.CSS3_COLOR, properties[self.CSS3_COLOR], macros[self.CSS3_COLOR]),
                (self.CSS3_FONTS, properties[self.CSS3_FONTS], macros[self.CSS3_FONTS]),
                # new object for font-face only?
                (
                    self.CSS3_FONT_FACE,
                    properties[self.CSS3_FONT_FACE],
                    macros[self.CSS3_FONTS],
                ),
                (
                    self.CSS3_PAGED_MEDIA,
                    properties[self.CSS3_PAGED_MEDIA],
                    macros[self.CSS3_PAGED_MEDIA],
                ),
                (self.CSS3_TEXT, properties[self.CSS3_TEXT], macros[self.CSS3_TEXT]),
            ]
        )

        self.__update_knownNames()

    def _expand_macros(self, dictionary, macros):
        """Expand macros in token dictionary"""

        def macro_value(m):
            return '(?:%s)' % macros[m.groupdict()['macro']]

        for key, value in list(dictionary.items()):
            if not hasattr(value, '__call__'):
                while re.search(r'{[a-z][a-z0-9-]*}', value):
                    value = re.sub(r'{(?P<macro>[a-z][a-z0-9-]*)}', macro_value, value)
            dictionary[key] = value

        return dictionary

    def _compile_regexes(self, dictionary):
        """Compile all regular expressions into callable objects"""
        for key, value in list(dictionary.items()):
            if not hasattr(value, '__call__'):
                # Compiling them now will slow down the cssutils import time,
                # even if cssutils is not needed. Thus, lazily compile them the
                # first time they're needed.
                # https://web.archive.org/web/20200701035537/https://bitbucket.org/cthedot/cssutils/issues/69)
                value = util.LazyRegex('^(?:%s)$' % value, re.I)
            dictionary[key] = value

        return dictionary

    def __update_knownNames(self):
        self._knownNames = []
        for properties in list(self._profilesProperties.values()):
            self._knownNames.extend(list(properties.keys()))

    def _getDefaultProfiles(self):
        "If not explicitly set same as Profiles.profiles but in reverse order."
        if not self._defaultProfiles:
            return self.profiles
        else:
            return self._defaultProfiles

    def _setDefaultProfiles(self, profiles):
        "profiles may be a single or a list of profile names"
        if isinstance(profiles, str):
            self._defaultProfiles = (profiles,)
        else:
            self._defaultProfiles = profiles

    defaultProfiles = property(
        _getDefaultProfiles,
        _setDefaultProfiles,
        doc="Names of profiles to use for validation."
        "To use e.g. the CSS2 profile set "
        "``cssutils.profile.defaultProfiles = "
        "cssutils.profile.CSS_LEVEL_2``",
    )

    profiles = property(
        lambda self: self._profileNames,
        doc='Names of all profiles in order as defined.',
    )

    @property
    def knownNames(self):
        """All known property names of all profiles."""
        return self._knownNames

    def _resetProperties(self, newMacros=None):
        "reset all props from raw values as changes in macros happened"
        # base
        macros = Profiles._TOKEN_MACROS.copy()
        macros.update(Profiles._MACROS.copy())

        # former
        for profile in self._profileNames:
            macros.update(self._rawProfiles[profile]['macros'])

        # new
        if newMacros:
            macros.update(newMacros)

        # reset properties
        self._profilesProperties.clear()
        for profile in self._profileNames:
            properties = self._expand_macros(
                # keep raw
                self._rawProfiles[profile]['properties'].copy(),
                macros,
            )
            self._profilesProperties[profile] = self._compile_regexes(properties)

        # save
        self._usedMacros = macros

    def addProfiles(self, profiles):
        """Add a list of profiles at once. Useful as if profiles define custom
        macros these are used in one go. Using `addProfile` instead my be
        **very** slow instead.
        """
        # add macros
        for profile, properties, macros in profiles:
            if macros:
                self._usedMacros.update(macros)
                self._rawProfiles[profile] = {'macros': macros.copy()}

        # only add new properties
        for profile, properties, macros in profiles:
            self.addProfile(profile, properties.copy(), None)

    def addProfile(self, profile, properties, macros=None):
        """Add a new profile with name `profile` (e.g. 'CSS level 2')
        and the given `properties`.

        :param profile:
            the new `profile`'s name
        :param properties:
            a dictionary of ``{ property-name: propery-value }`` items where
            property-value is a regex which may use macros defined in given
            ``macros`` or the standard macros Profiles.tokens and
            Profiles.generalvalues.

            ``propery-value`` may also be a function which takes a single
            argument which is the value to validate and which should return
            True or False.
            Any exceptions which may be raised during this custom validation
            are reported or raised as all other cssutils exceptions depending
            on cssutils.log.raiseExceptions which e.g during parsing normally
            is False so the exceptions would be logged only.
        :param macros:
            may be used in the given properties definitions. There are some
            predefined basic macros which may always be used in
            ``Profiles._TOKEN_MACROS`` and ``Profiles._MACROS``.
        """
        if macros:
            # check if known macros would change and if yes reset properties
            if len(set(macros.keys()).intersection(list(self._usedMacros.keys()))):
                self._resetProperties(newMacros=macros)

            else:
                # no replacement, simply continue
                self._usedMacros.update(macros)

        else:
            # might have been set by addProfiles before
            try:
                macros = self._rawProfiles[profile]['macros']
            except KeyError:
                macros = {}

        # save name and raw props/macros if macros change to completely reset
        self._profileNames.append(profile)
        self._rawProfiles[profile] = {
            'properties': properties.copy(),
            'macros': macros.copy(),
        }
        # prepare and save properties
        properties = self._expand_macros(properties, self._usedMacros)
        self._profilesProperties[profile] = self._compile_regexes(properties)

        self.__update_knownNames()

    def removeProfile(self, profile=None, all=False):
        """Remove `profile` or remove `all` profiles.

        If the removed profile used custom macros all remaining profiles
        are reset to reflect the macro changes. This may be quite an expensive
        operation!

        :param profile:
            profile name to remove
        :param all:
            if ``True`` removes all profiles to start with a clean state
        :exceptions:
            - ``cssutils.profiles.NoSuchProfileException``:
              If given `profile` cannot be found.
        """
        if all:
            self._profilesProperties.clear()
            self._rawProfiles.clear()
            del self._profileNames[:]
        else:
            reset = False

            try:
                if self._rawProfiles[profile]['macros']:
                    reset = True

                del self._profilesProperties[profile]
                del self._rawProfiles[profile]
                del self._profileNames[self._profileNames.index(profile)]
            except KeyError:
                raise NoSuchProfileException('No profile %r.' % profile)

            else:
                if reset:
                    # reset properties as macros were removed
                    self._resetProperties()

        self.__update_knownNames()

    def propertiesByProfile(self, profiles=None):
        """Generator: Yield property names, if no `profiles` is given all
        profile's properties are used.

        :param profiles:
            a single profile name or a list of names.
        """
        if not profiles:
            profiles = self.profiles
        elif isinstance(profiles, str):
            profiles = (profiles,)
        try:
            for profile in sorted(profiles):
                yield from sorted(self._profilesProperties[profile].keys())
        except KeyError as e:
            raise NoSuchProfileException(e)

    def validate(self, name, value):
        """Check if `value` is valid for given property `name` using **any**
        profile.

        :param name:
            a property name
        :param value:
            a CSS value (string)
        :returns:
            if the `value` is valid for the given property `name` in any
            profile
        """
        for profile in self.profiles:
            if name in self._profilesProperties[profile]:
                try:
                    # custom validation errors are caught
                    r = bool(self._profilesProperties[profile][name](value))
                except Exception as e:
                    # TODO: more specific exception?
                    # Validate should not be fatal though!
                    self._log.error(e, error=Exception)
                    r = False
                if r:
                    return r
        return False

    def validateWithProfile(self, name, value, profiles=None):  # noqa: C901
        """Check if `value` is valid for given property `name` returning
        ``(valid, profile)``.

        :param name:
            a property name
        :param value:
            a CSS value (string)
        :param profiles:
            internal parameter used by Property.validate only
        :returns:
            ``valid, matching, profiles`` where ``valid`` is if the `value`
            is valid for the given property `name` in any profile,
            ``matching==True`` if it is valid in the given `profiles`
            and ``profiles`` the profile names for which the value is valid
            (or ``[]`` if not valid at all)

        Example::

            > cssutils.profile.defaultProfiles = cssutils.profile.CSS_LEVEL_2
            > print(cssutils.profile.validateWithProfile('color', 'rgba(1,1,1,1)'))
            (True, False, Profiles.CSS3_COLOR)
        """
        if name not in self.knownNames:
            return False, False, []
        else:
            if not profiles:
                profiles = self.defaultProfiles
            elif isinstance(profiles, str):
                profiles = (profiles,)
            for profilename in reversed(profiles):
                # check given profiles
                if name in self._profilesProperties[profilename]:
                    validate = self._profilesProperties[profilename][name]
                    try:
                        if validate(value):
                            return True, True, [profilename]
                    except Exception as e:
                        self._log.error(e, error=Exception)

            for profilename in (p for p in self._profileNames if p not in profiles):
                # check remaining profiles as well
                if name in self._profilesProperties[profilename]:
                    validate = self._profilesProperties[profilename][name]
                    try:
                        if validate(value):
                            return True, False, [profilename]
                    except Exception as e:
                        self._log.error(e, error=Exception)

            names = []
            for profilename, properties in list(self._profilesProperties.items()):
                # return profile to which name belongs
                if name in list(properties.keys()):
                    names.append(profilename)
            names.sort()
            return False, False, names


properties = {}
macros = {}


"""
Define some regular expression fragments that will be used as
macros within the CSS property value regular expressions.
"""
macros[Profiles.CSS_LEVEL_2] = {
    'background-color': r'{color}|transparent|inherit',
    'background-image': r'{uri}|none|inherit',
    # 'background-position': r'({percentage}|{length})(\s*({percentage}|{length}))?|((top|center|bottom)\s*(left|center|right)?)|((left|center|right)\s*(top|center|bottom)?)|inherit',
    'background-position': r'({percentage}|{length}|left|center|right)(\s*({percentage}|{length}|top|center|bottom))?|((top|center|bottom)\s*(left|center|right)?)|((left|center|right)\s*(top|center|bottom)?)|inherit',
    'background-repeat': r'repeat|repeat-x|repeat-y|no-repeat|inherit',
    'background-attachment': r'scroll|fixed|inherit',
    'shape': r'rect\(({w}({length}|auto}){w},){3}{w}({length}|auto){w}\)',
    'counter': r'counter\({w}{ident}{w}(?:,{w}{list-style-type}{w})?\)',
    'identifier': r'{ident}',
    'family-name': r'{string}|({ident}(\s+{ident})*)',
    'generic-family': r'serif|sans-serif|cursive|fantasy|monospace',
    'absolute-size': r'(x?x-)?(small|large)|medium',
    'relative-size': r'smaller|larger',
    'font-family': r'({family-name}({w},{w}{family-name})*)|inherit',
    'font-size': r'{absolute-size}|{relative-size}|{positivelength}|{percentage}|inherit',
    'font-style': r'normal|italic|oblique|inherit',
    'font-variant': r'normal|small-caps|inherit',
    'font-weight': r'normal|bold|bolder|lighter|[1-9]00|inherit',
    'line-height': r'normal|{number}|{length}|{percentage}|inherit',
    'list-style-image': r'{uri}|none|inherit',
    'list-style-position': r'inside|outside|inherit',
    'list-style-type': r'disc|circle|square|decimal|decimal-leading-zero|lower-roman|upper-roman|lower-greek|lower-(latin|alpha)|upper-(latin|alpha)|armenian|georgian|none|inherit',
    'margin-width': r'{length}|{percentage}|auto',
    'padding-width': r'{length}|{percentage}',
    'specific-voice': r'{ident}',
    'generic-voice': r'male|female|child',
    'content': r'{string}|{uri}|{counter}|attr\({w}{ident}{w}\)|open-quote|close-quote|no-open-quote|no-close-quote',
    'background-attrs': r'{background-color}|{background-image}|{background-repeat}|{background-attachment}|{background-position}',
    'list-attrs': r'{list-style-type}|{list-style-position}|{list-style-image}',
    'font-attrs': r'{font-style}|{font-variant}|{font-weight}',
    'text-attrs': r'underline|overline|line-through|blink',
    'overflow': r'visible|hidden|scroll|auto|inherit',
}

"""
Define the regular expressions for validation all CSS values
"""
properties[Profiles.CSS_LEVEL_2] = {
    'azimuth': r'{angle}|(behind\s+)?(left-side|far-left|left|center-left|center|center-right|right|far-right|right-side)(\s+behind)?|behind|leftwards|rightwards|inherit',
    'background-attachment': r'{background-attachment}',
    'background-color': r'{background-color}',
    'background-image': r'{background-image}',
    'background-position': r'{background-position}',
    'background-repeat': r'{background-repeat}',
    # Each piece should only be allowed one time
    'background': r'{background-attrs}(\s+{background-attrs})*|inherit',
    'border-collapse': r'collapse|separate|inherit',
    'border-spacing': r'{length}(\s+{length})?|inherit',
    'bottom': r'{length}|{percentage}|auto|inherit',
    'caption-side': r'top|bottom|inherit',
    'clear': r'none|left|right|both|inherit',
    'clip': r'{shape}|auto|inherit',
    'color': r'{color}|inherit',
    'content': r'none|normal|{content}(\s+{content})*|inherit',
    'counter-increment': r'({ident}(\s+{integer})?)(\s+({ident}(\s+{integer})?))*|none|inherit',
    'counter-reset': r'({ident}(\s+{integer})?)(\s+({ident}(\s+{integer})?))*|none|inherit',
    'cue-after': r'{uri}|none|inherit',
    'cue-before': r'{uri}|none|inherit',
    'cue': r'({uri}|none|inherit){1,2}|inherit',
    # 'cursor': r'((({uri}{w},{w})*)?(auto|crosshair|default|pointer|move|(e|ne|nw|n|se|sw|s|w)-resize|text|wait|help|progress))|inherit',
    'direction': r'ltr|rtl|inherit',
    'display': r'inline|block|list-item|run-in|inline-block|table|inline-table|table-row-group|table-header-group|table-footer-group|table-row|table-column-group|table-column|table-cell|table-caption|none|inherit',
    'elevation': r'{angle}|below|level|above|higher|lower|inherit',
    'empty-cells': r'show|hide|inherit',
    'float': r'left|right|none|inherit',
    'font-family': r'{font-family}',
    'font-size': r'{font-size}',
    'font-style': r'{font-style}',
    'font-variant': r'{font-variant}',
    'font-weight': r'{font-weight}',
    'font': r'(({font-attrs}\s+)*{font-size}({w}/{w}{line-height})?\s+{font-family})|caption|icon|menu|message-box|small-caption|status-bar|inherit',
    'height': r'{length}|{percentage}|auto|inherit',
    'left': r'{length}|{percentage}|auto|inherit',
    'letter-spacing': r'normal|{length}|inherit',
    'line-height': r'{line-height}',
    'list-style-image': r'{list-style-image}',
    'list-style-position': r'{list-style-position}',
    'list-style-type': r'{list-style-type}',
    'list-style': r'{list-attrs}(\s+{list-attrs})*|inherit',
    'margin-right': r'{margin-width}|inherit',
    'margin-left': r'{margin-width}|inherit',
    'margin-top': r'{margin-width}|inherit',
    'margin-bottom': r'{margin-width}|inherit',
    'margin': r'{margin-width}(\s+{margin-width}){0,3}|inherit',
    'max-height': r'{length}|{percentage}|none|inherit',
    'max-width': r'{length}|{percentage}|none|inherit',
    'min-height': r'{length}|{percentage}|none|inherit',
    'min-width': r'{length}|{percentage}|none|inherit',
    'orphans': r'{integer}|inherit',
    'overflow': r'{overflow}',
    'padding-top': r'{padding-width}|inherit',
    'padding-right': r'{padding-width}|inherit',
    'padding-bottom': r'{padding-width}|inherit',
    'padding-left': r'{padding-width}|inherit',
    'padding': r'{padding-width}(\s+{padding-width}){0,3}|inherit',
    'page-break-after': r'auto|always|avoid|left|right|inherit',
    'page-break-before': r'auto|always|avoid|left|right|inherit',
    'page-break-inside': r'avoid|auto|inherit',
    'pause-after': r'{time}|{percentage}|inherit',
    'pause-before': r'{time}|{percentage}|inherit',
    'pause': r'({time}|{percentage}){1,2}|inherit',
    'pitch-range': r'{number}|inherit',
    'pitch': r'{frequency}|x-low|low|medium|high|x-high|inherit',
    'play-during': r'{uri}(\s+(mix|repeat))*|auto|none|inherit',
    'position': r'static|relative|absolute|fixed|inherit',
    'quotes': r'({string}\s+{string})(\s+{string}\s+{string})*|none|inherit',
    'richness': r'{number}|inherit',
    'right': r'{length}|{percentage}|auto|inherit',
    'speak-header': r'once|always|inherit',
    'speak-numeral': r'digits|continuous|inherit',
    'speak-punctuation': r'code|none|inherit',
    'speak': r'normal|none|spell-out|inherit',
    'speech-rate': r'{number}|x-slow|slow|medium|fast|x-fast|faster|slower|inherit',
    'stress': r'{number}|inherit',
    'table-layout': r'auto|fixed|inherit',
    'text-align': r'left|right|center|justify|inherit',
    'text-decoration': r'none|{text-attrs}(\s+{text-attrs})*|inherit',
    'text-indent': r'{length}|{percentage}|inherit',
    'text-transform': r'capitalize|uppercase|lowercase|none|inherit',
    'top': r'{length}|{percentage}|auto|inherit',
    'unicode-bidi': r'normal|embed|bidi-override|inherit',
    'vertical-align': r'baseline|sub|super|top|text-top|middle|bottom|text-bottom|{percentage}|{length}|inherit',
    'visibility': r'visible|hidden|collapse|inherit',
    'voice-family': r'({specific-voice}|{generic-voice}{w},{w})*({specific-voice}|{generic-voice})|inherit',
    'volume': r'{number}|{percentage}|silent|x-soft|soft|medium|loud|x-loud|inherit',
    'white-space': r'normal|pre|nowrap|pre-wrap|pre-line|inherit',
    'widows': r'{integer}|inherit',
    'width': r'{length}|{percentage}|auto|inherit',
    'word-spacing': r'normal|{length}|inherit',
    'z-index': r'auto|{integer}|inherit',
}


macros[Profiles.CSS3_BACKGROUNDS_AND_BORDERS] = {
    'border-style': 'none|hidden|dotted|dashed|solid|double|groove|ridge|inset|outset',
    'border-width': '{length}|thin|medium|thick',
    'b1': r'{border-width}?({w}{border-style})?({w}{color})?',
    'b2': r'{border-width}?({w}{color})?({w}{border-style})?',
    'b3': r'{border-style}?({w}{border-width})?({w}{color})?',
    'b4': r'{border-style}?({w}{color})?({w}{border-width})?',
    'b5': r'{color}?({w}{border-style})?({w}{border-width})?',
    'b6': r'{color}?({w}{border-width})?({w}{border-style})?',
    'border-attrs': r'{b1}|{b2}|{b3}|{b4}|{b5}|{b6}',
    'border-radius-part': r'({length}|{percentage})(\s+({length}|{percentage}))?',
}
properties[Profiles.CSS3_BACKGROUNDS_AND_BORDERS] = {
    'border-color': r'({color}|transparent)(\s+({color}|transparent)){0,3}|inherit',
    'border-style': r'{border-style}(\s+{border-style}){0,3}|inherit',
    'border-top': r'{border-attrs}|inherit',
    'border-right': r'{border-attrs}|inherit',
    'border-bottom': r'{border-attrs}|inherit',
    'border-left': r'{border-attrs}|inherit',
    'border-top-color': r'{color}|transparent|inherit',
    'border-right-color': r'{color}|transparent|inherit',
    'border-bottom-color': r'{color}|transparent|inherit',
    'border-left-color': r'{color}|transparent|inherit',
    'border-top-style': r'{border-style}|inherit',
    'border-right-style': r'{border-style}|inherit',
    'border-bottom-style': r'{border-style}|inherit',
    'border-left-style': r'{border-style}|inherit',
    'border-top-width': r'{border-width}|inherit',
    'border-right-width': r'{border-width}|inherit',
    'border-bottom-width': r'{border-width}|inherit',
    'border-left-width': r'{border-width}|inherit',
    'border-width': r'{border-width}(\s+{border-width}){0,3}|inherit',
    'border': r'{border-attrs}|inherit',
    'border-top-right-radius': '{border-radius-part}',
    'border-bottom-right-radius': '{border-radius-part}',
    'border-bottom-left-radius': '{border-radius-part}',
    'border-top-left-radius': '{border-radius-part}',
    'border-radius': '({length}{w}|{percentage}{w}){1,4}(/{w}({length}{w}|{percentage}{w}){1,4})?',
    'box-shadow': 'none|{shadow}({w},{w}{shadow})*',
}

# CSS3 Basic User Interface Module
macros[Profiles.CSS3_BASIC_USER_INTERFACE] = {
    'border-style': macros[Profiles.CSS3_BACKGROUNDS_AND_BORDERS]['border-style'],
    'border-width': macros[Profiles.CSS3_BACKGROUNDS_AND_BORDERS]['border-width'],
    'outline-1': r'{outline-color}(\s+{outline-style})?(\s+{outline-width})?',
    'outline-2': r'{outline-color}(\s+{outline-width})?(\s+{outline-style})?',
    'outline-3': r'{outline-style}(\s+{outline-color})?(\s+{outline-width})?',
    'outline-4': r'{outline-style}(\s+{outline-width})?(\s+{outline-color})?',
    'outline-5': r'{outline-width}(\s+{outline-color})?(\s+{outline-style})?',
    'outline-6': r'{outline-width}(\s+{outline-style})?(\s+{outline-color})?',
    'outline-color': r'{color}|invert|inherit',
    'outline-style': r'auto|{border-style}|inherit',
    'outline-width': r'{border-width}|inherit',
}
properties[Profiles.CSS3_BASIC_USER_INTERFACE] = {
    'box-sizing': r'content-box|border-box',
    'cursor': r'((({uri}{w}({number}{w}{number}{w})?,{w})*)?(auto|default|none|context-menu|help|pointer|progress|wait|cell|crosshair|text|vertical-text|alias|copy|move|no-drop|not-allowed|(e|n|ne|nw|s|se|sw|w|ew|ns|nesw|nwse|col|row)-resize|all-scroll))|inherit',
    'nav-index': r'auto|{number}|inherit',
    'outline-color': r'{outline-color}',
    'outline-style': r'{outline-style}',
    'outline-width': r'{outline-width}',
    'outline-offset': r'{length}|inherit',
    # 'outline': r'{outline-attrs}(\s+{outline-attrs})*|inherit',
    'outline': r'{outline-1}|{outline-2}|{outline-3}|{outline-4}|{outline-5}|{outline-6}|inherit',
    'resize': 'none|both|horizontal|vertical|inherit',
}

# CSS Box Module Level 3
macros[Profiles.CSS3_BOX] = {'overflow': macros[Profiles.CSS_LEVEL_2]['overflow']}
properties[Profiles.CSS3_BOX] = {
    'overflow': '{overflow}{w}{overflow}?|inherit',
    'overflow-x': '{overflow}|inherit',
    'overflow-y': '{overflow}|inherit',
}

# CSS Color Module Level 3
macros[Profiles.CSS3_COLOR] = {
    # orange and transparent in CSS 2.1
    'namedcolor': r'(currentcolor|transparent|aqua|black|blue|fuchsia|gray|green|lime|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow)',
    # orange?
    'rgbacolor': r'rgba\({w}{int}{w}\,{w}{int}{w}\,{w}{int}{w}\,{w}{num}{w}\)|rgba\({w}{num}%{w}\,{w}{num}%{w}\,{w}{num}%{w}\,{w}{num}{w}\)',
    'hslcolor': r'hsl\({w}{int}{w}\,{w}{num}%{w}\,{w}{num}%{w}\)|hsla\({w}{int}{w}\,{w}{num}%{w}\,{w}{num}%{w}\,{w}{num}{w}\)',
    'x11color': r'aliceblue|antiquewhite|aqua|aquamarine|azure|beige|bisque|black|blanchedalmond|blue|blueviolet|brown|burlywood|cadetblue|chartreuse|chocolate|coral|cornflowerblue|cornsilk|crimson|cyan|darkblue|darkcyan|darkgoldenrod|darkgray|darkgreen|darkgrey|darkkhaki|darkmagenta|darkolivegreen|darkorange|darkorchid|darkred|darksalmon|darkseagreen|darkslateblue|darkslategray|darkslategrey|darkturquoise|darkviolet|deeppink|deepskyblue|dimgray|dimgrey|dodgerblue|firebrick|floralwhite|forestgreen|fuchsia|gainsboro|ghostwhite|gold|goldenrod|gray|green|greenyellow|grey|honeydew|hotpink|indianred|indigo|ivory|khaki|lavender|lavenderblush|lawngreen|lemonchiffon|lightblue|lightcoral|lightcyan|lightgoldenrodyellow|lightgray|lightgreen|lightgrey|lightpink|lightsalmon|lightseagreen|lightskyblue|lightslategray|lightslategrey|lightsteelblue|lightyellow|lime|limegreen|linen|magenta|maroon|mediumaquamarine|mediumblue|mediumorchid|mediumpurple|mediumseagreen|mediumslateblue|mediumspringgreen|mediumturquoise|mediumvioletred|midnightblue|mintcream|mistyrose|moccasin|navajowhite|navy|oldlace|olive|olivedrab|orange|orangered|orchid|palegoldenrod|palegreen|paleturquoise|palevioletred|papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen',
    'uicolor': r'(ActiveBorder|ActiveCaption|AppWorkspace|Background|ButtonFace|ButtonHighlight|ButtonShadow|ButtonText|CaptionText|GrayText|Highlight|HighlightText|InactiveBorder|InactiveCaption|InactiveCaptionText|InfoBackground|InfoText|Menu|MenuText|Scrollbar|ThreeDDarkShadow|ThreeDFace|ThreeDHighlight|ThreeDLightShadow|ThreeDShadow|Window|WindowFrame|WindowText)',
    'color': r'{namedcolor}|{hexcolor}|{rgbcolor}|{rgbacolor}|{hslcolor}|{x11color}|inherit',
}
properties[Profiles.CSS3_COLOR] = {
    'opacity': r'{num}|inherit',
}

# CSS Fonts Module Level 3 http://www.w3.org/TR/css3-fonts/
macros[Profiles.CSS3_FONTS] = {
    # 'family-name': r'{string}|{ident}',
    'family-name': r'{string}|({ident}(\s+{ident})*)',
    'font-face-name': r'local\({w}{family-name}{w}\)',
    'font-stretch-names': r'(ultra-condensed|extra-condensed|condensed|semi-condensed|semi-expanded|expanded|extra-expanded|ultra-expanded)',
    'unicode-range': r'[uU]\+[0-9A-Fa-f?]{1,6}(\-[0-9A-Fa-f]{1,6})?',
}
properties[Profiles.CSS3_FONTS] = {
    'font-size-adjust': r'{number}|none|inherit',
    'font-stretch': r'normal|wider|narrower|{font-stretch-names}|inherit',
}
properties[Profiles.CSS3_FONT_FACE] = {
    'font-family': '{family-name}',
    'font-stretch': r'{font-stretch-names}',
    'font-style': r'normal|italic|oblique',
    'font-weight': r'normal|bold|[1-9]00',
    'src': r'({uri}{w}(format\({w}{string}{w}(\,{w}{string}{w})*\))?|{font-face-name})({w},{w}({uri}{w}(format\({w}{string}{w}(\,{w}{string}{w})*\))?|{font-face-name}))*',
    'unicode-range': '{unicode-range}({w},{w}{unicode-range})*',
}

# CSS3 Paged Media
macros[Profiles.CSS3_PAGED_MEDIA] = {
    'page-size': 'a5|a4|a3|b5|b4|letter|legal|ledger',
    'page-orientation': 'portrait|landscape',
    'page-1': '{page-size}(?:{w}{page-orientation})?',
    'page-2': '{page-orientation}(?:{w}{page-size})?',
    'page-size-orientation': '{page-1}|{page-2}',
    'pagebreak': 'auto|always|avoid|left|right',
}
properties[Profiles.CSS3_PAGED_MEDIA] = {
    'fit': 'fill|hidden|meet|slice',
    'fit-position': r'auto|(({percentage}|{length})(\s*({percentage}|{length}))?|((top|center|bottom)\s*(left|center|right)?)|((left|center|right)\s*(top|center|bottom)?))',
    'image-orientation': 'auto|{angle}',
    'orphans': r'{integer}|inherit',
    'page': 'auto|{ident}',
    'page-break-before': '{pagebreak}|inherit',
    'page-break-after': '{pagebreak}|inherit',
    'page-break-inside': 'auto|avoid|inherit',
    'size': '({length}{w}){1,2}|auto|{page-size-orientation}',
    'widows': r'{integer}|inherit',
}

macros[Profiles.CSS3_TEXT] = {}
properties[Profiles.CSS3_TEXT] = {
    'text-shadow': 'none|{shadow}({w},{w}{shadow})*',
}

env_pyrecodes/lib/python3.9/site-packages/cssutils/profiles.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSImportRule"""

import re
import xml.dom

import pytest

import cssutils
from . import test_cssrule


class TestCSSImportRule(test_cssrule.TestCSSRule):
    def _setup_rule(self):
        self.r = cssutils.css.CSSImportRule()
        self.rRO = cssutils.css.CSSImportRule(readonly=True)
        self.r_type = cssutils.css.CSSImportRule.IMPORT_RULE
        self.r_typeString = 'IMPORT_RULE'

    def test_init(self):
        "CSSImportRule.__init__()"
        super().test_init()

        # no init param
        assert self.r.href is None
        assert self.r.hreftype is None
        assert not self.r.hrefFound
        assert 'all' == self.r.media.mediaText
        assert isinstance(self.r.media, cssutils.stylesheets.MediaList)
        assert self.r.name is None
        assert isinstance(self.r.styleSheet, cssutils.css.CSSStyleSheet)
        assert 0 == self.r.styleSheet.cssRules.length
        assert '' == self.r.cssText

        # all
        r = cssutils.css.CSSImportRule(href='href', mediaText='tv', name='name')
        assert '@import url(href) tv "name";' == r.cssText
        assert "href" == r.href
        assert r.hreftype is None
        assert 'tv' == r.media.mediaText
        assert isinstance(r.media, cssutils.stylesheets.MediaList)
        assert 'name' == r.name
        assert r.parentRule is None  # see CSSRule
        assert r.parentStyleSheet is None  # see CSSRule
        assert isinstance(self.r.styleSheet, cssutils.css.CSSStyleSheet)
        assert 0 == self.r.styleSheet.cssRules.length

        # href
        r = cssutils.css.CSSImportRule('x')
        assert '@import url(x);' == r.cssText
        assert 'x' == r.href
        assert r.hreftype is None

        # href + mediaText
        r = cssutils.css.CSSImportRule('x', 'print')
        assert '@import url(x) print;' == r.cssText
        assert 'x' == r.href
        assert 'print' == r.media.mediaText

        # href + name
        r = cssutils.css.CSSImportRule('x', name='n')
        assert '@import url(x) "n";' == r.cssText
        assert 'x' == r.href
        assert 'n' == r.name

        # href + mediaText + name
        r = cssutils.css.CSSImportRule('x', 'print', 'n')
        assert '@import url(x) print "n";' == r.cssText
        assert 'x' == r.href
        assert 'print' == r.media.mediaText
        assert 'n' == r.name

        # media +name only
        self.r = cssutils.css.CSSImportRule(mediaText='print', name="n")
        assert isinstance(self.r.media, cssutils.stylesheets.MediaList)
        assert '' == self.r.cssText
        assert 'print' == self.r.media.mediaText
        assert 'n' == self.r.name

        # only possible to set @... similar name
        with pytest.raises(xml.dom.InvalidModificationErr):
            self.r._setAtkeyword('x')

    def test_cssText(self):
        "CSSImportRule.cssText"
        tests = {
            # href string
            '''@import "str";''': None,
            '''@import"str";''': '''@import "str";''',
            '''@\\import "str";''': '''@import "str";''',
            '''@IMPORT "str";''': '''@import "str";''',
            '''@import 'str';''': '''@import "str";''',
            '''@import 'str' ;''': '''@import "str";''',
            '''@import "str"  ;''': '''@import "str";''',
            r'''@import "\""  ;''': r'''@import "\"";''',
            '''@import '\\'';''': r'''@import "'";''',
            '''@import '"';''': r'''@import "\"";''',
            # href url
            '''@import url(x.css);''': None,
            # nospace
            '''@import url(")");''': '''@import url(")");''',
            '''@import url("\\"");''': '''@import url("\\"");''',
            '''@import url('\\'');''': '''@import url("'");''',
            # href + media
            # all is removed
            '''@import "str" all;''': '''@import "str";''',
            '''@import "str" tv, print;''': None,
            '''@import"str"tv,print;''': '''@import "str" tv, print;''',
            '''@import "str" tv, print, all;''': '''@import "str";''',
            '''@import "str" handheld, all;''': '''@import "str";''',
            '''@import "str" all, handheld;''': '''@import "str";''',
            '''@import "str" not tv;''': None,
            '''@import "str" only tv;''': None,
            '''@import "str" only tv and (color: 2);''': None,
            # href + name
            '''@import "str" "name";''': None,
            '''@import "str" 'name';''': '''@import "str" "name";''',
            '''@import url(x) "name";''': None,
            '''@import "str" "\\"";''': None,
            '''@import "str" '\\'';''': '''@import "str" "'";''',
            # href + media + name
            '''@import"str"tv"name";''': '''@import "str" tv "name";''',
            '''@import\t\r\f\n"str"\t\t\r\f\ntv\t\t\r\f\n"name"\t;''': '''@import "str" tv "name";''',
            # comments
            '''@import /*1*/ "str" /*2*/;''': None,
            '@import/*1*//*2*/"str"/*3*//*4*/all/*5*//*6*/"name"/*7*//*8*/ ;': '@import /*1*/ /*2*/ "str" /*3*/ /*4*/ all /*5*/ /*6*/ "name" /*7*/ /*8*/;',
            '@import/*1*//*2*/url(u)/*3*//*4*/all/*5*//*6*/"name"/*7*//*8*/ ;': '@import /*1*/ /*2*/ url(u) /*3*/ /*4*/ all /*5*/ /*6*/ "name" /*7*/ /*8*/;',
            '@import/*1*//*2*/url("u")/*3*//*4*/all/*5*//*6*/"name"/*7*//*8*/ ;': '@import /*1*/ /*2*/ url(u) /*3*/ /*4*/ all /*5*/ /*6*/ "name" /*7*/ /*8*/;',
            # WS
            '@import\n\t\f "str"\n\t\f tv\n\t\f "name"\n\t\f ;': '@import "str" tv "name";',
            '@import\n\t\f url(\n\t\f u\n\t\f )\n\t\f tv\n\t\f "name"\n\t\f ;': '@import url(u) tv "name";',
            '@import\n\t\f url("u")\n\t\f tv\n\t\f "name"\n\t\f ;': '@import url(u) tv "name";',
            '@import\n\t\f url(\n\t\f "u"\n\t\f )\n\t\f tv\n\t\f "name"\n\t\f ;': '@import url(u) tv "name";',
        }
        self.do_equal_r(tests)  # set cssText
        tests.update(
            {
                '@import "x.css" tv': '@import "x.css" tv;',
                '@import "x.css"': '@import "x.css";',  # no ;
                "@import 'x.css'": '@import "x.css";',  # no ;
                '@import url(x.css)': '@import url(x.css);',  # no ;
                '@import "x;': '@import "x;";',  # no "!
            }
        )
        self.do_equal_p(tests)  # parse

        tests = {
            '''@import;''': xml.dom.SyntaxErr,
            '''@import all;''': xml.dom.SyntaxErr,
            '''@import all"name";''': xml.dom.SyntaxErr,
            '''@import x";''': xml.dom.SyntaxErr,
            '''@import "str" ,all;''': xml.dom.SyntaxErr,
            '''@import "str" all,;''': xml.dom.SyntaxErr,
            '''@import "str" all tv;''': xml.dom.SyntaxErr,
            '''@import "str" "name" all;''': xml.dom.SyntaxErr,
        }
        self.do_raise_p(tests)  # parse
        tests.update(
            {
                '@import "x.css"': xml.dom.SyntaxErr,
                "@import 'x.css'": xml.dom.SyntaxErr,
                '@import url(x.css)': xml.dom.SyntaxErr,
                '@import "x.css" tv': xml.dom.SyntaxErr,
                '@import "x;': xml.dom.SyntaxErr,
                '''@import url("x);''': xml.dom.SyntaxErr,
                # trailing
                '''@import "x";"a"''': xml.dom.SyntaxErr,
                # trailing S or COMMENT
                '''@import "x";/**/''': xml.dom.SyntaxErr,
                '''@import "x"; ''': xml.dom.SyntaxErr,
            }
        )
        self.do_raise_r(tests)  # set cssText

    def test_href(self):
        "CSSImportRule.href"
        # set
        self.r.href = 'x'
        assert 'x' == self.r.href
        assert '@import url(x);' == self.r.cssText

        # http
        self.r.href = 'http://www.example.com/x?css=z&v=1'
        assert 'http://www.example.com/x?css=z&v=1' == self.r.href
        assert '@import url(http://www.example.com/x?css=z&v=1);' == self.r.cssText

        # also if hreftype changed
        self.r.hreftype = 'string'
        assert 'http://www.example.com/x?css=z&v=1' == self.r.href
        assert '@import "http://www.example.com/x?css=z&v=1";' == self.r.cssText

        # string escaping?
        self.r.href = '"'
        assert '@import "\\"";' == self.r.cssText
        self.r.hreftype = 'url'
        assert '@import url("\\"");' == self.r.cssText

        # url escaping?
        self.r.href = ')'
        assert '@import url(")");' == self.r.cssText

        self.r.hreftype = 'NOT VALID'  # using default
        assert '@import url(")");' == self.r.cssText

    def test_hrefFound(self):
        "CSSImportRule.hrefFound"

        def fetcher(url):
            if url == 'http://example.com/yes':
                return None, '/**/'
            else:
                return None, None

        parser = cssutils.CSSParser(fetcher=fetcher)
        sheet = parser.parseString('@import "http://example.com/yes" "name"')

        r = sheet.cssRules[0]
        assert b'/**/' == r.styleSheet.cssText
        assert r.hrefFound
        assert 'name' == r.name

        r.cssText = '@import url(http://example.com/none) "name2";'
        assert b'' == r.styleSheet.cssText
        assert not r.hrefFound
        assert 'name2' == r.name

        sheet.cssText = '@import url(http://example.com/none);'
        assert r != sheet.cssRules[0]

    def test_hreftype(self):
        "CSSImportRule.hreftype"
        self.r = cssutils.css.CSSImportRule()

        self.r.cssText = '@import /*1*/url(org) /*2*/;'
        assert 'uri' == self.r.hreftype
        assert '@import /*1*/ url(org) /*2*/;' == self.r.cssText

        self.r.cssText = '@import /*1*/"org" /*2*/;'
        assert 'string' == self.r.hreftype
        assert '@import /*1*/ "org" /*2*/;' == self.r.cssText

        self.r.href = 'new'
        assert '@import /*1*/ "new" /*2*/;' == self.r.cssText

        self.r.hreftype = 'uri'
        assert '@import /*1*/ url(new) /*2*/;' == self.r.cssText

    def test_media(self):
        "CSSImportRule.media"
        self.r.href = 'x'  # @import url(x)

        # media is readonly
        with pytest.raises(AttributeError):
            self.r.__setattr__('media', None)

        # but not static
        self.r.media.mediaText = 'print'
        assert '@import url(x) print;' == self.r.cssText
        self.r.media.appendMedium('tv')
        assert '@import url(x) print, tv;' == self.r.cssText

        tv_msg = re.escape(
            '''MediaList: Ignoring new medium '''
            '''cssutils.stylesheets.MediaQuery(mediaText='tv') '''
            '''as already specified "all" (set ``mediaText`` instead).'''
        )

        # for generated rule
        r = cssutils.css.CSSImportRule(href='x')
        with pytest.raises(xml.dom.InvalidModificationErr, match=tv_msg):
            r.media.appendMedium('tv')
        assert '@import url(x);' == r.cssText
        with pytest.raises(xml.dom.InvalidModificationErr, match=tv_msg):
            r.media.appendMedium('tv')
        assert '@import url(x);' == r.cssText
        r.media.mediaText = 'tv'
        assert '@import url(x) tv;' == r.cssText
        r.media.appendMedium('print')  # all + tv = all!
        assert '@import url(x) tv, print;' == r.cssText

        # for parsed rule without initial media
        s = cssutils.parseString('@import url(x);')
        r = s.cssRules[0]

        with pytest.raises(xml.dom.InvalidModificationErr, match=tv_msg):
            r.media.appendMedium('tv')
        assert '@import url(x);' == r.cssText
        with pytest.raises(xml.dom.InvalidModificationErr, match=tv_msg):
            r.media.appendMedium('tv')
        assert '@import url(x);' == r.cssText
        r.media.mediaText = 'tv'
        assert '@import url(x) tv;' == r.cssText
        r.media.appendMedium('print')  # all + tv = all!
        assert '@import url(x) tv, print;' == r.cssText

    def test_name(self):
        "CSSImportRule.name"
        r = cssutils.css.CSSImportRule('x', name='a000000')
        assert 'a000000' == r.name
        assert '@import url(x) "a000000";' == r.cssText

        r.name = "n"
        assert 'n' == r.name
        assert '@import url(x) "n";' == r.cssText
        r.name = '"'
        assert '"' == r.name
        assert '@import url(x) "\\"";' == r.cssText

        r.hreftype = 'string'
        assert '@import "x" "\\"";' == r.cssText
        r.name = "123"
        assert '@import "x" "123";' == r.cssText

        r.name = None
        assert r.name is None
        assert '@import "x";' == r.cssText

        r.name = ""
        assert r.name is None
        assert '@import "x";' == r.cssText

        with pytest.raises(xml.dom.SyntaxErr):
            r._setName(0)
        with pytest.raises(xml.dom.SyntaxErr):
            r._setName(123)

    def test_styleSheet(self):
        "CSSImportRule.styleSheet"

        def fetcher(url):
            if url == "/root/level1/anything.css":
                return None, '@import "level2/css.css" "title2";'
            else:
                return None, 'a { color: red }'

        parser = cssutils.CSSParser(fetcher=fetcher)
        sheet = parser.parseString(
            '''@charset "ascii";
                                   @import "level1/anything.css" tv "title";''',
            href='/root/',
        )

        assert sheet.href == '/root/'

        ir = sheet.cssRules[1]
        assert ir.href == 'level1/anything.css'
        assert ir.styleSheet.href == '/root/level1/anything.css'
        # inherits ascii as no self charset is set
        assert ir.styleSheet.encoding == 'ascii'
        assert ir.styleSheet.ownerRule == ir
        assert ir.styleSheet.media.mediaText == 'tv'
        assert ir.styleSheet.parentStyleSheet is None  # sheet
        assert ir.styleSheet.title == 'title'
        assert (
            ir.styleSheet.cssText
            == b'@charset "ascii";\n@import "level2/css.css" "title2";'
        )

        ir2 = ir.styleSheet.cssRules[1]
        assert ir2.href == 'level2/css.css'
        assert ir2.styleSheet.href == '/root/level1/level2/css.css'
        # inherits ascii as no self charset is set
        assert ir2.styleSheet.encoding == 'ascii'
        assert ir2.styleSheet.ownerRule == ir2
        assert ir2.styleSheet.media.mediaText == 'all'
        assert ir2.styleSheet.parentStyleSheet is None  # ir.styleSheet
        assert ir2.styleSheet.title == 'title2'
        assert (
            ir2.styleSheet.cssText == b'@charset "ascii";\na {\n    color: red\n    }'
        )

        sheet = cssutils.parseString('@import "CANNOT-FIND.css";')
        ir = sheet.cssRules[0]
        assert ir.href == "CANNOT-FIND.css"
        assert isinstance(ir.styleSheet, cssutils.css.CSSStyleSheet)

        def fetcher(url):
            if url.endswith('level1.css'):
                return None, b'@charset "ascii"; @import "level2.css";'
            else:
                return None, b'a { color: red }'

        parser = cssutils.CSSParser(fetcher=fetcher)

        sheet = parser.parseString('@charset "iso-8859-1";@import "level1.css";')
        assert sheet.encoding == 'iso-8859-1'

        sheet = sheet.cssRules[1].styleSheet
        assert sheet.encoding == 'ascii'

        sheet = sheet.cssRules[1].styleSheet
        assert sheet.encoding == 'ascii'

    def test_incomplete(self):
        "CSSImportRule (incomplete)"
        tests = {
            '@import "x.css': '@import "x.css";',
            "@import 'x": '@import "x";',
            # TODO:
            "@import url(x": '@import url(x);',
            "@import url('x": '@import url(x);',
            '@import url("x;': '@import url("x;");',
            '@import url( "x;': '@import url("x;");',
            '@import url("x ': '@import url("x ");',
            '@import url(x ': '@import url(x);',
            '''@import "a
                @import "b";
                @import "c";''': '@import "c";',
        }
        self.do_equal_p(tests, raising=False)  # parse

    def test_InvalidModificationErr(self):
        "CSSImportRule.cssText InvalidModificationErr"
        self._test_InvalidModificationErr('@import')

    def test_reprANDstr(self):
        "CSSImportRule.__repr__(), .__str__()"
        href = 'x.css'
        mediaText = 'tv, print'
        name = 'name'
        s = cssutils.css.CSSImportRule(href=href, mediaText=mediaText, name=name)

        # str(): mediaText nor name are present here
        assert href in str(s)

        # repr()
        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert href == s2.href
        assert mediaText == s2.media.mediaText
        assert name == s2.name

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssimportrule.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSMediaRule"""

# flake8: noqa

import xml.dom
from . import test_cssrule
import cssutils
import pytest


class TestCSSMediaRule(test_cssrule.TestCSSRule):
    def _setup_rule(self):
        self.r = cssutils.css.CSSMediaRule()
        self.rRO = cssutils.css.CSSMediaRule(readonly=True)
        self.r_type = cssutils.css.CSSMediaRule.MEDIA_RULE
        self.r_typeString = 'MEDIA_RULE'
        # for tests
        self.stylerule = cssutils.css.CSSStyleRule()
        self.stylerule.cssText = 'a {}'

    def test_init(self):
        "CSSMediaRule.__init__()"
        super().test_init()

        r = cssutils.css.CSSMediaRule()
        assert isinstance(r.cssRules, cssutils.css.CSSRuleList)
        assert [] == r.cssRules
        assert '' == r.cssText
        assert isinstance(r.media, cssutils.stylesheets.MediaList)
        assert 'all' == r.media.mediaText
        assert r.name is None

        r = cssutils.css.CSSMediaRule(mediaText='print', name='name')
        assert isinstance(r.cssRules, cssutils.css.CSSRuleList)
        assert [] == r.cssRules
        assert '' == r.cssText
        assert isinstance(r.media, cssutils.stylesheets.MediaList)
        assert 'print' == r.media.mediaText
        assert 'name' == r.name

        # only possible to set @... similar name
        with pytest.raises(xml.dom.InvalidModificationErr):
            self.r._setAtkeyword('x')

    def test_iter(self):
        "CSSMediaRule.__iter__()"
        m = cssutils.css.CSSMediaRule()
        m.cssText = '''@media all { /*1*/a { left: 0} b{ top:0} }'''
        types = [
            cssutils.css.CSSRule.COMMENT,
            cssutils.css.CSSRule.STYLE_RULE,
            cssutils.css.CSSRule.STYLE_RULE,
        ]
        for i, rule in enumerate(m):
            assert rule == m.cssRules[i]
            assert rule.type == types[i]
            assert rule.parentRule == m

    def test_refs(self):
        """CSSStylesheet references"""
        s = cssutils.parseString('@media all {a {color: red}}')
        r = s.cssRules[0]
        rules = r.cssRules
        assert r.cssRules[0].parentStyleSheet == s
        assert rules[0].parentStyleSheet == s

        # set cssText
        r.cssText = '@media all {a {color: blue}}'
        # not anymore: self.assertEqual(rules, r.cssRules)

        # set cssRules
        r.cssRules = cssutils.parseString(
            '''
            /**/
            @x;
            b {}').cssRules'''
        ).cssRules
        # new object
        assert rules != r.cssRules
        for i, sr in enumerate(r.cssRules):
            assert sr.parentStyleSheet == s
            assert sr.parentRule == r

    def test_cssRules(self):
        "CSSMediaRule.cssRules"
        r = cssutils.css.CSSMediaRule()
        assert [] == r.cssRules
        sr = cssutils.css.CSSStyleRule()
        r.cssRules.append(sr)
        assert [sr] == r.cssRules
        ir = cssutils.css.CSSImportRule()
        with pytest.raises(xml.dom.HierarchyRequestErr):
            r.cssRules.append(ir)

        s = cssutils.parseString('@media all { /*1*/a {x:1} }')
        m = s.cssRules[0]
        assert 2 == m.cssRules.length
        del m.cssRules[0]
        assert 1 == m.cssRules.length
        m.cssRules.append('/*2*/')
        assert 2 == m.cssRules.length
        m.cssRules.extend(cssutils.parseString('/*3*/x {y:2}').cssRules)
        assert 4 == m.cssRules.length
        assert (
            '@media all {\n    a {\n        x: 1\n        }\n    /*2*/\n    /*3*/\n    x {\n        y: 2\n        }\n    }'
            == m.cssText
        )

        for rule in m.cssRules:
            assert rule.parentStyleSheet == s
            assert rule.parentRule == m

    def test_cssText(self):
        "CSSMediaRule.cssText"
        style = '''{
    a {
        color: red
        }
    }'''

        mls = {
            ' (min-device-pixel-ratio: 1.3), (min-resolution: 1.3dppx) ': None,
            ' tv ': None,
            ' only tv ': None,
            ' not tv ': None,
            ' only tv and (color) ': None,
            ' only tv and(color)': ' only tv and (color) ',
            ' only tv and (color: red) ': None,
            ' only tv and (color: red) and (width: 100px) ': None,
            ' only tv and (color: red) and (width: 100px), tv ': None,
            ' only tv and (color: red) and (width: 100px), tv and (width: 20px) ': None,
            ' only tv and(color :red)and(  width :100px  )  ,tv and(width: 20px) ': ' only tv and (color: red) and (width: 100px), tv and (width: 20px) ',
            ' (color: red) and (width: 100px), (width: 20px) ': None,
            ' /*1*/ only /*2*/ tv /*3*/ and /*4*/ (/*5*/ width) /*5*/ /*6*/, (color) and (height) ': None,
            '(color)and(width),(height)': ' (color) and (width), (height) ',
        }
        tests = {}
        for b, a in list(mls.items()):
            if a is None:
                a = b
            tests[f'@media{b}{style}'] = f'@media{a}{style}'

        self.do_equal_p(tests)
        self.do_equal_r(tests)

        tests = {
            '@media only tv{}': '',
            '@media not tv{}': '',
            '@media only tv and (color){}': '',
            '@media only tv and (color: red){}': '',
            '@media only tv and (color: red) and (width: 100px){}': '',
            '@media only tv and (color: red) and (width: 100px), tv{}': '',
            '@media only tv and (color: red) and (width: 100px), tv and (width: 20px){}': '',
            '@media (color: red) and (width: 100px), (width: 20px){}': '',
            '@media (width){}': '',
            '@media (width:10px){}': '',
            '@media (width), (color){}': '',
            '@media (width)  ,  (color),(height){}': '',
            '@media (width)  ,  (color) and (height){}': '',
            '@media (width) and (color){}': '',
            '@media all and (width){}': '',
            '@media all and (width:10px){}': '',
            '@media all and (width), (color){}': '',
            '@media all and (width)  ,  (color),(height){}': '',
            '@media all and (width)  ,  (color) and (height){}': '',
            '@media all and (width) and (color){}': '',
            '@media only tv and (width){}': '',
            '@media only tv and (width:10px){}': '',
            '@media only tv and (width), (color){}': '',
            '@media only tv and (width)  ,  (color),(height){}': '',
            '@media only tv and (width)  ,  (color) and (height){}': '',
            '@media only tv and (width) and (color){}': '',
            '@media only tv and (width) "name" {}': '',
            '@media only tv and (width:10px) "name" {}': '',
            '@media only tv and (width), (color){}': '',
            '@media only tv and (width)  ,  (color),(height){}': '',
            '@media only tv and (width)  ,  (color) and (height){}': '',
            '@media only tv and (width) and (color){}': '',
            '@media all "name"{}': '',
            '@media all {}': '',
            '@media/*x*/all{}': '',
            '@media all { a{ x: 1} }': '@media all {\n    a {\n        x: 1\n        }\n    }',
            '@media all "name" { a{ x: 1} }': '@media all "name" {\n    a {\n        x: 1\n        }\n    }',
            '@MEDIA all { a{x:1} }': '@media all {\n    a {\n        x: 1\n        }\n    }',
            '@\\media all { a{x:1} }': '@media all {\n    a {\n        x: 1\n        }\n    }',
            '@media all {@x some;a{color: red;}b{color: green;}}': '''@media all {
    @x some;
    a {
        color: red
        }
    b {
        color: green
        }
    }''',
            '@media all { @x{}}': '@media all {\n    @x {\n        }\n    }',
            '@media all "n" /**/ { @x{}}': '@media all "n" /**/ {\n    @x {\n        }\n    }',
            # comments
            '@media/*1*//*2*/all/*3*//*4*/{/*5*/a{x:1}}': '@media /*1*/ /*2*/ all /*3*/ /*4*/ {\n    /*5*/\n    a {\n        x: 1\n        }\n    }',
            '@media  /*1*/  /*2*/  all  /*3*/  /*4*/  {  /*5*/  a{ x: 1} }': '@media /*1*/ /*2*/ all /*3*/ /*4*/ {\n    /*5*/\n    a {\n        x: 1\n        }\n    }',
            # WS
            '@media\n\t\f all\n\t\f {\n\t\f a{ x: 1}\n\t\f }': '@media all {\n    a {\n        x: 1\n        }\n    }',
            # @page rule inside @media
            '@media all { @page { margin: 0; } }': '@media all {\n    @page {\n        margin: 0\n        }\n    }',
            # nested media rules
            '@media all { @media all { p { color: red; } } }': '@media all {\n    @media all {\n        p {\n            '
            'color: red\n            }\n        }\n    }',
        }
        self.do_equal_p(tests)
        self.do_equal_r(tests)

        tests = {
            '@media {}': xml.dom.SyntaxErr,
            '@media;': xml.dom.SyntaxErr,
            '@media/*only comment*/{}': xml.dom.SyntaxErr,
            '@media all;': xml.dom.SyntaxErr,
            '@media all "n";': xml.dom.SyntaxErr,
            '@media all; @x{}': xml.dom.SyntaxErr,
            '@media { a{ x: 1} }': xml.dom.SyntaxErr,
            '@media "name" { a{ x: 1} }': xml.dom.SyntaxErr,
            '@media "name" all { a{ x: 1} }': xml.dom.SyntaxErr,
            '@media all { @charset "x"; a{}}': xml.dom.HierarchyRequestErr,
            '@media all { @import "x"; a{}}': xml.dom.HierarchyRequestErr,
            '@media all { , }': xml.dom.SyntaxErr,
            '@media all {}EXTRA': xml.dom.SyntaxErr,
            '@media ({}': xml.dom.SyntaxErr,
            '@media (color{}': xml.dom.SyntaxErr,
            '@media (color:{}': xml.dom.SyntaxErr,
            '@media (color:red{}': xml.dom.SyntaxErr,
            '@media (:red){}': xml.dom.SyntaxErr,
            '@media (:){}': xml.dom.SyntaxErr,
            '@media color:red){}': xml.dom.SyntaxErr,
        }
        self.do_raise_p(tests)
        self.do_raise_r(tests)

        tests = {
            # extra stuff
            '@media all { x{} } a{}': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests)

        m = cssutils.css.CSSMediaRule()
        m.cssText = '''@media all {@x; /*1*/a{color: red;}}'''
        for r in m.cssRules:
            assert m == r.parentRule
            assert m.parentStyleSheet == r.parentStyleSheet

        cssutils.ser.prefs.useDefaults()

    def test_media(self):
        "CSSMediaRule.media"
        # see CSSImportRule.media

        # setting not allowed
        with pytest.raises(AttributeError):
            self.r.__setattr__('media', None)
        with pytest.raises(AttributeError):
            self.r.__setattr__('media', 0)

        # set mediaText instead
        self.r.media.mediaText = 'print'
        self.r.insertRule(self.stylerule)
        assert '' == self.r.cssText
        cssutils.ser.prefs.keepEmptyRules = True
        assert '@media print {\n    a {}\n    }' == self.r.cssText
        cssutils.ser.prefs.useDefaults()

    def test_name(self):
        "CSSMediaRule.name"
        r = cssutils.css.CSSMediaRule()
        r.cssText = '@media all "\\n\\"ame" {a{left: 0}}'

        assert '\\n"ame' == r.name
        r.name = "n"
        assert 'n' == r.name
        assert (
            '@media all "n" {\n    a {\n        left: 0\n        }\n    }' == r.cssText
        )
        r.name = '"'
        assert '"' == r.name
        assert (
            '@media all "\\"" {\n    a {\n        left: 0\n        }\n    }'
            == r.cssText
        )

        r.name = ''
        assert r.name is None
        assert '@media all {\n    a {\n        left: 0\n        }\n    }' == r.cssText

        r.name = None
        assert r.name is None
        assert '@media all {\n    a {\n        left: 0\n        }\n    }' == r.cssText

        with pytest.raises(xml.dom.SyntaxErr):
            r._setName(0)
        with pytest.raises(xml.dom.SyntaxErr):
            r._setName(123)

    def test_deleteRuleIndex(self):
        "CSSMediaRule.deleteRule(index)"
        # see CSSStyleSheet.deleteRule
        m = cssutils.css.CSSMediaRule()
        m.cssText = '''@media all {
            @a;
            /* x */
            @b;
            @c;
            @d;
        }'''
        assert 5 == m.cssRules.length
        with pytest.raises(xml.dom.IndexSizeErr):
            m.deleteRule(5)

        # end -1
        # check parentRule
        r = m.cssRules[-1]
        assert m == r.parentRule
        m.deleteRule(-1)
        assert r.parentRule is None

        assert 4 == m.cssRules.length
        assert (
            '@media all {\n    @a;\n    /* x */\n    @b;\n    @c;\n    }' == m.cssText
        )
        # beginning
        m.deleteRule(0)
        assert 3 == m.cssRules.length
        assert '@media all {\n    /* x */\n    @b;\n    @c;\n    }' == m.cssText
        # middle
        m.deleteRule(1)
        assert 2 == m.cssRules.length
        assert '@media all {\n    /* x */\n    @c;\n    }' == m.cssText
        # end
        m.deleteRule(1)
        assert 1 == m.cssRules.length
        assert '@media all {\n    /* x */\n    }' == m.cssText

    def test_deleteRule(self):
        "CSSMediaRule.deleteRule(rule)"
        m = cssutils.css.CSSMediaRule()
        m.cssText = '''@media all {
            a { color: red; }
            b { color: blue; }
            c { color: green; }
        }'''
        s1, s2, s3 = m.cssRules

        r = cssutils.css.CSSStyleRule()
        with pytest.raises(xml.dom.IndexSizeErr):
            m.deleteRule(r)

        assert 3 == m.cssRules.length
        m.deleteRule(s2)
        assert 2 == m.cssRules.length
        assert (
            m.cssText
            == '@media all {\n    a {\n        color: red\n        }\n    c {\n        color: green\n        }\n    }'
        )
        with pytest.raises(xml.dom.IndexSizeErr):
            m.deleteRule(s2)

    def test_add(self):
        "CSSMediaRule.add()"
        # see CSSStyleSheet.add
        r = cssutils.css.CSSMediaRule()
        stylerule1 = cssutils.css.CSSStyleRule()
        stylerule2 = cssutils.css.CSSStyleRule()
        r.add(stylerule1)
        r.add(stylerule2)
        assert r.cssRules[0] == stylerule1
        assert r.cssRules[1] == stylerule2

    def test_insertRule(self):
        "CSSMediaRule.insertRule"
        # see CSSStyleSheet.insertRule
        r = cssutils.css.CSSMediaRule()
        charsetrule = cssutils.css.CSSCharsetRule('ascii')
        importrule = cssutils.css.CSSImportRule('x')
        namespacerule = cssutils.css.CSSNamespaceRule()
        unknownrule = cssutils.css.CSSUnknownRule('@x;')
        stylerule = cssutils.css.CSSStyleRule('a')
        stylerule.cssText = 'a { x: 1}'
        comment1 = cssutils.css.CSSComment('/*1*/')
        comment2 = cssutils.css.CSSComment('/*2*/')

        # hierarchy
        with pytest.raises(xml.dom.HierarchyRequestErr):
            r.insertRule(charsetrule, 0)
        with pytest.raises(xml.dom.HierarchyRequestErr):
            r.insertRule(importrule, 0)
        with pytest.raises(xml.dom.HierarchyRequestErr):
            r.insertRule(namespacerule, 0)

        # start insert
        r.insertRule(stylerule, 0)
        assert r == stylerule.parentRule
        assert r.parentStyleSheet == stylerule.parentStyleSheet
        # before
        r.insertRule(comment1, 0)
        assert r == comment1.parentRule
        assert r.parentStyleSheet == stylerule.parentStyleSheet
        # end explicit
        r.insertRule(unknownrule, 2)
        assert r == unknownrule.parentRule
        assert r.parentStyleSheet == stylerule.parentStyleSheet
        # end implicit
        r.insertRule(comment2)
        assert r == comment2.parentRule
        assert r.parentStyleSheet == stylerule.parentStyleSheet
        assert (
            '@media all {\n    /*1*/\n    a {\n        x: 1\n        }\n    @x;\n    /*2*/\n    }'
            == r.cssText
        )

        # index
        with pytest.raises(xml.dom.IndexSizeErr):
            r.insertRule(stylerule, -1)
        with pytest.raises(xml.dom.IndexSizeErr):
            r.insertRule(stylerule, r.cssRules.length + 1)

    def test_InvalidModificationErr(self):
        "CSSMediaRule.cssText InvalidModificationErr"
        self._test_InvalidModificationErr('@media')

    def test_incomplete(self):
        "CSSMediaRule (incomplete)"
        tests = {
            '@media all { @unknown;': '@media all {\n    @unknown;\n    }',  # no }
            '@media all { a {x:"1"}': '@media all {\n    a {\n        x: "1"\n        }\n    }',  # no }
            '@media all { a {x:"1"': '@media all {\n    a {\n        x: "1"\n        }\n    }',  # no }}
            '@media all { a {x:"1': '@media all {\n    a {\n        x: "1"\n        }\n    }',  # no "}}
        }
        self.do_equal_p(tests)  # parse

    def test_reprANDstr(self):
        "CSSMediaRule.__repr__(), .__str__()"
        mediaText = 'tv, print'

        s = cssutils.css.CSSMediaRule(mediaText=mediaText)

        assert mediaText in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert mediaText == s2.media.mediaText

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssmediarule.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSRule"""

import xml.dom
from . import basetest
import cssutils.css
import pytest


class TestCSSRule(basetest.BaseTestCase):
    """
    base class for all CSSRule subclass tests

    overwrite setUp with the appriopriate values, will be used in
    test_init and test_readonly
    overwrite all tests as you please, use::

        super(CLASSNAME, self).test_TESTNAME(params)

    to use the base class tests too
    """

    def setup_method(self):
        """
        self.r is the rule
        self.rRO the readonly rule
        relf.r_type the type as defined in CSSRule
        """
        self._setup_rule()
        self.sheet = cssutils.css.CSSStyleSheet()

    def _setup_rule(self):
        self.r = cssutils.css.CSSRule()
        self.rRO = cssutils.css.CSSRule()
        self.rRO._readonly = True  # must be set here!
        self.r_type = cssutils.css.CSSRule.UNKNOWN_RULE
        self.r_typeString = 'UNKNOWN_RULE'

    def teardown_method(self):
        cssutils.ser.prefs.useDefaults()

    def test_init(self):
        "CSSRule.type and init"
        assert self.r_type == self.r.type
        assert self.r_typeString == self.r.typeString
        assert '' == self.r.cssText
        assert self.r.parentRule is None
        assert self.r.parentStyleSheet is None

    def test_parentRule_parentStyleSheet_type(self):  # noqa: C901
        "CSSRule.parentRule .parentStyleSheet .type"
        rules = [
            ('@charset "ascii";', cssutils.css.CSSRule.CHARSET_RULE),
            ('@import "x";', cssutils.css.CSSRule.IMPORT_RULE),
            ('@namespace "x";', cssutils.css.CSSRule.NAMESPACE_RULE),
            ('@font-face { src: url(x) }', cssutils.css.CSSRule.FONT_FACE_RULE),
            (
                '''@media all {
                    @x;
                    a { color: red }
                    /* c  */
                }''',
                cssutils.css.CSSRule.MEDIA_RULE,
            ),
            ('@page :left { color: red }', cssutils.css.CSSRule.PAGE_RULE),
            ('@unknown;', cssutils.css.CSSRule.UNKNOWN_RULE),
            ('b { left: 0 }', cssutils.css.CSSRule.STYLE_RULE),
            ('/*1*/', cssutils.css.CSSRule.COMMENT),  # must be last for add test
        ]
        mrt = [
            cssutils.css.CSSRule.UNKNOWN_RULE,
            cssutils.css.CSSRule.STYLE_RULE,
            cssutils.css.CSSRule.COMMENT,
        ]

        def test(s):
            for i, rule in enumerate(s):
                assert rule.parentRule is None
                assert rule.parentStyleSheet == s
                # self.assertEqual(rule.type, rules[i][1])
                if rule.MEDIA_RULE == rule.type:
                    for j, r in enumerate(rule):
                        assert r.parentRule == rule
                        assert r.parentStyleSheet == s
                        assert r.type == mrt[j]

                if i == 0:  # check encoding
                    assert 'ascii' == s.encoding
                elif i == 2:  # check namespaces
                    assert 'x' == s.namespaces['']

        cssText = ''.join(r[0] for r in rules)
        # parsing
        s = cssutils.parseString(cssText)
        test(s)
        # sheet.cssText
        s = cssutils.css.CSSStyleSheet()
        s.cssText = cssText
        test(s)
        # sheet.add CSS
        s = cssutils.css.CSSStyleSheet()
        for css, type_ in rules:
            s.add(css)
        test(s)
        # sheet.insertRule CSS
        s = cssutils.css.CSSStyleSheet()
        for css, type_ in rules:
            s.insertRule(css)
        test(s)

        types = [
            cssutils.css.CSSCharsetRule,
            cssutils.css.CSSImportRule,
            cssutils.css.CSSNamespaceRule,
            cssutils.css.CSSFontFaceRule,
            cssutils.css.CSSMediaRule,
            cssutils.css.CSSPageRule,
            cssutils.css.CSSUnknownRule,
            cssutils.css.CSSStyleRule,
            cssutils.css.CSSComment,
        ]
        # sheet.add CSSRule
        s = cssutils.css.CSSStyleSheet()
        for i, (css, type_) in enumerate(rules):
            rule = types[i]()
            rule.cssText = css
            s.add(rule)
        test(s)
        # sheet.insertRule CSSRule
        s = cssutils.css.CSSStyleSheet()
        for i, (css, type_) in enumerate(rules):
            rule = types[i]()
            rule.cssText = css
            s.insertRule(rule)
        test(s)

    def test_CSSMediaRule_cssRules_parentRule_parentStyleSheet_type(self):
        "CSSMediaRule.cssRules.parentRule .parentStyleSheet .type"
        rules = [
            ('b { left: 0 }', cssutils.css.CSSRule.STYLE_RULE),
            ('/*1*/', cssutils.css.CSSRule.COMMENT),
            ('@x;', cssutils.css.CSSRule.UNKNOWN_RULE),
        ]

        def test(s):
            mr = s.cssRules[0]
            for i, rule in enumerate(mr):
                assert rule.parentRule == mr
                assert rule.parentStyleSheet == s
                assert rule.parentStyleSheet == mr.parentStyleSheet
                assert rule.type == rules[i][1]

        cssText = '@media all { %s }' % ''.join(r[0] for r in rules)
        # parsing
        s = cssutils.parseString(cssText)
        test(s)
        # sheet.cssText
        s = cssutils.css.CSSStyleSheet()
        s.cssText = cssText
        test(s)

        def getMediaSheet():
            s = cssutils.css.CSSStyleSheet()
            s.cssText = '@media all {}'
            return s, s.cssRules[0]

        # sheet.add CSS
        s, mr = getMediaSheet()
        for css, type_ in rules:
            mr.add(css)
        test(s)
        # sheet.insertRule CSS
        s, mr = getMediaSheet()
        for css, type_ in rules:
            mr.insertRule(css)
        test(s)

        types = [
            cssutils.css.CSSStyleRule,
            cssutils.css.CSSComment,
            cssutils.css.CSSUnknownRule,
        ]
        # sheet.add CSSRule
        s, mr = getMediaSheet()
        for i, (css, type_) in enumerate(rules):
            rule = types[i]()
            rule.cssText = css
            mr.add(rule)
        test(s)
        # sheet.insertRule CSSRule
        s, mr = getMediaSheet()
        for i, (css, type_) in enumerate(rules):
            rule = types[i]()
            rule.cssText = css
            mr.insertRule(rule)
        test(s)

    def test_readonly(self):
        "CSSRule readonly"
        self.rRO = cssutils.css.CSSRule()
        self.rRO._readonly = True
        assert self.rRO._readonly
        assert '' == self.rRO.cssText
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            self.rRO._setCssText('x')
        assert '' == self.rRO.cssText

    def _test_InvalidModificationErr(self, startwithspace):
        """
        CSSRule.cssText InvalidModificationErr

        called by subclasses

        startwithspace

        for test starting with this not the test but " test" is tested
        e.g. " @page {}"
        exception is the style rule test
        """
        tests = (
            '',
            '/* comment */',
            '@charset "utf-8";',
            '@font-face {}',
            '@import url(x);',
            '@media all {}',
            '@namespace "x";' '@page {}',
            '@unknown;',
            '@variables;',
            # TODO:
            # u'@top-left {}'
            'a style rule {}',
        )
        for test in tests:
            if startwithspace in ('a style rule',) and test in (
                '/* comment */',
                'a style rule {}',
            ):
                continue

            if test.startswith(startwithspace):
                test = ' %s' % test

            with pytest.raises(xml.dom.InvalidModificationErr):
                self.r._setCssText(test)

        # check that type is readonly
        with pytest.raises(AttributeError):
            self.r.__setattr__('parentRule', None)
        with pytest.raises(AttributeError):
            self.r.__setattr__('parentStyleSheet', None)
        with pytest.raises(AttributeError):
            self.r.__setattr__('type', 1)
        with pytest.raises(AttributeError):
            self.r.__setattr__('typeString', "")

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssrule.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.cssstyledelaration.CSSStyleDeclaration."""

import xml.dom
from . import basetest
import cssutils
import pytest


class TestCSSStyleDeclaration(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.css.CSSStyleDeclaration()

    def test_init(self):
        "CSSStyleDeclaration.__init__()"
        s = cssutils.css.CSSStyleDeclaration()
        assert '' == s.cssText
        assert 0 == s.length
        assert s.parentRule is None

        s = cssutils.css.CSSStyleDeclaration(cssText='left: 0')
        assert 'left: 0' == s.cssText
        assert '0' == s.getPropertyValue('left')

        sheet = cssutils.css.CSSStyleRule()
        s = cssutils.css.CSSStyleDeclaration(parentRule=sheet)
        assert sheet == s.parentRule

        # should not be used but ordered parameter test
        s = cssutils.css.CSSStyleDeclaration('top: 0', sheet)
        assert 'top: 0' == s.cssText
        assert sheet == s.parentRule

    def test_items(self):
        "CSSStyleDeclaration[CSSName]"
        s = cssutils.css.CSSStyleDeclaration()
        name, value, priority = 'color', 'name', ''
        s[name] = value
        assert value == s[name]
        assert value == s.__getattribute__(name)
        assert value == s.getProperty(name).value
        assert priority == s.getProperty(name).priority

        name, value, priority = 'UnKnown-ProPERTY', 'unknown value', 'important'
        s[name] = (value, priority)
        assert value == s[name]
        assert value == s[name.lower()]  # will be normalized
        with pytest.raises(AttributeError):
            s.__getattribute__(name)
        assert value == s.getProperty(name).value
        assert priority == s.getProperty(name).priority

        name, value, priority = 'item', '1', ''
        s[name] = value
        assert value == s[name]
        assert value == s.getProperty(name).value
        assert priority == s.getProperty(name).priority

        del s[name]
        assert '' == s[name]
        assert '' == s['never set']

    def test__contains__(self):
        "CSSStyleDeclaration.__contains__(nameOrProperty)"
        s = cssutils.css.CSSStyleDeclaration(cssText=r'x: 1;\y: 2')
        for test in ('x', r'x', 'y', r'y'):
            assert test in s
            assert test.upper() in s
            assert cssutils.css.Property(test, '1') in s
        assert 'z' not in s
        assert cssutils.css.Property('z', '1') not in s

    def test__iter__item(self):
        "CSSStyleDeclaration.__iter__ and .item"
        s = cssutils.css.CSSStyleDeclaration()
        s.cssText = r'''
            color: red; c\olor: blue; CO\lor: green;
            left: 1px !important; left: 0;
            border: 0;
        '''
        # __iter__
        ps = []
        for p in s:
            ps.append((p.literalname, p.value, p.priority))
        assert len(ps) == 3
        assert ps[0] == (r'co\lor', 'green', '')
        assert ps[1] == (r'left', '1px', 'important')
        assert ps[2] == (r'border', '0', '')

        # item
        assert s.length == 3
        assert s.item(0) == 'color'
        assert s.item(1) == 'left'
        assert s.item(2) == 'border'
        assert s.item(10) == ''

    def test_keys(self):
        "CSSStyleDeclaration.keys()"
        s = cssutils.parseStyle('x:1; x:2; y:1')
        assert ['x', 'y'] == list(s.keys())
        assert s['x'] == '2'
        assert s['y'] == '1'

    def test_parse(self):
        "CSSStyleDeclaration parse"
        # error but parse
        tests = {
            # property names are caseinsensitive
            'TOP:0': 'top: 0',
            'top:0': 'top: 0',
            # simple escape
            'c\\olor: red; color:green': 'color: green',
            'color:g\\reen': 'color: g\\reen',
            # http://www.w3.org/TR/2009/CR-CSS2-20090423/syndata.html#illegalvalues
            'color:green': 'color: green',
            'color:green; color': 'color: green',
            'color:red;   color; color:green': 'color: green',
            'color:green; color:': 'color: green',
            'color:red;   color:; color:green': 'color: green',
            'color:green; color{;color:maroon}': 'color: green',
            'color:red; color{;color:maroon}; color:green': 'color: green',
            # tantek hack
            r'''color: red;
voice-family: "\"}\"";
voice-family:inherit;
color: green;''': 'voice-family: inherit;\ncolor: green',
            r'''col\or: blue;
                font-family: 'Courier New Times
                color: red;
                color: green;''': 'color: green',
            # special IE hacks are not preserved anymore (>=0.9.5b3)
            '/color: red; color: green': 'color: green',
            '/ color: red; color: green': 'color: green',
            '1px: red; color: green': 'color: green',
            '0: red; color: green': 'color: green',
            '1px:: red; color: green': 'color: green',
            r'$top: 0': '',
            r'$: 0': '',  # really invalid!
            # unknown rule but valid
            '@x;\ncolor: red': None,
            '@x {\n    }\ncolor: red': None,
            '/**/\ncolor: red': None,
            '/**/\ncolor: red;\n/**/': None,
            # issue #28
            ';color: red': 'color: red',
            ';;color: red;;': 'color: red',
        }
        cssutils.ser.prefs.keepAllProperties = False
        for test, exp in list(tests.items()):
            sh = cssutils.parseString('a { %s }' % test)
            if exp is None:
                exp = '%s' % test
            elif exp != '':
                exp = '%s' % exp
            assert exp == sh.cssRules[0].style.cssText

        cssutils.ser.prefs.useDefaults()

    def test_serialize(self):
        "CSSStyleDeclaration serialize"
        s = cssutils.css.CSSStyleDeclaration()
        tests = {
            'a:1 !important; a:2;b:1': (
                'a: 1 !important;\nb: 1',
                'a: 1 !important;\na: 2;\nb: 1',
            )
        }
        for test, exp in list(tests.items()):
            s.cssText = test
            cssutils.ser.prefs.keepAllProperties = False
            assert exp[0] == s.cssText
            cssutils.ser.prefs.keepAllProperties = True
            assert exp[1] == s.cssText

        cssutils.ser.prefs.useDefaults()

    def test_children(self):
        "CSSStyleDeclaration.children()"
        style = '/*1*/color: red; color: green; @x;'
        types = [
            cssutils.css.CSSComment,
            cssutils.css.Property,
            cssutils.css.Property,
            cssutils.css.CSSUnknownRule,
        ]

        def t(s):
            for i, x in enumerate(s.children()):
                assert types[i] == type(x)
                assert x.parent == s

        t(cssutils.parseStyle(style))
        t(cssutils.parseString('a {' + style + '}').cssRules[0].style)
        t(
            cssutils.parseString('@media all {a {' + style + '}}')
            .cssRules[0]
            .cssRules[0]
            .style
        )

        s = cssutils.parseStyle(style)
        s['x'] = '0'
        assert s == s.getProperty('x').parent
        s.setProperty('y', '1')
        assert s == s.getProperty('y').parent

    def test_cssText(self):
        "CSSStyleDeclaration.cssText"
        # empty
        s = cssutils.css.CSSStyleDeclaration()
        tests = {'': '', ' ': '', ' \t \n  ': '', '/*x*/': '/*x*/'}
        for test, exp in list(tests.items()):
            s.cssText = 'left: 0;'  # dummy to reset s
            s.cssText = test
            assert exp == s.cssText

        # normal
        s = cssutils.css.CSSStyleDeclaration()
        tests = {
            ';': '',
            'left: 0': 'left: 0',
            'left:0': 'left: 0',
            ' left : 0 ': 'left: 0',
            'left: 0;': 'left: 0',
            'left: 0 !important ': 'left: 0 !important',
            'left:0!important': 'left: 0 !important',
            'left: 0; top: 1': 'left: 0;\ntop: 1',
            # comments
            # TODO: spaces?
            '/*1*//*2*/left/*3*//*4*/:/*5*//*6*/0/*7*//*8*/!/*9*//*a*/important/*b*//*c*/;': '/*1*/\n/*2*/\nleft/*3*//*4*/: /*5*/ /*6*/ 0 /*7*/ /*8*/ !/*9*//*a*/important/*b*//*c*/',
            '/*1*/left: 0;/*2*/ top: 1/*3*/': '/*1*/\nleft: 0;\n/*2*/\ntop: 1 /*3*/',
            'left:0; top:1;': 'left: 0;\ntop: 1',
            '/*1*/left: 0;/*2*/ top: 1;/*3*/': '/*1*/\nleft: 0;\n/*2*/\ntop: 1;\n/*3*/',
            # WS
            'left:0!important;margin:1px 2px 3px 4px!important;': 'left: 0 !important;\nmargin: 1px 2px 3px 4px !important',
            '\n\r\f\t left\n\r\f\t :\n\r\f\t 0\n\r\f\t !\n\r\f\t important\n\r\f\t ;\n\r\f\t margin\n\r\f\t :\n\r\f\t 1px\n\r\f\t 2px\n\r\f\t 3px\n\r\f\t 4px;': 'left: 0 !important;\nmargin: 1px 2px 3px 4px',
        }
        for test, exp in list(tests.items()):
            s.cssText = test
            assert exp == s.cssText

        # exception
        tests = {
            'color: #xyz': xml.dom.SyntaxErr,
            'top': xml.dom.SyntaxErr,
            'top:': xml.dom.SyntaxErr,
            'top : ': xml.dom.SyntaxErr,
            'top:!important': xml.dom.SyntaxErr,
            'top:!important;': xml.dom.SyntaxErr,
            'top:;': xml.dom.SyntaxErr,
            'top 0': xml.dom.SyntaxErr,
            'top 0;': xml.dom.SyntaxErr,
            ':': xml.dom.SyntaxErr,
            ':0': xml.dom.SyntaxErr,
            ':0;': xml.dom.SyntaxErr,
            ':0!important': xml.dom.SyntaxErr,
            ':;': xml.dom.SyntaxErr,
            ': ;': xml.dom.SyntaxErr,
            ':!important;': xml.dom.SyntaxErr,
            ': !important;': xml.dom.SyntaxErr,
            '0': xml.dom.SyntaxErr,
            '0!important': xml.dom.SyntaxErr,
            '0!important;': xml.dom.SyntaxErr,
            '0;': xml.dom.SyntaxErr,
            '!important': xml.dom.SyntaxErr,
            '!important;': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests)

    def test_getCssText(self):
        "CSSStyleDeclaration.getCssText(separator)"
        s = cssutils.css.CSSStyleDeclaration(cssText='a:1;b:2')
        assert 'a: 1;\nb: 2' == s.getCssText()
        assert 'a: 1;b: 2' == s.getCssText(separator='')
        assert 'a: 1;/*x*/b: 2' == s.getCssText(separator='/*x*/')

    def test_parentRule(self):
        "CSSStyleDeclaration.parentRule"
        s = cssutils.css.CSSStyleDeclaration()
        sheet = cssutils.css.CSSStyleRule()
        s.parentRule = sheet
        assert sheet == s.parentRule

        sheet = cssutils.parseString('a{x:1}')
        s = sheet.cssRules[0]
        d = s.style
        assert s == d.parentRule

        s = cssutils.parseString(
            '''
        @font-face {
            font-weight: bold;
            }
        a {
            font-weight: bolder;
            }
        @page {
            font-weight: bolder;
            }
        '''
        )
        for r in s:
            assert r.style.parentRule == r

    def test_getProperty(self):
        "CSSStyleDeclaration.getProperty"
        s = cssutils.css.CSSStyleDeclaration()
        s.cssText = r'''
            color: red; c\olor: blue; CO\lor: green;
            left: 1px !important; left: 0;
            border: 0;
        '''
        assert s.getProperty('color').cssText == r'co\lor: green'
        assert s.getProperty(r'COLO\r').cssText == r'co\lor: green'
        assert s.getProperty('left').cssText == r'left: 1px !important'
        assert s.getProperty('border').cssText == r'border: 0'

    def test_getProperties(self):
        "CSSStyleDeclaration.getProperties()"
        s = cssutils.css.CSSStyleDeclaration(
            cssText='/*1*/y:0;x:a !important;y:1; \\x:b;'
        )
        tests = {
            # name, all
            (None, False): [('y', '1', ''), ('x', 'a', 'important')],
            (None, True): [
                ('y', '0', ''),
                ('x', 'a', 'important'),
                ('y', '1', ''),
                ('\\x', 'b', ''),
            ],
            ('x', False): [('x', 'a', 'important')],
            ('\\x', False): [('x', 'a', 'important')],
            ('x', True): [('x', 'a', 'important'), ('\\x', 'b', '')],
            ('\\x', True): [('x', 'a', 'important'), ('\\x', 'b', '')],
        }
        for test in tests:
            name, all = test
            expected = tests[test]
            actual = s.getProperties(name, all)
            assert len(expected) == len(actual)
            for i, ex in enumerate(expected):
                a = actual[i]
                assert ex == (a.literalname, a.value, a.priority)

        # order is be effective properties set
        s = cssutils.css.CSSStyleDeclaration(cssText='a:0;b:1;a:1')
        assert 'ba' == ''.join([p.name for p in s])

    def test_getPropertyCSSValue(self):
        "CSSStyleDeclaration.getPropertyCSSValue()"
        s = cssutils.css.CSSStyleDeclaration(cssText='color: red;c\\olor: green')
        assert 'green' == s.getPropertyCSSValue('color').cssText
        assert 'green' == s.getPropertyCSSValue('c\\olor').cssText
        assert 'red' == s.getPropertyCSSValue('color', False).cssText
        assert 'green' == s.getPropertyCSSValue('c\\olor', False).cssText

    #        # shorthand CSSValue should be None
    #        SHORTHAND = [
    #            u'background',
    #            u'border',
    #            u'border-left', u'border-right',
    #            u'border-top', u'border-bottom',
    #            u'border-color', u'border-style', u'border-width',
    #            u'cue',
    #            u'font',
    #            u'list-style',
    #            u'margin',
    #            u'outline',
    #            u'padding',
    #            u'pause']
    #        for short in SHORTHAND:
    #            s.setProperty(short, u'inherit')
    #            self.assertEqual(None, s.getPropertyCSSValue(short))

    def test_getPropertyValue(self):
        "CSSStyleDeclaration.getPropertyValue()"
        s = cssutils.css.CSSStyleDeclaration()
        assert '' == s.getPropertyValue('unset')

        s.setProperty('left', '0')
        assert '0' == s.getPropertyValue('left')

        s.setProperty('border', '1px  solid  green')
        assert '1px solid green' == s.getPropertyValue('border')

        s = cssutils.css.CSSStyleDeclaration(cssText='color: red;c\\olor: green')
        assert 'green' == s.getPropertyValue('color')
        assert 'green' == s.getPropertyValue('c\\olor')
        assert 'red' == s.getPropertyValue('color', False)
        assert 'green' == s.getPropertyValue('c\\olor', False)

        tests = {
            r'color: red; color: green': 'green',
            r'c\olor: red; c\olor: green': 'green',
            r'color: red; c\olor: green': 'green',
            r'color: red !important; color: green !important': 'green',
            r'color: green !important; color: red': 'green',
        }
        for test in tests:
            s = cssutils.css.CSSStyleDeclaration(cssText=test)
            assert tests[test] == s.getPropertyValue('color')

    def test_getPropertyPriority(self):
        "CSSStyleDeclaration.getPropertyPriority()"
        s = cssutils.css.CSSStyleDeclaration()
        assert '' == s.getPropertyPriority('unset')

        s.setProperty('left', '0', '!important')
        assert 'important' == s.getPropertyPriority('left')

        s = cssutils.css.CSSStyleDeclaration(
            cssText='x: 1 !important;\\x: 2;x: 3 !important;\\x: 4'
        )
        assert 'important' == s.getPropertyPriority('x')
        assert 'important' == s.getPropertyPriority('\\x')
        assert 'important' == s.getPropertyPriority('x', True)
        assert '' == s.getPropertyPriority('\\x', False)

    def test_removeProperty(self):
        "CSSStyleDeclaration.removeProperty()"
        cssutils.ser.prefs.useDefaults()
        s = cssutils.css.CSSStyleDeclaration()
        css = r'\x:0 !important; x:1; \x:2; x:3'

        # normalize=True DEFAULT
        s.cssText = css
        assert '0' == s.removeProperty('x')
        assert '' == s.cssText

        # normalize=False
        s.cssText = css
        assert '3' == s.removeProperty('x', normalize=False)
        assert r'\x: 0 !important;\x: 2' == s.getCssText(separator='')
        assert '0' == s.removeProperty(r'\x', normalize=False)
        assert '' == s.cssText

        s.cssText = css
        assert '0' == s.removeProperty(r'\x', normalize=False)
        assert r'x: 1;x: 3' == s.getCssText(separator='')
        assert '3' == s.removeProperty('x', normalize=False)
        assert '' == s.cssText

    def test_setProperty(self):
        "CSSStyleDeclaration.setProperty()"
        s = cssutils.css.CSSStyleDeclaration()
        s.setProperty('top', '0', '!important')
        assert '0' == s.getPropertyValue('top')
        assert 'important' == s.getPropertyPriority('top')
        s.setProperty('top', '1px')
        assert '1px' == s.getPropertyValue('top')
        assert '' == s.getPropertyPriority('top')

        s.setProperty('top', '2px')
        assert '2px' == s.getPropertyValue('top')

        s.setProperty('\\top', '3px')
        assert '3px' == s.getPropertyValue('top')

        s.setProperty('\\top', '4px', normalize=False)
        assert '4px' == s.getPropertyValue('top')
        assert '4px' == s.getPropertyValue('\\top', False)
        assert '3px' == s.getPropertyValue('top', False)

        # case insensitive
        s.setProperty('TOP', '0', '!IMPORTANT')
        assert '0' == s.getPropertyValue('top')
        assert 'important' == s.getPropertyPriority('top')

        tests = {
            ('left', '0', ''): 'left: 0',
            ('left', '0', 'important'): 'left: 0 !important',
            ('LEFT', '0', 'important'): 'left: 0 !important',
            ('left', '0', 'important'): 'left: 0 !important',
        }
        for test, exp in list(tests.items()):
            s = cssutils.css.CSSStyleDeclaration()
            n, v, p = test
            s.setProperty(n, v, p)
            assert exp == s.cssText
            assert v == s.getPropertyValue(n)
            assert p == s.getPropertyPriority(n)

        # empty
        s = cssutils.css.CSSStyleDeclaration()
        assert '' == s.top
        s.top = '0'
        assert '0' == s.top
        s.top = ''
        assert '' == s.top
        s.top = None
        assert '' == s.top

    def test_setProperty2(self):
        "CSSStyleDeclaration.setProperty(replace=)"
        s = cssutils.css.CSSStyleDeclaration()
        s.setProperty('top', '1px')
        assert len(s.getProperties('top', all=True)) == 1

        s.setProperty('top', '2px')
        assert len(s.getProperties('top', all=True)) == 1
        assert s.getPropertyValue('top') == '2px'

        s.setProperty('top', '3px', replace=False)
        assert len(s.getProperties('top', all=True)) == 2
        assert s.getPropertyValue('top') == '3px'

    def test_length(self):
        "CSSStyleDeclaration.length"
        s = cssutils.css.CSSStyleDeclaration()

        # cssText
        s.cssText = 'left: 0'
        assert 1 == s.length
        assert 1 == len(s.seq)
        s.cssText = '/*1*/left/*x*/:/*x*/0/*x*/;/*2*/ top: 1;/*3*/'
        assert 2 == s.length
        assert 5 == len(s.seq)

        # set
        s = cssutils.css.CSSStyleDeclaration()
        s.setProperty('top', '0', '!important')
        assert 1 == s.length
        s.setProperty('top', '1px')
        assert 1 == s.length
        s.setProperty('left', '1px')

    def test_nameParameter(self):
        "CSSStyleDeclaration.XXX(name)"
        s = cssutils.css.CSSStyleDeclaration()
        s.setProperty('top', '1px', '!important')

        assert '1px' == s.getPropertyValue('top')
        assert '1px' == s.getPropertyValue('TOP')
        assert '1px' == s.getPropertyValue(r'T\op')

        assert 'important' == s.getPropertyPriority('top')
        assert 'important' == s.getPropertyPriority('TOP')
        assert 'important' == s.getPropertyPriority(r'T\op')

        s.setProperty('top', '2px', '!important')
        assert '2px' == s.removeProperty('top')
        s.setProperty('top', '2px', '!important')
        assert '2px' == s.removeProperty('TOP')
        s.setProperty('top', '2px', '!important')
        assert '2px' == s.removeProperty(r'T\op')

    def test_css2properties(self):
        "CSSStyleDeclaration.$css2property get set del"
        s = cssutils.css.CSSStyleDeclaration(
            cssText='left: 1px;color: red; font-style: italic'
        )

        s.color = 'green'
        s.fontStyle = 'normal'
        assert 'green' == s.color
        assert 'normal' == s.fontStyle
        assert 'green' == s.getPropertyValue('color')
        assert 'normal' == s.getPropertyValue('font-style')
        assert '''left: 1px;\ncolor: green;\nfont-style: normal''' == s.cssText

        del s.color
        assert '''left: 1px;\nfont-style: normal''' == s.cssText
        del s.fontStyle
        assert 'left: 1px' == s.cssText

        with pytest.raises(AttributeError):
            s.__setattr__('UNKNOWN', 'red')
        # unknown properties must be set with setProperty!
        s.setProperty('UNKNOWN', 'red')
        # but are still not usable as property!
        with pytest.raises(AttributeError):
            s.__getattribute__('UNKNOWN')
        with pytest.raises(AttributeError):
            s.__delattr__('UNKNOWN')
        # but are kept
        assert 'red' == s.getPropertyValue('UNKNOWN')
        assert '''left: 1px;\nunknown: red''' == s.cssText

    def test_reprANDstr(self):
        "CSSStyleDeclaration.__repr__(), .__str__()"
        s = cssutils.css.CSSStyleDeclaration(cssText='a:1;b:2')

        assert "2" in str(s)  # length

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)

    def test_valid(self):
        "CSSStyleDeclaration.valid"
        cases = [
            ('color: red;', True),
            ('color: asd;', False),
            ('foo: red;', False),
            ('color: red; foo: red;', False),
        ]
        for case, expected in cases:
            s = cssutils.css.CSSStyleDeclaration(cssText=case)
            msg = '{!r} should be {}'.format(case, 'valid' if expected else 'invalid')
            assert s.valid == expected, msg

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssstyledeclaration.py
NikolaBlagojevic/pyrecodes
"""Tests for css.CSSStyleSheet"""

import xml.dom
from . import basetest
import cssutils.css
import pytest


class TestCSSStyleSheet(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.css.CSSStyleSheet()  # used by basetest
        self.s = self.r  # used here
        self.rule = cssutils.css.CSSStyleRule()

    def teardown_method(self):
        cssutils.ser.prefs.useDefaults()

    def test_init(self):
        "CSSStyleSheet.__init__()"
        assert 'text/css' == self.s.type
        assert not self.s._readonly
        assert [] == self.s.cssRules
        assert not self.s.disabled
        assert self.s.href is None
        assert self.s.media is None
        assert self.s.ownerNode is None
        assert self.s.parentStyleSheet is None
        assert '' == self.s.title

        # check that type is readonly
        with pytest.raises(AttributeError):
            self.r.__setattr__('href', 'x')
        with pytest.raises(AttributeError):
            self.r.__setattr__('parentStyleSheet', 'x')
        with pytest.raises(AttributeError):
            self.r.__setattr__('ownerNode', 'x')
        with pytest.raises(AttributeError):
            self.r.__setattr__('type', 'x')

    def test_iter(self):
        "CSSStyleSheet.__iter__()"
        s = cssutils.css.CSSStyleSheet()
        s.cssText = '''@import "x";@import "y";@namespace "u";'''
        types = [
            cssutils.css.CSSRule.IMPORT_RULE,
            cssutils.css.CSSRule.IMPORT_RULE,
            cssutils.css.CSSRule.NAMESPACE_RULE,
        ]
        for i, rule in enumerate(s):
            assert rule == s.cssRules[i]
            assert rule.type == types[i]

    def test_refs(self):
        """CSSStylesheet references"""
        s = cssutils.parseString('a {}')
        rules = s.cssRules
        assert s.cssRules[0].parentStyleSheet == s
        assert rules[0].parentStyleSheet == s

        # set cssText
        s.cssText = 'b{}'

        # from 0.9.7b1
        assert rules != s.cssRules

        # set cssRules
        s.cssRules = cssutils.parseString(
            '''
            @charset "ascii";
            /**/
            @import "x";
            @namespace "http://example.com/ns0";
            @media all {
                a { color: green; }
                }
            @font-face {
                font-family: x;
                }
            @page {
                font-family: Arial;
                }
            @x;
            b {}').cssRules'''
        ).cssRules
        # new object
        assert rules != s.cssRules
        for i, r in enumerate(s.cssRules):
            assert r.parentStyleSheet == s

        # namespaces
        s = cssutils.parseString('@namespace "http://example.com/ns1"; a {}')
        namespaces = s.namespaces
        assert list(s.namespaces.items()) == [('', 'http://example.com/ns1')]
        s.cssText = '@namespace x "http://example.com/ns2"; x|a {}'
        # not anymore!
        assert namespaces != s.namespaces
        assert list(s.namespaces.items()) == [('x', 'http://example.com/ns2')]

        # variables
        s = cssutils.parseString('@variables { a:1}')
        vars1 = s.variables
        assert vars1['a'] == '1'

        s = cssutils.parseString('@variables { a:2}')
        vars2 = s.variables
        assert vars1 != vars2
        assert vars1['a'] == '1'
        assert vars2['a'] == '2'

    def test_cssRules(self):
        "CSSStyleSheet.cssRules"
        s = cssutils.parseString('/*1*/a {x:1}')
        assert 2 == s.cssRules.length
        del s.cssRules[0]
        assert 1 == s.cssRules.length
        s.cssRules.append('/*2*/')
        assert 2 == s.cssRules.length
        s.cssRules.extend(cssutils.parseString('/*3*/x {y:2}').cssRules)
        assert 4 == s.cssRules.length
        assert b'a {\n    x: 1\n    }\n/*2*/\n/*3*/\nx {\n    y: 2\n    }' == s.cssText

        for r in s.cssRules:
            assert r.parentStyleSheet == s

    def test_cssText(self):
        "CSSStyleSheet.cssText"
        tests = {
            '': b'',
            # @charset
            '@charset "ascii";\n@import "x";': b'@charset "ascii";\n@import "x";',
            '@charset "ascii";\n@media all {}': b'@charset "ascii";',
            '@charset "ascii";\n@x;': b'@charset "ascii";\n@x;',
            '@charset "ascii";\na {\n    x: 1\n    }': b'@charset "ascii";\na {\n    x: 1\n    }',
            # @import
            '@x;\n@import "x";': b'@x;\n@import "x";',
            '@import "x";\n@import "y";': b'@import "x";\n@import "y";',
            '@import "x";\n@media all {}': b'@import "x";',
            '@import "x";\n@x;': b'@import "x";\n@x;',
            '@import "x";\na {\n    x: 1\n    }': b'@import "x";\na {\n    x: 1\n    }',
            # @namespace
            '@x;\n@namespace a "x";': b'@x;\n@namespace a "x";',
            '@namespace a "x";\n@namespace b "y";': b'@namespace a "x";\n@namespace b "y";',
            '@import "x";\n@namespace a "x";\n@media all {}': b'@import "x";\n@namespace a "x";',
            '@namespace a "x";\n@x;': b'@namespace a "x";\n@x;',
            '@namespace a "x";\na {\n    x: 1\n    }': b'@namespace a "x";\na {\n    x: 1\n    }',
            """@namespace url("e1");
                @namespace url("e2");
                @namespace x url("x1");
                @namespace x url("x2");
                test{color: green}
                x|test {color: green}""": b"""@namespace "e2";
@namespace x "x2";
test {
    color: green
    }
x|test {
    color: green
    }"""
            #            ur'\1 { \2: \3 }': ur'''\x01 {
            #    \x02: \x03
            #    }''',
            #            ur'''
            #            \@ { \@: \@ }
            #            \1 { \2: \3 }
            #            \{{\::\;;}
            #            ''': ur'''\@ {
            #    \@: \@
            #    }
            # \1 {
            #    \2: \3
            #    }
            # \{
            #    {\:: \;
            #    }'''
        }
        self.do_equal_r(tests)

        tests = {
            '': None,
            # @charset
            '@charset "ascii";\n@import "x";': None,
            '@charset "ascii";\n@media all {}': '@charset "ascii";',
            '@charset "ascii";\n@x;': None,
            '@charset "ascii";\na {\n    x: 1\n    }': None,
            # @import
            '@x;\n@import "x";': None,
            '@import "x";\n@import "y";': None,
            '@import "x";\n@media all {}': '@import "x";',
            '@import "x";\n@x;': None,
            '@import "x";\na {\n    x: 1\n    }': None,
            # @namespace
            '@x;\n@namespace a "x";': None,
            '@namespace a "x";\n@namespace b "y";': None,
            '@import "x";\n@namespace a "x";\n@media all {}': '@import "x";\n@namespace a "x";',
            '@namespace a "x";\n@x;': None,
            '@namespace a "x";\na {\n    x: 1\n    }': None,
            """@namespace url("e1");
                @namespace url("e2");
                @namespace x url("x1");
                @namespace x url("x2");
                test{color: green}
                x|test {color: green}""": """@namespace "e2";
@namespace x "x2";
test {
    color: green
    }
x|test {
    color: green
    }"""
            #            ur'\1 { \2: \3 }': ur'''\x01 {
            #    \x02: \x03
            #    }''',
            #            ur'''
            #            \@ { \@: \@ }
            #            \1 { \2: \3 }
            #            \{{\::\;;}
            #            ''': ur'''\@ {
            #    \@: \@
            #    }
            # \1 {
            #    \2: \3
            #    }
            # \{
            #    {\:: \;
            #    }'''
        }
        self.do_equal_p(tests)

        s = cssutils.css.CSSStyleSheet()
        s.cssText = '''@charset "ascii";@import "x";@namespace a "x";
        @media all {/*1*/}@page {margin: 0}a {\n    x: 1\n    }@unknown;/*comment*/'''
        for r in s.cssRules:
            assert s == r.parentStyleSheet

    def test_cssText_HierarchyRequestErr(self):
        "CSSStyleSheet.cssText HierarchyRequestErr"
        tests = {
            # @charset: only one and always 1st
            ' @charset "utf-8";': xml.dom.HierarchyRequestErr,
            '@charset "ascii";@charset "ascii";': xml.dom.HierarchyRequestErr,
            '/*c*/@charset "ascii";': xml.dom.HierarchyRequestErr,
            '@import "x"; @charset "ascii";': xml.dom.HierarchyRequestErr,
            '@namespace a "x"; @charset "ascii";': xml.dom.HierarchyRequestErr,
            '@media all {} @charset "ascii";': xml.dom.HierarchyRequestErr,
            '@page {} @charset "ascii";': xml.dom.HierarchyRequestErr,
            'a {} @charset "ascii";': xml.dom.HierarchyRequestErr,
            # @import: before @namespace, @media, @page, sr
            '@namespace a "x"; @import "x";': xml.dom.HierarchyRequestErr,
            '@media all {} @import "x";': xml.dom.HierarchyRequestErr,
            '@page {} @import "x";': xml.dom.HierarchyRequestErr,
            'a {} @import "x";': xml.dom.HierarchyRequestErr,
            # @namespace: before @media, @page, sr
            '@media all {} @namespace a "x";': xml.dom.HierarchyRequestErr,
            '@page {} @namespace a "x";': xml.dom.HierarchyRequestErr,
            'a {} @namespace a "x";': xml.dom.HierarchyRequestErr,
        }
        self.do_raise_r(tests)
        self.do_raise_p(tests)

    def test_cssText_SyntaxErr(self):
        """CSSStyleSheet.cssText SyntaxErr

        for single {, } or ;
        """
        tests = {
            '{': xml.dom.SyntaxErr,
            '}': xml.dom.SyntaxErr,
            ';': xml.dom.SyntaxErr,
            '@charset "ascii";{': xml.dom.SyntaxErr,
            '@charset "ascii";}': xml.dom.SyntaxErr,
            '@charset "ascii";;': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests)
        self.do_raise_p(tests)

    def test_encoding(self):
        "CSSStyleSheet.encoding"
        self.s.cssText = ''
        assert 'utf-8' == self.s.encoding

        self.s.encoding = 'ascii'
        assert 'ascii' == self.s.encoding
        assert 1 == self.s.cssRules.length
        assert 'ascii' == self.s.cssRules[0].encoding

        self.s.encoding = None
        assert 'utf-8' == self.s.encoding
        assert 0 == self.s.cssRules.length

        self.s.encoding = 'UTF-8'
        assert 'utf-8' == self.s.encoding
        assert 1 == self.s.cssRules.length

        with pytest.raises(xml.dom.SyntaxErr):
            self.s._setEncoding('INVALID ENCODING')
        assert 'utf-8' == self.s.encoding
        assert 1 == self.s.cssRules.length

    def test_namespaces1(self):
        "CSSStyleSheet.namespaces.namespaces"
        # tests for namespaces internal methods
        s = cssutils.css.CSSStyleSheet()
        assert 0 == len(s.namespaces)

        css = '''@namespace "default";
@namespace ex "example";
@namespace ex2 "example";
ex2|x { top: 0 }'''
        expcss = '''@namespace "default";
@namespace ex2 "example";
ex2|x {
    top: 0
    }'''
        s.cssText = css
        assert s.cssText == expcss.encode()
        assert s.namespaces.namespaces == {'': 'default', 'ex2': 'example'}

        # __contains__
        assert '' in s.namespaces
        assert 'ex2' in s.namespaces
        assert "NOTSET" not in s.namespaces
        # __delitem__
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            s.namespaces.__delitem__('ex2')
        s.namespaces['del'] = 'del'
        del s.namespaces['del']
        with pytest.raises(xml.dom.NamespaceErr):
            s.namespaces.__getitem__('del')
        # __getitem__
        assert 'default' == s.namespaces['']
        assert 'example' == s.namespaces['ex2']
        with pytest.raises(xml.dom.NamespaceErr):
            s.namespaces.__getitem__('UNSET')
        # __iter__
        assert ['', 'ex2'] == sorted(list(s.namespaces))
        # __len__
        assert 2 == len(s.namespaces)
        # __setitem__
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            s.namespaces.__setitem__('ex2', 'NEWURI')
        s.namespaces['n1'] = 'new'
        assert s.namespaces.namespaces == {'': 'default', 'ex2': 'example', 'n1': 'new'}
        s.namespaces['n'] = 'new'  # replaces prefix!
        assert s.namespaces.namespaces == {'': 'default', 'ex2': 'example', 'n': 'new'}
        # prefixForNamespaceURI
        assert '' == s.namespaces.prefixForNamespaceURI('default')
        assert 'ex2' == s.namespaces.prefixForNamespaceURI('example')
        with pytest.raises(IndexError):
            s.namespaces.prefixForNamespaceURI('UNSET')
        # .keys
        assert set(s.namespaces.keys()) == {'', 'ex2', 'n'}
        # .get
        assert 'x' == s.namespaces.get('UNKNOWN', 'x')
        assert 'example' == s.namespaces.get('ex2', 'not used defa')

    def test_namespaces2(self):
        "CSSStyleSheet.namespaces"
        # tests using CSSStyleSheet.namespaces

        s = cssutils.css.CSSStyleSheet()
        css = '@namespace n "new";'
        # doubles will be removed
        s.insertRule(css + css)
        assert s.cssText == css.encode()
        r = cssutils.css.CSSNamespaceRule(prefix='ex2', namespaceURI='example')
        s.insertRule(r)
        r = cssutils.css.CSSNamespaceRule(namespaceURI='default')
        s.insertRule(r)

        expcss = '''@namespace n "new";
@namespace ex2 "example";
@namespace "default";'''
        assert s.cssText == expcss.encode()
        r.prefix = 'DEFAULT'
        expcss = '''@namespace n "new";
@namespace ex2 "example";
@namespace DEFAULT "default";'''
        assert s.cssText == expcss.encode()

        # CSSMediaRule
        with pytest.raises(xml.dom.NamespaceErr):
            s.add('@media all {x|a {left: 0}}')
        mcss = '@media all {\n    ex2|SEL1 {\n        left: 0\n        }\n    }'
        s.add(mcss)
        expcss += '\n' + mcss
        assert s.cssText == expcss.encode()

        # CSSStyleRule
        with pytest.raises(xml.dom.NamespaceErr):
            s.add('x|a {top: 0}')
        scss = 'n|SEL2 {\n    top: 0\n    }'
        s.add(scss)
        expcss += '\n' + scss
        assert s.cssText == expcss.encode()

        mr = s.cssRules[3]
        sr = s.cssRules[4]

        # SelectorList @media
        with pytest.raises(xml.dom.NamespaceErr):
            mr.cssRules[0]._setSelectorText('x|b')
        oldsel, newsel = mr.cssRules[0].selectorText, 'n|SEL3, a'
        mr.cssRules[0].selectorText = newsel
        expcss = expcss.replace(oldsel, newsel)
        assert s.cssText == expcss.encode()
        # SelectorList stylerule
        with pytest.raises(xml.dom.NamespaceErr):
            sr._setSelectorText('x|b')
        oldsel, newsel = sr.selectorText, 'ex2|SEL4, a'
        sr.selectorText = newsel
        expcss = expcss.replace(oldsel, newsel)
        assert s.cssText == expcss.encode()

        # Selector @media
        with pytest.raises(xml.dom.NamespaceErr):
            mr.cssRules[0].selectorList.append('x|b')
        oldsel, newsel = mr.cssRules[0].selectorText, 'ex2|SELMR'
        mr.cssRules[0].selectorList.append(newsel)
        expcss = expcss.replace(oldsel, oldsel + ', ' + newsel)
        assert s.cssText == expcss.encode()
        # Selector stylerule
        with pytest.raises(xml.dom.NamespaceErr):
            sr.selectorList.append('x|b')
        oldsel, newsel = sr.selectorText, 'ex2|SELSR'
        sr.selectorList.append(newsel)
        expcss = expcss.replace(oldsel, oldsel + ', ' + newsel)
        assert s.cssText == expcss.encode()

        assert (
            s.cssText
            == b'''@namespace n "new";
@namespace ex2 "example";
@namespace DEFAULT "default";
@media all {
    n|SEL3, a, ex2|SELMR {
        left: 0
        }
    }
ex2|SEL4, a, ex2|SELSR {
    top: 0
    }'''
        )

    def test_namespaces3(self):
        "CSSStyleSheet.namespaces 3"
        # tests setting namespaces with new {}
        s = cssutils.css.CSSStyleSheet()
        css = 'h|a { top: 0 }'
        with pytest.raises(xml.dom.NamespaceErr):
            s.add(css)

        s.add('@namespace x "html";')
        assert s.namespaces['x'] == 'html'

        r = cssutils.css.CSSStyleRule()
        r.cssText = (css, {'h': 'html'})
        s.add(r)  # uses x as set before! h is temporary only
        assert s.cssText == b'@namespace x "html";\nx|a {\n    top: 0\n    }'

        # prefix is now "x"!
        with pytest.raises(xml.dom.NamespaceErr):
            r.selectorList.append('h|b')
        with pytest.raises(xml.dom.NamespaceErr):
            r.selectorList.append('y|b')
        s.namespaces['y'] = 'html'
        r.selectorList.append('y|b')
        assert s.cssText == b'@namespace y "html";\ny|a, y|b {\n    top: 0\n    }'

        with pytest.raises(xml.dom.NoModificationAllowedErr):
            s.namespaces.__delitem__('y')
        assert s.cssText == b'@namespace y "html";\ny|a, y|b {\n    top: 0\n    }'

        s.cssRules[0].prefix = ''
        assert s.cssText == b'@namespace "html";\na, b {\n    top: 0\n    }'

        # remove need of namespace
        s.cssRules[0].prefix = 'x'
        s.cssRules[1].selectorText = 'a, b'
        assert s.cssText == b'@namespace x "html";\na, b {\n    top: 0\n    }'

    def test_namespaces4(self):
        "CSSStyleSheet.namespaces 4"
        # tests setting namespaces with new {}
        s = cssutils.css.CSSStyleSheet()
        assert {} == s.namespaces.namespaces

        s.namespaces.namespaces['a'] = 'no setting possible'
        assert {} == s.namespaces.namespaces

        s.namespaces[None] = 'default'
        assert {'': 'default'} == s.namespaces.namespaces

        del s.namespaces['']
        assert {} == s.namespaces.namespaces

        s.namespaces[''] = 'default'
        assert {'': 'default'} == s.namespaces.namespaces

        del s.namespaces[None]
        assert {} == s.namespaces.namespaces

        s.namespaces['p'] = 'uri'
        # cannot use namespaces.namespaces
        del s.namespaces.namespaces['p']
        assert {'p': 'uri'} == s.namespaces.namespaces

        with pytest.raises(xml.dom.NamespaceErr, match=r"Prefix undefined not found\."):
            del s.namespaces['undefined']

    def test_namespaces5(self):
        "CSSStyleSheet.namespaces 5"
        # unknown namespace
        s = cssutils.parseString('a|a { color: red }')
        assert s.cssText == b''

        s = cssutils.css.CSSStyleSheet()
        with pytest.raises(xml.dom.NamespaceErr, match=r"Prefix a not found\."):
            s._setCssText('a|a { color: red }')

    def test_deleteRuleIndex(self):
        "CSSStyleSheet.deleteRule(index)"
        self.s.cssText = '@charset "ascii"; @import "x"; @x; a {\n    x: 1\n    }@y;'
        assert 5 == self.s.cssRules.length

        with pytest.raises(xml.dom.IndexSizeErr):
            self.s.deleteRule(5)

        # end -1
        # check parentStyleSheet
        r = self.s.cssRules[-1]
        assert self.s == r.parentStyleSheet
        self.s.deleteRule(-1)
        assert r.parentStyleSheet is None

        assert 4 == self.s.cssRules.length
        assert (
            b'@charset "ascii";\n@import "x";\n@x;\na {\n    x: 1\n    }'
            == self.s.cssText
        )
        # beginning
        self.s.deleteRule(0)
        assert 3 == self.s.cssRules.length
        assert b'@import "x";\n@x;\na {\n    x: 1\n    }' == self.s.cssText
        # middle
        self.s.deleteRule(1)
        assert 2 == self.s.cssRules.length
        assert b'@import "x";\na {\n    x: 1\n    }' == self.s.cssText
        # end
        self.s.deleteRule(1)
        assert 1 == self.s.cssRules.length
        assert b'@import "x";' == self.s.cssText

    def test_deleteRule(self):
        "CSSStyleSheet.deleteRule(rule)"
        s = cssutils.css.CSSStyleSheet()
        s.cssText = '''
        @namespace x "http://example.com";
        a { color: red; }
        b { color: blue; }
        c { color: green; }
        '''
        n, s1, s2, s3 = s.cssRules

        r = cssutils.css.CSSStyleRule()
        with pytest.raises(xml.dom.IndexSizeErr):
            s.deleteRule(r)

        assert 4 == s.cssRules.length
        s.deleteRule(n)
        assert 3 == s.cssRules.length
        assert (
            s.cssText
            == b'a {\n    color: red\n    }\nb {\n    color: blue\n    }\nc {\n    color: green\n    }'
        )
        with pytest.raises(xml.dom.IndexSizeErr):
            s.deleteRule(n)
        s.deleteRule(s2)
        assert 2 == s.cssRules.length
        assert s.cssText == b'a {\n    color: red\n    }\nc {\n    color: green\n    }'
        with pytest.raises(xml.dom.IndexSizeErr):
            s.deleteRule(s2)

    def _gets(self):
        # complete
        self.cr = cssutils.css.CSSCharsetRule('ascii')
        self.c = cssutils.css.CSSComment('/*c*/')
        self.ur = cssutils.css.CSSUnknownRule('@x;')
        self.ir = cssutils.css.CSSImportRule('x')
        self.nr = cssutils.css.CSSNamespaceRule('uri', 'p')
        self.mr = cssutils.css.CSSMediaRule()
        self.mr.cssText = '@media all { @m; }'
        self.pr = cssutils.css.CSSPageRule()
        self.pr.style = 'margin: 0;'
        self.sr = cssutils.css.CSSStyleRule()
        self.sr.cssText = 'a {\n    x: 1\n    }'

        s = cssutils.css.CSSStyleSheet()
        s.insertRule(self.cr)  # 0
        s.insertRule(self.ir)  # 1
        s.insertRule(self.nr)  # 2
        s.insertRule(self.mr)  # 3
        s.insertRule(self.sr)  # 4
        s.insertRule(self.mr)  # 5
        s.insertRule(self.pr)  # 6
        s.insertRule(self.sr)  # 7
        assert (
            b'@charset "ascii";\n@import url(x);\n@namespace p "uri";\n@media all {\n    @m;\n    }\na {\n    x: 1\n    }\n@media all {\n    @m;\n    }\n@page {\n    margin: 0\n    }\na {\n    x: 1\n    }'
            == s.cssText
        )
        return s, s.cssRules.length

    def test_add(self):
        "CSSStyleSheet.add()"
        full = cssutils.css.CSSStyleSheet()
        sheet = cssutils.css.CSSStyleSheet()
        css = [
            '@charset "ascii";',
            '@import "x";',
            '@namespace p "u";',
            '@page {\n    left: 0\n    }',
            '@font-face {\n    src: local(x)\n    }',
            '@media all {\n    a {\n        color: red\n        }\n    }',
            'a {\n    color: green\n    }',
            '/*comment*/',
            '@x;',
        ]

        fullcss = '\n'.join(css)
        full.cssText = fullcss
        assert full.cssText == fullcss.encode()
        for i, line in enumerate(css):
            # sheet without same ruletype
            before = css[:i]
            after = css[i + 1 :]
            sheet.cssText = ''.join(before + after)

            index = sheet.add(line)
            if i < 3:
                # specific insertion point
                assert fullcss.encode() == sheet.cssText
                assert i == index
            else:
                # end of sheet
                expected = before
                expected.extend(after)
                expected.append(line)
                assert '\n'.join(expected).encode() == sheet.cssText
                assert len(expected) - 1 == index  # no same rule present

            # sheet with the same ruletype
            if i == 1:
                line = '@import "x2";'
            if i == 2:
                line = '@namespace p2 "u2";'

            full.cssText = fullcss
            index = full.add(line)
            if i < 1:
                assert fullcss.encode() == sheet.cssText
                assert i == index  # no same rule present
            else:
                if i < 3:
                    # in order
                    expected = css[: i + 1]  # including same rule
                    expected.append(line)
                    expected.extend(css[i + 1 :])
                    expectedindex = i + 1
                else:
                    # just appended as no order needed
                    expected = css[:]
                    expected.append(line)
                    expectedindex = len(expected) - 1

                assert '\n'.join(expected).encode() == full.cssText
                assert expectedindex == index  # no same rule present

    def test_addimport(self):
        p = cssutils.CSSParser(fetcher=lambda url: (None, '/**/'))

        cssrulessheet = p.parseString('@import "example.css";')
        imports = (
            '@import "example.css";',  # string
            cssutils.css.CSSImportRule(href="example.css"),  # CSSRule
            cssrulessheet.cssRules,  # CSSRuleList
        )
        for imp in imports:
            sheet = p.parseString('', href='http://example.com')
            sheet.add(imp)
            added = sheet.cssRules[0]
            assert sheet == added.parentStyleSheet
            assert 'example.css' == added.href
            assert 'utf-8' == added.styleSheet.encoding
            assert b'/**/' == added.styleSheet.cssText

        cssrulessheet = p.parseString('@import "example.css";')
        imports = (
            ('@import "example.css";', 'ascii'),  # string
            (cssutils.css.CSSImportRule(href="example.css"), 'ascii'),  # CSSRule
            # got encoding from old parent already
            (cssrulessheet.cssRules, 'utf-8'),  # CSSRuleList
        )
        for imp, enc in imports:
            sheet = p.parseString('', href='http://example.com', encoding='ascii')
            sheet.add(imp)
            added = sheet.cssRules[1]
            assert sheet == added.parentStyleSheet
            assert 'example.css' == added.href
            assert enc == added.styleSheet.encoding
            if enc == 'ascii':
                assert b'@charset "ascii";\n/**/' == added.styleSheet.cssText
            else:
                assert b'/**/' == added.styleSheet.cssText

        # if styleSheet is already there encoding is not set new
        impsheet = p.parseString('@import "example.css";')
        imp = impsheet.cssRules[0]
        sheet = p.parseString('', href='http://example.com', encoding='ascii')
        sheet.add(imp)
        added = sheet.cssRules[1]
        assert sheet == added.parentStyleSheet
        assert 'example.css' == added.href
        assert 'utf-8' == added.styleSheet.encoding
        assert b'/**/' == added.styleSheet.cssText

    def test_insertRule(self):
        "CSSStyleSheet.insertRule()"
        s, L = self._gets()

        # INVALID index
        with pytest.raises(xml.dom.IndexSizeErr):
            s.insertRule(self.sr, -1)
        with pytest.raises(xml.dom.IndexSizeErr):
            s.insertRule(self.sr, s.cssRules.length + 1)
        #   check if rule is really not in
        assert L == s.cssRules.length

        # insert string
        s.insertRule('a {}')
        assert L + 1 == s.cssRules.length
        # insert rule
        s.insertRule(self.sr)
        assert L + 2 == s.cssRules.length
        # insert rulelist
        s2, L2 = self._gets()
        rulelist = s2.cssRules
        del rulelist[:-2]
        s.insertRule(rulelist)
        assert L + 2 + 2 == s.cssRules.length

    def _insertRule(self, rules, notbefore, notafter, anywhere, checkdoubles=True):
        """
        helper
        test if any rule in rules cannot be inserted before rules in before
        or after rules in after but before and after rules in anywhere
        """
        for rule in rules:
            for r in notbefore:
                s = cssutils.css.CSSStyleSheet()
                s.insertRule(r)
                with pytest.raises(xml.dom.HierarchyRequestErr):
                    s.insertRule(rule, 0)
                s = cssutils.css.CSSStyleSheet()
                s.add(r)
                with pytest.raises(xml.dom.HierarchyRequestErr):
                    s.insertRule(rule, 0)
            for r in notafter:
                s = cssutils.css.CSSStyleSheet()
                s.insertRule(r)
                with pytest.raises(xml.dom.HierarchyRequestErr):
                    s.insertRule(rule, 1)
                s = cssutils.css.CSSStyleSheet()
                s.add(r)
                s.add(rule)  # never raises
                assert s == r.parentStyleSheet

            for r in anywhere:
                s = cssutils.css.CSSStyleSheet()
                s.insertRule(r)
                s.insertRule(rule, 0)  # before
                s.insertRule(rule)  # after
                if checkdoubles:
                    assert s.cssRules.length == 3
                assert s == r.parentStyleSheet

    def test_insertRule_charset(self):
        "CSSStyleSheet.insertRule(@charset)"
        s, L = self._gets()
        notbefore = (self.cr,)
        notafter = (
            self.cr,
            self.ir,
            self.nr,
            self.mr,
            self.pr,
            self.sr,
            self.c,
            self.ur,
        )
        anywhere = ()
        self._insertRule((self.cr,), notbefore, notafter, anywhere)

    def test_insertRule_import(self):
        "CSSStyleSheet.insertRule(@import)"
        s, L = self._gets()
        notbefore = (self.cr,)
        notafter = (self.nr, self.pr, self.mr, self.sr)
        anywhere = (self.c, self.ur, self.ir)
        self._insertRule((self.ir,), notbefore, notafter, anywhere)

    def test_insertRule_namespace(self):
        "CSSStyleSheet.insertRule(@namespace)"
        s, L = self._gets()
        notbefore = (self.cr, self.ir)
        notafter = (self.pr, self.mr, self.sr)
        anywhere = (self.c, self.ur, self.nr)
        self._insertRule((self.nr,), notbefore, notafter, anywhere, checkdoubles=False)

    def test_insertRule_media_page_style(self):
        "CSSStyleSheet.insertRule(@media, @page, stylerule)"
        s, L = self._gets()
        notbefore = (self.cr, self.ir, self.nr)
        notafter = ()
        anywhere = (self.c, self.ur, self.mr, self.pr, self.sr)
        self._insertRule((self.pr, self.mr, self.sr), notbefore, notafter, anywhere)

    def test_insertRule_unknownandcomment(self):
        "CSSStyleSheet.insertRule(@ unknown, comment)"
        s, L = self._gets()
        notbefore = (self.cr,)
        notafter = ()
        anywhere = (self.c, self.ur, self.ir, self.nr, self.mr, self.pr, self.sr)
        self._insertRule((self.ur,), notbefore, notafter, anywhere)

    def test_HTMLComments(self):
        "CSSStyleSheet CDO CDC"
        css = '''body { color: red }
<!-- comment -->
body { color: blue }
body { color: pink }
<!-- comment -->
body { color: green }
'''
        exp = '''body {
    color: red
    }
body {
    color: pink
    }'''
        sheet = cssutils.parseString(css)
        assert sheet.cssText == exp.encode()

    def test_incomplete(self):
        "CSSStyleRule (incomplete)"
        tests = {
            '@import "a': '@import "a";',  # no }
            'a { x: 1': 'a {\n    x: 1\n    }',  # no }
            'a { font-family: "arial sans': 'a {\n    font-family: "arial sans"\n    }',  # no "
        }
        self.do_equal_p(tests)  # parse

    def test_NoModificationAllowedErr(self):
        "CSSStyleSheet NoModificationAllowedErr"
        css = cssutils.css.CSSStyleSheet(readonly=True)

        assert css._readonly  # internal...

        with pytest.raises(xml.dom.NoModificationAllowedErr):
            css._setCssText('@x;')
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            css.insertRule(self.rule)
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            css.insertRule(self.rule, 0)
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            css.deleteRule(0)

    def test_reprANDstr(self):
        "CSSStyleSheet.__repr__(), .__str__()"
        href = 'file:foo.css'
        title = 'title-of-css'

        s = cssutils.css.CSSStyleSheet(href=href, title=title)

        assert href in str(s)
        assert title in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert href == s2.href
        assert title == s2.title

    def test_valid(self):
        cases = [
            ('body { color: red; }', True),
            ('body { color: asd; }', False),
            ('body { foo: 12px; }', False),
        ]
        for case, expected in cases:
            sheet = cssutils.parseString(case)
            msg = "{!r} should be {}".format(case, 'valid' if expected else 'invalid')
            assert sheet.valid == expected, msg

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssstylesheet.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSCharsetRule"""


from . import basetest
import codecs
import cssutils
import os
import sys
import tempfile
from unittest import mock

import pytest


@pytest.fixture
def serializer(monkeypatch):
    monkeypatch.setattr(cssutils, 'ser', cssutils.serialize.CSSSerializer())


class TestCSSutils(basetest.BaseTestCase):
    def setup_method(self):
        cssutils.ser.prefs.useDefaults()

    def teardown_method(self):
        cssutils.ser.prefs.useDefaults()

    exp = '''@import "import/import2.css";
.import {
    /* ./import.css */
    background-image: url(images/example.gif)
    }'''

    def test_parseString(self):
        "cssutils.parseString()"
        s = cssutils.parseString(
            self.exp, media='handheld, screen', title='from string'
        )
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        assert s.href is None
        assert self.exp.encode() == s.cssText
        assert 'utf-8' == s.encoding
        assert 'handheld, screen' == s.media.mediaText
        assert 'from string' == s.title
        assert self.exp.encode() == s.cssText

        ir = s.cssRules[0]
        assert 'import/import2.css' == ir.href
        irs = ir.styleSheet
        assert isinstance(irs, cssutils.css.CSSStyleSheet)

        href = basetest.get_sheet_filename('import.css')
        href = cssutils.helper.path2url(href)
        s = cssutils.parseString(self.exp, href=href)
        assert href == s.href

        ir = s.cssRules[0]
        assert 'import/import2.css' == ir.href
        irs = ir.styleSheet
        assert isinstance(irs, cssutils.css.CSSStyleSheet)
        assert (
            irs.cssText == b'@import "../import3.css";\n'
            b'@import "import-impossible.css" print;\n.import2 {\n'
            b'    /* sheets/import2.css */\n'
            b'    background: url(http://example.com/images/example.gif);\n'
            b'    background: url(//example.com/images/example.gif);\n'
            b'    background: url(/images/example.gif);\n'
            b'    background: url(images2/example.gif);\n'
            b'    background: url(./images2/example.gif);\n'
            b'    background: url(../images/example.gif);\n'
            b'    background: url(./../images/example.gif)\n'
            b'    }'
        )

        tests = {
            'a {color: red}': 'a {\n    color: red\n    }',
            'a {color: rgb(1,2,3)}': 'a {\n    color: rgb(1, 2, 3)\n    }',
        }
        self.do_equal_p(tests)

    def test_parseFile(self):
        "cssutils.parseFile()"
        # name if used with open, href used for @import resolving
        name = basetest.get_sheet_filename('import.css')
        href = cssutils.helper.path2url(name)

        s = cssutils.parseFile(name, href=href, media='screen', title='from file')
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        if sys.platform.startswith('java'):
            # on Jython only file:
            assert s.href.startswith('file:')
        else:
            # normally file:/// on win and file:/ on unix
            assert s.href.startswith('file:/')
        assert s.href.endswith('/sheets/import.css')
        assert 'utf-8' == s.encoding
        assert 'screen' == s.media.mediaText
        assert 'from file' == s.title
        assert self.exp.encode() == s.cssText

        ir = s.cssRules[0]
        assert 'import/import2.css' == ir.href
        irs = ir.styleSheet
        assert isinstance(irs, cssutils.css.CSSStyleSheet)
        assert (
            irs.cssText == b'@import "../import3.css";\n'
            b'@import "import-impossible.css" print;\n'
            b'.import2 {\n'
            b'    /* sheets/import2.css */\n'
            b'    background: url(http://example.com/images/example.gif);\n'
            b'    background: url(//example.com/images/example.gif);\n'
            b'    background: url(/images/example.gif);\n'
            b'    background: url(images2/example.gif);\n'
            b'    background: url(./images2/example.gif);\n'
            b'    background: url(../images/example.gif);\n'
            b'    background: url(./../images/example.gif)\n'
            b'    }'
        )

        # name is used for open and setting of href automatically
        # test needs to be relative to this test file!
        os.chdir(os.path.dirname(__file__))
        name = basetest.get_sheet_filename('import.css')

        s = cssutils.parseFile(name, media='screen', title='from file')
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        if sys.platform.startswith('java'):
            # on Jython only file:
            assert s.href.startswith('file:')
        else:
            # normally file:/// on win and file:/ on unix
            assert s.href.startswith('file:/')
        assert s.href.endswith('/sheets/import.css')
        assert 'utf-8' == s.encoding
        assert 'screen' == s.media.mediaText
        assert 'from file' == s.title
        assert self.exp.encode() == s.cssText

        ir = s.cssRules[0]
        assert 'import/import2.css' == ir.href
        irs = ir.styleSheet
        assert isinstance(irs, cssutils.css.CSSStyleSheet)
        assert (
            irs.cssText == b'@import "../import3.css";\n'
            b'@import "import-impossible.css" print;\n'
            b'.import2 {\n'
            b'    /* sheets/import2.css */\n'
            b'    background: url(http://example.com/images/example.gif);\n'
            b'    background: url(//example.com/images/example.gif);\n'
            b'    background: url(/images/example.gif);\n'
            b'    background: url(images2/example.gif);\n'
            b'    background: url(./images2/example.gif);\n'
            b'    background: url(../images/example.gif);\n'
            b'    background: url(./../images/example.gif)\n'
            b'    }'
        )

        # next test
        css = 'a:after { content: "羊蹄€\u2020" }'

        fd, name = tempfile.mkstemp('_cssutilstest.css')
        t = os.fdopen(fd, 'wb')
        t.write(css.encode('utf-8'))
        t.close()

        with pytest.raises(UnicodeDecodeError):
            cssutils.parseFile(name, 'ascii')

        # ???
        s = cssutils.parseFile(name, encoding='iso-8859-1')
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        assert s.cssRules[1].selectorText == 'a:after'

        s = cssutils.parseFile(name, encoding='utf-8')
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        assert s.cssRules[1].selectorText == 'a:after'

        css = '@charset "iso-8859-1"; a:after { content: "ä" }'
        t = codecs.open(name, 'w', 'iso-8859-1')
        t.write(css)
        t.close()

        with pytest.raises(UnicodeDecodeError):
            cssutils.parseFile(name, 'ascii')

        s = cssutils.parseFile(name, encoding='iso-8859-1')
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        assert s.cssRules[1].selectorText == 'a:after'

        with pytest.raises(UnicodeDecodeError):
            cssutils.parseFile(name, 'utf-8')

        # clean up
        try:
            os.remove(name)
        except OSError:
            pass

    def test_parseUrl(self):
        "cssutils.parseUrl()"
        href = basetest.get_sheet_filename('import.css')
        # href = u'file:' + urllib.pathname2url(href)
        href = cssutils.helper.path2url(href)
        # href = 'http://seewhatever.de/sheets/import.css'
        s = cssutils.parseUrl(href, media='tv, print', title='from url')
        assert isinstance(s, cssutils.css.CSSStyleSheet)
        assert href == s.href
        assert self.exp.encode() == s.cssText
        assert 'utf-8' == s.encoding
        assert 'tv, print' == s.media.mediaText
        assert 'from url' == s.title

        sr = s.cssRules[1]
        img = sr.style.getProperty('background-image').propertyValue[0].value
        assert img == 'images/example.gif'

        ir = s.cssRules[0]
        assert 'import/import2.css' == ir.href
        irs = ir.styleSheet
        assert (
            irs.cssText == b'@import "../import3.css";\n'
            b'@import "import-impossible.css" print;\n'
            b'.import2 {\n'
            b'    /* sheets/import2.css */\n'
            b'    background: url(http://example.com/images/example.gif);\n'
            b'    background: url(//example.com/images/example.gif);\n'
            b'    background: url(/images/example.gif);\n'
            b'    background: url(images2/example.gif);\n'
            b'    background: url(./images2/example.gif);\n'
            b'    background: url(../images/example.gif);\n'
            b'    background: url(./../images/example.gif)\n'
            b'    }'
        )

        ir2 = irs.cssRules[0]
        assert '../import3.css' == ir2.href
        irs2 = ir2.styleSheet
        assert (
            irs2.cssText == b'/* import3 */\n'
            b'.import3 {\n'
            b'    /* from ./import/../import3.css */\n'
            b'    background: url(images/example3.gif);\n'
            b'    background: url(./images/example3.gif);\n'
            b'    background: url(import/images2/example2.gif);\n'
            b'    background: url(./import/images2/example2.gif);\n'
            b'    background: url(import/images2/../../images/example3.gif)\n'
            b'    }'
        )

    def test_setCSSSerializer(self):
        "cssutils.setSerializer() and cssutils.ser"
        s = cssutils.parseString('a { left: 0 }')
        exp4 = '''a {
    left: 0
    }'''
        exp1 = '''a {
 left: 0
 }'''
        assert exp4.encode() == s.cssText
        newser = cssutils.CSSSerializer(cssutils.serialize.Preferences(indent=' '))
        cssutils.setSerializer(newser)
        assert exp1.encode() == s.cssText
        newser = cssutils.CSSSerializer(cssutils.serialize.Preferences(indent='    '))
        cssutils.ser = newser
        assert exp4.encode() == s.cssText

    def test_parseStyle(self):
        "cssutils.parseStyle()"
        s = cssutils.parseStyle('x:0; y:red')
        assert isinstance(s, cssutils.css.CSSStyleDeclaration)
        assert s.cssText == 'x: 0;\ny: red'

        s = cssutils.parseStyle('@import "x";')
        assert isinstance(s, cssutils.css.CSSStyleDeclaration)
        assert s.cssText == ''

        tests = [('content: "ä"', 'iso-8859-1'), ('content: "€"', 'utf-8')]
        for v, e in tests:
            s = cssutils.parseStyle(v.encode(e), encoding=e)
            assert s.cssText == v

        with pytest.raises(UnicodeDecodeError):
            cssutils.parseStyle(
                'content: "ä"'.encode(),
                'ascii',
            )

    def test_getUrls(self):
        "cssutils.getUrls()"
        cssutils.ser.prefs.keepAllProperties = True

        css = r'''
        @import "im1";
        @import url(im2);
        @import url( im3 );
        @import url( "im4" );
        @import url( 'im5' );
        a {
            background-image: url(a) !important;
            background-\image: url(b);
            background: url(c) no-repeat !important;
            /* issue #46 */
            src: local("xx"),
                 url("f.woff") format("woff"),
                 url("f.otf") format("opentype"),
                 url("f.svg#f") format("svg");
            }'''
        urls = set(cssutils.getUrls(cssutils.parseString(css)))
        assert urls == {
            "im1",
            "im2",
            "im3",
            "im4",
            "im5",
            "a",
            "b",
            "c",
            'f.woff',
            'f.svg#f',
            'f.otf',
        }
        cssutils.ser.prefs.keepAllProperties = False

    def test_replaceUrls(self):
        "cssutils.replaceUrls()"
        cssutils.ser.prefs.keepAllProperties = True

        css = r'''
        @import "im1";
        @import url(im2);
        a {
            background-image: url(c) !important;
            background-\image: url(b);
            background: url(a) no-repeat !important;
            }'''
        s = cssutils.parseString(css)
        cssutils.replaceUrls(s, lambda old: "NEW" + old)
        assert '@import "NEWim1";' == s.cssRules[0].cssText
        assert 'NEWim2' == s.cssRules[1].href
        assert (
            '''background-image: url(NEWc) !important;
background-\\image: url(NEWb);
background: url(NEWa) no-repeat !important'''
            == s.cssRules[2].style.cssText
        )

        cssutils.ser.prefs.keepAllProperties = False

        # CSSStyleDeclaration
        style = cssutils.parseStyle(
            '''color: red;
                                        background-image:
                                            url(1.png),
                                            url('2.png')'''
        )
        cssutils.replaceUrls(style, lambda url: 'prefix/' + url)
        assert (
            style.cssText
            == '''color: red;
background-image: url(prefix/1.png), url(prefix/2.png)'''
        )

    @pytest.mark.usefixtures('serializer')
    def test_resolveImports(self):
        "cssutils.resolveImports(sheet)"
        cssutils.ser.prefs.useMinified()

        a = '@charset "iso-8859-1";@import"b.css";\xe4{color:green}'.encode(
            'iso-8859-1'
        )
        b = b'@charset "ascii";\\E4 {color:red}'

        # normal
        m = mock.Mock()
        with mock.patch('cssutils.util._defaultFetcher', m):
            m.return_value = (None, b)
            s = cssutils.parseString(a)

            # py3 TODO
            assert a == s.cssText
            assert b == s.cssRules[1].styleSheet.cssText

            c = cssutils.resolveImports(s)

            # py3 TODO
            assert b'\xc3\xa4{color:red}\xc3\xa4{color:green}' == c.cssText

            c.encoding = 'ascii'
            assert br'@charset "ascii";\E4 {color:red}\E4 {color:green}' == c.cssText

        # b cannot be found
        m = mock.Mock()
        with mock.patch('cssutils.util._defaultFetcher', m):
            m.return_value = (None, None)
            s = cssutils.parseString(a)

            # py3 TODO
            assert a == s.cssText
            assert isinstance(s.cssRules[1].styleSheet, cssutils.css.CSSStyleSheet)
            c = cssutils.resolveImports(s)
            # py3 TODO
            assert b'@import"b.css";\xc3\xa4{color:green}' == c.cssText

        # @import with media
        a = '@import"b.css";@import"b.css" print, tv ;@import"b.css" all;'
        b = 'a {color: red}'
        m = mock.Mock()
        with mock.patch('cssutils.util._defaultFetcher', m):
            m.return_value = (None, b)
            s = cssutils.parseString(a)

            c = cssutils.resolveImports(s)

            assert b'a{color:red}@media print,tv{a{color:red}}a{color:red}' == c.cssText

        # cannot resolve with media => keep original
        a = '@import"b.css"print;'
        b = '@namespace "http://example.com";'
        m = mock.Mock()
        with mock.patch('cssutils.util._defaultFetcher', m):
            m.return_value = (None, b)
            s = cssutils.parseString(a)
            c = cssutils.resolveImports(s)
            assert a.encode() == c.cssText

        # urls are adjusted too, layout:
        # a.css
        # c.css
        # img/img.gif
        # b/
        #     b.css
        #     subimg/subimg.gif
        a = '''
             @import"b/b.css";
             a {
                 x: url(/img/abs.gif);
                 y: url(img/img.gif);
                 z: url(b/subimg/subimg.gif);
                 }'''

        def fetcher(url):
            c = {
                'b.css': '''
                     @import"../c.css";
                     b {
                         x: url(/img/abs.gif);
                         y: url(../img/img.gif);
                         z: url(subimg/subimg.gif);
                         }''',
                'c.css': '''
                     c {
                         x: url(/img/abs.gif);
                         y: url(./img/img.gif);
                         z: url(./b/subimg/subimg.gif);
                         }''',
            }
            return 'utf-8', c[os.path.split(url)[1]]

        @mock.patch.object(cssutils.util, '_defaultFetcher', new=fetcher)
        def do():
            s = cssutils.parseString(a)
            r = cssutils.resolveImports(s)
            return s, r

        s, r = do()

        cssutils.ser.prefs.useDefaults()
        cssutils.ser.prefs.keepComments = False
        expected = b'''c {
    x: url(/img/abs.gif);
    y: url(img/img.gif);
    z: url(b/subimg/subimg.gif)
    }
b {
    x: url(/img/abs.gif);
    y: url(img/img.gif);
    z: url(b/subimg/subimg.gif)
    }
a {
    x: url(/img/abs.gif);
    y: url(img/img.gif);
    z: url(b/subimg/subimg.gif)
    }'''
        assert expected == r.cssText

        cssutils.ser.prefs.useDefaults()

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssutils.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSValue and CSSPrimitiveValue."""


import xml.dom
from . import basetest
import cssutils

import pytest


pytestmark = pytest.mark.xfail(reason="incomplete")


class TestCSSValue(basetest.BaseTestCase):
    def setup(self):
        self.r = cssutils.css.CSSValue()  # needed for tests

    def test_init(self):
        "CSSValue.__init__()"
        v = cssutils.css.CSSValue()
        assert '' == v.cssText
        assert None is v.cssValueType
        assert None is v.cssValueTypeString

    def test_escapes(self):
        "CSSValue Escapes"
        v = cssutils.css.CSSValue()
        v.cssText = '1px'
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_PX == v.primitiveType
        assert '1px' == v.cssText

        v.cssText = '1PX'
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_PX == v.primitiveType
        assert '1px' == v.cssText

        v.cssText = '1p\\x'
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_PX == v.primitiveType
        assert '1px' == v.cssText

    def test_cssText(self):
        "CSSValue.cssText"
        v = cssutils.css.CSSValue()
        v.cssText = '1px'
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_PX == v.primitiveType
        assert '1px' == v.cssText

        v = cssutils.css.CSSValue()
        v.cssText = '1px'
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_PX == v.primitiveType
        assert '1px' == v.cssText

        v = cssutils.css.CSSValue()
        v.cssText = 'a  ,b,  c  ,"d or d", "e, " '
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_STRING == v.primitiveType
        assert 'a, b, c, "d or d", "e, "' == v.cssText

        v.cssText = '  1   px    '
        assert v.CSS_VALUE_LIST == v.cssValueType
        assert '1 px' == v.cssText

        v.cssText = '  normal 1px a, b, "c" end   '
        assert v.CSS_VALUE_LIST == v.cssValueType
        assert 'normal 1px a, b, "c" end' == v.cssText

        for i, x in enumerate(v):
            assert x.CSS_PRIMITIVE_VALUE == x.cssValueType
            if x == 0:
                assert x.CSS_IDENT == x.primitiveType
                assert 'normal' == x.cssText
            elif x == 1:
                assert x.CSS_PX == x.primitiveType
                assert '1px' == x.cssText
            if x == 2:
                assert x.CSS_STRING == x.primitiveType
                assert 'a, b, "c"' == x.cssText
            if x == 3:
                assert x.CSS_IDENT == x.primitiveType
                assert 'end' == x.cssText

        v = cssutils.css.CSSValue()
        v.cssText = '  1   px    '
        assert v.CSS_VALUE_LIST == v.cssValueType
        assert '1 px' == v.cssText

        v.cssText = 'expression(document.body.clientWidth > 972 ? "1014px": "100%" )'
        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert v.CSS_UNKNOWN == v.primitiveType
        assert (
            'expression(document.body.clientWidth > 972?"1014px": "100%")' == v.cssText
        )

    def test_cssText2(self):
        "CSSValue.cssText 2"
        tests = {
            # mix
            'a()1,-1,+1,1%,-1%,1px,-1px,"a",a,url(a),#aabb44': 'a() 1, -1, 1, 1%, -1%, 1px, -1px, "a", a, url(a), #ab4',
            # S or COMMENT
            'red': 'red',
            'red ': 'red',
            ' red ': 'red',
            '/**/red': '/**/ red',
            'red/**/': 'red /**/',
            '/**/red/**/': '/**/ red /**/',
            '/**/ red': '/**/ red',
            'red /**/': 'red /**/',
            '/**/ red /**/': '/**/ red /**/',
            'red-': 'red-',
            # num / dimension
            '.0': '0',
            '0': '0',
            '0.0': '0',
            '00': '0',
            '0%': '0%',
            '0px': '0',
            '-.0': '0',
            '-0': '0',
            '-0.0': '0',
            '-00': '0',
            '-0%': '0%',
            '-0px': '0',
            '+.0': '0',
            '+0': '0',
            '+0.0': '0',
            '+00': '0',
            '+0%': '0%',
            '+0px': '0',
            '1': '1',
            '1.0': '1',
            '1px': '1px',
            '1%': '1%',
            '1px1': '1px1',
            '+1': '1',
            '-1': '-1',
            '+1.0': '1',
            '-1.0': '-1',
            # string, escaped nl is removed during tokenizing
            '"x"': '"x"',
            "'x'": '"x"',
            # ur''' "1\'2" ''': u'''"1'2"''', #???
            # ur"'x\"'": ur'"x\""', #???
            r'''"x\
y"''': '''"xy"''',
            # hash and rgb/a
            '#112234': '#112234',
            '#112233': '#123',
            'rgb(1,2,3)': 'rgb(1, 2, 3)',
            'rgb(  1  ,  2  ,  3  )': 'rgb(1, 2, 3)',
            'rgb(-1,+2,0)': 'rgb(-1, 2, 0)',
            'rgba(1,2,3,4)': 'rgba(1, 2, 3, 4)',
            'rgba(  1  ,  2  ,  3  ,  4 )': 'rgba(1, 2, 3, 4)',
            'rgba(-1,+2,0, 0)': 'rgba(-1, 2, 0, 0)',
            # FUNCTION
            'f(1,2)': 'f(1, 2)',
            'f(  1  ,  2  )': 'f(1, 2)',
            'f(-1,+2)': 'f(-1, 2)',
            'f(  -1  ,  +2  )': 'f(-1, 2)',
            'fun(  -1  ,  +2  )': 'fun(-1, 2)',
            'local( x )': 'local(x)',
            'test(1px, #111, y, 1, 1%, "1", y(), var(x))': 'test(1px, #111, y, 1, 1%, "1", y(), var(x))',
            'test(-1px, #111, y, -1, -1%, "1", -y())': 'test(-1px, #111, y, -1, -1%, "1", -y())',
            'url(y)  format( "x" ,  "y" )': 'url(y) format("x", "y")',
            'f(1 2,3 4)': 'f(1 2, 3 4)',
            # IE expression
            r'Expression()': 'Expression()',
            r'expression(-1 < +2)': 'expression(-1< + 2)',
            r'expression(document.width == "1")': 'expression(document.width=="1")',
            'alpha(opacity=80)': 'alpha(opacity=80)',
            'alpha( opacity = 80 , x=2  )': 'alpha(opacity=80, x=2)',
            # unicode-range
            'u+f': 'u+f',
            'U+ABCdef': 'u+abcdef',
            # url
            'url(a)': 'url(a)',
            'uRl(a)': 'url(a)',
            'u\\rl(a)': 'url(a)',
            'url("a")': 'url(a)',
            'url(  "a"  )': 'url(a)',
            'url(";")': 'url(";")',
            'url(",")': 'url(",")',
            'url(")")': 'url(")")',
            '''url("'")''': '''url("'")''',
            '''url('"')''': '''url("\\"")''',
            '1 2': '1 2',
            '1   2': '1 2',
            '1,2': '1, 2',
            '1,  2': '1, 2',
            '1  ,2': '1, 2',
            '1  ,  2': '1, 2',
            '1/2': '1/2',
            '1/  2': '1/2',
            '1  /2': '1/2',
            '1  /  2': '1/2',
            # comment
            '1/**/2': '1 /**/ 2',
            '1 /**/2': '1 /**/ 2',
            '1/**/ 2': '1 /**/ 2',
            '1 /**/ 2': '1 /**/ 2',
            '1  /*a*/  /*b*/  2': '1 /*a*/ /*b*/ 2',
            # , before
            '1,/**/2': '1, /**/ 2',
            '1 ,/**/2': '1, /**/ 2',
            '1, /**/2': '1, /**/ 2',
            '1 , /**/2': '1, /**/ 2',
            # , after
            '1/**/,2': '1 /**/, 2',
            '1/**/ ,2': '1 /**/, 2',
            '1/**/, 2': '1 /**/, 2',
            '1/**/ , 2': '1 /**/, 2',
            # all
            '1/*a*/  ,/*b*/  2': '1 /*a*/, /*b*/ 2',
            '1  /*a*/,  /*b*/2': '1 /*a*/, /*b*/ 2',
            '1  /*a*/  ,  /*b*/  2': '1 /*a*/, /*b*/ 2',
            # list
            'a b1,b2 b2,b3,b4': 'a b1, b2 b2, b3, b4',
            'a b1  ,   b2   b2  ,  b3  ,   b4': 'a b1, b2 b2, b3, b4',
            'u+1  ,   u+2-5': 'u+1, u+2-5',
            'local( x ),  url(y)  format( "x" ,  "y" )': 'local(x), url(y) format("x", "y")',
            # FUNCTION
            'attr( href )': 'attr(href)',
            # PrinceXML extende FUNC syntax with nested FUNC
            'target-counter(attr(href),page)': 'target-counter(attr(href), page)',
        }

        self.do_equal_r(tests)

        tests = {
            'a+': xml.dom.SyntaxErr,
            '-': xml.dom.SyntaxErr,
            '+': xml.dom.SyntaxErr,
            '-%': xml.dom.SyntaxErr,
            '+a': xml.dom.SyntaxErr,
            '--1px': xml.dom.SyntaxErr,
            '++1px': xml.dom.SyntaxErr,
            '#': xml.dom.SyntaxErr,
            '#00': xml.dom.SyntaxErr,
            '#0000': xml.dom.SyntaxErr,
            '#00000': xml.dom.SyntaxErr,
            '#0000000': xml.dom.SyntaxErr,
            '-#0': xml.dom.SyntaxErr,
            # operator
            ',': xml.dom.SyntaxErr,
            '1,,2': xml.dom.SyntaxErr,
            '1,/**/,2': xml.dom.SyntaxErr,
            '1  ,  /**/  ,  2': xml.dom.SyntaxErr,
            '1,': xml.dom.SyntaxErr,
            '1, ': xml.dom.SyntaxErr,
            '1 ,': xml.dom.SyntaxErr,
            '1 , ': xml.dom.SyntaxErr,
            '1  ,  ': xml.dom.SyntaxErr,
            '1//2': xml.dom.SyntaxErr,
            # URL
            'url(x))': xml.dom.SyntaxErr,
            # string
            '"': xml.dom.SyntaxErr,
            "'": xml.dom.SyntaxErr,
            # function
            'f(-)': xml.dom.SyntaxErr,
            'f(x))': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests)

    def test_incomplete(self):
        "CSSValue (incomplete)"
        tests = {'url("a': 'url(a)', 'url(a': 'url(a)'}
        for v, exp in list(tests.items()):
            s = cssutils.parseString('a { background: %s' % v)
            v = s.cssRules[0].style.background
            assert v == exp

    def test_cssValueType(self):
        "CSSValue.cssValueType .cssValueTypeString"
        tests = [
            (['inherit', 'INhe\\rit'], 'CSS_INHERIT', cssutils.css.CSSValue),
            (
                [
                    '1',
                    '1%',
                    '1em',
                    '1ex',
                    '1px',
                    '1cm',
                    '1mm',
                    '1in',
                    '1pt',
                    '1pc',
                    '1deg',
                    '1rad',
                    '1grad',
                    '1ms',
                    '1s',
                    '1hz',
                    '1khz',
                    '1other',
                    '"string"',
                    "'string'",
                    'url(x)',
                    'red',
                    'attr(a)',
                    'counter(x)',
                    'rect(1px, 2px, 3px, 4px)',
                    'rgb(0, 0, 0)',
                    '#000',
                    '#123456',
                    'rgba(0, 0, 0, 0)',
                    'hsl(0, 0, 0)',
                    'hsla(0, 0, 0, 0)',
                ],
                'CSS_PRIMITIVE_VALUE',
                cssutils.css.CSSPrimitiveValue,
            ),
            (
                ['1px 1px', 'red blue green x'],
                'CSS_VALUE_LIST',
                cssutils.css.CSSValueList,
            ),
            # what is a custom value?
            # ([], 'CSS_CUSTOM', cssutils.css.CSSValue)
        ]
        for values, name, cls in tests:
            for value in values:
                v = cssutils.css.CSSValue(cssText=value)
                if value == "'string'":
                    # will be changed to " always
                    value = '"string"'
                assert value == v.cssText
                assert name == v.cssValueTypeString
                assert getattr(v, name) == v.cssValueType
                assert cls == type(v)

    def test_readonly(self):
        "(CSSValue._readonly)"
        v = cssutils.css.CSSValue(cssText='inherit')
        assert False is v._readonly

        v = cssutils.css.CSSValue(cssText='inherit', readonly=True)
        assert True is v._readonly
        assert 'inherit', v.cssText
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            v._setCssText('x')
        assert 'inherit', v.cssText

    def test_reprANDstr(self):
        "CSSValue.__repr__(), .__str__()"
        cssText = 'inherit'

        s = cssutils.css.CSSValue(cssText=cssText)

        assert cssText in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert cssText == s2.cssText


class TestCSSPrimitiveValue:
    def test_init(self):
        "CSSPrimitiveValue.__init__()"
        v = cssutils.css.CSSPrimitiveValue('1')
        assert '1' == v.cssText

        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert "CSS_PRIMITIVE_VALUE" == v.cssValueTypeString

        assert v.CSS_NUMBER == v.primitiveType
        assert "CSS_NUMBER" == v.primitiveTypeString

        # DUMMY to be able to test empty constructor call
        # self.assertRaises(xml.dom.SyntaxErr, v.__init__, None)

        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getCounterValue()
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getRGBColorValue()
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getRectValue()
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getStringValue()

    def test_CSS_UNKNOWN(self):
        "CSSPrimitiveValue.CSS_UNKNOWN"
        v = cssutils.css.CSSPrimitiveValue('expression(false)')
        assert v.CSS_UNKNOWN == v.primitiveType
        assert 'CSS_UNKNOWN' == v.primitiveTypeString

    def test_CSS_NUMBER_AND_OTHER_DIMENSIONS(self):
        "CSSPrimitiveValue.CSS_NUMBER .. CSS_DIMENSION"
        defs = [
            ('', 'CSS_NUMBER'),
            ('%', 'CSS_PERCENTAGE'),
            ('em', 'CSS_EMS'),
            ('ex', 'CSS_EXS'),
            ('px', 'CSS_PX'),
            ('cm', 'CSS_CM'),
            ('mm', 'CSS_MM'),
            ('in', 'CSS_IN'),
            ('pt', 'CSS_PT'),
            ('pc', 'CSS_PC'),
            ('deg', 'CSS_DEG'),
            ('rad', 'CSS_RAD'),
            ('grad', 'CSS_GRAD'),
            ('ms', 'CSS_MS'),
            ('s', 'CSS_S'),
            ('hz', 'CSS_HZ'),
            ('khz', 'CSS_KHZ'),
            ('other_dimension', 'CSS_DIMENSION'),
        ]
        for dim, name in defs:
            for n in (0, 1, 1.1, -1, -1.1, -0):
                v = cssutils.css.CSSPrimitiveValue('%i%s' % (n, dim))
                assert name == v.primitiveTypeString
                assert getattr(v, name) == v.primitiveType

    def test_CSS_STRING_AND_OTHER(self):
        "CSSPrimitiveValue.CSS_STRING .. CSS_RGBCOLOR"
        defs = [
            (
                (
                    '""',
                    "''",
                    '"some thing"',
                    "' A\\ND '",
                    # comma separated lists are STRINGS FOR NOW!
                    'a, b',
                    '"a", "b"',
                ),
                'CSS_STRING',
            ),
            (('url(a)', 'url("a b")', "url(' ')"), 'CSS_URI'),
            (('some', 'or_anth-er'), 'CSS_IDENT'),
            (('attr(a)', 'attr(b)'), 'CSS_ATTR'),
            (('counter(1)', 'counter(2)'), 'CSS_COUNTER'),
            (('rect(1,2,3,4)',), 'CSS_RECT'),
            (('rgb(1,2,3)', 'rgb(10%, 20%, 30%)', '#123', '#123456'), 'CSS_RGBCOLOR'),
            (
                (
                    'rgba(1,2,3,4)',
                    'rgba(10%, 20%, 30%, 40%)',
                ),
                'CSS_RGBACOLOR',
            ),
            (('U+0', 'u+ffffff', 'u+000000-f', 'u+0-f, U+ee-ff'), 'CSS_UNICODE_RANGE'),
        ]

        for examples, name in defs:
            for x in examples:
                v = cssutils.css.CSSPrimitiveValue(x)
                assert getattr(v, name) == v.primitiveType
                assert name == v.primitiveTypeString

    def test_getFloat(self):
        "CSSPrimitiveValue.getFloatValue()"
        # NOT TESTED are float values as it seems difficult to
        # compare these. Maybe use decimal.Decimal?

        v = cssutils.css.CSSPrimitiveValue('1px')
        tests = {
            '0': (v.CSS_NUMBER, 0),
            '-1.1': (v.CSS_NUMBER, -1.1),
            '1%': (v.CSS_PERCENTAGE, 1),
            '-1%': (v.CSS_PERCENTAGE, -1),
            '1em': (v.CSS_EMS, 1),
            '-1.1em': (v.CSS_EMS, -1.1),
            '1ex': (v.CSS_EXS, 1),
            '1px': (v.CSS_PX, 1),
            '1cm': (v.CSS_CM, 1),
            # '1cm': (v.CSS_MM, 10),
            '254cm': (v.CSS_IN, 100),
            '1mm': (v.CSS_MM, 1),
            '10mm': (v.CSS_CM, 1),
            '254mm': (v.CSS_IN, 10),
            '1in': (v.CSS_IN, 1),
            '100in': (v.CSS_CM, 254),  # ROUNDED!!!
            '10in': (v.CSS_MM, 254),  # ROUNDED!!!
            '1pt': (v.CSS_PT, 1),
            '1pc': (v.CSS_PC, 1),
            '1deg': (v.CSS_DEG, 1),
            '1rad': (v.CSS_RAD, 1),
            '1grad': (v.CSS_GRAD, 1),
            '1ms': (v.CSS_MS, 1),
            '1000ms': (v.CSS_S, 1),
            '1s': (v.CSS_S, 1),
            # '1s': (v.CSS_MS, 1000),
            '1hz': (v.CSS_HZ, 1),
            '1000hz': (v.CSS_KHZ, 1),
            '1khz': (v.CSS_KHZ, 1),
            # '1khz': (v.CSS_HZ, 1000),
            '1DIMENSION': (v.CSS_DIMENSION, 1),
        }
        for cssText in tests:
            v.cssText = cssText
            unitType, exp = tests[cssText]
            val = v.getFloatValue(unitType)
            if unitType in (v.CSS_IN, v.CSS_CM):
                val = round(val)
            assert val == exp

    def test_setFloat(self):
        "CSSPrimitiveValue.setFloatValue()"
        V = cssutils.css.CSSPrimitiveValue

        tests = {
            # unitType, value
            (V.CSS_NUMBER, 1): [
                # unitType, setvalue,
                #    getvalue or expected exception, msg or cssText
                (V.CSS_NUMBER, 0, 0, '0'),
                (V.CSS_NUMBER, 0.1, 0.1, '0.1'),
                (V.CSS_NUMBER, -0, 0, '0'),
                (V.CSS_NUMBER, 2, 2, '2'),
                (V.CSS_NUMBER, 2.0, 2, '2'),
                (V.CSS_NUMBER, 2.1, 2.1, '2.1'),
                (V.CSS_NUMBER, -2.1, -2.1, '-2.1'),
                # setting with string does work
                (V.CSS_NUMBER, '1', 1, '1'),
                (V.CSS_NUMBER, '1.1', 1.1, '1.1'),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DEG, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_RAD, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_GRAD, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_S, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_MS, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_KHZ, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_HZ, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DIMENSION, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_MM, 2, xml.dom.InvalidAccessErr, None),
                (
                    V.CSS_NUMBER,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: floatValue 'x' is not a float",
                ),
                (
                    V.CSS_NUMBER,
                    '1x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: floatValue '1x' is not a float",
                ),
                (
                    V.CSS_STRING,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_STRING' is not a float type",
                ),
                (
                    V.CSS_URI,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_URI' is not a float type",
                ),
                (
                    V.CSS_ATTR,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_ATTR' is not a float type",
                ),
                (
                    V.CSS_IDENT,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_IDENT' is not a float type",
                ),
                (
                    V.CSS_RGBCOLOR,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_RGBCOLOR' is not a float type",
                ),
                (
                    V.CSS_RGBACOLOR,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_RGBACOLOR' is not a float type",
                ),
                (
                    V.CSS_RECT,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_RECT' is not a float type",
                ),
                (
                    V.CSS_COUNTER,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_COUNTER' is not a float type",
                ),
                (
                    V.CSS_EMS,
                    1,
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: Cannot coerce "
                    "primitiveType 'CSS_NUMBER' to 'CSS_EMS'",
                ),
                (
                    V.CSS_EXS,
                    1,
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: Cannot coerce primitiveType "
                    "'CSS_NUMBER' to 'CSS_EXS'",
                ),
            ],
            (V.CSS_MM, '1mm'): [
                (V.CSS_MM, 2, 2, '2mm'),
                (V.CSS_MM, 0, 0, '0mm'),
                (V.CSS_MM, 0.1, 0.1, '0.1mm'),
                (V.CSS_MM, -0, -0, '0mm'),
                (V.CSS_MM, 3.0, 3, '3mm'),
                (V.CSS_MM, 3.1, 3.1, '3.1mm'),
                (V.CSS_MM, -3.1, -3.1, '-3.1mm'),
                (V.CSS_CM, 1, 10, '10mm'),
                (V.CSS_IN, 10, 254, '254mm'),
                (V.CSS_PT, 1, 1828.8, '1828.8mm'),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_NUMBER, 2, xml.dom.InvalidAccessErr, None),
            ],
            (V.CSS_PT, '1pt'): [
                (V.CSS_PT, 2, 2, '2pt'),
                (V.CSS_PC, 12, 1, '1pt'),
                (V.CSS_NUMBER, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DEG, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
            ],
            (V.CSS_KHZ, '1khz'): [
                (V.CSS_HZ, 2000, 2, '2khz'),
                (V.CSS_NUMBER, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DEG, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
            ],
        }
        for test in tests:
            initialType, initialValue = test
            pv = cssutils.css.CSSPrimitiveValue(initialValue)
            for setType, setValue, exp, cssText in tests[test]:
                if type(exp) == type or type(exp) == type:  # 2.4 compatibility
                    if cssText:
                        with pytest.raises(exp, match=cssText):
                            pv.setFloatValue(setType, setValue)
                    else:
                        with pytest.raises(exp):
                            pv.setFloatValue(setType, setValue)
                else:
                    pv.setFloatValue(setType, setValue)
                    assert pv._value[0] == cssText
                    if cssText == '0mm':
                        cssText = '0'
                    assert pv.cssText == cssText
                    assert pv.getFloatValue(initialType) == exp

    def test_getString(self):
        "CSSPrimitiveValue.getStringValue()"
        v = cssutils.css.CSSPrimitiveValue('1px')
        assert v.primitiveType == v.CSS_PX
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getStringValue()

        pv = cssutils.css.CSSPrimitiveValue
        tests = {
            pv.CSS_STRING: ("'red'", 'red'),
            pv.CSS_STRING: ('"red"', 'red'),
            pv.CSS_URI: ('url(http://example.com)', None),
            pv.CSS_URI: ("url('http://example.com')", "http://example.com"),
            pv.CSS_URI: ('url("http://example.com")', 'http://example.com'),
            pv.CSS_URI: ('url("http://example.com?)")', 'http://example.com?)'),
            pv.CSS_IDENT: ('red', None),
            pv.CSS_ATTR: ('attr(att-name)', 'att-name'),  # the name of the attrr
        }
        for t in tests:
            val, exp = tests[t]
            if not exp:
                exp = val

            v = cssutils.css.CSSPrimitiveValue(val)
            assert v.primitiveType == t
            assert v.getStringValue() == exp

    def test_setString(self):
        "CSSPrimitiveValue.setStringValue()"
        # CSS_STRING
        v = cssutils.css.CSSPrimitiveValue('"a"')
        assert v.CSS_STRING == v.primitiveType
        v.setStringValue(v.CSS_STRING, 'b')
        assert ('b', 'STRING') == v._value
        assert 'b' == v.getStringValue()
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_STRING' to 'CSS_URI'",
        ):
            v.setStringValue(v.CSS_URI, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_STRING' to 'CSS_IDENT'",
        ):
            v.setStringValue(v.CSS_IDENT, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_STRING' to 'CSS_ATTR'",
        ):
            v.setStringValue(v.CSS_ATTR, 'x')

        # CSS_IDENT
        v = cssutils.css.CSSPrimitiveValue('new')
        v.setStringValue(v.CSS_IDENT, 'ident')
        assert v.CSS_IDENT == v.primitiveType
        assert ('ident', 'IDENT') == v._value
        assert 'ident' == v.getStringValue()
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_IDENT' to 'CSS_URI'",
        ):
            v.setStringValue(v.CSS_URI, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_IDENT' to 'CSS_STRING'",
        ):
            v.setStringValue(v.CSS_STRING, '"x"')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_IDENT' to 'CSS_ATTR'",
        ):
            v.setStringValue(v.CSS_ATTR, 'x')

        # CSS_URI
        v = cssutils.css.CSSPrimitiveValue('url(old)')
        v.setStringValue(v.CSS_URI, '(')
        assert ('(', 'URI') == v._value
        assert '(' == v.getStringValue()

        v.setStringValue(v.CSS_URI, ')')
        assert (')', 'URI') == v._value
        assert ')' == v.getStringValue()

        v.setStringValue(v.CSS_URI, '"')
        assert r'"' == v.getStringValue()
        assert (r'"', 'URI') == v._value

        v.setStringValue(v.CSS_URI, "''")
        assert r"''" == v.getStringValue()
        assert (r"''", 'URI') == v._value

        v.setStringValue(v.CSS_URI, ',')
        assert r',' == v.getStringValue()
        assert (r',', 'URI') == v._value

        v.setStringValue(v.CSS_URI, ' ')
        assert (' ', 'URI') == v._value
        assert ' ' == v.getStringValue()

        v.setStringValue(v.CSS_URI, 'a)')
        assert ('a)', 'URI') == v._value
        assert 'a)' == v.getStringValue()

        v.setStringValue(v.CSS_URI, 'a')
        assert v.CSS_URI == v.primitiveType
        assert ('a', 'URI') == v._value
        assert 'a' == v.getStringValue()

        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_URI' to 'CSS_IDENT'",
        ):
            v.setStringValue(v.CSS_IDENT, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_URI' to 'CSS_STRING'",
        ):
            v.setStringValue(v.CSS_STRING, '"x"')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_URI' to 'CSS_ATTR'",
        ):
            v.setStringValue(v.CSS_ATTR, 'x')

        # CSS_ATTR
        v = cssutils.css.CSSPrimitiveValue('attr(old)')
        v.setStringValue(v.CSS_ATTR, 'a')
        assert v.CSS_ATTR == v.primitiveType
        assert 'a' == v.getStringValue()
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_ATTR' to 'CSS_IDENT'",
        ):
            v.setStringValue(v.CSS_IDENT, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_ATTR' to 'CSS_STRING'",
        ):
            v.setStringValue(v.CSS_STRING, '"x"')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_ATTR' to 'CSS_URI'",
        ):
            v.setStringValue(v.CSS_URI, 'x')

        # TypeError as 'x' is no valid type
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: stringType 'x' (UNKNOWN TYPE) is not a string type",
        ):
            v.setStringValue('x', 'brown')
        # IndexError as 111 is no valid type
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: stringType 111 (UNKNOWN TYPE) is not a string type",
        ):
            v.setStringValue(111, 'brown')
        # CSS_PX is no string type
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: stringType CSS_PX is not a string type",
        ):
            v.setStringValue(v.CSS_PX, 'brown')

    def test_typeRGBColor(self):
        "RGBColor"
        v = cssutils.css.CSSPrimitiveValue('RGB(1, 5, 10)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        assert 'rgb(1, 5, 10)' == v.cssText

        v = cssutils.css.CSSPrimitiveValue('rgb(1, 5, 10)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        assert 'rgb(1, 5, 10)' == v.cssText

        v = cssutils.css.CSSPrimitiveValue('rgb(1%, 5%, 10%)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        assert 'rgb(1%, 5%, 10%)' == v.cssText

        v = cssutils.css.CSSPrimitiveValue('  rgb(  1 ,5,  10  )')
        assert v.CSS_RGBCOLOR == v.primitiveType
        v = cssutils.css.CSSPrimitiveValue('rgb(1,5,10)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        v = cssutils.css.CSSPrimitiveValue('rgb(1%, .5%, 10.1%)')
        assert v.CSS_RGBCOLOR == v.primitiveType

    def test_reprANDstr(self):
        "CSSPrimitiveValue.__repr__(), .__str__()"
        v = '111'

        s = cssutils.css.CSSPrimitiveValue(v)

        assert v in str(s)
        assert 'CSS_NUMBER' in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert v == s2.cssText


class TestCSSValueList:
    def test_init(self):
        "CSSValueList.__init__()"
        v = cssutils.css.CSSValue(cssText='red blue')
        assert v.CSS_VALUE_LIST == v.cssValueType
        assert 'red blue' == v.cssText

        assert 2 == v.length

        item = v.item(0)
        item.setStringValue(item.CSS_IDENT, 'green')
        assert 'green blue' == v.cssText

    def test_numbers(self):
        "CSSValueList.cssText"
        tests = {
            '0 0px -0px +0px': ('0 0 0 0', 4),
            '1 2 3 4': (None, 4),
            '-1 -2 -3 -4': (None, 4),
            '-1 2': (None, 2),
            '-1px red "x"': (None, 3),
            'a, b c': (None, 2),
            '1px1 2% 3': ('1px1 2% 3', 3),
            'f(+1pX, -2, 5%) 1': ('f(1px, -2, 5%) 1', 2),
            '0 f()0': ('0 f() 0', 3),
            'f()0': ('f() 0', 2),
            'f()1%': ('f() 1%', 2),
            'f()1px': ('f() 1px', 2),
            'f()"str"': ('f() "str"', 2),
            'f()ident': ('f() ident', 2),
            'f()#123': ('f() #123', 2),
            'f()url()': ('f() url()', 2),
            'f()f()': ('f() f()', 2),
            'url(x.gif)0 0': ('url(x.gif) 0 0', 3),
            'url(x.gif)no-repeat': ('url(x.gif) no-repeat', 2),
        }
        for test in tests:
            exp, num = tests[test]
            if not exp:
                exp = test
            v = cssutils.css.CSSValue(cssText=test)
            assert v.CSS_VALUE_LIST == v.cssValueType
            assert num == v.length
            assert exp == v.cssText

    def test_reprANDstr(self):
        "CSSValueList.__repr__(), .__str__()"
        v = '1px 2px'

        s = cssutils.css.CSSValue(v)
        assert isinstance(s, cssutils.css.CSSValueList)

        assert 'length=2' in str(s)
        assert v in str(s)

        # not "eval()"able!
        # s2 = eval(repr(s))

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssvalue.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.cssvariablesdelaration.CSSVariablesDeclaration."""

import xml.dom

from . import basetest
import cssutils


class TestCSSVariablesDeclaration(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.css.CSSVariablesDeclaration()
        cssutils.ser.prefs.useDefaults()

    def teardown_method(self):
        cssutils.ser.prefs.useDefaults()

    def test_init(self):
        "CSSVariablesDeclaration.__init__()"
        v = cssutils.css.CSSVariablesDeclaration()
        assert '' == v.cssText
        assert 0 == v.length
        assert v.parentRule is None

        v = cssutils.css.CSSVariablesDeclaration(cssText='x: 0')
        assert 'x: 0' == v.cssText
        assert '0' == v.getVariableValue('x')

        rule = cssutils.css.CSSVariablesRule()
        v = cssutils.css.CSSVariablesDeclaration(cssText='x: 0', parentRule=rule)
        assert rule == v.parentRule

    def test__contains__(self):
        "CSSVariablesDeclaration.__contains__(name)"
        v = cssutils.css.CSSVariablesDeclaration(cssText='x: 0; y: 2')
        for test in ('x', 'y'):
            assert test in v
            assert test.upper() in v

        assert 'z' not in v

    def test_items(self):
        "CSSVariablesDeclaration[variableName]"
        v = cssutils.css.CSSVariablesDeclaration()

        value = '0'
        v['X'] = value
        assert value == v['X']
        assert value == v.getVariableValue('X')
        assert value == v['x']
        assert value == v.getVariableValue('x')

        assert '' == v['y']
        assert '' == v.getVariableValue('y')

        v['z'] = '1'
        assert 2 == v.length

        # unsorted!
        assert sorted(v) == ['x', 'z']

        del v['z']
        assert 1 == v.length
        assert 1 == v.length

        assert '0' == v.removeVariable('x')
        assert '' == v.removeVariable('z')
        assert 0 == v.length

        v.cssText = 'x:0; y:1'
        keys = []
        # unsorted!
        for i in range(0, v.length):
            keys.append(v.item(i))
        assert sorted(keys) == ['x', 'y']

    def test_keys(self):
        "CSSVariablesDeclaration.keys()"
        v = cssutils.css.CSSVariablesDeclaration(cssText='x: 0; Y: 2')
        assert ['x', 'y'] == sorted(v.keys())

    def test_cssText(self):
        "CSSVariablesDeclaration.cssText"
        # empty
        tests = {
            '': '',
            ' ': '',
            ' \t \n  ': '',
            'x: 1': None,
            'x: "a"': None,
            'x: rgb(1, 2, 3)': None,
            'x: 1px 2px 3px': None,
            'x:1': 'x: 1',
            'x:1;': 'x: 1',
            'x  :  1  ': 'x: 1',
            'x  :  1  ;  ': 'x: 1',
            'x:1;y:2': 'x: 1;\ny: 2',
            'x:1;y:2;': 'x: 1;\ny: 2',
            'x  :  1  ;  y  :  2  ': 'x: 1;\ny: 2',
            'x  :  1  ;  y  :  2  ;  ': 'x: 1;\ny: 2',
            '/*x*/': '/*x*/',
            'x555: 5': None,
            'xxx:1;yyy:2': 'xxx: 1;\nyyy: 2',
            'xxx : 1; yyy : 2': 'xxx: 1;\nyyy: 2',
            'x:1;x:2;X:2': 'x: 2',
            'same:1;SAME:2;': 'same: 2',
            '/**/x/**/:/**/1/**/;/**/y/**/:/**/2/**/': '/**/ \n /**/ \n /**/ \n x: 1 /**/;\n/**/ \n /**/ \n /**/ \n y: 2 /**/',
        }
        self.do_equal_r(tests)

        def test_cssText2(self):
            "CSSVariablesDeclaration.cssText"

            tests = {
                'top': xml.dom.SyntaxErr,
                'top:': xml.dom.SyntaxErr,
                'top : ': xml.dom.SyntaxErr,
                'top:;': xml.dom.SyntaxErr,
                'top 0': xml.dom.SyntaxErr,
                'top 0;': xml.dom.SyntaxErr,
                ':': xml.dom.SyntaxErr,
                ':0': xml.dom.SyntaxErr,
                ':0;': xml.dom.SyntaxErr,
                ':;': xml.dom.SyntaxErr,
                ': ;': xml.dom.SyntaxErr,
                '0': xml.dom.SyntaxErr,
                '0;': xml.dom.SyntaxErr,
                ';': xml.dom.SyntaxErr,
            }
            self.do_raise_r(tests)

    def test_xVariable(self):
        "CSSVariablesDeclaration.xVariable()"
        v = cssutils.css.CSSVariablesDeclaration()
        # unset
        assert '' == v.getVariableValue('x')
        # set
        v.setVariable('x', '0')
        assert '0' == v.getVariableValue('x')
        assert '0' == v.getVariableValue('X')
        assert 'x: 0' == v.cssText
        v.setVariable('X', '0')
        assert '0' == v.getVariableValue('x')
        assert '0' == v.getVariableValue('X')
        assert 'x: 0' == v.cssText
        # remove
        assert '0' == v.removeVariable('x')
        assert '' == v.removeVariable('x')
        assert '' == v.getVariableValue('x')
        assert '' == v.cssText

    def test_imports(self):
        "CSSVariables imports"

        def fetcher(url):
            url = url.replace('\\', '/')
            url = url[url.rfind('/') + 1 :]
            return (
                None,
                {
                    '3.css': '''
                    @variables {
                        over3-2-1-0: 3;
                        over3-2-1: 3;
                        over3-2: 3;
                        over3-2-0: 3;
                        over3-1: 3;
                        over3-1-0: 3;
                        over3-0: 3;
                        local3: 3;
                    }

                ''',
                    '2.css': '''
                    @variables {
                        over3-2-1-0: 2;
                        over3-2-1: 2;
                        over3-2-0: 2;
                        over3-2: 2;
                        over2-1: 2;
                        over2-1-0: 2;
                        over2-0: 2;
                        local2: 2;
                    }

                ''',
                    '1.css': '''
                    @import "3.css";
                    @import "2.css";
                    @variables {
                        over3-2-1-0: 1;
                        over3-2-1: 1;
                        over3-1: 1;
                        over3-1-0: 1;
                        over2-1: 1;
                        over2-1-0: 1;
                        over1-0: 1;
                        local1: 1;
                    }

                ''',
                }[url],
            )

        css = '''
            @import "1.css";
            @variables {
                over3-2-1-0: 0;
                over3-2-0: 0;
                over3-1-0: 0;
                over2-1-0: 0;
                over3-0: 0;
                over2-0: 0;
                over1-0: 0;
                local0: 0;
            }
            a {
                local0: var(local0);
                local1: var(local1);
                local2: var(local2);
                local3: var(local3);
                over1-0: var(over1-0);
                over2-0: var(over2-0);
                over3-0: var(over3-0);
                over2-1: var(over2-1);
                over3-1: var(over3-1);
                over3-2: var(over3-2);
                over2-1-0: var(over2-1-0);
                over3-2-0: var(over3-2-0);
                over3-2-1: var(over3-2-1);
                over3-2-1-0: var(over3-2-1-0);
            }
        '''
        p = cssutils.CSSParser(fetcher=fetcher)
        s = p.parseString(css)

        # only these in rule of this sheet
        assert s.cssRules[1].variables.length == 8
        # but all vars in s available!
        assert s.variables.length == 15
        assert [
            'local0',
            'local1',
            'local2',
            'local3',
            'over1-0',
            'over2-0',
            'over2-1',
            'over2-1-0',
            'over3-0',
            'over3-1',
            'over3-1-0',
            'over3-2',
            'over3-2-0',
            'over3-2-1',
            'over3-2-1-0',
        ] == sorted(s.variables.keys())

        # test with variables rule
        cssutils.ser.prefs.resolveVariables = False
        assert (
            s.cssText
            == b'''@import "1.css";
@variables {
    over3-2-1-0: 0;
    over3-2-0: 0;
    over3-1-0: 0;
    over2-1-0: 0;
    over3-0: 0;
    over2-0: 0;
    over1-0: 0;
    local0: 0
    }
a {
    local0: var(local0);
    local1: var(local1);
    local2: var(local2);
    local3: var(local3);
    over1-0: var(over1-0);
    over2-0: var(over2-0);
    over3-0: var(over3-0);
    over2-1: var(over2-1);
    over3-1: var(over3-1);
    over3-2: var(over3-2);
    over2-1-0: var(over2-1-0);
    over3-2-0: var(over3-2-0);
    over3-2-1: var(over3-2-1);
    over3-2-1-0: var(over3-2-1-0)
    }'''
        )

        # test with resolved vars
        cssutils.ser.prefs.resolveVariables = True
        assert (
            s.cssText
            == b'''@import "1.css";
a {
    local0: 0;
    local1: 1;
    local2: 2;
    local3: 3;
    over1-0: 0;
    over2-0: 0;
    over3-0: 0;
    over2-1: 1;
    over3-1: 1;
    over3-2: 2;
    over2-1-0: 0;
    over3-2-0: 0;
    over3-2-1: 1;
    over3-2-1-0: 0
    }'''
        )

        s = cssutils.resolveImports(s)
        assert (
            s.cssText
            == b'''/* START @import "1.css" */
/* START @import "3.css" */
/* START @import "2.css" */
a {
    local0: 0;
    local1: 1;
    local2: 2;
    local3: 3;
    over1-0: 0;
    over2-0: 0;
    over3-0: 0;
    over2-1: 1;
    over3-1: 1;
    over3-2: 2;
    over2-1-0: 0;
    over3-2-0: 0;
    over3-2-1: 1;
    over3-2-1-0: 0
    }'''
        )

    def test_parentRule(self):
        "CSSVariablesDeclaration.parentRule"
        s = cssutils.parseString('@variables { a:1}')
        r = s.cssRules[0]
        d = r.variables
        assert r == d.parentRule

        d2 = cssutils.css.CSSVariablesDeclaration('b: 2')
        r.variables = d2
        assert r == d2.parentRule

    def test_reprANDstr(self):
        "CSSVariablesDeclaration.__repr__(), .__str__()"
        s = cssutils.css.CSSVariablesDeclaration(cssText='a:1;b:2')

        assert "2" in str(s)  # length

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_cssvariablesdeclaration.py
NikolaBlagojevic/pyrecodes
"""Tests for parsing which does not raise Exceptions normally"""

import logging
import io
import sys
import xml.dom
import socket

import cssutils


class TestErrorHandler:
    def setup_method(self):
        "replace default log and ignore its output"
        self._oldlog = cssutils.log._log
        self._saved = cssutils.log.raiseExceptions

        cssutils.log.raiseExceptions = False
        cssutils.log.setLog(logging.getLogger('IGNORED-CSSUTILS-TEST'))

    def teardown_method(self):
        "reset default log"
        cssutils.log.setLog(self._oldlog)
        # for tests only
        cssutils.log.setLevel(logging.FATAL)
        cssutils.log.raiseExceptions = self._saved

    def _setHandler(self):
        "sets new handler and returns StringIO instance to getvalue"
        s = io.StringIO()
        h = logging.StreamHandler(s)
        h.setFormatter(logging.Formatter('%(levelname)s    %(message)s'))
        # remove if present already
        cssutils.log.removeHandler(h)
        cssutils.log.addHandler(h)
        return s

    def test_calls(self):
        "cssutils.log.*"
        s = self._setHandler()
        cssutils.log.setLevel(logging.DEBUG)
        cssutils.log.debug('msg', neverraise=True)
        assert s.getvalue() == 'DEBUG    msg\n'

        s = self._setHandler()
        cssutils.log.setLevel(logging.INFO)
        cssutils.log.info('msg', neverraise=True)
        assert s.getvalue() == 'INFO    msg\n'

        s = self._setHandler()
        cssutils.log.setLevel(logging.WARNING)
        cssutils.log.warn('msg', neverraise=True)
        assert s.getvalue() == 'WARNING    msg\n'

        s = self._setHandler()
        cssutils.log.setLevel(logging.ERROR)
        cssutils.log.error('msg', neverraise=True)
        assert s.getvalue() == 'ERROR    msg\n'

        s = self._setHandler()
        cssutils.log.setLevel(logging.FATAL)
        cssutils.log.fatal('msg', neverraise=True)
        assert s.getvalue() == 'CRITICAL    msg\n'

        s = self._setHandler()
        cssutils.log.setLevel(logging.CRITICAL)
        cssutils.log.critical('msg', neverraise=True)
        assert s.getvalue() == 'CRITICAL    msg\n'

        s = self._setHandler()
        cssutils.log.setLevel(logging.CRITICAL)
        cssutils.log.error('msg', neverraise=True)
        assert s.getvalue() == ''

    def test_linecol(self):
        "cssutils.log line col"
        o = cssutils.log.raiseExceptions
        cssutils.log.raiseExceptions = True

        s = cssutils.css.CSSStyleSheet()
        try:
            s.cssText = '@import x;'
        except xml.dom.DOMException as e:
            assert str(e) == 'CSSImportRule: Unexpected ident. [1:9: x]'
            assert e.line == 1
            assert e.col == 9
            if sys.platform.startswith('java'):
                assert e.msg == 'CSSImportRule: Unexpected ident. [1:9: x]'
            else:
                assert e.args == ('CSSImportRule: Unexpected ident. [1:9: x]',)

        cssutils.log.raiseExceptions = o

    def test_handlers(self):
        "cssutils.log"
        s = self._setHandler()

        cssutils.log.setLevel(logging.FATAL)
        assert cssutils.log.getEffectiveLevel() == logging.FATAL

        cssutils.parseString('a { color: 1 }')
        assert s.getvalue() == ''

        cssutils.log.setLevel(logging.DEBUG)
        cssutils.parseString('a { color: 1 }')
        # TODO: Fix?
        # self.assertEqual(
        #     s.getvalue(),
        #     u'ERROR    Property: Invalid value for "CSS Color Module '
        #     'Level 3/CSS Level 2.1" property: 1 [1:5: color]\n')
        assert (
            s.getvalue() == 'ERROR    Property: Invalid value for "CSS Level 2.1" '
            'property: 1 [1:5: color]\n'
        )

        s = self._setHandler()

        try:
            socket.getaddrinfo('example.com', 80)
        except OSError:
            # skip the test as the name can't resolve
            return

        cssutils.log.setLevel(logging.ERROR)
        cssutils.parseUrl('http://example.com')
        assert s.getvalue()[:38] == 'ERROR    Expected "text/css" mime type'

    def test_parsevalidation(self):
        style = 'color: 1'
        t = 'a { %s }' % style

        cssutils.log.setLevel(logging.DEBUG)

        # sheet
        s = self._setHandler()
        cssutils.parseString(t)
        assert len(s.getvalue()) != 0

        s = self._setHandler()
        cssutils.parseString(t, validate=False)
        assert s.getvalue() == ''

        # style
        s = self._setHandler()
        cssutils.parseStyle(style)
        assert len(s.getvalue()) != 0

        s = self._setHandler()
        cssutils.parseStyle(style, validate=True)
        assert len(s.getvalue()) != 0

        s = self._setHandler()
        cssutils.parseStyle(style, validate=False)
        assert s.getvalue() == ''

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_errorhandler.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.stylesheets.MediaList"""

import re
import xml.dom

import pytest

from . import basetest
import cssutils.stylesheets


class TestMediaList(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.stylesheets.MediaList()

    def test_set(self):
        "MediaList.mediaText 1"
        ml = cssutils.stylesheets.MediaList()

        assert 0 == ml.length
        assert 'all' == ml.mediaText

        ml.mediaText = ' print   , screen '
        assert 2 == ml.length
        assert 'print, screen' == ml.mediaText

        # with pytest.raises(xml.dom.InvalidModificationErr, match=self.media_msg('tv')):
        #     ml._setMediaText(u' print , all  , tv ')

        # self.assertEqual(u'all', ml.mediaText)
        # self.assertEqual(1, ml.length)

        with pytest.raises(xml.dom.SyntaxErr):
            ml.appendMedium('test')

    def test_appendMedium(self):
        "MediaList.appendMedium() 1"
        ml = cssutils.stylesheets.MediaList()

        ml.appendMedium('print')
        assert 1 == ml.length
        assert 'print' == ml.mediaText

        ml.appendMedium('screen')
        assert 2 == ml.length
        assert 'print, screen' == ml.mediaText

        # automatic del and append!
        ml.appendMedium('print')
        assert 2 == ml.length
        assert 'screen, print' == ml.mediaText

        # automatic del and append!
        ml.appendMedium('SCREEN')
        assert 2 == ml.length
        assert 'print, SCREEN' == ml.mediaText

        # append invalid MediaQuery
        mq = cssutils.stylesheets.MediaQuery()
        ml.appendMedium(mq)
        assert 2 == ml.length
        assert 'print, SCREEN' == ml.mediaText

        # append()
        mq = cssutils.stylesheets.MediaQuery('tv')
        ml.append(mq)
        assert 3 == ml.length
        assert 'print, SCREEN, tv' == ml.mediaText

        # __setitem__
        with pytest.raises(IndexError):
            ml.__setitem__(10, 'all')
        ml[0] = 'handheld'
        assert 3 == ml.length
        assert 'handheld, SCREEN, tv' == ml.mediaText

    def test_appendAll(self):
        "MediaList.append() 2"
        ml = cssutils.stylesheets.MediaList()
        ml.appendMedium('print')
        ml.appendMedium('tv')
        assert 2 == ml.length
        assert 'print, tv' == ml.mediaText

        ml.appendMedium('all')
        assert 1 == ml.length
        assert 'all' == ml.mediaText

        with pytest.raises(xml.dom.InvalidModificationErr, match=self.media_msg('tv')):
            ml.appendMedium('tv')
        assert 1 == ml.length
        assert 'all' == ml.mediaText

        with pytest.raises(xml.dom.SyntaxErr):
            ml.appendMedium('test')

    @staticmethod
    def media_msg(text):
        return re.escape(
            'MediaList: Ignoring new medium '
            f'cssutils.stylesheets.MediaQuery(mediaText={text!r}) '
            'as already specified "all" (set ``mediaText`` instead).'
        )

    def test_append2All(self):
        "MediaList all"
        ml = cssutils.stylesheets.MediaList()
        ml.appendMedium('all')
        with pytest.raises(
            xml.dom.InvalidModificationErr, match=self.media_msg('print')
        ):
            ml.appendMedium('print')

        sheet = cssutils.parseString('@media all, print { /**/ }')
        assert b'@media all {\n    /**/\n    }' == sheet.cssText

    def test_delete(self):
        "MediaList.deleteMedium()"
        ml = cssutils.stylesheets.MediaList()

        with pytest.raises(xml.dom.NotFoundErr):
            ml.deleteMedium('all')
        with pytest.raises(xml.dom.NotFoundErr):
            ml.deleteMedium('test')

        ml.appendMedium('print')
        ml.deleteMedium('print')
        ml.appendMedium('tV')
        ml.deleteMedium('Tv')
        assert 0 == ml.length
        assert 'all' == ml.mediaText

    def test_item(self):
        "MediaList.item()"
        ml = cssutils.stylesheets.MediaList()
        ml.appendMedium('print')
        ml.appendMedium('screen')

        assert 'print' == ml.item(0)
        assert 'screen' == ml.item(1)
        assert ml.item(2) is None

    # REMOVED special case!
    # def test_handheld(self):
    #    "MediaList handheld"
    #    ml = cssutils.stylesheets.MediaList()

    #    ml.mediaText = u' handheld , all  '
    #    self.assertEqual(2, ml.length)
    #    self.assertEqual(u'handheld, all', ml.mediaText)

    #    with pytest.raises(xml.dom.InvalidModificationErr, match=self.media_msg('handheld')):
    #        ml._setMediaText(' handheld , all  , tv ')

    def test_mediaText(self):
        "MediaList.mediaText 2"
        tests = {
            'ALL': 'ALL',
            'Tv': 'Tv',
            'all': None,
            'all, handheld': 'all',
            'tv': None,
            'tv, handheld, print': None,
            'tv and (color), handheld and (width: 1px) and (color)': None,
        }
        self.do_equal_r(tests, att='mediaText')

        tests = {
            '': xml.dom.SyntaxErr,
            'UNKNOWN': xml.dom.SyntaxErr,
            'a,b': xml.dom.SyntaxErr,
            'a and (color)': xml.dom.SyntaxErr,
            'not': xml.dom.SyntaxErr,  # known but need media
            'only': xml.dom.SyntaxErr,  # known but need media
            'not tv,': xml.dom.SyntaxErr,  # known but need media
            'all;': xml.dom.SyntaxErr,
            'all, and(color)': xml.dom.SyntaxErr,
            'all,': xml.dom.SyntaxErr,
            'all, ': xml.dom.SyntaxErr,
            'all ,': xml.dom.SyntaxErr,
            'all, /*1*/': xml.dom.SyntaxErr,
            'all and (color),': xml.dom.SyntaxErr,
            'all tv, print': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests, att='_setMediaText')

    def test_comments(self):
        "MediaList.mediaText comments"
        tests = {
            '/*1*/ tv /*2*/, /*3*/ handheld /*4*/, print': '/*1*/ tv /*2*/ /*3*/, handheld /*4*/, print',
        }
        self.do_equal_r(tests, att='mediaText')

    def test_reprANDstr(self):
        "MediaList.__repr__(), .__str__()"
        mediaText = 'tv, print'

        s = cssutils.stylesheets.MediaList(mediaText=mediaText)

        assert mediaText in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert mediaText == s2.mediaText

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_medialist.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.stylesheets.MediaQuery"""

import xml.dom
from . import basetest
import cssutils.stylesheets
import pytest


class TestMediaQuery(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.stylesheets.MediaQuery()

    def test_mediaText(self):
        "MediaQuery.mediaText"
        tests = {
            'all': None,
            'braille': None,
            'embossed': None,
            'handheld': None,
            'print': None,
            'projection': None,
            'screen': None,
            'speech': None,
            'tty': None,
            'tv': None,
            'ALL': None,
            'a\\ll': None,
            'not tv': None,
            'n\\ot t\\v': None,
            'only tv': None,
            '\\only \\tv': None,
            'PRINT': None,
            'NOT PRINT': None,
            'ONLY PRINT': None,
            'tv and (color)': None,
            'not tv and (color)': None,
            'only tv and (color)': None,
            'print and(color)': 'print and (color)',
        }
        self.do_equal_r(tests, att='mediaText')

        tests = {
            '': xml.dom.SyntaxErr,
            'two values': xml.dom.SyntaxErr,
            'or even three': xml.dom.SyntaxErr,
            'aural': xml.dom.SyntaxErr,  # a dimension
            '3d': xml.dom.SyntaxErr,  # a dimension
        }
        self.do_raise_r(tests, att='_setMediaText')

    def test_mediaType(self):
        "MediaQuery.mediaType"
        mq = cssutils.stylesheets.MediaQuery()

        assert '' == mq.mediaText

        for mt in cssutils.stylesheets.MediaQuery.MEDIA_TYPES:
            mq.mediaType = mt
            assert mq.mediaType == mt
            mq.mediaType = mt.upper()
            assert mq.mediaType == mt.upper()

        mt = '3D-UNKOwn-MEDIAtype0123'
        # mq.mediaType = mt
        with pytest.raises(xml.dom.SyntaxErr):
            mq._setMediaType(mt)
        # self.assertRaises(xml.dom.InvalidCharacterErr, mq._setMediaType, mt)

    def test_comments(self):
        "MediaQuery.mediaText comments"
        tests = {
            'all': None,
            'print': None,
            'not print': None,
            'only print': None,
            'print and (color)': None,
            'print and (color) and (width)': None,
            'print and (color: 2)': None,
            'print and (min-width: 100px)': None,
            'print and (min-width: 100px) and (color: red)': None,
            'not print and (min-width: 100px)': None,
            'only print and (min-width: 100px)': None,
            '/*1*/ tv /*2*/': None,
            '/*0*/ only /*1*/ tv /*2*/': None,
            '/*0* /not /*1*/ tv /*2*/': None,
            '/*x*/ only /*x*/ print /*x*/ and /*x*/ (/*x*/ '
            'min-width /*x*/: /*x*/ 100px /*x*/)': None,
            'print and/*1*/(color)': 'print and /*1*/ (color)',
        }
        self.do_equal_r(tests, att='mediaText')

    def test_reprANDstr(self):
        "MediaQuery.__repr__(), .__str__()"
        mediaText = 'tv and (color)'
        s = cssutils.stylesheets.MediaQuery(mediaText=mediaText)
        assert mediaText in str(s)
        s2 = eval(repr(s))
        assert mediaText == s2.mediaText
        assert isinstance(s2, s.__class__)

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_mediaquery.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSCharsetRule"""

import sys
import re
import xml.dom
from cssutils.prodparser import (
    Prod,
    Sequence,
    Choice,
    PreDef,
    ParseError,
    Exhausted,
    ProdParser,
)
import pytest


class TestProd:
    def test_init(self):
        "Prod.__init__(...)"
        p = Prod('min', lambda t, v: t == 1 and v == 2)

        assert str(p) == 'min'
        assert p.toStore is None
        assert p.optional is False

        p = Prod('optional', lambda t, v: True, optional=True)
        assert p.optional is True

    def test_initMatch(self):
        "Prod.__init__(...match=...)"
        p = Prod('min', lambda t, v: t == 1 and v == 2)
        assert p.match(1, 2) is True
        assert p.match(2, 2) is False
        assert p.match(1, 1) is False

    def test_initToSeq(self):
        "Prod.__init__(...toSeq=...)"
        # simply saves
        p = Prod('all', lambda t, tokens: True, toSeq=None)
        assert p.toSeq([1, 2], None) == (1, 2)  # simply saves
        assert p.toSeq(['s1', 's2'], None) == ('s1', 's2')  # simply saves

        # saves callback(val)
        p = Prod(
            'all', lambda t, v: True, toSeq=lambda t, tokens: (1 == t[0], 3 == t[1])
        )
        assert p.toSeq([1, 3], None) == (True, True)
        assert p.toSeq([2, 4], None) == (False, False)

    def test_initToStore(self):
        "Prod.__init__(...toStore=...)"
        p = Prod('all', lambda t, v: True, toStore='key')

        # save as key
        s = {}
        p.toStore(s, 1)
        assert s['key'] == 1

        # append to key
        s = {'key': []}
        p.toStore(s, 1)
        p.toStore(s, 2)
        assert s['key'] == [1, 2]

        # callback
        def doubleToStore(key):
            def toStore(store, item):
                store[key] = item * 2

            return toStore

        p = Prod('all', lambda t, v: True, toStore=doubleToStore('key'))
        s = {'key': []}
        p.toStore(s, 1)
        assert s['key'] == 2

    def test_matches(self):
        "Prod.matches(token)"
        p1 = Prod('p1', lambda t, v: t == 1 and v == 2)
        p2 = Prod('p2', lambda t, v: t == 1 and v == 2, optional=True)
        assert p1.matches([1, 2, 0, 0]) is True
        assert p2.matches([1, 2, 0, 0]) is True
        assert p1.matches([0, 0, 0, 0]) is False
        assert p2.matches([0, 0, 0, 0]) is False


class TestSequence:
    def test_init(self):
        "Sequence.__init__()"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2)
        seq = Sequence(p1, p2)

        assert 1 == seq._min
        assert 1 == seq._max

    def test_initminmax(self):
        "Sequence.__init__(...minmax=...)"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2)

        s = Sequence(p1, p2, minmax=lambda: (2, 3))
        assert 2 == s._min
        assert 3 == s._max

        s = Sequence(p1, p2, minmax=lambda: (0, None))
        assert 0 == s._min

        try:
            # py2.6/3
            m = sys.maxsize
        except AttributeError:
            # py<1.6
            m = sys.maxsize
        assert m == s._max

    def test_optional(self):
        "Sequence.optional"
        p1 = Prod('p1', lambda t, v: t == 1)

        s = Sequence(p1, minmax=lambda: (1, 3))
        assert not s.optional
        s = Sequence(p1, minmax=lambda: (0, 3))
        assert s.optional
        s = Sequence(p1, minmax=lambda: (0, None))
        assert s.optional

    def test_reset(self):
        "Sequence.reset()"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2)
        seq = Sequence(p1, p2)
        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)
        assert p1 == seq.nextProd(t1)
        assert p2 == seq.nextProd(t2)
        with pytest.raises(Exhausted):
            seq.nextProd(t1)
        seq.reset()
        assert p1 == seq.nextProd(t1)

    def test_matches(self):
        "Sequence.matches()"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2, optional=True)

        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)
        t3 = (3, 0, 0, 0)

        s = Sequence(p1, p2)
        assert s.matches(t1)
        assert not s.matches(t2)

        s = Sequence(p2, p1)
        assert s.matches(t1)
        assert s.matches(t2)

        s = Sequence(Choice(p1, p2))
        assert s.matches(t1)
        assert s.matches(t2)
        assert not s.matches(t3)

    def test_nextProd(self):
        "Sequence.nextProd()"
        p1 = Prod('p1', lambda t, v: t == 1, optional=True)
        p2 = Prod('p2', lambda t, v: t == 2)
        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)

        tests = {
            # seq: list of list of (token, prod or error msg)
            (p1,): (
                [(t1, p1)],
                [(t2, 'Extra token')],  # as p1 optional
                [(t1, p1), (t1, 'Extra token')],
                [(t1, p1), (t2, 'Extra token')],
            ),
            (p2,): (
                [(t2, p2)],
                [(t2, p2), (t2, 'Extra token')],
                [(t2, p2), (t1, 'Extra token')],
                [(t1, 'Missing token for production p2')],
            ),
            (p1, p2): (
                [(t1, p1), (t2, p2)],
                [(t1, p1), (t1, 'Missing token for production p2')],
            ),
        }
        for seqitems, results in list(tests.items()):
            for result in results:
                seq = Sequence(*seqitems)
                for t, p in result:
                    if isinstance(p, str):
                        with pytest.raises(ParseError, match=p):
                            seq.nextProd(t)
                    else:
                        assert p == seq.nextProd(t)

        tests = {
            # seq: list of list of (token, prod or error msg)
            # as p1 optional!
            (p1, p1): (
                [(t1, p1)],
                [(t1, p1), (t1, p1)],
                [(t1, p1), (t1, p1)],
                [(t1, p1), (t1, p1), (t1, p1)],
                [(t1, p1), (t1, p1), (t1, p1), (t1, p1)],
                [(t1, p1), (t1, p1), (t1, p1), (t1, p1), (t1, 'Extra token')],
            ),
            (p1,): (
                [(t1, p1)],
                [(t2, 'Extra token')],
                [(t1, p1), (t1, p1)],
                [(t1, p1), (t2, 'Extra token')],
                [(t1, p1), (t1, p1), (t1, 'Extra token')],
                [(t1, p1), (t1, p1), (t2, 'Extra token')],
            ),
            # as p2 NOT optional
            (p2,): (
                [(t2, p2)],
                [(t1, 'Missing token for production p2')],
                [(t2, p2), (t2, p2)],
                [(t2, p2), (t1, 'No match for (1, 0, 0, 0) in Sequence(p2)')],
                [(t2, p2), (t2, p2), (t2, 'Extra token')],
                [(t2, p2), (t2, p2), (t1, 'Extra token')],
            ),
            (p1, p2): (
                [(t1, p1), (t1, 'Missing token for production p2')],
                [(t2, p2), (t2, p2)],
                [(t2, p2), (t1, p1), (t2, p2)],
                [(t1, p1), (t2, p2), (t2, p2)],
                [(t1, p1), (t2, p2), (t1, p1), (t2, p2)],
                [(t2, p2), (t2, p2), (t2, 'Extra token')],
                [(t2, p2), (t1, p1), (t2, p2), (t1, 'Extra token')],
                [(t2, p2), (t1, p1), (t2, p2), (t2, 'Extra token')],
                [(t1, p1), (t2, p2), (t2, p2), (t1, 'Extra token')],
                [(t1, p1), (t2, p2), (t2, p2), (t2, 'Extra token')],
                [(t1, p1), (t2, p2), (t1, p1), (t2, p2), (t1, 'Extra token')],
                [(t1, p1), (t2, p2), (t1, p1), (t2, p2), (t2, 'Extra token')],
            ),
        }
        for seqitems, results in list(tests.items()):
            for result in results:
                seq = Sequence(minmax=lambda: (1, 2), *seqitems)
                for t, p in result:
                    if isinstance(p, str):
                        with pytest.raises(ParseError, match=re.escape(p)):
                            seq.nextProd(t)
                    else:
                        assert p == seq.nextProd(t)


class TestChoice:
    def test_init(self):
        "Choice.__init__()"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2)
        t0 = (0, 0, 0, 0)
        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)

        ch = Choice(p1, p2)
        with pytest.raises(
            ParseError, match=re.escape('No match for (0, 0, 0, 0) in Choice(p1, p2)')
        ):
            ch.nextProd(t0)
        assert p1 == ch.nextProd(t1)
        with pytest.raises(Exhausted, match='Extra token'):
            ch.nextProd(t1)

        ch = Choice(p1, p2)
        assert p2 == ch.nextProd(t2)
        with pytest.raises(Exhausted, match='Extra token'):
            ch.nextProd(t2)

        ch = Choice(p2, p1)
        with pytest.raises(
            ParseError, match=re.escape('No match for (0, 0, 0, 0) in Choice(p2, p1)')
        ):
            ch.nextProd(t0)
        assert p1 == ch.nextProd(t1)
        with pytest.raises(Exhausted, match='Extra token'):
            ch.nextProd(t1)

        ch = Choice(p2, p1)
        assert p2 == ch.nextProd(t2)
        with pytest.raises(Exhausted, match='Extra token'):
            ch.nextProd(t2)

    def test_matches(self):
        "Choice.matches()"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2, optional=True)

        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)
        t3 = (3, 0, 0, 0)

        c = Choice(p1, p2)
        assert c.matches(t1)
        assert c.matches(t2)
        assert not c.matches(t3)

        c = Choice(Sequence(p1), Sequence(p2))
        assert c.matches(t1)
        assert c.matches(t2)
        assert not c.matches(t3)

    def test_nested(self):
        "Choice with nested Sequence"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2)
        s1 = Sequence(p1, p1)
        s2 = Sequence(p2, p2)
        t0 = (0, 0, 0, 0)
        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)

        ch = Choice(s1, s2)
        expected = (
            'No match for (0, 0, 0, 0) in Choice(Sequence(p1, p1), Sequence(p2, p2))'
        )
        with pytest.raises(ParseError, match=re.escape(expected)):
            ch.nextProd(t0)
        assert s1 == ch.nextProd(t1)
        with pytest.raises(Exhausted, match='Extra token'):
            ch.nextProd(t1)

        ch = Choice(s1, s2)
        assert s2 == ch.nextProd(t2)
        with pytest.raises(Exhausted, match='Extra token'):
            ch.nextProd(t1)

    def test_reset(self):
        "Choice.reset()"
        p1 = Prod('p1', lambda t, v: t == 1)
        p2 = Prod('p2', lambda t, v: t == 2)
        t1 = (1, 0, 0, 0)
        t2 = (2, 0, 0, 0)

        ch = Choice(p1, p2)
        assert p1 == ch.nextProd(t1)
        with pytest.raises(Exhausted):
            ch.nextProd(t1)
        ch.reset()
        assert p2 == ch.nextProd(t2)


class TestProdParser:
    def setup_method(self):
        pass

    def test_parse_keepS(self):
        "ProdParser.parse(keepS)"
        p = ProdParser()

        # text, name, productions, store=None
        def prods():
            return Sequence(PreDef.char(';', ';'), PreDef.char(':', ':'))

        w, seq, store, unused = p.parse('; :', 'test', prods(), keepS=True)
        assert w
        assert 3 == len(seq)

        w, seq, store, unused = p.parse('; :', 'test', prods(), keepS=False)
        assert w
        assert 2 == len(seq)

    def test_combi(self):
        "ProdParser.parse() 2"
        p1 = Prod('p1', lambda t, v: v == '1')
        p2 = Prod('p2', lambda t, v: v == '2')
        p3 = Prod('p3', lambda t, v: v == '3')

        tests = {
            '1 2': True,
            '1 2 1 2': True,
            '3': True,
            # '': 'No match in Choice(Sequence(p1, p2), p3)',
            '1': 'Missing token for production p2',
            '1 2 1': 'Missing token for production p2',
            '1 2 1 2 x': "No match: ('IDENT', 'x', 1, 9)",
            '1 2 1 2 1': "No match: ('NUMBER', '1', 1, 9)",
            '3 x': "No match: ('IDENT', 'x', 1, 3)",
            '3 3': "No match: ('NUMBER', '3', 1, 3)",
        }
        for text, exp in list(tests.items()):
            prods = Choice(Sequence(p1, p2, minmax=lambda: (1, 2)), p3)
            if exp is True:
                wellformed, seq, store, unused = ProdParser().parse(text, 'T', prods)
                assert wellformed == exp
            else:
                with pytest.raises(xml.dom.SyntaxErr, match=re.escape(f'T: {exp}')):
                    ProdParser().parse(
                        text,
                        'T',
                        prods,
                    )

        tests = {
            '1 3': True,
            '1 1 3': True,
            '2 3': True,
            '1': 'Missing token for production p3',
            '1 1': 'Missing token for production p3',
            '1 3 3': "No match: ('NUMBER', '3', 1, 5)",
            '1 1 3 3': "No match: ('NUMBER', '3', 1, 7)",
            '2 3 3': "No match: ('NUMBER', '3', 1, 5)",
            '2': 'Missing token for production p3',
            '3': "Missing token for production Choice(Sequence(p1), p2): "
            "('NUMBER', '3', 1, 1)",
        }
        for text, exp in list(tests.items()):
            prods = Sequence(Choice(Sequence(p1, minmax=lambda: (1, 2)), p2), p3)
            if exp is True:
                wellformed, seq, store, unused = ProdParser().parse(text, 'T', prods)
                assert wellformed == exp
            else:
                with pytest.raises(xml.dom.SyntaxErr, match=re.escape(f'T: {exp}')):
                    ProdParser().parse(
                        text,
                        'T',
                        prods,
                    )

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_prodparser.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.property._Property."""

import copy
import cssutils
from cssutils.css.property import Property

debug = False


class TestProperties:
    def setup_method(self):
        "init test values"
        V = {
            '0': ('0', '-0'),  # , '+0'),
            'NUMBER': ('0', '-0', '100.1', '-100.1'),  # , '+0', '+100.1'),
            'PERCENTAGE': ('0%', '-0%', '100.1%', '-100.1%'),  # , '+0%', '+100.1%'),
            'EM': '1.2em',
            'EX': '1.2ex',
            'PX': '1.2px',
            'CM': '1.2cm',
            'MM': '1.2mm',
            'IN': '1.2in',
            'PT': '1.2pt',
            'PC': '1.2pc',
            'ANGLES': ('1deg', '1rad', '1grad'),
            'TIMES': ('1s', '1ms'),
            'FREQUENCIES': ('1hz', '1khz'),
            'DIMENSION': ('1dimension', '1_dimension', '1dimension2'),
            'STRING': ('"string"', "'STRING'"),
            'URI': ('url(x)', 'URL("x")', "url(')')"),
            'IDENT': ('ident', 'IDENT', '_IDENT', '_2', 'i-2'),
            # 'AUTO': 'auto', # explicit in list as an IDENT too
            # 'INHERIT': 'inherit', # explicit in list as an IDENT too
            'ATTR': ('attr(x)'),
            'RECT': ('rect(1,2,3,4)'),
            # ?
            'CLIP': ('rect(1,2,3,4)'),
            'FUNCTION': (),
            'HEX3': '#123',
            'HEX6': '#123abc',
            'RGB': 'rgb(1,2,3)',
            'RGB100': 'rgb(1%,2%,100%)',
            'RGBA': 'rgba(1,2,3, 1)',
            'RGBA100': 'rgba(1%,2%,100%, 0)',
            'HSL': 'hsl(1,2%,3%)',
            'HSLA': 'hsla(1,2%,3%, 1.0)',
        }

        def expanded(*keys):
            r = []
            for k in keys:
                if isinstance(V[k], str):
                    r.append(V[k])
                else:
                    r.extend(list(V[k]))
            return r

        # before adding combined
        self.V = V
        self.ALL = list(self._valuesofkeys(list(V.keys())))

        # combined values, only keys of V may be used!
        self.V['LENGTHS'] = expanded(
            '0', 'EM', 'EX', 'PX', 'CM', 'MM', 'IN', 'PT', 'PC'
        )
        self.V['COLORS'] = expanded('HEX3', 'HEX6', 'RGB', 'RGB100')
        self.V['COLORS3'] = expanded('RGBA', 'RGBA100', 'HSL', 'HSLA')

    def _allvalues(self):
        "Return list of **all** possible values as simple list"
        return copy.copy(self.ALL)

    def _valuesofkeys(self, keys):
        "Generate all distinct values in given keys of self.V"
        done = []
        for key in keys:
            if isinstance(key, list):
                # not a key but a list of values, return directly
                for v in key:
                    yield v
            else:
                v = self.V[key]
                if isinstance(v, str):
                    # single value
                    if v not in done:
                        done.append(v)
                        yield v
                else:
                    # a list of values
                    for value in v:
                        if value not in done:
                            done.append(value)
                            yield value

    def _check(self, name, keys):
        """
        Check each value in values if for property name p.name==exp.
        """
        notvalid = self._allvalues()

        for value in self._valuesofkeys(keys):
            if name == debug:
                print('+True?', Property(name, value).valid, value)
            assert Property(name, value).valid
            if value in notvalid:
                notvalid.remove(value)
        for value in notvalid:
            if name == debug:
                print('-False?', Property(name, value).valid, value)
            assert not Property(name, value).valid

    def test_properties(self):
        "properties"
        tests = {
            # propname: key or [list of values]
            'color': ('COLORS', 'COLORS3', ['inherit', 'red']),
            'fit': (['fill', 'hidden', 'meet', 'slice'],),
            'fit-position': (
                'LENGTHS',
                'PERCENTAGE',
                ['auto', 'top left', '0% 50%', '1cm 5em', 'bottom'],
            ),
            'font-family': (
                'STRING',
                'IDENT',
                [
                    'a, b',
                    '"a", "b"',
                    'a, "b"',
                    '"a", b',
                    r'a\{b',
                    r'a\ b',
                    'a b' 'a b, c  d  , e',
                ],
            ),
            # 'src': ('STRING',),
            'font-weight': (
                [
                    'normal',
                    'bold',
                    'bolder',
                    'lighter',
                    'inherit',
                    '100',
                    '200',
                    '300',
                    '400',
                    '500',
                    '600',
                    '700',
                    '800',
                    '900',
                ],
            ),
            'font-stretch': (
                [
                    'normal',
                    'wider',
                    'narrower',
                    'ultra-condensed',
                    'extra-condensed',
                    'condensed',
                    'semi-condensed',
                    'semi-expanded',
                    'expanded',
                    'extra-expanded',
                    'ultra-expanded',
                    'inherit',
                ],
            ),
            'font-style': (['normal', 'italic', 'oblique', 'inherit'],),
            'font-variant': (['normal', 'small-caps', 'inherit'],),
            'font-size': (
                'LENGTHS',
                'PERCENTAGE',
                [
                    'xx-small',
                    'x-small',
                    'small',
                    'medium',
                    'large',
                    'x-large',
                    'xx-large',
                    'larger',
                    'smaller',
                    '1em',
                    '1%',
                    'inherit',
                ],
            ),
            'font-size-adjust': ('NUMBER', ['none', 'inherit']),
            # 'font': (['italic small-caps bold 1px/3 a, "b", serif',
            #           'caption', 'icon', 'menu', 'message-box', 'small-caption',
            #           'status-bar', 'inherit'],),
            'image-orientation': ('0', 'ANGLES', ['auto']),
            'left': ('LENGTHS', 'PERCENTAGE', ['inherit', 'auto']),
            'opacity': ('NUMBER', ['inherit']),
            'orphans': ('0', ['1', '99999', 'inherit']),
            'page': ('IDENT',),
            'page-break-inside': (['auto', 'inherit', 'avoid'],),
            'size': (
                'LENGTHS',
                ['auto', '1em 1em', 'a4 portrait', 'b4 landscape', 'A5 PORTRAIT'],
            ),
            'widows': ('0', ['1', '99999', 'inherit']),
        }
        for name, keys in list(tests.items()):
            # keep track of valid keys
            self._check(name, keys)

    def test_validate(self):
        "Property.validate() and Property.valid"
        tests = {
            # (default L2, no default, no profile, L2, Color L3)
            'red': (True, True, True, True, True),
            'rgba(1,2,3,1)': (False, True, True, False, True),
            '1': (False, False, False, False, False),
        }
        for v, rs in list(tests.items()):
            p = Property('color', v)

            # TODO: Fix
            #            cssutils.profile.defaultProfiles = \
            #                cssutils.profile.CSS_LEVEL_2
            #            self.assertEqual(rs[0], p.valid)

            cssutils.profile.defaultProfiles = None
            assert rs[1] == p.valid

            assert rs[2] == p.validate()


#            self.assertEqual(rs[3], p.validate(
#                profiles=cssutils.profile.CSS_LEVEL_2))
#            self.assertEqual(rs[4], p.validate(
#                cssutils.profile.CSS3_COLOR))


if __name__ == '__main__':
    debug = 'font-family'
    import logging
    import unittest

    cssutils.log.setLevel(logging.FATAL)
    # debug = True
    unittest.main()

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_properties.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.property._Property."""

import re
import xml.dom

import pytest
from jaraco.test import property_error

from . import basetest
import cssutils


class TestProperty(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.css.property.Property('top', '1px')  # , 'important')

    def test_init(self):
        "Property.__init__()"
        p = cssutils.css.property.Property('top', '1px')
        assert 'top: 1px' == p.cssText
        assert 'top' == p.literalname
        assert 'top' == p.name
        assert '1px' == p.value
        # self.assertEqual('1px', p.cssValue.cssText)
        assert '1px' == p.propertyValue.cssText
        assert '' == p.priority
        assert p.valid
        assert p.wellformed

        assert ['top'] == p.seqs[0]
        assert isinstance(cssutils.css.PropertyValue(cssText="2px"), type(p.seqs[1]))
        assert [] == p.seqs[2]

        assert p.valid

        # Prop of MediaQuery
        p = cssutils.css.property.Property('top', _mediaQuery=True)
        assert 'top' == p.cssText
        assert 'top' == p.literalname
        assert 'top' == p.name
        assert '' == p.value
        # self.assertEqual('', p.cssValue.cssText)
        assert '' == p.propertyValue.cssText
        assert '' == p.priority
        assert not p.valid
        # p.cssValue.cssText = '1px'
        p.propertyValue.cssText = '1px'
        assert 'top: 1px' == p.cssText
        # p.cssValue = ''
        p.propertyValue = ''
        assert 'top' == p.cssText

        with pytest.raises(xml.dom.SyntaxErr):
            cssutils.css.property.Property('top', '')
        with pytest.raises(xml.dom.SyntaxErr):
            cssutils.css.property.Property('top')
        p = cssutils.css.property.Property('top', '0')
        assert '0' == p.value
        assert p.wellformed
        with pytest.raises(xml.dom.SyntaxErr):
            p._setValue('')
        assert '0' == p.value
        assert p.wellformed

    #        self.assertEqual(True, p.valid)

    #    def test_valid(self):
    #        "Property.valid"
    #        # context property must be set
    #        tests = [
    #            ('color', r'INHe\rIT', True),
    #            ('color', '1', False),
    #            ('color', 'red', True),
    #            ('left', '1', False),
    #            ('left', '1px', True),
    #            ('font', 'normal 1em/1.5 serif', True),
    #            ('background', 'url(x.gif) 1 0', False)
    #            ]
    #        for n, v, exp in tests:
    #            v = cssutils.css.CSSValue(cssText=v)
    #            self.assertTrue(v.wellformed, True)

    def test_cssText(self):
        "Property.cssText"
        p = cssutils.css.property.Property()

        tests = {
            'a: 1': None,
            'a: 1px 2px': None,
            'a: 1 !important': None,
            'a: 1 !IMPORTANT': 'a: 1 !important',
            'a: 1 !impor\\tant': 'a: 1 !important',
            # TODO: important with unicode escapes!
            'font: normal 1em/1.5 serif': None,
            'font: normal 1em/serif': None,
        }
        self.do_equal_r(tests)

        tests = {
            '': (xml.dom.SyntaxErr, 'Property: No property name found: '),
            ':': (xml.dom.SyntaxErr, 'Property: No property name found: : [1:1: :]'),
            'a': (xml.dom.SyntaxErr, 'Property: No ":" after name found: a [1:1: a]'),
            'b !': (
                xml.dom.SyntaxErr,
                'Property: No ":" after name found: b ! [1:3: !]',
            ),
            '/**/x': (
                xml.dom.SyntaxErr,
                'Property: No ":" after name found: /**/x [1:5: x]',
            ),
            'c:': (xml.dom.SyntaxErr, "Property: No property value found: c: [1:2: :]"),
            'd: ': (xml.dom.SyntaxErr, "No content to parse."),
            'e:!important': (xml.dom.SyntaxErr, "No content to parse."),
            'f: 1!': (xml.dom.SyntaxErr, 'Property: Invalid priority: !'),
            'g: 1!importantX': (
                xml.dom.SyntaxErr,
                "Property: No CSS priority value: importantx",
            ),
            # TODO?
            # u'a: 1;': (xml.dom.SyntaxErr,
            #       u'''CSSValue: No match: ('CHAR', u';', 1, 5)''')
        }
        for test in tests:
            ecp, msg = tests[test]
            with pytest.raises(ecp, match=re.escape(msg)):
                p._setCssText(test)

    def test_name(self):
        "Property.name"
        p = cssutils.css.property.Property('top', '1px')
        p.name = 'left'
        assert 'left' == p.name

        tests = {
            'top': None,
            ' top': 'top',
            'top ': 'top',
            ' top ': 'top',
            '/*x*/ top ': 'top',
            ' top /*x*/': 'top',
            '/*x*/top/*x*/': 'top',
            '\\x': 'x',
            'a\\010': 'a\x10',
            'a\\01': 'a\x01',
        }
        self.do_equal_r(tests, att='name')

        tests = {
            '': xml.dom.SyntaxErr,
            ' ': xml.dom.SyntaxErr,
            '"\n': xml.dom.SyntaxErr,
            '/*x*/': xml.dom.SyntaxErr,
            ':': xml.dom.SyntaxErr,
            ';': xml.dom.SyntaxErr,
            'top:': xml.dom.SyntaxErr,
            'top;': xml.dom.SyntaxErr,
            'color: #xyz': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests, att='_setName')

        p = cssutils.css.property.Property(r'c\olor', 'red')
        assert r'c\olor' == p.literalname
        assert 'color' == p.name

    def test_literalname(self):
        "Property.literalname"
        p = cssutils.css.property.Property(r'c\olor', 'red')
        assert r'c\olor' == p.literalname
        with pytest.raises(
            AttributeError, match=property_error("Property.literalname")
        ):
            p.literalname = 'color'

    def test_validate(self):
        "Property.valid"
        p = cssutils.css.property.Property('left', '1px', '')

        assert p.valid

        p.name = 'color'
        assert p.valid is False

        p.name = 'top'
        assert p.valid

        p.value = 'red'
        assert p.valid is False

    # def test_cssValue(self):
    #    "Property.cssValue"
    #    pass
    #    # DEPRECATED

    def test_priority(self):
        "Property.priority"
        p = cssutils.css.property.Property('top', '1px', 'important')

        for prio in (None, ''):
            p.priority = prio
            assert '' == p.priority
            assert '' == p.literalpriority

        for prio in (
            '!important',
            '! important',
            '!/* x */ important',
            '!/* x */ important /**/',
            'important',
            'IMPORTANT',
            r'im\portant',
        ):
            p.priority = prio
            assert 'important' == p.priority
            if prio.startswith('!'):
                prio = prio[1:].strip()
            if '/*' in prio:
                check = 'important'
            else:
                check = prio
            assert check == p.literalpriority

        tests = {
            ' ': xml.dom.SyntaxErr,
            '"\n': xml.dom.SyntaxErr,
            # u'important': xml.dom.SyntaxErr,
            ';': xml.dom.SyntaxErr,
            '!important !important': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests, att='_setPriority')

    def test_value(self):
        "Property.value"
        p = cssutils.css.property.Property('top', '1px')
        assert '1px' == p.value
        p.value = '2px'
        assert '2px' == p.value

        tests = {
            '1px': None,
            ' 2px': '2px',
            '3px ': '3px',
            ' 4px ': '4px',
            '5px 1px': '5px 1px',
        }
        self.do_equal_r(tests, att='value')

        tests = {
            # no value
            None: xml.dom.SyntaxErr,
            '': xml.dom.SyntaxErr,
            ' ': xml.dom.SyntaxErr,
            '"\n': xml.dom.SyntaxErr,
            '/*x*/': xml.dom.SyntaxErr,
            # not allowed:
            ':': xml.dom.SyntaxErr,
            ';': xml.dom.SyntaxErr,
            '!important': xml.dom.SyntaxErr,
        }
        self.do_raise_r(tests, att='_setValue')

    def test_reprANDstr(self):
        "Property.__repr__(), .__str__()"
        name = "color"
        value = "red"
        priority = "important"

        s = cssutils.css.property.Property(name=name, value=value, priority=priority)

        assert name in str(s)
        assert value in str(s)
        assert priority in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert name == s2.name
        assert value == s2.value
        assert priority == s2.priority

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_property.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.selector.Selector.

what should happen here?
    - star 7 hack::
        x*
        does not validate but works in IE>5 and FF, does it???

"""
import xml.dom

import pytest
from jaraco.test import property_error

from . import basetest
import cssutils


class TestSelector(basetest.BaseTestCase):
    def setup_method(self):
        self.r = cssutils.css.Selector('*')

    def test_init(self):
        "Selector.__init__()"
        s = cssutils.css.Selector('*')
        assert (None, '*') == s.element
        assert {} == s._namespaces.namespaces
        assert s.parent is None
        assert '*' == s.selectorText
        assert (0, 0, 0, 0) == s.specificity
        assert s.wellformed

        s = cssutils.css.Selector(('p|b', {'p': 'URI'}))
        assert ('URI', 'b') == s.element
        assert {'p': 'URI'} == s._namespaces.namespaces
        assert s.parent is None
        assert 'p|b' == s.selectorText
        assert (0, 0, 0, 1) == s.specificity
        assert s.wellformed

        with pytest.raises(xml.dom.NamespaceErr):
            cssutils.css.Selector('p|b')

    def test_element(self):
        "Selector.element (TODO: RESOLVE)"
        tests = {
            '*': (None, '*'),
            'x': (None, 'x'),
            '\\x': (None, '\\x'),
            '|x': ('', 'x'),
            '*|x': (cssutils._ANYNS, 'x'),
            'ex|x': ('example', 'x'),
            'a x': (None, 'x'),
            'a+x': (None, 'x'),
            'a>x': (None, 'x'),
            'a~x': (None, 'x'),
            'a+b~c x': (None, 'x'),
            'x[href]': (None, 'x'),
            'x[href="123"]': (None, 'x'),
            'x:hover': (None, 'x'),
            'x:first-letter': (None, 'x'),  # TODO: Really?
            'x::first-line': (None, 'x'),  # TODO: Really?
            'x:not(href)': (None, 'x'),  # TODO: Really?
            '#id': None,
            '.c': None,
            'x#id': (None, 'x'),
            'x.c': (None, 'x'),
        }
        for test, ele in list(tests.items()):
            s = cssutils.css.Selector((test, {'ex': 'example'}))
            assert ele == s.element

    def test_namespaces(self):
        "Selector.namespaces"
        namespaces = [
            {'p': 'other'},  # no default
            {'': 'default', 'p': 'other'},  # with default
            {'': 'default', 'p': 'default'},  # same default
        ]
        tests = {
            # selector: with default, no default, same default
            '*': ('*', '*', '*'),
            'x': ('x', 'x', 'x'),
            '|*': ('|*', '|*', '|*'),
            '|x': ('|x', '|x', '|x'),
            '*|*': ('*|*', '*|*', '*|*'),
            '*|x': ('*|x', '*|x', '*|x'),
            'p|*': ('p|*', 'p|*', '*'),
            'p|x': ('p|x', 'p|x', 'x'),
            'x[a][|a][*|a][p|a]': (
                'x[a][a][*|a][p|a]',
                'x[a][a][*|a][p|a]',
                'x[a][a][*|a][a]',
            ),
        }
        for sel, exp in list(tests.items()):
            for i, result in enumerate(exp):
                s = cssutils.css.Selector((sel, namespaces[i]))
                assert result == s.selectorText

        # add to CSSStyleSheet
        sheet = cssutils.css.CSSStyleSheet()
        sheet.cssText = '@namespace p "u"; a { color: green }'

        r = sheet.cssRules[1]

        assert r.selectorText == 'a'

        # add default namespace
        sheet.namespaces[''] = 'a'
        assert r.selectorText == '|a'

        del sheet.namespaces['']
        assert r.selectorText == 'a'

    #        r.selectorList.append('a')
    #        self.assertEqual(r.selectorText, u'|a, a')
    #        r.selectorList.append('*|a')
    #        self.assertEqual(r.selectorText, u'|a, a, *|a')

    def test_default_namespace(self):
        "Selector.namespaces default"
        css = '''@namespace "default";
                a[att] { color:green; }
        '''
        sheet = cssutils.css.CSSStyleSheet()
        sheet.cssText = css
        assert (
            sheet.cssText == b'@namespace "default";\na[att] {\n    color: green\n    }'
        )
        # use a prefix for default namespace, does not goes for atts!
        sheet.namespaces['p'] = 'default'
        assert (
            sheet.cssText
            == b'@namespace p "default";\np|a[att] {\n    color: green\n    }'
        )

    def test_parent(self):
        "Selector.parent"
        sl = cssutils.css.SelectorList('a, b')
        for sel in sl:
            assert sl == sel.parent

        newsel = cssutils.css.Selector('x')
        sl.append(newsel)
        assert sl == newsel.parent

        newsel = cssutils.css.Selector('y')
        sl.appendSelector(newsel)
        assert sl == newsel.parent

    def test_selectorText(self):
        "Selector.selectorText"
        tests = {
            # combinators
            'a+b>c~e f': 'a + b > c ~ e f',
            'a  +  b  >  c  ~  e   f': 'a + b > c ~ e f',
            'a+b': 'a + b',
            'a  +  b': 'a + b',
            'a\n  +\t  b': 'a + b',
            'a~b': 'a ~ b',
            'a b': None,
            'a   b': 'a b',
            'a\nb': 'a b',
            'a\tb': 'a b',
            'a   #b': 'a #b',
            'a   .b': 'a .b',
            'a * b': None,
            # >
            'a>b': 'a > b',
            'a> b': 'a > b',
            'a >b': 'a > b',
            'a > b': 'a > b',
            'a+ b': 'a + b',
            'a +b': 'a + b',
            'a + b': 'a + b',
            'a~ b': 'a ~ b',
            'a ~b': 'a ~ b',
            'a ~ b': 'a ~ b',
            # type selector
            'a': None,
            'h1-a_x__--': None,
            'a-a': None,
            'a_a': None,
            '-a': None,
            '_': None,
            '-_': None,
            r'-\72': '-r',
            # ur'\25': u'%', # TODO: should be escaped!
            '.a a': None,
            'a1': None,
            'a1-1': None,
            '.a1-1': None,
            # universal
            '*': None,
            '*/*x*/': None,
            '* /*x*/': None,
            '*:hover': None,
            '* :hover': None,
            '*:lang(fr)': None,
            '* :lang(fr)': None,
            '*::first-line': None,
            '* ::first-line': None,
            '*[lang=fr]': None,
            '[lang=fr]': None,
            # HASH
            '''#a''': None,
            '''#a1''': None,
            '''#1a''': None,  # valid to grammar but not for HTML
            '''#1''': None,  # valid to grammar but not for HTML
            '''a#b''': None,
            '''a #b''': None,
            '''a#b.c''': None,
            '''a.c#b''': None,
            '''a #b.c''': None,
            '''a .c#b''': None,
            # class
            'ab': 'ab',
            'a.b': None,
            'a.b.c': None,
            '.a1._1': None,
            # attrib
            '''[x]''': None,
            '''*[x]''': None,
            '''a[x]''': None,
            '''a[ x]''': 'a[x]',
            '''a[x ]''': 'a[x]',
            '''a [x]''': 'a [x]',
            '''* [x]''': None,  # is really * *[x]
            '''a[x="1"]''': None,
            '''a[x ="1"]''': 'a[x="1"]',
            '''a[x= "1"]''': 'a[x="1"]',
            '''a[x = "1"]''': 'a[x="1"]',
            '''a[ x = "1"]''': 'a[x="1"]',
            '''a[x = "1" ]''': 'a[x="1"]',
            '''a[ x = "1" ]''': 'a[x="1"]',
            '''a [ x = "1" ]''': 'a [x="1"]',
            '''a[x~=a1]''': None,
            '''a[x ~=a1]''': 'a[x~=a1]',
            '''a[x~= a1]''': 'a[x~=a1]',
            '''a[x ~= a1]''': 'a[x~=a1]',
            '''a[ x ~= a1]''': 'a[x~=a1]',
            '''a[x ~= a1 ]''': 'a[x~=a1]',
            '''a[ x ~= a1 ]''': 'a[x~=a1]',
            '''a [ x ~= a1 ]''': 'a [x~=a1]',  # same as next!
            '''a *[ x ~= a1 ]''': 'a *[x~=a1]',
            '''a[x|=en]''': None,
            '''a[x|= en]''': 'a[x|=en]',
            '''a[x |=en]''': 'a[x|=en]',
            '''a[x |= en]''': 'a[x|=en]',
            '''a[ x |= en]''': 'a[x|=en]',
            '''a[x |= en ]''': 'a[x|=en]',
            '''a [ x |= en]''': 'a [x|=en]',
            # CSS3
            '''a[x^=en]''': None,
            '''a[x$=en]''': None,
            '''a[x*=en]''': None,
            '''a[/*1*/x/*2*/]''': None,
            '''a[/*1*/x/*2*/=/*3*/a/*4*/]''': None,
            '''a[/*1*/x/*2*/~=/*3*/a/*4*/]''': None,
            '''a[/*1*/x/*2*/|=/*3*/a/*4*/]''': None,
            # pseudo-elements
            'a x:first-line': None,
            'a x:first-letter': None,
            'a x:before': None,
            'a x:after': None,
            'a x::selection': None,
            'a:hover+b:hover>c:hover~e:hover f:hover': 'a:hover + b:hover > c:hover ~ e:hover f:hover',
            'a:hover  +  b:hover  >  c:hover  ~  e:hover   f:hover': 'a:hover + b:hover > c:hover ~ e:hover f:hover',
            'a::selection+b::selection>c::selection~e::selection f::selection': 'a::selection + b::selection > c::selection ~ e::selection f::selection',
            'a::selection  +  b::selection  >  c::selection  ~  e::selection   '
            'f::selection': 'a::selection + b::selection > c::selection ~ e::selection f::selection',
            'x:lang(de) y': None,
            'x:nth-child(odd) y': None,
            # functional pseudo
            'x:func(a + b-2px22.3"s"i)': None,
            'x:func(1 + 1)': None,
            'x:func(1+1)': 'x:func(1+1)',
            'x:func(1   +   1)': 'x:func(1 + 1)',
            'x:func(1-1)': 'x:func(1-1)',
            'x:func(1  -  1)': 'x:func(1 -1)',  # TODO: FIX!
            'x:func(a-1)': 'x:func(a-1)',
            'x:func(a -1px)': 'x:func(a -1px)',
            'x:func(1px)': None,
            'x:func(23.4)': None,
            'x:func("s")': None,
            'x:func(i)': None,
            # negation
            ':not(y)': None,
            ':not(   y  \t\n)': ':not(y)',
            '*:not(y)': None,
            'x:not(y)': None,
            '.x:not(y)': None,
            ':not(*)': None,
            ':not(#a)': None,
            ':not(.a)': None,
            ':not([a])': None,
            ':not(:first-letter)': None,
            ':not(::first-letter)': None,
            # escapes
            r'\74\72 td': 'trtd',
            r'\74\72  td': 'tr td',
            r'\74\000072 td': 'trtd',
            r'\74\000072  td': 'tr td',
            # comments
            'a/**/ b': None,
            'a /**/b': None,
            'a /**/ b': None,
            'a  /**/ b': 'a /**/ b',
            'a /**/  b': 'a /**/ b',
            # namespaces
            '|e': None,
            '*|e': None,
            '*|*': None,
            ('p|*', (('p', 'uri'),)): 'p|*',
            ('p|e', (('p', 'uri'),)): 'p|e',
            ('-a_x12|e', (('-a_x12', 'uri'),)): '-a_x12|e',
            ('*|b[p|a]', (('p', 'uri'),)): '*|b[p|a]',
            # case
            'elemenT.clasS#iD[atT="valuE"]:noT(x)::firsT-linE': 'elemenT.clasS#iD[atT="valuE"]:not(x)::first-line',
        }
        # do not parse as not complete
        self.do_equal_r(tests, att='selectorText')

        tests = {
            'x|a': xml.dom.NamespaceErr,
            ('p|*', (('x', 'uri'),)): xml.dom.NamespaceErr,
            '': xml.dom.SyntaxErr,
            '1': xml.dom.SyntaxErr,
            '-1': xml.dom.SyntaxErr,
            'a*b': xml.dom.SyntaxErr,
            'a *b': xml.dom.SyntaxErr,
            'a* b': xml.dom.SyntaxErr,
            'a/**/b': xml.dom.SyntaxErr,
            '#': xml.dom.SyntaxErr,
            '|': xml.dom.SyntaxErr,
            ':': xml.dom.SyntaxErr,
            '::': xml.dom.SyntaxErr,
            ': a': xml.dom.SyntaxErr,
            ':: a': xml.dom.SyntaxErr,
            ':a()': xml.dom.SyntaxErr,  # no value
            '::a()': xml.dom.SyntaxErr,  # no value
            ':::a': xml.dom.SyntaxErr,
            ':1': xml.dom.SyntaxErr,
            '#.x': xml.dom.SyntaxErr,
            '.': xml.dom.SyntaxErr,
            '.1': xml.dom.SyntaxErr,
            '.a.1': xml.dom.SyntaxErr,
            '[a': xml.dom.SyntaxErr,
            'a]': xml.dom.SyntaxErr,
            '[a b]': xml.dom.SyntaxErr,
            '[=b]': xml.dom.SyntaxErr,
            '[a=]': xml.dom.SyntaxErr,
            '[a|=]': xml.dom.SyntaxErr,
            '[a~=]': xml.dom.SyntaxErr,
            '[a=1]': xml.dom.SyntaxErr,
            'a +': xml.dom.SyntaxErr,
            'a >': xml.dom.SyntaxErr,
            'a ++ b': xml.dom.SyntaxErr,
            'a + > b': xml.dom.SyntaxErr,
            # functional pseudo
            '*:lang(': xml.dom.SyntaxErr,
            '*:lang()': xml.dom.SyntaxErr,  # no arg
            # negation
            'not(x)': xml.dom.SyntaxErr,  # no valid function
            ':not()': xml.dom.SyntaxErr,  # no arg
            ':not(x': xml.dom.SyntaxErr,  # no )
            ':not(-': xml.dom.SyntaxErr,  # not allowed
            ':not(+': xml.dom.SyntaxErr,  # not allowed
            # only one selector!
            ',': xml.dom.InvalidModificationErr,
            ',a': xml.dom.InvalidModificationErr,
            'a,': xml.dom.InvalidModificationErr,
            # @
            'p @here': xml.dom.SyntaxErr,  # not allowed
        }
        # only set as not complete
        self.do_raise_r(tests, att='_setSelectorText')

    def test_specificity(self):
        "Selector.specificity"
        selector = cssutils.css.Selector()

        # property is read-only
        with pytest.raises(
            AttributeError, match=property_error('Selector.specificity')
        ):
            selector.specificity = 1

        tests = {
            '*': (0, 0, 0, 0),
            'li': (0, 0, 0, 1),
            'li:first-line': (0, 0, 0, 2),
            'ul li': (0, 0, 0, 2),
            'ul ol+li': (0, 0, 0, 3),
            'h1 + *[rel=up]': (0, 0, 1, 1),
            'ul ol li.red': (0, 0, 1, 3),
            'li.red.level': (0, 0, 2, 1),
            '#x34y': (0, 1, 0, 0),
            'UL OL LI.red': (0, 0, 1, 3),
            'LI.red.level': (0, 0, 2, 1),
            '#s12:not(FOO)': (0, 1, 0, 1),
            'button:not([DISABLED])': (0, 0, 1, 1),  # ?
            '*:not(FOO)': (0, 0, 0, 1),
            # elements
            'a+b': (0, 0, 0, 2),
            'a>b': (0, 0, 0, 2),
            'a b': (0, 0, 0, 2),
            '* a': (0, 0, 0, 1),
            'a *': (0, 0, 0, 1),
            'a * b': (0, 0, 0, 2),
            'a:hover': (0, 0, 0, 1),
            'a:first-line': (0, 0, 0, 2),
            'a:first-letter': (0, 0, 0, 2),
            'a:before': (0, 0, 0, 2),
            'a:after': (0, 0, 0, 2),
            # classes and attributes
            '.a': (0, 0, 1, 0),
            '*.a': (0, 0, 1, 0),
            'a.a': (0, 0, 1, 1),
            '.a.a': (0, 0, 2, 0),  # IE<7 False (0,0,1,0)
            'a.a.a': (0, 0, 2, 1),
            '.a.b': (0, 0, 2, 0),
            'a.a.b': (0, 0, 2, 1),
            '.a .a': (0, 0, 2, 0),
            '*[x]': (0, 0, 1, 0),
            '*[x=a]': (0, 0, 1, 0),
            '*[x~=a]': (0, 0, 1, 0),
            '*[x|=a]': (0, 0, 1, 0),
            '*[x^=a]': (0, 0, 1, 0),
            '*[x*=a]': (0, 0, 1, 0),
            '*[x$=a]': (0, 0, 1, 0),
            '*[x][y]': (0, 0, 2, 0),
            # ids
            '#a': (0, 1, 0, 0),
            '*#a': (0, 1, 0, 0),
            'x#a': (0, 1, 0, 1),
            '.x#a': (0, 1, 1, 0),
            'a.x#a': (0, 1, 1, 1),
            '#a#a': (0, 2, 0, 0),  # e.g. html:id + xml:id
            '#a#b': (0, 2, 0, 0),
            '#a #b': (0, 2, 0, 0),
        }
        for text in tests:
            selector.selectorText = text
            assert tests[text] == selector.specificity

    def test_reprANDstr(self):
        "Selector.__repr__(), .__str__()"
        sel = 'a + b'

        s = cssutils.css.Selector(selectorText=sel)

        assert sel in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert sel == s2.selectorText

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_selector.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.selectorlist.SelectorList."""

import xml.dom
from . import basetest
import cssutils
from cssutils.css.selectorlist import SelectorList
import pytest


class TestSelectorList(basetest.BaseTestCase):
    def setup_method(self):
        self.r = SelectorList()

    def test_init(self):
        "SelectorList.__init__() and .length"
        s = SelectorList()
        assert 0 == s.length

        s = SelectorList('a, b')
        assert 2 == s.length
        assert 'a, b' == s.selectorText

        s = SelectorList(selectorText='a')
        assert 1 == s.length
        assert 'a' == s.selectorText

        s = SelectorList(selectorText=('p|a', {'p': 'uri'}))  # n-dict
        assert 1 == s.length
        assert 'p|a' == s.selectorText

        s = SelectorList(selectorText=('p|a', (('p', 'uri'),)))  # n-tuples
        assert 1 == s.length
        assert 'p|a' == s.selectorText

    def test_parentRule(self):
        "Selector.parentRule"

        def check(style):
            assert style == style.selectorList.parentRule
            for sel in style.selectorList:
                assert style.selectorList == sel.parent

        style = cssutils.css.CSSStyleRule('a, b')
        check(style)

        # add new selector
        style.selectorList.append(cssutils.css.Selector('x'))
        check(style)

        # replace selectorList
        style.selectorList = cssutils.css.SelectorList('x')
        check(style)

        # replace selectorText
        style.selectorText = 'x, y'
        check(style)

    def test_appendSelector(self):
        "SelectorList.appendSelector() and .length"
        s = SelectorList()
        s.appendSelector('a')
        assert 1 == s.length

        with pytest.raises(xml.dom.InvalidModificationErr):
            s.appendSelector('b,')
        assert 1 == s.length

        assert 'a' == s.selectorText

        s.append('b')
        assert 2 == s.length
        assert 'a, b' == s.selectorText

        s.append('a')
        assert 2 == s.length
        assert 'b, a' == s.selectorText

        # __setitem__
        with pytest.raises(IndexError):
            s.__setitem__(4, 'x')
        s[1] = 'c'
        assert 2 == s.length
        assert 'b, c' == s.selectorText
        # TODO: remove duplicates?
        #        s[0] = 'c'
        #        self.assertEqual(1, s.length)
        #        self.assertEqual(u'c', s.selectorText)

        s = SelectorList()
        s.appendSelector(('p|a', {'p': 'uri', 'x': 'xxx'}))
        assert 'p|a' == s.selectorText
        # x gets lost as not used
        with pytest.raises(xml.dom.NamespaceErr):
            s.append('x|a')
        # not set at all
        with pytest.raises(xml.dom.NamespaceErr):
            s.append('y|a')
        # but p is retained
        s.append('p|b')
        assert 'p|a, p|b' == s.selectorText

    def test_selectorText(self):
        "SelectorList.selectorText"
        s = SelectorList()
        s.selectorText = 'a, b'
        assert 'a, b' == s.selectorText
        with pytest.raises(xml.dom.SyntaxErr):
            s._setSelectorText(',')
        # not changed as invalid!
        assert 'a, b' == s.selectorText

        tests = {
            '*': None,
            '/*1*/*': None,
            '/*1*/*, a': None,
            'a, b': None,
            'a ,b': 'a, b',
            'a , b': 'a, b',
            'a, b, c': 'a, b, c',
            '#a, x#a, .b, x.b': '#a, x#a, .b, x.b',
            ('[p|a], p|*', (('p', 'uri'),)): '[p|a], p|*',
        }
        # do not parse as not complete
        self.do_equal_r(tests, att='selectorText')

        tests = {
            'x|*': xml.dom.NamespaceErr,
            '': xml.dom.SyntaxErr,
            ' ': xml.dom.SyntaxErr,
            ',': xml.dom.SyntaxErr,
            'a,': xml.dom.SyntaxErr,
            ',a': xml.dom.SyntaxErr,
            '/* 1 */,a': xml.dom.SyntaxErr,
        }
        # only set as not complete
        self.do_raise_r(tests, att='_setSelectorText')

    def test_reprANDstr(self):
        "SelectorList.__repr__(), .__str__()"
        sel = ('a, p|b', {'p': 'uri'})

        s = cssutils.css.SelectorList(selectorText=sel)

        assert sel[0] in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert sel[0] == s2.selectorText

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_selectorlist.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.CSSSerializer"""

from . import basetest
import cssutils


class TestPreferences(basetest.BaseTestCase):
    """
    testcases for cssutils.serialize.Preferences
    """

    def setup_method(self):
        cssutils.ser.prefs.useDefaults()

    def teardown_method(self):
        cssutils.ser.prefs.useDefaults()

    #    def testkeepUnkownAtRules(self):
    #        "Preferences.keepUnkownAtRules"
    #        from warnings import catch_warnings
    #        with catch_warnings(record=True) as log:
    #            x = cssutils.ser.prefs.keepUnkownAtRules
    #
    #        if log:
    #            # unpack the only member of log
    #            warning, = log
    #            self.assertEqual(warning.category, DeprecationWarning)

    def test_resolveVariables(self):
        "Preferences.resolveVariables"
        assert cssutils.ser.prefs.resolveVariables

        cssutils.ser.prefs.resolveVariables = False

        vars = '''
            @variables {
                c1: red;
                c2: #0f0;
                px: 1px 2px;
            }
        '''
        tests = {
            '''a {\n    color: var(c1)\n    }''': '''a {\n    color: red\n    }''',
            '''a {\n    color: var(c1)\n; color: var(  c2   )    }''': '''a {\n    color: red;\n    color: #0f0\n    }''',
            '''a {\n    margin: var(px)\n    }''': '''a {\n    margin: 1px 2px\n    }''',
            '''@media all {
                a {
                    margin: var(px) var(px);
                    color: var(c1);
                    left: var(unknown)
                    }
            }''': '''@media all {\n    a {\n        margin: 1px 2px 1px 2px;\n'''
            '''        color: red;\n        left: var(unknown)\n        }\n    }''',
        }
        cssutils.ser.prefs.resolveVariables = True

        for test, exp in list(tests.items()):
            s = cssutils.parseString(vars + test)
            assert exp.encode() == s.cssText

        cssutils.ser.prefs.resolveVariables = True

    def test_useDefaults(self):
        "Preferences.useDefaults()"
        cssutils.ser.prefs.useMinified()
        cssutils.ser.prefs.useDefaults()
        assert cssutils.ser.prefs.defaultAtKeyword
        assert cssutils.ser.prefs.defaultPropertyName
        assert cssutils.ser.prefs.defaultPropertyPriority
        assert cssutils.ser.prefs.importHrefFormat is None
        assert cssutils.ser.prefs.indent == 4 * ' '
        assert cssutils.ser.prefs.indentClosingBrace
        assert cssutils.ser.prefs.keepAllProperties
        assert cssutils.ser.prefs.keepComments
        assert cssutils.ser.prefs.keepEmptyRules is False
        assert cssutils.ser.prefs.keepUnknownAtRules
        assert cssutils.ser.prefs.keepUsedNamespaceRulesOnly is False
        assert cssutils.ser.prefs.lineNumbers is False
        assert cssutils.ser.prefs.lineSeparator == '\n'
        assert cssutils.ser.prefs.listItemSpacer == ' '
        assert cssutils.ser.prefs.minimizeColorHash
        assert cssutils.ser.prefs.omitLastSemicolon
        assert cssutils.ser.prefs.omitLeadingZero is False
        assert cssutils.ser.prefs.paranthesisSpacer == ' '
        assert cssutils.ser.prefs.propertyNameSpacer == ' '
        assert cssutils.ser.prefs.selectorCombinatorSpacer == ' '
        assert cssutils.ser.prefs.spacer == ' '
        assert cssutils.ser.prefs.validOnly is False
        css = '''
    /*1*/
    @import url(x) tv , print;
    @namespace prefix "uri";
    @namespace unused "unused";
    @media all {}
    @media all {
        a {}
    }
    @media   all  {
    a { color: red; }
        }
    @page     { left: 0; }
    a {}
    prefix|x, a  +  b  >  c  ~  d  ,  b { top : 1px ;
        font-family : arial ,'some'
        }
    '''
        parsedcss = '''/*1*/
@import url(x) tv, print;
@namespace prefix "uri";
@namespace unused "unused";
@media all {
    a {
        color: red
        }
    }
@page {
    left: 0
    }
prefix|x, a + b > c ~ d, b {
    top: 1px;
    font-family: arial, "some"
    }'''
        s = cssutils.parseString(css)
        assert s.cssText == parsedcss.encode()

        tests = {
            '0.1 .1 0.1px .1px 0.1% .1% +0.1 +.1 +0.1px +.1px +0.1% +.1% '
            '-0.1 -.1 -0.1px -.1px -0.1% -.1%': '0.1 0.1 0.1px 0.1px 0.1% 0.1% +0.1 +0.1 +0.1px +0.1px +0.1% '
            '+0.1% -0.1 -0.1 -0.1px -0.1px -0.1% -0.1%'
        }
        cssutils.ser.prefs.useDefaults()
        for test, exp in list(tests.items()):
            s = cssutils.parseString('a{x:%s}' % test)
            assert ('a {\n    x: %s\n    }' % exp).encode() == s.cssText

    def test_useMinified(self):
        "Preferences.useMinified()"
        cssutils.ser.prefs.useDefaults()
        cssutils.ser.prefs.useMinified()
        assert cssutils.ser.prefs.defaultAtKeyword
        assert cssutils.ser.prefs.defaultPropertyName
        assert cssutils.ser.prefs.importHrefFormat == 'string'
        assert cssutils.ser.prefs.indent == ''
        assert cssutils.ser.prefs.keepAllProperties
        assert cssutils.ser.prefs.keepComments is False
        assert cssutils.ser.prefs.keepEmptyRules is False
        assert cssutils.ser.prefs.keepUnknownAtRules is False
        assert cssutils.ser.prefs.keepUsedNamespaceRulesOnly
        assert cssutils.ser.prefs.lineNumbers is False
        assert cssutils.ser.prefs.lineSeparator == ''
        assert cssutils.ser.prefs.listItemSpacer == ''
        assert cssutils.ser.prefs.omitLastSemicolon
        assert cssutils.ser.prefs.omitLeadingZero
        assert cssutils.ser.prefs.paranthesisSpacer == ''
        assert cssutils.ser.prefs.propertyNameSpacer == ''
        assert cssutils.ser.prefs.selectorCombinatorSpacer == ''
        assert cssutils.ser.prefs.spacer == ''
        assert cssutils.ser.prefs.validOnly is False

        css = '''
    /*1*/
    @import   url(x) tv , print;
    @namespace   prefix "uri";
    @namespace   unused "unused";
    @media  all {}
    @media  all {
        a {}
    }
    @media all "name" {
        a { color: red; }
    }
    @page:left {
    left: 0
    }
    a {}
    prefix|x, a + b > c ~ d , b { top : 1px ;
        font-family : arial ,  'some'
        }
    @x  x;
    '''
        s = cssutils.parseString(css)
        cssutils.ser.prefs.keepUnknownAtRules = True
        assert (
            s.cssText
            == b'''@import"x"tv,print;@namespace prefix"uri";@media all"name"'''
            b'''{a{color:red}}@page :left{left:0}prefix|x,a+b>c~d,b{top:1px;'''
            b'''font-family:arial,"some"}@x x;'''
        )
        cssutils.ser.prefs.keepUnknownAtRules = False
        assert (
            s.cssText
            == b'''@import"x"tv,print;@namespace prefix"uri";@media all"name"'''
            b'''{a{color:red}}@page :left{left:0}prefix|x,a+b>c~d,b{top:1px;'''
            b'''font-family:arial,"some"}'''
        )
        # Values
        valuetests = {
            '  a  a1  a-1  a-1a  ': 'a a1 a-1 a-1a',
            'a b 1 c 1em d -1em e': 'a b 1 c 1em d -1em e',
            '  1em  /  5  ': '1em/5',
            '1em/5': '1em/5',
            'a 0 a .0 a 0.0 a -0 a -.0 a -0.0 a +0 a +.0 a +0.0': 'a 0 a 0 a 0 a 0 a 0 a 0 a 0 a 0 a 0',
            'a  0px  a  .0px  a  0.0px  a  -0px  a  -.0px  a  -0.0px  a  +0px  '
            'a  +.0px  a  +0.0px ': 'a 0 a 0 a 0 a 0 a 0 a 0 a 0 a 0 a 0',
            'a  1  a  .1  a  1.0  a  0.1  a  -1  a  -.1  a  -1.0  a  -0.1  a  '
            '+1  a  +.1  a  +1.0': 'a 1 a .1 a 1 a .1 a -1 a -.1 a -1 a -.1 a +1 a +.1 a +1',
            '  url(x)  f()': 'url(x) f()',
            '#112233': '#123',
            '#112234': '#112234',
            '#123': '#123',
            '#123 url() f()': '#123 url() f()',
            '1 +2 +3 -4': '1 +2 +3 -4',  # ?
            '0.1 .1 0.1px .1px 0.1% .1% +0.1 +.1 +0.1px +.1px +0.1% '
            '+.1% -0.1 -.1 -0.1px -.1px -0.1% -.1%': '.1 .1 .1px .1px .1% .1% +.1 +.1 +.1px +.1px +.1% +.1% '
            '-.1 -.1 -.1px -.1px -.1% -.1%',
        }
        for test, exp in list(valuetests.items()):
            s = cssutils.parseString('a{x:%s}' % test)
            assert ('a{x:%s}' % exp).encode() == s.cssText

    def test_defaultAtKeyword(self):
        "Preferences.defaultAtKeyword"
        s = cssutils.parseString('@im\\port "x";')
        assert b'@import "x";' == s.cssText
        cssutils.ser.prefs.defaultAtKeyword = True
        assert b'@import "x";' == s.cssText
        cssutils.ser.prefs.defaultAtKeyword = False
        assert b'@im\\port "x";' == s.cssText

    def test_defaultPropertyName(self):
        "Preferences.defaultPropertyName"
        cssutils.ser.prefs.keepAllProperties = False

        # does not actually work as once the name is set it is used also
        # if used with a backslash in it later...

        s = cssutils.parseString(r'a { c\olor: green; }')
        assert b'a {\n    color: green\n    }' == s.cssText
        cssutils.ser.prefs.defaultPropertyName = True
        assert b'a {\n    color: green\n    }' == s.cssText
        cssutils.ser.prefs.defaultPropertyName = False
        assert b'a {\n    c\\olor: green\n    }' == s.cssText

        s = cssutils.parseString(r'a { color: red; c\olor: green; }')
        assert b'a {\n    c\\olor: green\n    }' == s.cssText
        cssutils.ser.prefs.defaultPropertyName = False
        assert b'a {\n    c\\olor: green\n    }' == s.cssText
        cssutils.ser.prefs.defaultPropertyName = True
        assert b'a {\n    color: green\n    }' == s.cssText

    def test_defaultPropertyPriority(self):
        "Preferences.defaultPropertyPriority"
        css = 'a {\n    color: green !IM\\portant\n    }'
        s = cssutils.parseString(css)
        assert s.cssText == b'a {\n    color: green !important\n    }'
        cssutils.ser.prefs.defaultPropertyPriority = False
        assert s.cssText == css.encode()

    def test_importHrefFormat(self):
        "Preferences.importHrefFormat"
        r0 = cssutils.css.CSSImportRule()
        r0.cssText = '@import url("not");'
        r1 = cssutils.css.CSSImportRule()
        r1.cssText = '@import "str";'
        assert '@import url(not);' == r0.cssText
        assert '@import "str";' == r1.cssText

        cssutils.ser.prefs.importHrefFormat = 'string'
        assert '@import "not";' == r0.cssText
        assert '@import "str";' == r1.cssText

        cssutils.ser.prefs.importHrefFormat = 'uri'
        assert '@import url(not);' == r0.cssText
        assert '@import url(str);' == r1.cssText

        cssutils.ser.prefs.importHrefFormat = 'not defined'
        assert '@import url(not);' == r0.cssText
        assert '@import "str";' == r1.cssText

    def test_indent(self):
        "Preferences.ident"
        s = cssutils.parseString('a { left: 0 }')
        exp4 = '''a {
    left: 0
    }'''
        exp1 = '''a {
 left: 0
 }'''
        cssutils.ser.prefs.indent = ' '
        assert exp1.encode() == s.cssText
        cssutils.ser.prefs.indent = 4 * ' '
        assert exp4.encode() == s.cssText

    def test_indentClosingBrace(self):
        "Preferences.indentClosingBrace"
        s = cssutils.parseString('@media all {a {left: 0}} b { top: 0 }')
        expT = '''@media all {
    a {
        left: 0
        }
    }
b {
    top: 0
    }'''
        expF = '''@media all {
    a {
        left: 0
    }
}
b {
    top: 0
}'''
        cssutils.ser.prefs.useDefaults()
        assert expT.encode() == s.cssText
        cssutils.ser.prefs.indentClosingBrace = False
        assert expF.encode() == s.cssText

    def test_keepAllProperties(self):
        "Preferences.keepAllProperties"
        css = r'''a {
            color: pink;
            color: red;
            c\olor: blue;
            c\olor: green;
            }'''
        s = cssutils.parseString(css)
        # keep only last
        cssutils.ser.prefs.keepAllProperties = False
        assert b'a {\n    color: green\n    }' == s.cssText
        # keep all
        cssutils.ser.prefs.keepAllProperties = True
        assert (
            b'a {\n    color: pink;\n    color: red;\n    c\\olor: blue;\n    '
            b'c\\olor: green\n    }' == s.cssText
        )

    def test_keepComments(self):
        "Preferences.keepComments"
        s = cssutils.parseString('/*1*/ a { /*2*/ }')
        cssutils.ser.prefs.keepComments = False
        assert b'' == s.cssText
        cssutils.ser.prefs.keepEmptyRules = True
        assert b'a {}' == s.cssText

    def test_keepEmptyRules(self):
        "Preferences.keepEmptyRules"
        # CSSStyleRule
        css = '''a {}
a {
    /*1*/
    }
a {
    color: red
    }'''
        s = cssutils.parseString(css)
        cssutils.ser.prefs.useDefaults()
        cssutils.ser.prefs.keepEmptyRules = True
        assert css.encode() == s.cssText
        cssutils.ser.prefs.keepEmptyRules = False
        assert b'a {\n    /*1*/\n    }\na {\n    color: red\n    }' == s.cssText
        cssutils.ser.prefs.keepComments = False
        assert b'a {\n    color: red\n    }' == s.cssText

        # CSSMediaRule
        css = '''@media tv {
    }
@media all {
    /*1*/
    }
@media print {
    a {}
    }
@media print {
    a {
        /*1*/
        }
    }
@media all {
    a {
        color: red
        }
    }'''
        s = cssutils.parseString(css)
        cssutils.ser.prefs.useDefaults()
        cssutils.ser.prefs.keepEmptyRules = True
        #     self.assertEqual(css, s.cssText)
        cssutils.ser.prefs.keepEmptyRules = False
        assert (
            b'''@media all {
    /*1*/
    }
@media print {
    a {
        /*1*/
        }
    }
@media all {
    a {
        color: red
        }
    }'''
            == s.cssText
        )
        cssutils.ser.prefs.keepComments = False
        assert (
            b'''@media all {
    a {
        color: red
        }
    }'''
            == s.cssText
        )

    def test_keepUnknownAtRules(self):
        "Preferences.keepUnknownAtRules"
        tests = {
            '''@three-dee {
              @background-lighting {
                azimuth: 30deg;
                elevation: 190deg;
              }
              h1 { color: red }
            }
            h1 { color: blue }''': (
                '''@three-dee {
    @background-lighting {
        azimuth: 30deg;
        elevation: 190deg;
        } h1 {
        color: red
        }
    }
h1 {
    color: blue
    }''',
                '''h1 {
    color: blue
    }''',
            )
        }
        for test in tests:
            s = cssutils.parseString(test)
            expwith, expwithout = tests[test]
            cssutils.ser.prefs.keepUnknownAtRules = True
            assert s.cssText == expwith.encode()
            cssutils.ser.prefs.keepUnknownAtRules = False
            assert s.cssText == expwithout.encode()

    def test_keepUsedNamespaceRulesOnly(self):
        "Preferences.keepUsedNamespaceRulesOnly"
        tests = {
            # default == prefix => both are combined
            '@namespace p "u"; @namespace "u"; p|a, a {top: 0}': (
                '@namespace "u";\na, a {\n    top: 0\n    }',
                '@namespace "u";\na, a {\n    top: 0\n    }',
            ),
            '@namespace "u"; @namespace p "u"; p|a, a {top: 0}': (
                '@namespace p "u";\np|a, p|a {\n    top: 0\n    }',
                '@namespace p "u";\np|a, p|a {\n    top: 0\n    }',
            ),
            # default and prefix
            '@namespace p "u"; @namespace "d"; p|a, a {top: 0}': (
                '@namespace p "u";\n@namespace "d";\np|a, a {\n    top: 0\n    }',
                '@namespace p "u";\n@namespace "d";\np|a, a {\n    top: 0\n    }',
            ),
            # prefix only
            '@namespace p "u"; @namespace "d"; p|a {top: 0}': (
                '@namespace p "u";\n@namespace "d";\np|a {\n    top: 0\n    }',
                '@namespace p "u";\np|a {\n    top: 0\n    }',
            ),
            # default only
            '@namespace p "u"; @namespace "d"; a {top: 0}': (
                '@namespace p "u";\n@namespace "d";\na {\n    top: 0\n    }',
                '@namespace "d";\na {\n    top: 0\n    }',
            ),
            # prefix-ns only
            '@namespace p "u"; @namespace d "d"; p|a {top: 0}': (
                '@namespace p "u";\n@namespace d "d";\np|a {\n    top: 0\n    }',
                '@namespace p "u";\np|a {\n    top: 0\n    }',
            ),
        }
        for test in tests:
            s = cssutils.parseString(test)
            expwith, expwithout = tests[test]
            cssutils.ser.prefs.keepUsedNamespaceRulesOnly = False
            assert s.cssText == expwith.encode()
            cssutils.ser.prefs.keepUsedNamespaceRulesOnly = True
            assert s.cssText == expwithout.encode()

    def test_lineNumbers(self):
        "Preferences.lineNumbers"

        s = cssutils.parseString('a {top: 1; left: 2}')
        exp0 = '''a {
    top: 1;
    left: 2
    }'''
        exp1 = '''1: a {
2:     top: 1;
3:     left: 2
4:     }'''
        assert not cssutils.ser.prefs.lineNumbers
        assert exp0.encode() == s.cssText
        cssutils.ser.prefs.lineNumbers = True
        assert cssutils.ser.prefs.lineNumbers
        assert exp1.encode() == s.cssText

    def test_lineSeparator(self):
        "Preferences.lineSeparator"
        s = cssutils.parseString('a { x:1;y:2}')
        assert b'a {\n    x: 1;\n    y: 2\n    }' == s.cssText
        # cannot be indented as no split possible
        cssutils.ser.prefs.lineSeparator = ''
        assert b'a {x: 1;y: 2    }' == s.cssText
        # no valid css but should work
        cssutils.ser.prefs.lineSeparator = 'XXX'
        assert b'a {XXX    x: 1;XXX    y: 2XXX    }' == s.cssText

    def test_listItemSpacer(self):
        "Preferences.listItemSpacer"
        cssutils.ser.prefs.keepEmptyRules = True

        css = '''
        @import "x" print, tv;
a, b {}'''
        s = cssutils.parseString(css)
        assert b'@import "x" print, tv;\na, b {}' == s.cssText
        cssutils.ser.prefs.listItemSpacer = ''
        assert b'@import "x" print,tv;\na,b {}' == s.cssText

    def test_minimizeColorHash(self):
        "Preferences.minimizeColorHash"
        css = 'a { color: #ffffff }'
        s = cssutils.parseString(css)
        assert b'a {\n    color: #fff\n    }' == s.cssText
        cssutils.ser.prefs.minimizeColorHash = False
        assert b'a {\n    color: #ffffff\n    }' == s.cssText

    def test_omitLastSemicolon(self):
        "Preferences.omitLastSemicolon"
        css = 'a { x: 1; y: 2 }'
        s = cssutils.parseString(css)
        assert b'a {\n    x: 1;\n    y: 2\n    }' == s.cssText
        cssutils.ser.prefs.omitLastSemicolon = False
        assert b'a {\n    x: 1;\n    y: 2;\n    }' == s.cssText

    def test_normalizedVarNames(self):
        "Preferences.normalizedVarNames"
        cssutils.ser.prefs.resolveVariables = False

        css = '@variables { A: 1 }'
        s = cssutils.parseString(css)
        assert b'@variables {\n    a: 1\n    }' == s.cssText
        cssutils.ser.prefs.normalizedVarNames = False
        assert b'@variables {\n    A: 1\n    }' == s.cssText

        cssutils.ser.prefs.resolveVariables = True

    def test_paranthesisSpacer(self):
        "Preferences.paranthesisSpacer"
        css = 'a { x: 1; y: 2 }'
        s = cssutils.parseString(css)
        assert b'a {\n    x: 1;\n    y: 2\n    }' == s.cssText
        cssutils.ser.prefs.paranthesisSpacer = ''
        assert b'a{\n    x: 1;\n    y: 2\n    }' == s.cssText

    def test_propertyNameSpacer(self):
        "Preferences.propertyNameSpacer"
        css = 'a { x: 1; y: 2 }'
        s = cssutils.parseString(css)
        assert b'a {\n    x: 1;\n    y: 2\n    }' == s.cssText
        cssutils.ser.prefs.propertyNameSpacer = ''
        assert b'a {\n    x:1;\n    y:2\n    }' == s.cssText

    def test_selectorCombinatorSpacer(self):
        "Preferences.selectorCombinatorSpacer"
        s = cssutils.css.Selector(selectorText='a+b>c~d  e')
        assert 'a + b > c ~ d e' == s.selectorText
        cssutils.ser.prefs.selectorCombinatorSpacer = ''
        assert 'a+b>c~d e' == s.selectorText

    def test_spacer(self):
        cssutils.ser.prefs.spacer = ''
        tests = {
            '@font-face {a:1}': '@font-face {\n    a: 1\n    }',
            '@import  url( a );': '@import url(a);',
            '@media  all{a{color:red}}': '@media all {\n    a {\n        color: red\n        }\n    }',
            '@namespace "a";': '@namespace"a";',
            '@namespace a  "a";': '@namespace a"a";',
            '@page  :left {   a  :1  }': '@page :left {\n    a: 1\n    }',
            '@x  x;': '@x x;',
            '@import"x"tv': '@import"x"tv;',  # ?
        }
        for css, exp in list(tests.items()):
            assert exp.encode() == cssutils.parseString(css).cssText

    def test_validOnly(self):
        "Preferences.validOnly"
        # Property
        p = cssutils.css.Property('color', '1px')
        assert p.cssText == 'color: 1px'
        p.value = '1px'
        cssutils.ser.prefs.validOnly = True
        assert p.cssText == ''
        cssutils.ser.prefs.validOnly = False
        assert p.cssText == 'color: 1px'

        # CSSStyleDeclaration has no actual property valid
        # but is empty if containing invalid Properties only
        s = cssutils.css.CSSStyleDeclaration()
        s.cssText = 'left: x;top: x'
        assert s.cssText == 'left: x;\ntop: x'
        cssutils.ser.prefs.validOnly = True
        assert s.cssText == ''

        cssutils.ser.prefs.useDefaults()
        cssutils.ser.prefs.keepComments = False
        cssutils.ser.prefs.validOnly = True
        tests = {
            'h1 { color: red; rotation: 70minutes }': 'h1 {\n    color: red;\n    }',
            '''img { float: left }       /* correct CSS 2.1 */
img { float: left here }  /* "here" is not a value of 'float' */
img { background: "red" } /* keywords cannot be quoted */
img { border-width: 3 }   /* a unit must be specified for length values */''': 'img {\n    float: left\n    }',
        }
        self.do_equal_p(tests, raising=False)


class TestCSSSerializer:
    """
    testcases for cssutils.CSSSerializer
    """

    def setup_method(self):
        cssutils.ser.prefs.useDefaults()

    def teardown_method(self):
        cssutils.ser.prefs.useDefaults()

    def test_canonical(self):
        tests = {
            '''1''': '''1''',
            # => remove +
            '''+1''': '''+1''',
            # 0 => remove unit
            '''0''': '''0''',
            '''+0''': '''0''',
            '''-0''': '''0''',
            '''0.0''': '''0''',
            '''00.0''': '''0''',
            '''00.0px''': '''0''',
            '''00.0pc''': '''0''',
            '''00.0em''': '''0''',
            '''00.0ex''': '''0''',
            '''00.0cm''': '''0''',
            '''00.0mm''': '''0''',
            '''00.0in''': '''0''',
            # 0 => keep unit
            '''00.0%''': '''0%''',
            '''00.0ms''': '''0ms''',
            '''00.0s''': '''0s''',
            '''00.0khz''': '''0khz''',
            '''00.0hz''': '''0hz''',
            '''00.0deg''': '''0deg''',
            '''00.0rad''': '''0rad''',
            '''00.0grad''': '''0grad''',
            '''00.0xx''': '''0xx''',
            # 11.
            '''a, 'b"', serif''': r'''a, "b\"", serif''',
            # SHOULD: \[ => [ but keep!
            r"""url('h)i') '\[\]'""": r'''url("h)i") "\[\]"''',
            '''rgb(18, 52, 86)''': '''rgb(18, 52, 86)''',
            '''#123456''': '''#123456''',
            # SHOULD => #112233
            '''#112233''': '''#123''',
            # SHOULD => #000000
            #            u'rgba(000001, 0, 0, 1)': u'#000'
        }
        for test, exp in list(tests.items()):
            v = cssutils.css.PropertyValue(test)
            assert exp == v.cssText

    def test_CSSStyleSheet(self):
        "CSSSerializer.do_CSSStyleSheet"
        css = '/* κουρος */'
        sheet = cssutils.parseString(css)
        assert css == str(sheet.cssText, 'utf-8')

        css = '@charset "utf-8";\n/* κουρος */'
        sheet = cssutils.parseString(css)
        assert css == str(sheet.cssText, 'utf-8')
        sheet.cssRules[0].encoding = 'ascii'
        assert (
            b'@charset "ascii";\n/* \\3BA \\3BF \\3C5 \\3C1 \\3BF \\3C2  */'
            == sheet.cssText
        )

    def test_Property(self):
        "CSSSerializer.do_Property"

        name = "color"
        value = "red"
        priority = "!important"

        s = cssutils.css.property.Property(name=name, value=value, priority=priority)
        assert 'color: red !important' == cssutils.ser.do_Property(s)

        s = cssutils.css.property.Property(name=name, value=value)
        assert 'color: red' == cssutils.ser.do_Property(s)

    def test_escapestring(self):
        "CSSSerializer._escapestring"
        # '"\a\22\27"'
        css = r'''@import url("ABC\a");
@import "ABC\a";
@import 'ABC\a';
a[href='"\a\22\27"'] {
    a: "\a\d\c";
    b: "\a \d \c ";
    c: "\"";
    d: "\22";
    e: '\'';
    f: "\\";
    g: "2\\ 1\ 2\\";
    content: '\27';
    }'''
        #        exp = ur'''@import url("ABC\a ");
        # @import "ABC\a";
        # @import "ABC\a";
        # a[href="\"\a\22\27\""] {
        #    a: "\a\d\c";
        #    b: "\a \d \c ";
        #    c: "\"";
        #    d: "\22";
        #    e: "'";
        #    f: "\\";
        #    g: "2\\ 1\ 2\\";
        #    content: "\27"
        #    }'''
        exp = r'''@import url("ABC\a ");
@import "ABC\a ";
@import "ABC\a ";
a[href="\"\a \"'\""] {
    a: "\a \d \c ";
    b: "\a \d \c ";
    c: "\"";
    d: "\"";
    e: "'";
    f: "\\";
    g: "2\\ 1\ 2\\";
    content: "'"
    }'''
        sheet = cssutils.parseString(css)
        assert sheet.cssText == exp.encode()

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_serialize.py
NikolaBlagojevic/pyrecodes
"""Testcases for new cssutils.tokenize.Tokenizer

TODO: old tests as new ones are **not complete**!
"""

import sys

import pytest

import cssutils.tokenize2 as tokenize2
from cssutils.tokenize2 import Tokenizer


class TestTokenizer:
    testsall = {
        # IDENT
        'äöüß€': [('IDENT', 'äöüß€', 1, 1)],
        ' a ': [('S', ' ', 1, 1), ('IDENT', 'a', 1, 2), ('S', ' ', 1, 3)],
        '_a': [('IDENT', '_a', 1, 1)],
        '-a': [('IDENT', '-a', 1, 1)],
        '--a': [('IDENT', '--a', 1, 1)],
        'aA-_\200\377': [('IDENT', 'aA-_\200\377', 1, 1)],
        'a1': [('IDENT', 'a1', 1, 1)],
        # escapes must end with S or max 6 digits:
        '\\44 b': [('IDENT', 'Db', 1, 1)],
        '\\44  b': [('IDENT', 'D', 1, 1), ('S', ' ', 1, 5), ('IDENT', 'b', 1, 6)],
        '\\44\nb': [('IDENT', 'Db', 1, 1)],
        '\\44\rb': [('IDENT', 'Db', 1, 1)],
        '\\44\fb': [('IDENT', 'Db', 1, 1)],
        '\\44\n*': [('IDENT', 'D', 1, 1), ('CHAR', '*', 2, 1)],
        '\\44  a': [('IDENT', 'D', 1, 1), ('S', ' ', 1, 5), ('IDENT', 'a', 1, 6)],
        # TODO:
        # Note that this means that a "real" space after the escape sequence
        # must itself either be escaped or doubled:
        '\\44\\ x': [('IDENT', 'D\\ x', 1, 1)],
        '\\44  ': [('IDENT', 'D', 1, 1), ('S', ' ', 1, 5)],
        r'\44': [('IDENT', 'D', 1, 1)],
        r'\\': [('IDENT', r'\\', 1, 1)],
        r'\{': [('IDENT', r'\{', 1, 1)],
        r'\"': [('IDENT', r'\"', 1, 1)],
        r'\(': [('IDENT', r'\(', 1, 1)],
        r'\1 \22 \333 \4444 \55555 \666666 \777777 7 \7777777': [
            (
                ('IDENT', '\x01"\u0333\u4444\U00055555\\666666 \\777777 7', 1, 1)
                if sys.maxunicode > 0x10000
                else ('IDENT', '\x01"\u0333\u4444\\55555 \\666666 \\777777 7', 1, 1)
            ),
            ('S', ' ', 1, 43),
            ('IDENT', '\\7777777', 1, 44),
        ],
        # Not a function, important for media queries
        'and(': [('IDENT', 'and', 1, 1), ('CHAR', '(', 1, 4)],
        '\\1 b': [('IDENT', '\x01b', 1, 1)],
        # repeat: '\\44 b': [('IDENT', 'Db', 1, 1)],
        '\\123 b': [('IDENT', '\u0123b', 1, 1)],
        '\\1234 b': [('IDENT', '\u1234b', 1, 1)],
        '\\12345 b': [
            (
                ('IDENT', '\U00012345b', 1, 1)
                if sys.maxunicode > 0x10000
                else ('IDENT', '\\12345 b', 1, 1)
            )
        ],
        '\\123456 b': [('IDENT', '\\123456 b', 1, 1)],
        '\\1234567 b': [
            ('IDENT', '\\1234567', 1, 1),
            ('S', ' ', 1, 9),
            ('IDENT', 'b', 1, 10),
        ],
        '\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,': [
            ('IDENT', '\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,', 1, 1)
        ],
        # STRING
        ' "" ': [('S', ' ', 1, 1), ('STRING', '""', 1, 2), ('S', ' ', 1, 4)],
        ' "\'" ': [('S', ' ', 1, 1), ('STRING', '"\'"', 1, 2), ('S', ' ', 1, 5)],
        " '' ": [('S', ' ', 1, 1), ('STRING', "''", 1, 2), ('S', ' ', 1, 4)],
        "'\\\n'": [('STRING', "''", 1, 1)],
        "'\\\n\\\n\\\n'": [('STRING', "''", 1, 1)],
        "'\\\f'": [('STRING', "''", 1, 1)],
        "'\\\r'": [('STRING', "''", 1, 1)],
        "'1\\\n2'": [('STRING', "'12'", 1, 1)],
        "'1\\\r\n2'": [('STRING', "'12'", 1, 1)],
        # r'"\0020|\0020"': [('STRING', '"\\0020|\\0020"', 1, 1)],
        r'"\61|\0061"': [('STRING', '"a|a"', 1, 1)],
        # HASH
        ' #a ': [('S', ' ', 1, 1), ('HASH', '#a', 1, 2), ('S', ' ', 1, 4)],
        '#ccc': [('HASH', '#ccc', 1, 1)],
        '#111': [('HASH', '#111', 1, 1)],
        '#a1a1a1': [('HASH', '#a1a1a1', 1, 1)],
        '#1a1a1a': [('HASH', '#1a1a1a', 1, 1)],
        # NUMBER, for plus see CSS3
        ' 0 ': [('S', ' ', 1, 1), ('NUMBER', '0', 1, 2), ('S', ' ', 1, 3)],
        ' 0.1 ': [('S', ' ', 1, 1), ('NUMBER', '0.1', 1, 2), ('S', ' ', 1, 5)],
        ' .0 ': [('S', ' ', 1, 1), ('NUMBER', '.0', 1, 2), ('S', ' ', 1, 4)],
        ' -0 ': [
            ('S', ' ', 1, 1),
            # ('CHAR', '-', 1, 2),
            # ('NUMBER', '0', 1, 3),
            ('NUMBER', '-0', 1, 2),
            ('S', ' ', 1, 4),
        ],
        # PERCENTAGE
        ' 0% ': [('S', ' ', 1, 1), ('PERCENTAGE', '0%', 1, 2), ('S', ' ', 1, 4)],
        ' .5% ': [('S', ' ', 1, 1), ('PERCENTAGE', '.5%', 1, 2), ('S', ' ', 1, 5)],
        # URI
        ' url() ': [('S', ' ', 1, 1), ('URI', 'url()', 1, 2), ('S', ' ', 1, 7)],
        ' url(a) ': [('S', ' ', 1, 1), ('URI', 'url(a)', 1, 2), ('S', ' ', 1, 8)],
        ' url("a") ': [('S', ' ', 1, 1), ('URI', 'url("a")', 1, 2), ('S', ' ', 1, 10)],
        ' url( a ) ': [('S', ' ', 1, 1), ('URI', 'url( a )', 1, 2), ('S', ' ', 1, 10)],
        # UNICODE-RANGE
        # CDO
        ' <!-- ': [('S', ' ', 1, 1), ('CDO', '<!--', 1, 2), ('S', ' ', 1, 6)],
        '"<!--""-->"': [('STRING', '"<!--"', 1, 1), ('STRING', '"-->"', 1, 7)],
        # CDC
        ' --> ': [('S', ' ', 1, 1), ('CDC', '-->', 1, 2), ('S', ' ', 1, 5)],
        # S
        ' ': [('S', ' ', 1, 1)],
        '  ': [('S', '  ', 1, 1)],
        '\r': [('S', '\r', 1, 1)],
        '\n': [('S', '\n', 1, 1)],
        '\r\n': [('S', '\r\n', 1, 1)],
        '\f': [('S', '\f', 1, 1)],
        '\t': [('S', '\t', 1, 1)],
        '\r\n\r\n\f\t ': [('S', '\r\n\r\n\f\t ', 1, 1)],
        # COMMENT, for incomplete see later
        '/*x*/ ': [('COMMENT', '/*x*/', 1, 1), ('S', ' ', 1, 6)],
        # FUNCTION
        ' x( ': [('S', ' ', 1, 1), ('FUNCTION', 'x(', 1, 2), ('S', ' ', 1, 4)],
        # INCLUDES
        ' ~= ': [('S', ' ', 1, 1), ('INCLUDES', '~=', 1, 2), ('S', ' ', 1, 4)],
        '~==': [('INCLUDES', '~=', 1, 1), ('CHAR', '=', 1, 3)],
        # DASHMATCH
        ' |= ': [('S', ' ', 1, 1), ('DASHMATCH', '|=', 1, 2), ('S', ' ', 1, 4)],
        '|==': [('DASHMATCH', '|=', 1, 1), ('CHAR', '=', 1, 3)],
        # CHAR
        ' @ ': [('S', ' ', 1, 1), ('CHAR', '@', 1, 2), ('S', ' ', 1, 3)],
        # --- overwritten for CSS 2.1 ---
        # LBRACE
        ' { ': [('S', ' ', 1, 1), ('CHAR', '{', 1, 2), ('S', ' ', 1, 3)],
        # PLUS
        ' + ': [('S', ' ', 1, 1), ('CHAR', '+', 1, 2), ('S', ' ', 1, 3)],
        # GREATER
        ' > ': [('S', ' ', 1, 1), ('CHAR', '>', 1, 2), ('S', ' ', 1, 3)],
        # COMMA
        ' , ': [('S', ' ', 1, 1), ('CHAR', ',', 1, 2), ('S', ' ', 1, 3)],
        # class
        ' . ': [('S', ' ', 1, 1), ('CHAR', '.', 1, 2), ('S', ' ', 1, 3)],
    }

    tests3 = {
        # UNICODE-RANGE
        ' u+0 ': [('S', ' ', 1, 1), ('UNICODE-RANGE', 'u+0', 1, 2), ('S', ' ', 1, 5)],
        ' u+01 ': [('S', ' ', 1, 1), ('UNICODE-RANGE', 'u+01', 1, 2), ('S', ' ', 1, 6)],
        ' u+012 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+012', 1, 2),
            ('S', ' ', 1, 7),
        ],
        ' u+0123 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+0123', 1, 2),
            ('S', ' ', 1, 8),
        ],
        ' u+01234 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+01234', 1, 2),
            ('S', ' ', 1, 9),
        ],
        ' u+012345 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+012345', 1, 2),
            ('S', ' ', 1, 10),
        ],
        ' u+0123456 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+012345', 1, 2),
            ('NUMBER', '6', 1, 10),
            ('S', ' ', 1, 11),
        ],
        ' U+123456 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'U+123456', 1, 2),
            ('S', ' ', 1, 10),
        ],
        ' \\55+abcdef ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'U+abcdef', 1, 2),
            ('S', ' ', 1, 12),
        ],
        ' \\75+abcdef ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+abcdef', 1, 2),
            ('S', ' ', 1, 12),
        ],
        ' u+0-1 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+0-1', 1, 2),
            ('S', ' ', 1, 7),
        ],
        ' u+0-1, u+123456-abcdef ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+0-1', 1, 2),
            ('CHAR', ',', 1, 7),
            ('S', ' ', 1, 8),
            ('UNICODE-RANGE', 'u+123456-abcdef', 1, 9),
            ('S', ' ', 1, 24),
        ],
        # specials
        'c\\olor': [('IDENT', 'c\\olor', 1, 1)],
        # '-1': [('CHAR', '-', 1, 1), ('NUMBER', '1', 1, 2)],
        # '-1px': [('CHAR', '-', 1, 1), ('DIMENSION', '1px', 1, 2)],
        '-1': [('NUMBER', '-1', 1, 1)],
        '-1px': [('DIMENSION', '-1px', 1, 1)],
        # ATKEYWORD
        ' @x ': [('S', ' ', 1, 1), ('ATKEYWORD', '@x', 1, 2), ('S', ' ', 1, 4)],
        '@X': [('ATKEYWORD', '@X', 1, 1)],
        '@\\x': [('ATKEYWORD', '@\\x', 1, 1)],
        # -
        '@1x': [('CHAR', '@', 1, 1), ('DIMENSION', '1x', 1, 2)],
        # DIMENSION
        ' 0px ': [('S', ' ', 1, 1), ('DIMENSION', '0px', 1, 2), ('S', ' ', 1, 5)],
        ' 1s ': [('S', ' ', 1, 1), ('DIMENSION', '1s', 1, 2), ('S', ' ', 1, 4)],
        '0.2EM': [('DIMENSION', '0.2EM', 1, 1)],
        '1p\\x': [('DIMENSION', '1p\\x', 1, 1)],
        '1PX': [('DIMENSION', '1PX', 1, 1)],
        # NUMBER
        ' - 0 ': [
            ('S', ' ', 1, 1),
            ('CHAR', '-', 1, 2),
            ('S', ' ', 1, 3),
            ('NUMBER', '0', 1, 4),
            ('S', ' ', 1, 5),
        ],
        ' + 0 ': [
            ('S', ' ', 1, 1),
            ('CHAR', '+', 1, 2),
            ('S', ' ', 1, 3),
            ('NUMBER', '0', 1, 4),
            ('S', ' ', 1, 5),
        ],
        # PREFIXMATCH
        ' ^= ': [('S', ' ', 1, 1), ('PREFIXMATCH', '^=', 1, 2), ('S', ' ', 1, 4)],
        '^==': [('PREFIXMATCH', '^=', 1, 1), ('CHAR', '=', 1, 3)],
        # SUFFIXMATCH
        ' $= ': [('S', ' ', 1, 1), ('SUFFIXMATCH', '$=', 1, 2), ('S', ' ', 1, 4)],
        '$==': [('SUFFIXMATCH', '$=', 1, 1), ('CHAR', '=', 1, 3)],
        # SUBSTRINGMATCH
        ' *= ': [('S', ' ', 1, 1), ('SUBSTRINGMATCH', '*=', 1, 2), ('S', ' ', 1, 4)],
        '*==': [('SUBSTRINGMATCH', '*=', 1, 1), ('CHAR', '=', 1, 3)],
        # BOM only at start
        '\xfe\xff ': [('BOM', '\xfe\xff', 1, 1), ('S', ' ', 1, 1)],
        ' \xfe\xff ': [('S', ' ', 1, 1), ('IDENT', '\xfe\xff', 1, 2), ('S', ' ', 1, 4)],
        '\xef\xbb\xbf ': [('BOM', '\xef\xbb\xbf', 1, 1), ('S', ' ', 1, 1)],
        ' \xef\xbb\xbf ': [
            ('S', ' ', 1, 1),
            ('IDENT', '\xef\xbb\xbf', 1, 2),
            ('S', ' ', 1, 5),
        ],
    }

    tests2 = {
        # escapes work not for a-f!
        # IMPORT_SYM
        ' @import ': [
            ('S', ' ', 1, 1),
            ('IMPORT_SYM', '@import', 1, 2),
            ('S', ' ', 1, 9),
        ],
        '@IMPORT': [('IMPORT_SYM', '@IMPORT', 1, 1)],
        '@\\49\r\nMPORT': [('IMPORT_SYM', '@\\49\r\nMPORT', 1, 1)],
        r'@\i\m\p\o\r\t': [('IMPORT_SYM', r'@\i\m\p\o\r\t', 1, 1)],
        r'@\I\M\P\O\R\T': [('IMPORT_SYM', r'@\I\M\P\O\R\T', 1, 1)],
        r'@\49 \04d\0050\0004f\000052\54': [
            ('IMPORT_SYM', r'@\49 \04d\0050\0004f\000052\54', 1, 1)
        ],
        r'@\69 \06d\0070\0006f\000072\74': [
            ('IMPORT_SYM', r'@\69 \06d\0070\0006f\000072\74', 1, 1)
        ],
        # PAGE_SYM
        ' @page ': [('S', ' ', 1, 1), ('PAGE_SYM', '@page', 1, 2), ('S', ' ', 1, 7)],
        '@PAGE': [('PAGE_SYM', '@PAGE', 1, 1)],
        r'@\pa\ge': [('PAGE_SYM', r'@\pa\ge', 1, 1)],
        r'@\PA\GE': [('PAGE_SYM', r'@\PA\GE', 1, 1)],
        r'@\50\41\47\45': [('PAGE_SYM', r'@\50\41\47\45', 1, 1)],
        r'@\70\61\67\65': [('PAGE_SYM', r'@\70\61\67\65', 1, 1)],
        # MEDIA_SYM
        ' @media ': [('S', ' ', 1, 1), ('MEDIA_SYM', '@media', 1, 2), ('S', ' ', 1, 8)],
        '@MEDIA': [('MEDIA_SYM', '@MEDIA', 1, 1)],
        r'@\med\ia': [('MEDIA_SYM', r'@\med\ia', 1, 1)],
        r'@\MED\IA': [('MEDIA_SYM', r'@\MED\IA', 1, 1)],
        '@\\4d\n\\45\r\\44\t\\49\r\nA': [
            ('MEDIA_SYM', '@\\4d\n\\45\r\\44\t\\49\r\nA', 1, 1)
        ],
        '@\\4d\n\\45\r\\44\t\\49\r\\41\f': [
            ('MEDIA_SYM', '@\\4d\n\\45\r\\44\t\\49\r\\41\f', 1, 1)
        ],
        '@\\6d\n\\65\r\\64\t\\69\r\\61\f': [
            ('MEDIA_SYM', '@\\6d\n\\65\r\\64\t\\69\r\\61\f', 1, 1)
        ],
        # FONT_FACE_SYM
        ' @font-face ': [
            ('S', ' ', 1, 1),
            ('FONT_FACE_SYM', '@font-face', 1, 2),
            ('S', ' ', 1, 12),
        ],
        '@FONT-FACE': [('FONT_FACE_SYM', '@FONT-FACE', 1, 1)],
        r'@f\o\n\t\-face': [('FONT_FACE_SYM', r'@f\o\n\t\-face', 1, 1)],
        r'@F\O\N\T\-FACE': [('FONT_FACE_SYM', r'@F\O\N\T\-FACE', 1, 1)],
        # TODO: "-" as hex!
        r'@\46\4f\4e\54\-\46\41\43\45': [
            ('FONT_FACE_SYM', r'@\46\4f\4e\54\-\46\41\43\45', 1, 1)
        ],
        r'@\66\6f\6e\74\-\66\61\63\65': [
            ('FONT_FACE_SYM', r'@\66\6f\6e\74\-\66\61\63\65', 1, 1)
        ],
        # CHARSET_SYM only if "@charset "!
        '@charset  ': [('CHARSET_SYM', '@charset ', 1, 1), ('S', ' ', 1, 10)],
        ' @charset  ': [
            ('S', ' ', 1, 1),
            ('CHARSET_SYM', '@charset ', 1, 2),  # not at start
            ('S', ' ', 1, 11),
        ],
        '@charset': [('ATKEYWORD', '@charset', 1, 1)],  # no ending S
        '@CHARSET ': [('ATKEYWORD', '@CHARSET', 1, 1), ('S', ' ', 1, 9)],  # uppercase
        '@cha\\rset ': [
            ('ATKEYWORD', '@cha\\rset', 1, 1),  # not literal
            ('S', ' ', 1, 10),
        ],
        # NAMESPACE_SYM
        ' @namespace ': [
            ('S', ' ', 1, 1),
            ('NAMESPACE_SYM', '@namespace', 1, 2),
            ('S', ' ', 1, 12),
        ],
        r'@NAMESPACE': [('NAMESPACE_SYM', r'@NAMESPACE', 1, 1)],
        r'@\na\me\s\pace': [('NAMESPACE_SYM', r'@\na\me\s\pace', 1, 1)],
        r'@\NA\ME\S\PACE': [('NAMESPACE_SYM', r'@\NA\ME\S\PACE', 1, 1)],
        r'@\4e\41\4d\45\53\50\41\43\45': [
            ('NAMESPACE_SYM', r'@\4e\41\4d\45\53\50\41\43\45', 1, 1)
        ],
        r'@\6e\61\6d\65\73\70\61\63\65': [
            ('NAMESPACE_SYM', r'@\6e\61\6d\65\73\70\61\63\65', 1, 1)
        ],
        # ATKEYWORD
        ' @unknown ': [
            ('S', ' ', 1, 1),
            ('ATKEYWORD', '@unknown', 1, 2),
            ('S', ' ', 1, 10),
        ],
        # STRING
        # strings with linebreak in it
        ' "\\na"\na': [
            ('S', ' ', 1, 1),
            ('STRING', '"\\na"', 1, 2),
            ('S', '\n', 1, 7),
            ('IDENT', 'a', 2, 1),
        ],
        " '\\na'\na": [
            ('S', ' ', 1, 1),
            ('STRING', "'\\na'", 1, 2),
            ('S', '\n', 1, 7),
            ('IDENT', 'a', 2, 1),
        ],
        ' "\\r\\n\\t\\n\\ra"a': [
            ('S', ' ', 1, 1),
            ('STRING', '"\\r\\n\\t\\n\\ra"', 1, 2),
            ('IDENT', 'a', 1, 15),
        ],
        # IMPORTANT_SYM is not IDENT!!!
        ' !important ': [
            ('S', ' ', 1, 1),
            ('CHAR', '!', 1, 2),
            ('IDENT', 'important', 1, 3),
            ('S', ' ', 1, 12),
        ],
        '! /*1*/ important ': [
            ('CHAR', '!', 1, 1),
            ('S', ' ', 1, 2),
            ('COMMENT', '/*1*/', 1, 3),
            ('S', ' ', 1, 8),
            ('IDENT', 'important', 1, 9),
            ('S', ' ', 1, 18),
        ],
        '! important': [
            ('CHAR', '!', 1, 1),
            ('S', ' ', 1, 2),
            ('IDENT', 'important', 1, 3),
        ],
        '!\n\timportant': [
            ('CHAR', '!', 1, 1),
            ('S', '\n\t', 1, 2),
            ('IDENT', 'important', 2, 2),
        ],
        '!IMPORTANT': [('CHAR', '!', 1, 1), ('IDENT', 'IMPORTANT', 1, 2)],
        r'!\i\m\p\o\r\ta\n\t': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'\i\m\p\o\r\ta\n\t', 1, 2),
        ],
        r'!\I\M\P\O\R\Ta\N\T': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'\I\M\P\O\R\Ta\N\T', 1, 2),
        ],
        r'!\49\4d\50\4f\52\54\41\4e\54': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'IMPORTANT', 1, 2),
        ],
        r'!\69\6d\70\6f\72\74\61\6e\74': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'important', 1, 2),
        ],
    }

    # overwriting tests in testsall
    tests2only = {
        # LBRACE
        ' { ': [('S', ' ', 1, 1), ('LBRACE', '{', 1, 2), ('S', ' ', 1, 3)],
        # PLUS
        ' + ': [('S', ' ', 1, 1), ('PLUS', '+', 1, 2), ('S', ' ', 1, 3)],
        # GREATER
        ' > ': [('S', ' ', 1, 1), ('GREATER', '>', 1, 2), ('S', ' ', 1, 3)],
        # COMMA
        ' , ': [('S', ' ', 1, 1), ('COMMA', ',', 1, 2), ('S', ' ', 1, 3)],
        # class
        ' . ': [('S', ' ', 1, 1), ('CLASS', '.', 1, 2), ('S', ' ', 1, 3)],
    }

    testsfullsheet = {
        # escape ends with explicit space but \r\n as single space
        '\\65\r\nb': [('IDENT', 'eb', 1, 1)],
        # STRING
        r'"\""': [('STRING', r'"\""', 1, 1)],
        r'"\" "': [('STRING', r'"\" "', 1, 1)],
        """'\\''""": [('STRING', """'\\''""", 1, 1)],
        ' "\na': [
            ('S', ' ', 1, 1),
            ('INVALID', '"', 1, 2),
            ('S', '\n', 1, 3),
            ('IDENT', 'a', 2, 1),
        ],
        # strings with linebreak in it
        ' "\\na\na': [
            ('S', ' ', 1, 1),
            ('INVALID', '"\\na', 1, 2),
            ('S', '\n', 1, 6),
            ('IDENT', 'a', 2, 1),
        ],
        ' "\\r\\n\\t\\n\\ra\na': [
            ('S', ' ', 1, 1),
            ('INVALID', '"\\r\\n\\t\\n\\ra', 1, 2),
            ('S', '\n', 1, 14),
            ('IDENT', 'a', 2, 1),
        ],
        # URI
        'ur\\l(a)': [('URI', 'ur\\l(a)', 1, 1)],
        'url(a)': [('URI', 'url(a)', 1, 1)],
        '\\55r\\4c(a)': [('URI', 'UrL(a)', 1, 1)],
        '\\75r\\6c(a)': [('URI', 'url(a)', 1, 1)],
        ' url())': [('S', ' ', 1, 1), ('URI', 'url()', 1, 2), ('CHAR', ')', 1, 7)],
        'url("x"))': [('URI', 'url("x")', 1, 1), ('CHAR', ')', 1, 9)],
        "url('x'))": [('URI', "url('x')", 1, 1), ('CHAR', ')', 1, 9)],
    }

    # tests if fullsheet=False is set on tokenizer
    testsfullsheetfalse = {
        # COMMENT incomplete
        '/*': [('CHAR', '/', 1, 1), ('CHAR', '*', 1, 2)],
        # INVALID incomplete
        ' " ': [('S', ' ', 1, 1), ('INVALID', '" ', 1, 2)],
        " 'abc\"with quote\" in it": [
            ('S', ' ', 1, 1),
            ('INVALID', "'abc\"with quote\" in it", 1, 2),
        ],
        # URI incomplete
        'url(a': [('FUNCTION', 'url(', 1, 1), ('IDENT', 'a', 1, 5)],
        'url("a': [('FUNCTION', 'url(', 1, 1), ('INVALID', '"a', 1, 5)],
        "url('a": [('FUNCTION', 'url(', 1, 1), ('INVALID', "'a", 1, 5)],
        "UR\\l('a": [('FUNCTION', 'UR\\l(', 1, 1), ('INVALID', "'a", 1, 6)],
    }

    # tests if fullsheet=True is set on tokenizer
    testsfullsheettrue = {
        # COMMENT incomplete
        '/*': [('COMMENT', '/**/', 1, 1)],
        #        # INVALID incomplete => STRING
        ' " ': [('S', ' ', 1, 1), ('STRING', '" "', 1, 2)],
        " 'abc\"with quote\" in it": [
            ('S', ' ', 1, 1),
            ('STRING', "'abc\"with quote\" in it'", 1, 2),
        ],
        # URI incomplete FUNC => URI
        'url(a': [('URI', 'url(a)', 1, 1)],
        'url( a': [('URI', 'url( a)', 1, 1)],
        'url("a': [('URI', 'url("a")', 1, 1)],
        'url( "a ': [('URI', 'url( "a ")', 1, 1)],
        "url('a": [('URI', "url('a')", 1, 1)],
        'url("a"': [('URI', 'url("a")', 1, 1)],
        "url('a'": [('URI', "url('a')", 1, 1)],
    }

    def setup_method(self):
        # log = cssutils.errorhandler.ErrorHandler()
        self.tokenizer = Tokenizer()

    def test_tokenize(self):
        "cssutils Tokenizer().tokenize()"
        import cssutils.cssproductions

        tokenizer = Tokenizer(
            cssutils.cssproductions.MACROS, cssutils.cssproductions.PRODUCTIONS
        )
        tests = {}
        tests.update(self.testsall)
        tests.update(self.tests2)
        tests.update(self.tests3)
        tests.update(self.testsfullsheet)
        tests.update(self.testsfullsheetfalse)
        for css in tests:
            # check token format
            tokens = tokenizer.tokenize(css)
            for i, actual in enumerate(tokens):
                expected = tests[css][i]
                assert expected == actual

            # check if all same number of tokens
            tokens = list(tokenizer.tokenize(css))
            assert len(tokens) == len(tests[css])

    def test_tokenizefullsheet(self):
        "cssutils Tokenizer().tokenize(fullsheet=True)"
        import cssutils.cssproductions

        tokenizer = Tokenizer(
            cssutils.cssproductions.MACROS, cssutils.cssproductions.PRODUCTIONS
        )
        tests = {}
        tests.update(self.testsall)
        tests.update(self.tests2)
        tests.update(self.tests3)
        tests.update(self.testsfullsheet)
        tests.update(self.testsfullsheettrue)
        for css in tests:
            # check token format
            tokens = tokenizer.tokenize(css, fullsheet=True)
            for i, actual in enumerate(tokens):
                try:
                    expected = tests[css][i]
                except IndexError:
                    # EOF is added
                    assert actual[0] == 'EOF'
                else:
                    assert expected == actual

            # check if all same number of tokens
            tokens = list(tokenizer.tokenize(css, fullsheet=True))
            # EOF is added so -1
            assert len(tokens) - 1 == len(tests[css])


class TestTokenizerUtils:
    """Tests for the util functions of tokenize"""

    has_at_cases = [
        ('foo', 0, 'foo', True),
        ('foo', 0, 'f', True),
        ('foo', 1, 'o', True),
        ('foo', 1, 'oo', True),
        ('foo', 4, 'foo', False),
        ('foo', 0, 'bar', False),
        ('foo', 0, 'foobar', False),
    ]

    @pytest.mark.parametrize("string, pos, text, expected", has_at_cases)
    def test_has_at(self, string, pos, text, expected):
        assert tokenize2.has_at(string, pos, text) == expected

    suffix_eq_cases = [
        ('foobar', 0, 'foobar', True),
        ('foobar', 3, 'bar', True),
        ('foobar', 3, 'foo', False),
        ('foobar', 10, 'bar', False),
    ]

    @pytest.mark.parametrize("string, pos, suffix, expected", suffix_eq_cases)
    def test_suffix_eq(self, string, pos, suffix, expected):
        assert tokenize2.suffix_eq(string, pos, suffix) == expected

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_tokenize2.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.util"""


from email.message import Message
import re
import urllib.request
import urllib.error
import urllib.parse
from unittest import mock

from cssutils.util import Base, ListSeq, _readUrl, _defaultFetcher, LazyRegex
import pytest


class ListSeqTestCase:
    def test_all(self):
        "util.ListSeq"
        ls = ListSeq()
        assert 0 == len(ls)
        # append()
        with pytest.raises(NotImplementedError):
            ls.append(1)
        # set
        with pytest.raises(NotImplementedError):
            ls.__setitem__(0, 1)

        # hack:
        ls.seq.append(1)
        ls.seq.append(2)

        # len
        assert 2 == len(ls)
        # __contains__
        assert True is (1 in ls)
        # get
        assert 1 == ls[0]
        assert 2 == ls[1]
        # del
        del ls[0]
        assert 1 == len(ls)
        assert False is (1 in ls)
        # for in
        for x in ls:
            assert 2 == x


class BaseTestCase:
    def test_normalize(self):
        "Base._normalize()"
        b = Base()
        tests = {
            'abcdefg ABCDEFG äöüß€ AÖÜ': 'abcdefg abcdefg äöüß€ aöü',
            r'\ga\Ga\\\ ': r'gaga\ ',
            r'0123456789': '0123456789',
            # unicode escape seqs should have been done by
            # the tokenizer...
        }
        for test, exp in list(tests.items()):
            assert b._normalize(test) == exp
            # static too
            assert Base._normalize(test) == exp

    def test_tokenupto(self):  # noqa: C901
        "Base._tokensupto2()"

        # tests nested blocks of {} [] or ()
        b = Base()

        tests = [
            ('default', 'a[{1}]({2}) { } NOT', 'a[{1}]({2}) { }', False),
            ('default', 'a[{1}]({2}) { } NOT', 'a[{1}]func({2}) { }', True),
            ('blockstartonly', 'a[{1}]({2}) { NOT', 'a[{1}]({2}) {', False),
            ('blockstartonly', 'a[{1}]({2}) { NOT', 'a[{1}]func({2}) {', True),
            ('propertynameendonly', 'a[(2)1] { }2 : a;', 'a[(2)1] { }2 :', False),
            ('propertynameendonly', 'a[(2)1] { }2 : a;', 'a[func(2)1] { }2 :', True),
            (
                'propertyvalueendonly',
                'a{;{;}[;](;)}[;{;}[;](;)](;{;}[;](;)) 1; NOT',
                'a{;{;}[;](;)}[;{;}[;](;)](;{;}[;](;)) 1;',
                False,
            ),
            (
                'propertyvalueendonly',
                'a{;{;}[;](;)}[;{;}[;](;)](;{;}[;](;)) 1; NOT',
                'a{;{;}[;]func(;)}[;{;}[;]func(;)]func(;{;}[;]func(;)) 1;',
                True,
            ),
            (
                'funcendonly',
                'a{[1]}([3])[{[1]}[2]([3])]) NOT',
                'a{[1]}([3])[{[1]}[2]([3])])',
                False,
            ),
            (
                'funcendonly',
                'a{[1]}([3])[{[1]}[2]([3])]) NOT',
                'a{[1]}func([3])[{[1]}[2]func([3])])',
                True,
            ),
            (
                'selectorattendonly',
                '[a[()]{()}([()]{()}())] NOT',
                '[a[()]{()}([()]{()}())]',
                False,
            ),
            (
                'selectorattendonly',
                '[a[()]{()}([()]{()}())] NOT',
                '[a[func()]{func()}func([func()]{func()}func())]',
                True,
            ),
            # issue 50
            ('withstarttoken [', 'a];x', '[a];', False),
        ]

        for typ, values, exp, paransasfunc in tests:

            def maketokens(valuelist):
                # returns list of tuples
                return [('TYPE', v, 0, 0) for v in valuelist]

            tokens = maketokens(list(values))
            if paransasfunc:
                for i, t in enumerate(tokens):
                    if '(' == t[1]:
                        tokens[i] = ('FUNCTION', 'func(', t[2], t[3])

            if 'default' == typ:
                restokens = b._tokensupto2(tokens)
            elif 'blockstartonly' == typ:
                restokens = b._tokensupto2(tokens, blockstartonly=True)
            elif 'propertynameendonly' == typ:
                restokens = b._tokensupto2(tokens, propertynameendonly=True)
            elif 'propertyvalueendonly' == typ:
                restokens = b._tokensupto2(tokens, propertyvalueendonly=True)
            elif 'funcendonly' == typ:
                restokens = b._tokensupto2(tokens, funcendonly=True)
            elif 'selectorattendonly' == typ:
                restokens = b._tokensupto2(tokens, selectorattendonly=True)
            elif 'withstarttoken [' == typ:
                restokens = b._tokensupto2(tokens, ('CHAR', '[', 0, 0))

            res = ''.join([t[1] for t in restokens])
            assert exp == res


class _readUrl_TestCase:
    """needs mock"""

    def test_readUrl(self):
        """util._readUrl()"""
        # for additional tests see test_parse.py
        url = 'http://example.com/test.css'

        def make_fetcher(r):
            # normally r == encoding, content
            def fetcher(url):
                return r

            return fetcher

        tests = {
            # defaultFetcher returns: readUrl returns
            None: (None, None, None),
            (None, ''): ('utf-8', 5, ''),
            (None, '€'.encode()): ('utf-8', 5, '€'),
            ('utf-8', '€'.encode()): ('utf-8', 1, '€'),
            ('ISO-8859-1', 'ä'.encode('iso-8859-1')): ('ISO-8859-1', 1, 'ä'),
            ('ASCII', b'a'): ('ASCII', 1, 'a'),
        }

        for r, exp in list(tests.items()):
            assert _readUrl(url, fetcher=make_fetcher(r)) == exp

        tests = {
            # (overrideEncoding, parentEncoding, (httpencoding, content)):
            #                        readUrl returns
            # ===== 0. OVERRIDE WINS =====
            # override + parent + http
            ('latin1', 'ascii', ('utf-16', b'')): ('latin1', 0, ''),
            ('latin1', 'ascii', ('utf-16', b'123')): ('latin1', 0, '123'),
            ('latin1', 'ascii', ('utf-16', 'ä'.encode('iso-8859-1'))): (
                'latin1',
                0,
                'ä',
            ),
            ('latin1', 'ascii', ('utf-16', b'a')): ('latin1', 0, 'a'),
            # + @charset
            ('latin1', 'ascii', ('utf-16', b'@charset "ascii";')): (
                'latin1',
                0,
                '@charset "latin1";',
            ),
            ('latin1', 'ascii', ('utf-16', '@charset "utf-8";ä'.encode('latin1'))): (
                'latin1',
                0,
                '@charset "latin1";ä',
            ),
            ('latin1', 'ascii', ('utf-16', '@charset "utf-8";ä'.encode())): (
                'latin1',
                0,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # override only
            ('latin1', None, None): (None, None, None),
            ('latin1', None, (None, b'')): ('latin1', 0, ''),
            ('latin1', None, (None, b'123')): ('latin1', 0, '123'),
            ('latin1', None, (None, 'ä'.encode('iso-8859-1'))): ('latin1', 0, 'ä'),
            ('latin1', None, (None, b'a')): ('latin1', 0, 'a'),
            # + @charset
            ('latin1', None, (None, b'@charset "ascii";')): (
                'latin1',
                0,
                '@charset "latin1";',
            ),
            ('latin1', None, (None, '@charset "utf-8";ä'.encode('latin1'))): (
                'latin1',
                0,
                '@charset "latin1";ä',
            ),
            ('latin1', None, (None, '@charset "utf-8";ä'.encode())): (
                'latin1',
                0,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # override + parent
            ('latin1', 'ascii', None): (None, None, None),
            ('latin1', 'ascii', (None, b'')): ('latin1', 0, ''),
            ('latin1', 'ascii', (None, b'123')): ('latin1', 0, '123'),
            ('latin1', 'ascii', (None, 'ä'.encode('iso-8859-1'))): ('latin1', 0, 'ä'),
            ('latin1', 'ascii', (None, b'a')): ('latin1', 0, 'a'),
            # + @charset
            ('latin1', 'ascii', (None, b'@charset "ascii";')): (
                'latin1',
                0,
                '@charset "latin1";',
            ),
            ('latin1', 'ascii', (None, '@charset "utf-8";ä'.encode('latin1'))): (
                'latin1',
                0,
                '@charset "latin1";ä',
            ),
            ('latin1', 'ascii', (None, '@charset "utf-8";ä'.encode())): (
                'latin1',
                0,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # override + http
            ('latin1', None, ('utf-16', b'')): ('latin1', 0, ''),
            ('latin1', None, ('utf-16', b'123')): ('latin1', 0, '123'),
            ('latin1', None, ('utf-16', 'ä'.encode('iso-8859-1'))): ('latin1', 0, 'ä'),
            ('latin1', None, ('utf-16', b'a')): ('latin1', 0, 'a'),
            # + @charset
            ('latin1', None, ('utf-16', b'@charset "ascii";')): (
                'latin1',
                0,
                '@charset "latin1";',
            ),
            ('latin1', None, ('utf-16', '@charset "utf-8";ä'.encode('latin1'))): (
                'latin1',
                0,
                '@charset "latin1";ä',
            ),
            ('latin1', None, ('utf-16', '@charset "utf-8";ä'.encode())): (
                'latin1',
                0,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # override ü @charset
            ('latin1', None, (None, b'@charset "ascii";')): (
                'latin1',
                0,
                '@charset "latin1";',
            ),
            ('latin1', None, (None, '@charset "utf-8";ä'.encode('latin1'))): (
                'latin1',
                0,
                '@charset "latin1";ä',
            ),
            ('latin1', None, (None, '@charset "utf-8";ä'.encode())): (
                'latin1',
                0,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # ===== 1. HTTP WINS =====
            (None, 'ascii', ('latin1', b'')): ('latin1', 1, ''),
            (None, 'ascii', ('latin1', b'123')): ('latin1', 1, '123'),
            (None, 'ascii', ('latin1', 'ä'.encode('iso-8859-1'))): ('latin1', 1, 'ä'),
            (None, 'ascii', ('latin1', b'a')): ('latin1', 1, 'a'),
            # + @charset
            (None, 'ascii', ('latin1', b'@charset "ascii";')): (
                'latin1',
                1,
                '@charset "latin1";',
            ),
            (None, 'ascii', ('latin1', '@charset "utf-8";ä'.encode('latin1'))): (
                'latin1',
                1,
                '@charset "latin1";ä',
            ),
            (None, 'ascii', ('latin1', '@charset "utf-8";ä'.encode())): (
                'latin1',
                1,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # ===== 2. @charset WINS =====
            (None, 'ascii', (None, b'@charset "latin1";')): (
                'latin1',
                2,
                '@charset "latin1";',
            ),
            (None, 'ascii', (None, '@charset "latin1";ä'.encode('latin1'))): (
                'latin1',
                2,
                '@charset "latin1";ä',
            ),
            (None, 'ascii', (None, '@charset "latin1";ä'.encode())): (
                'latin1',
                2,
                '@charset "latin1";\xc3\xa4',
            ),  # read as latin1!
            # ===== 2. BOM WINS =====
            (None, 'ascii', (None, 'ä'.encode('utf-8-sig'))): (
                'utf-8-sig',
                2,
                '\xe4',
            ),  # read as latin1!
            (None, 'ascii', (None, '@charset "utf-8";ä'.encode('utf-8-sig'))): (
                'utf-8-sig',
                2,
                '@charset "utf-8";\xe4',
            ),  # read as latin1!
            (None, 'ascii', (None, '@charset "latin1";ä'.encode('utf-8-sig'))): (
                'utf-8-sig',
                2,
                '@charset "utf-8";\xe4',
            ),  # read as latin1!
            # ===== 4. parentEncoding WINS =====
            (None, 'latin1', (None, b'')): ('latin1', 4, ''),
            (None, 'latin1', (None, b'123')): ('latin1', 4, '123'),
            (None, 'latin1', (None, 'ä'.encode('iso-8859-1'))): ('latin1', 4, 'ä'),
            (None, 'latin1', (None, b'a')): ('latin1', 4, 'a'),
            (None, 'latin1', (None, 'ä'.encode())): (
                'latin1',
                4,
                '\xc3\xa4',
            ),  # read as latin1!
            # ===== 5. default WINS which in this case is None! =====
            (None, None, (None, b'')): ('utf-8', 5, ''),
            (None, None, (None, b'123')): ('utf-8', 5, '123'),
            (None, None, (None, b'a')): ('utf-8', 5, 'a'),
            (None, None, (None, 'ä'.encode())): (
                'utf-8',
                5,
                'ä',
            ),  # read as utf-8
            (
                None,
                None,
                (None, 'ä'.encode('iso-8859-1')),
            ): (  # trigger UnicodeDecodeError!
                'utf-8',
                5,
                None,
            ),
        }
        for (override, parent, r), exp in list(tests.items()):
            assert (
                _readUrl(
                    url,
                    overrideEncoding=override,
                    parentEncoding=parent,
                    fetcher=make_fetcher(r),
                )
                == exp
            )

    def test_defaultFetcher(self):  # noqa: C901
        """util._defaultFetcher"""

        class Response:
            """urllib2.Reponse mock"""

            def __init__(self, url, contenttype, content, exception=None, args=None):
                self.url = url

                m = Message()
                m['content-type'] = contenttype

                self.mimetype = m.get_content_type()
                self.charset = m.get_param('charset', None)

                self.text = content

                self.exception = exception
                self.args = args

            def geturl(self):
                return self.url

            def info(self):
                mimetype, charset = self.mimetype, self.charset

                class Info:
                    # py2x
                    def gettype(self):
                        return mimetype

                    def getparam(self, name=None):
                        return charset

                    # py 3x
                    get_content_type = gettype
                    get_content_charset = getparam  # here always charset!

                return Info()

            def read(self):
                # returns fake text or raises fake exception
                if not self.exception:
                    return self.text
                else:
                    raise self.exception(*self.args)

        def urlopen(url, contenttype=None, content=None, exception=None, args=None):
            # return an mock which returns parameterized Response
            def x(*ignored):
                if exception:
                    raise exception(*args)
                else:
                    return Response(
                        url, contenttype, content, exception=exception, args=args
                    )

            return x

        urlopenpatch = 'urllib.request.urlopen'

        # positive tests
        tests = {
            # content-type, contentstr: encoding, contentstr
            ('text/css', '€'.encode()): (None, '€'.encode()),
            ('text/css;charset=utf-8', '€'.encode()): (
                'utf-8',
                '€'.encode(),
            ),
            ('text/css;charset=ascii', 'a'): ('ascii', 'a'),
        }
        url = 'http://example.com/test.css'
        for (contenttype, content), exp in list(tests.items()):

            @mock.patch(urlopenpatch, new=urlopen(url, contenttype, content))
            def do(url):
                return _defaultFetcher(url)

            assert exp == do(url)

        # wrong mimetype
        @mock.patch(urlopenpatch, new=urlopen(url, 'text/html', 'a'))
        def do(url):
            return _defaultFetcher(url)

        with pytest.raises(ValueError):
            do(url)

        # calling url results in fake exception

        # py2 ~= py3 raises error earlier than urlopen!
        tests = {
            '1': (ValueError, ['invalid value for url']),
            # _readUrl('mailto:a.css')
            'mailto:e4': (urllib.error.URLError, ['urlerror']),
            # cannot resolve x, IOError
            'http://x': (urllib.error.URLError, ['ioerror']),
        }
        for url, (exception, args) in list(tests.items()):

            @mock.patch(urlopenpatch, new=urlopen(url, exception=exception, args=args))
            def do(url):
                return _defaultFetcher(url)

            with pytest.raises(exception):
                do(url)

        urlrequestpatch = 'urllib.request.Request'
        tests = {
            # _readUrl('http://cthedot.de/__UNKNOWN__.css')
            'e2': (urllib.error.HTTPError, ['u', 500, 'server error', {}, None]),
            'e3': (urllib.error.HTTPError, ['u', 404, 'not found', {}, None]),
        }
        for url, (exception, args) in list(tests.items()):

            @mock.patch(
                urlrequestpatch, new=urlopen(url, exception=exception, args=args)
            )
            def do(url):
                return _defaultFetcher(url)

            with pytest.raises(exception):
                do(url)


class TestLazyRegex:
    """Tests for cssutils.util.LazyRegex."""

    def setup_method(self):
        self.lazyre = LazyRegex('f.o')

    def test_public_interface(self):
        methods = [
            'search',
            'match',
            'split',
            'sub',
            'subn',
            'findall',
            'finditer',
            'pattern',
            'flags',
            'groups',
            'groupindex',
        ]
        for method in methods:
            assert hasattr(self.lazyre, method), 'expected %r public attribute' % method

    def test_ensure(self):
        assert self.lazyre.matcher is None
        self.lazyre.ensure()
        assert self.lazyre.matcher is not None

    def test_calling(self):
        assert self.lazyre('bar') is None
        match = self.lazyre('foobar')
        assert match.group() == 'foo'

    def test_matching(self):
        assert self.lazyre.match('bar') is None
        match = self.lazyre.match('foobar')
        assert match.group() == 'foo'

    def test_matching_with_position_parameters(self):
        assert self.lazyre.match('foo', 1) is None
        assert self.lazyre.match('foo', 0, 2) is None

    def test_searching(self):
        assert self.lazyre.search('rafuubar') is None
        match = self.lazyre.search('rafoobar')
        assert match.group() == 'foo'

    def test_searching_with_position_parameters(self):
        assert self.lazyre.search('rafoobar', 3) is None
        assert self.lazyre.search('rafoobar', 0, 4) is None
        match = self.lazyre.search('rafoofuobar', 4)
        assert match.group() == 'fuo'

    def test_split(self):
        assert self.lazyre.split('rafoobarfoobaz') == ['ra', 'bar', 'baz']
        assert self.lazyre.split('rafoobarfoobaz', 1) == ['ra', 'barfoobaz']

    def test_findall(self):
        assert self.lazyre.findall('rafoobarfuobaz') == ['foo', 'fuo']

    def test_finditer(self):
        result = self.lazyre.finditer('rafoobarfuobaz')
        assert [m.group() for m in result] == ['foo', 'fuo']

    def test_sub(self):
        assert self.lazyre.sub('bar', 'foofoo') == 'barbar'
        assert self.lazyre.sub(lambda x: 'baz', 'foofoo') == 'bazbaz'

    def test_subn(self):
        subbed = self.lazyre.subn('bar', 'foofoo')
        assert subbed == ('barbar', 2)
        subbed = self.lazyre.subn(lambda x: 'baz', 'foofoo')
        assert subbed == ('bazbaz', 2)

    def test_groups(self):
        lazyre = LazyRegex('(.)(.)')
        assert lazyre.groups is None
        lazyre.ensure()
        assert lazyre.groups == 2

    def test_groupindex(self):
        lazyre = LazyRegex('(?P<foo>.)')
        assert lazyre.groupindex is None
        lazyre.ensure()
        assert lazyre.groupindex == {'foo': 1}

    def test_flags(self):
        self.lazyre.ensure()
        assert self.lazyre.flags == re.compile('.').flags

    def test_pattern(self):
        assert self.lazyre.pattern == 'f.o'

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_util.py
NikolaBlagojevic/pyrecodes
"""Testcases for cssutils.css.CSSValue and CSSPrimitiveValue."""

import xml.dom
import cssutils
import types
import pytest


class PropertyValueTestCase:
    def setup(self):
        self.r = cssutils.css.PropertyValue()

    def test_init(self):
        "PropertyValue.__init__() .item() .length"
        pv = cssutils.css.PropertyValue()
        assert '' == pv.cssText
        assert 0 == pv.length
        assert '' == pv.value

        cssText = '0, 0/0 1px var(x) url(x)'
        items = ['0', '0', '0', '1px', 'var(x)', 'url(x)']
        pv = cssutils.css.PropertyValue(cssText)
        assert cssText == pv.cssText
        assert 6 == len(pv)
        assert 6 == pv.length

        # __iter__
        for i, x in enumerate(pv):
            assert x.cssText == items[i]

        # cssText
        for i, item in enumerate(items):
            assert item == pv[i].cssText
            assert item == pv.item(i).cssText

    def test_cssText(self):
        "PropertyValue.cssText"
        tests = {
            '0': (None, 1, None),
            '0 0': (None, 2, None),
            '0, 0': (None, 2, None),
            '0,0': ('0, 0', 2, None),
            '0  ,   0': ('0, 0', 2, None),
            '0/0': (None, 2, None),
            '/**/ 0 /**/': (None, 1, '0'),
            '0 /**/ 0 /**/ 0': (None, 3, '0 0 0'),
            '0, /**/ 0, /**/ 0': (None, 3, '0, 0, 0'),
            '0//**/ 0//**/ 0': (None, 3, '0/0/0'),
            '/**/ red': (None, 1, 'red'),
            '/**/red': ('/**/ red', 1, 'red'),
            'red /**/': (None, 1, 'red'),
            'red/**/': ('red /**/', 1, 'red'),
            'a()1,-1,+1,1%,-1%,1px,-1px,"a",a,url(a),#aabb44': (
                'a() 1, -1, +1, 1%, -1%, 1px, -1px, "a", a, url(a), #ab4',
                12,
                'a() 1, -1, +1, 1%, -1%, 1px, -1px, "a", a, url(a), #ab4',
            ),
            # calc values
            'calc(1)': (None, 1, 'calc(1)'),
            'calc( 1)': ('calc(1)', 1, 'calc(1)'),
            'calc(1 )': ('calc(1)', 1, 'calc(1)'),
            'calc(1px)': (None, 1, 'calc(1px)'),
            'calc(1p-x-)': (None, 1, 'calc(1p-x-)'),
            'calc(1%)': (None, 1, 'calc(1%)'),
            'calc(-1)': (None, 1, 'calc(-1)'),
            'calc(+1)': (None, 1, 'calc(+1)'),
            'calc(1  +   1px)': ('calc(1 + 1px)', 1, 'calc(1 + 1px)'),
            'calc(1 - 1px)': (None, 1, 'calc(1 - 1px)'),
            'calc(1*1px)': ('calc(1 * 1px)', 1, 'calc(1 * 1px)'),
            'calc(1  /  1px)': ('calc(1 / 1px)', 1, 'calc(1 / 1px)'),
            'calc( 1*1px)': ('calc(1 * 1px)', 1, 'calc(1 * 1px)'),
            'calc( 1  /  1px)': ('calc(1 / 1px)', 1, 'calc(1 / 1px)'),
            'calc(1*1px )': ('calc(1 * 1px)', 1, 'calc(1 * 1px)'),
            'calc(1  /  1px )': ('calc(1 / 1px)', 1, 'calc(1 / 1px)'),
            'calc( 1*1px )': ('calc(1 * 1px)', 1, 'calc(1 * 1px)'),
            'calc( 1  /  1px )': ('calc(1 / 1px)', 1, 'calc(1 / 1px)'),
            'calc(calc(1px + 5px) * 4)': (
                'calc(calc(1px + 5px) * 4)',
                1,
                'calc(calc(1px + 5px) * 4)',
            ),
            'calc( calc(1px + 5px)*4 )': (
                'calc(calc(1px + 5px) * 4)',
                1,
                'calc(calc(1px + 5px) * 4)',
            ),
            'calc(var(X))': (None, 1, None),
            'calc(2 * var(X))': (None, 1, None),
            'calc(2px + var(X))': (None, 1, None),
            # issue #24
            'rgb(0, 10, 255)': (None, 1, 'rgb(0, 10, 255)'),
            'hsl(10, 10%, 25%)': (None, 1, 'hsl(10, 10%, 25%)'),
            'rgba(0, 10, 255, 0.5)': (None, 1, 'rgba(0, 10, 255, 0.5)'),
            'hsla(10, 10%, 25%, 0.5)': (None, 1, 'hsla(10, 10%, 25%, 0.5)'),
            # issue #27
            'matrix(0.000092, 0.2500010, -0.250000, 0.000092, 0, 0)': (
                'matrix(0.000092, 0.250001, -0.25, 0.000092, 0, 0)',
                1,
                'matrix(0.000092, 0.250001, -0.25, 0.000092, 0, 0)',
            ),
        }
        for cssText, (c, len, v) in list(tests.items()):
            if c is None:
                c = cssText
            if v is None:
                v = c

            pv = cssutils.css.PropertyValue(cssText)
            assert c == pv.cssText
            assert len == pv.length
            assert v == pv.value

        tests = {
            '0 0px -0px +0px': ('0 0 0 0', 4),
            '1 2 3 4': (None, 4),
            '-1 -2 -3 -4': (None, 4),
            '-1 2': (None, 2),
            '-1px red "x"': (None, 3),
            'a, b c': (None, 3),
            '1px1 2% 3': ('1px1 2% 3', 3),
            'f(+1pX, -2, 5%) 1': ('f(+1px, -2, 5%) 1', 2),
            '0 f()0': ('0 f() 0', 3),
            'f()0': ('f() 0', 2),
            'f()1%': ('f() 1%', 2),
            'f()1px': ('f() 1px', 2),
            'f()"str"': ('f() "str"', 2),
            'f()ident': ('f() ident', 2),
            'f()#123': ('f() #123', 2),
            'f()url()': ('f() url()', 2),
            'f()f()': ('f() f()', 2),
            'url(x.gif)0 0': ('url(x.gif) 0 0', 3),
            'url(x.gif)no-repeat': ('url(x.gif) no-repeat', 2),
        }
        for cssText, (c, len) in list(tests.items()):
            if c is None:
                c = cssText
            pv = cssutils.css.PropertyValue(cssText)
            assert c == pv.cssText
            assert len == pv.length

        tests = {
            # hash and rgb/a
            '#112234': '#112234',
            '#112233': '#123',
            'rgb(1,2,3)': 'rgb(1, 2, 3)',
            'rgb(  1  ,  2  ,  3  )': 'rgb(1, 2, 3)',
            'rgba(1,2,3,4)': 'rgba(1, 2, 3, 4)',
            'rgba(  1  ,  2  ,  3  ,  4 )': 'rgba(1, 2, 3, 4)',
            'rgb(-1,+2,0)': 'rgb(-1, +2, 0)',
            'rgba(-1,+2,0, 0)': 'rgba(-1, +2, 0, 0)',
            # FUNCTION
            'f(1,2)': 'f(1, 2)',
            'f(  1  ,  2  )': 'f(1, 2)',
            'f(-1,+2)': 'f(-1, +2)',
            'f(  -1  ,  +2  )': 'f(-1, +2)',
            'fun(  -1  ,  +2  )': 'fun(-1, +2)',
            'local( x )': 'local(x)',
            'test(1px, #111, y, 1, 1%, "1", y(), var(x))': 'test(1px, #111, y, 1, 1%, "1", y(), var(x))',
            'test(-1px, #111, y, -1, -1%, "1", -y())': 'test(-1px, #111, y, -1, -1%, "1", -y())',
            'url(y)  format( "x" ,  "y" )': 'url(y) format("x", "y")',
            'f(1 2,3 4)': 'f(1 2, 3 4)',
            # IE expression
            r'Expression()': 'Expression()',
            r'expression(-1 < +2)': 'expression(-1<+2)',
            r'expression(document.width == "1")': 'expression(document.width=="1")',
            'alpha(opacity=80)': 'alpha(opacity=80)',
            'alpha( opacity = 80 , x=2  )': 'alpha(opacity=80, x=2)',
            'expression(eval(document.documentElement.scrollTop))': 'expression(eval(document.documentElement.scrollTop))',
            # TODO
            #            u'expression((function(ele){ele.style.behavior="none";})(this))':
            #                u'expression((function(ele){ele.style.behavior="none";})(this))',
            # unicode-range
            'u+f': 'u+f',
            'U+ABCdef': 'u+abcdef',
            # url
            'url(a)': 'url(a)',
            'uRl(a)': 'url(a)',
            'u\\rl(a)': 'url(a)',
            'url("a")': 'url(a)',
            'url(  "a"  )': 'url(a)',
            'url(";")': 'url(";")',
            'url(",")': 'url(",")',
            'url(")")': 'url(")")',
            '''url("'")''': '''url("'")''',
            '''url('"')''': '''url("\\"")''',
            # operator
            '1': '1',
            '1 2': '1 2',
            '1   2': '1 2',
            '1,2': '1, 2',
            '1,  2': '1, 2',
            '1  ,2': '1, 2',
            '1  ,  2': '1, 2',
            '1/2': '1/2',
            '1/  2': '1/2',
            '1  /2': '1/2',
            '1  /  2': '1/2',
            # comment
            '1/**/2': '1 /**/ 2',
            '1 /**/2': '1 /**/ 2',
            '1/**/ 2': '1 /**/ 2',
            '1 /**/ 2': '1 /**/ 2',
            '1  /*a*/  /*b*/  2': '1 /*a*/ /*b*/ 2',
            # , before
            '1,/**/2': '1, /**/ 2',
            '1 ,/**/2': '1, /**/ 2',
            '1, /**/2': '1, /**/ 2',
            '1 , /**/2': '1, /**/ 2',
            # , after
            '1/**/,2': '1 /**/, 2',
            '1/**/ ,2': '1 /**/, 2',
            '1/**/, 2': '1 /**/, 2',
            '1/**/ , 2': '1 /**/, 2',
            # all
            '1/*a*/  ,/*b*/  2': '1 /*a*/, /*b*/ 2',
            '1  /*a*/,  /*b*/2': '1 /*a*/, /*b*/ 2',
            '1  /*a*/  ,  /*b*/  2': '1 /*a*/, /*b*/ 2',
            # list
            'a b1,b2 b2,b3,b4': 'a b1, b2 b2, b3, b4',
            'a b1  ,   b2   b2  ,  b3  ,   b4': 'a b1, b2 b2, b3, b4',
            'u+1  ,   u+2-5': 'u+1, u+2-5',
            'local( x ),  url(y)  format( "x" ,  "y" )': 'local(x), url(y) format("x", "y")',
            # FUNCTION
            'attr( href )': 'attr(href)',
            # PrinceXML extende FUNC syntax with nested FUNC
            'target-counter(attr(href),page)': 'target-counter(attr(href), page)',
        }
        self.do_equal_r(tests)

        tests = [
            'a+',
            '-',
            '+',
            '-%',
            '+a',
            '--1px',
            '++1px',
            '#',
            '#00',
            '#12x',
            '#xyz',
            '#0000',
            '#00000',
            '#0000000',
            '-#0',
            # operator
            ',',
            '1,,2',
            '1,/**/,2',
            '1  ,  /**/  ,  2',
            '1,',
            '1, ',
            '1 ,',
            '1 , ',
            '1  ,  ',
            '1//2',
            # URL
            'url(x))',
            # string
            '"',
            "'",
            # function
            'f(-)',
            'f(x))',
            # calc
            'calc(',
            'calc(1',
            'calc(1 + 1',
            'calc(1+1)',
            'calc(1-1)',
            'calc(1 +1)',
            'calc(1+ 1)',
            'calc(1 -1)',
            'calc(1- 1)',
            'calc(+)',
            'calc(+ 1)',
            'calc(-)',
            'calc(- 1)',
            'calc(*)',
            'calc(*1)',
            'calc(* 2)',
            'calc(/)',
            'calc(/1)',
            'calc(/ 2)',
            'calc(1+)',
            'calc(1 +)',
            'calc(1 + )',
            'calc(2px -)',
            'calc(3px*)',
            'calc(3px *)',
            'calc(3px * )',
            'calc(4em/)',
            'calc(4em /)',
            'calc(4em / )',
            'calc(1 + + 1)',
            'calc(1 ++ 1)',
        ]
        self.do_raise_r_list(tests, xml.dom.SyntaxErr)

    def test_list(self):
        "PropertyValue[index]"
        # issue #41
        css = """div.one {color: rgb(255, 0, 0);}   """
        sheet = cssutils.parseString(css)
        pv = sheet.cssRules[0].style.getProperty('color').propertyValue
        assert pv.value == 'rgb(255, 0, 0)'
        assert pv[0].value == 'rgb(255, 0, 0)'

        # issue #42
        sheet = cssutils.parseString('body { font-family: "A", b, serif }')
        pv = sheet.cssRules[0].style.getProperty('font-family').propertyValue
        assert 3 == pv.length
        assert pv[0].value == 'A'
        assert pv[1].value == 'b'
        assert pv[2].value == 'serif'

    def test_comments(self):
        "PropertyValue with comment"
        # issue #45
        for t in (
            'green',
            'green /* comment */',
            '/* comment */green',
            '/* comment */green/* comment */',
            '/* comment */  green  /* comment */',
            '/* comment *//**/  green  /* comment *//**/',
        ):
            sheet = cssutils.parseString('body {color: %s; }' % t)
            p = sheet.cssRules[0].style.getProperties()[0]
            assert p.valid
        for t in (
            'gree',
            'gree /* comment */',
            '/* comment */gree',
            '/* comment */gree/* comment */',
            '/* comment */  gree  /* comment */',
            '/* comment *//**/  gree  /* comment *//**/',
        ):
            sheet = cssutils.parseString('body {color: %s; }' % t)
            p = sheet.cssRules[0].style.getProperties()[0]
            assert not p.valid

    def test_incomplete(self):
        "PropertyValue (incomplete)"
        tests = {'url("a': 'url(a)', 'url(a': 'url(a)'}
        for v, exp in list(tests.items()):
            s = cssutils.parseString('a { background: %s' % v)
            v = s.cssRules[0].style.background
            assert v == exp

    def test_readonly(self):
        "PropertyValue._readonly"
        v = cssutils.css.PropertyValue(cssText='inherit')
        assert False is v._readonly

        v = cssutils.css.PropertyValue(cssText='inherit', readonly=True)
        assert True is v._readonly
        assert 'inherit', v.cssText
        with pytest.raises(xml.dom.NoModificationAllowedErr):
            v._setCssText('x')
        assert 'inherit', v.cssText

    def test_reprANDstr(self):
        "PropertyValue.__repr__(), .__str__()"
        cssText = 'inherit'

        s = cssutils.css.PropertyValue(cssText=cssText)

        assert cssText in str(s)

        s2 = eval(repr(s))
        assert isinstance(s2, s.__class__)
        assert cssText == s2.cssText


class ValueTestCase:
    def test_init(self):
        "Value.__init__()"
        v = cssutils.css.Value()
        assert '' == v.cssText
        assert '' == v.value
        assert None is v.type

    def test_cssText(self):
        "Value.cssText"
        # HASH IDENT STRING UNICODE-RANGE
        tests = {
            '#123': ('#123', '#123', 'HASH'),
            '#123456': ('#123456', '#123456', 'HASH'),
            '#112233': ('#123', '#112233', 'HASH'),
            '  #112233  ': ('#123', '#112233', 'HASH'),
            'red': ('red', 'red', 'IDENT'),
            '  red  ': ('red', 'red', 'IDENT'),
            'red  ': ('red', 'red', 'IDENT'),
            '  red': ('red', 'red', 'IDENT'),
            'red-': ('red-', 'red-', 'IDENT'),
            '-red': ('-red', '-red', 'IDENT'),
            '"red"': ('"red"', 'red', 'STRING'),
            "'red'": ('"red"', 'red', 'STRING'),
            '  "red"  ': ('"red"', 'red', 'STRING'),
            r'"red\""': (r'"red\""', r'red"', 'STRING'),
            r"'x\"'": (r'"x\\""', r'x\"', 'STRING'),  # ???
            '''"x\
y"''': (
                '"xy"',
                'xy',
                'STRING',
            ),
        }
        for p, (r, n, t) in list(tests.items()):
            v = cssutils.css.Value(p)
            assert r == v.cssText
            assert t == v.type
            assert n == v.value


class ColorValueTestCase:
    def test_init(self):
        "ColorValue.__init__()"
        v = cssutils.css.ColorValue()
        assert v.COLOR_VALUE == v.type
        assert '' == v.cssText
        assert '' == v.value
        assert 'transparent' == v.name
        assert v.colorType is None

    def test_cssText(self):
        "ColorValue.cssText"
        tests = {
            # HASH
            '#123': ('#123',),
            '#112233': ('#123',),
            # rgb
            'rgb(1,2,3)': ('rgb(1, 2, 3)',),
            'rgb(1%,2%,3%)': ('rgb(1%, 2%, 3%)',),
            'rgb(-1,-1,-1)': ('rgb(-1, -1, -1)',),
            'rgb(-1%,-2%,-3%)': ('rgb(-1%, -2%, -3%)',),
            # rgba
            'rgba(1,2,3, 0)': ('rgba(1, 2, 3, 0)',),
            # hsl
            'hsl(1,2%,3%)': ('hsl(1, 2%, 3%)',),
            'hsla(1,2%,3%, 1.0)': ('hsla(1, 2%, 3%, 1)',),
        }
        for p, (r,) in list(tests.items()):
            v = cssutils.css.ColorValue(p)
            assert v.COLOR_VALUE == v.type
            assert r == v.cssText
            assert r == v.value

            v = cssutils.css.ColorValue()
            v.cssText = p
            assert v.COLOR_VALUE == v.type
            assert r == v.cssText
            assert r == v.value

        tests = {
            '1': xml.dom.SyntaxErr,
            'a': xml.dom.SyntaxErr,
            '#12': xml.dom.SyntaxErr,
            '#1234': xml.dom.SyntaxErr,
            '#1234567': xml.dom.SyntaxErr,
            '#12345678': xml.dom.SyntaxErr,
            'rgb(1,1%,1%)': xml.dom.SyntaxErr,
            'rgb(1%,1,1)': xml.dom.SyntaxErr,
            'rgb(-1,-1%,-1%)': xml.dom.SyntaxErr,
            'rgb(-1%,-1,-1)': xml.dom.SyntaxErr,
            'rgb(1,1,1, 0)': xml.dom.SyntaxErr,
            'rgb(1%,1%,1%, 0)': xml.dom.SyntaxErr,
            'rgba(1,1,1)': xml.dom.SyntaxErr,
            'rgba(1%,1%,1%)': xml.dom.SyntaxErr,
            'rgba(1,1,1, 0%)': xml.dom.SyntaxErr,
            'rgba(1%,1%,1%, 0%)': xml.dom.SyntaxErr,
            'hsl(1,2%,3%, 1)': xml.dom.SyntaxErr,
            'hsla(1,2%,3%)': xml.dom.SyntaxErr,
            'hsl(1,2,3)': xml.dom.SyntaxErr,
            'hsl(1%,2,3)': xml.dom.SyntaxErr,
            'hsl(1%,2,3%)': xml.dom.SyntaxErr,
            'hsl(1%,2%,3)': xml.dom.SyntaxErr,
            'hsla(1,2%,3%, 0%)': xml.dom.SyntaxErr,
            'hsla(1,2,3, 0.0)': xml.dom.SyntaxErr,
            'hsla(1%,2,3, 0.0)': xml.dom.SyntaxErr,
            'hsla(1%,2,3%, 0.0)': xml.dom.SyntaxErr,
            'hsla(1%,2%,3, 0.0)': xml.dom.SyntaxErr,
        }
        self.r = cssutils.css.ColorValue()
        self.do_raise_r(tests)

    def test_rgb(self):
        "ColorValue.red .green .blue"
        tests = {
            ('#0A0AD2', 'rgb(10, 10, 210)'): (10, 10, 210, 1.0),
            # TODO: Fix rounding?
            ('hsl(240, 91%, 43%)',): (10, 10, 209, 1.0),
            ('#ff8800', '#f80', 'rgb(255, 136, 0)', 'rgba(255, 136, 0, 1.0)'): (
                255,
                136,
                0,
                1.0,
            ),
            (
                'red',
                '#ff0000',
                '#f00',
                'hsl(0, 100%, 50%)',
                'hsla(0, 100%, 50%, 1.0)',
            ): (255, 0, 0, 1.0),
            ('lime', '#00ff00', '#0f0', 'hsl(120, 100%, 50%)'): (0, 255, 0, 1.0),
            ('rgba(255, 127, 0, .1)', 'rgba(100%, 50%, 0%, .1)'): (255, 127, 0, 0.1),
            ('transparent', 'rgba(0, 0, 0, 0)'): (0, 0, 0, 0),
            ('aqua',): (0, 255, 255, 1.0),
        }
        for colors, rgba in list(tests.items()):
            for color in colors:
                c = cssutils.css.ColorValue(color)
                assert c.red == rgba[0]
                assert c.green == rgba[1]
                assert c.blue == rgba[2]
                assert c.alpha == rgba[3]


class URIValueTestCase:
    def test_init(self):
        "URIValue.__init__()"
        v = cssutils.css.URIValue()
        assert 'url()' == v.cssText
        assert '' == v.value
        assert '' == v.uri
        assert v.URI is v.type

        v.uri = '1'
        assert '1' == v.value
        assert '1' == v.uri
        assert 'url(1)' == v.cssText

        v.value = '2'
        assert '2' == v.value
        assert '2' == v.uri
        assert 'url(2)' == v.cssText

    def test_absoluteUri(self):
        "URIValue.absoluteUri"
        s = cssutils.parseString(
            'a { background-image: url(x.gif)}', href="/path/to/x.css"
        )
        v = s.cssRules[0].style.getProperty('background-image').propertyValue[0]
        assert 'x.gif' == v.uri
        assert '/path/to/x.gif' == v.absoluteUri

        v = cssutils.css.URIValue('url(x.gif)')
        assert 'x.gif' == v.uri
        assert 'x.gif' == v.absoluteUri

    def test_cssText(self):
        "URIValue.cssText"
        tests = {
            'url()': ('url()', '', 'URI'),
            # comments are part of the url!
            'url(/**/)': ('url(/**/)', '/**/', 'URI'),
            'url(/**/1)': ('url(/**/1)', '/**/1', 'URI'),
            'url(1/**/)': ('url(1/**/)', '1/**/', 'URI'),
            'url(/**/1/**/)': ('url(/**/1/**/)', '/**/1/**/', 'URI'),
            'url(some.gif)': ('url(some.gif)', 'some.gif', 'URI'),
            '  url(some.gif)  ': ('url(some.gif)', 'some.gif', 'URI'),
            'url(   some.gif  )': ('url(some.gif)', 'some.gif', 'URI'),
        }
        for p, (r, n, t) in list(tests.items()):
            v = cssutils.css.URIValue(p)
            assert r == v.cssText
            assert t == v.type
            assert n == v.value
            assert n == v.uri

            v = cssutils.css.URIValue()
            v.cssText = p
            assert r == v.cssText
            assert t == v.type
            assert n == v.value
            assert n == v.uri

        tests = {
            'a()': xml.dom.SyntaxErr,
            '1': xml.dom.SyntaxErr,
            'url(': xml.dom.SyntaxErr,
            'url("': xml.dom.SyntaxErr,
            'url(\'': xml.dom.SyntaxErr,
        }
        self.r = cssutils.css.URIValue()
        self.do_raise_r(tests)


class DimensionValueTestCase:
    def test_init(self):
        "DimensionValue.__init__()"
        v = cssutils.css.DimensionValue()
        assert '' == v.cssText
        assert '' == v.value
        assert None is v.type
        assert None is v.dimension

    def test_cssText(self):
        "DimensionValue.cssText"
        # NUMBER DIMENSION PERCENTAGE
        tests = {
            '0': ('0', 0, None, 'NUMBER'),
            '00': ('0', 0, None, 'NUMBER'),
            '.0': ('0', 0, None, 'NUMBER'),
            '0.0': ('0', 0, None, 'NUMBER'),
            '+0': ('0', 0, None, 'NUMBER'),
            '+00': ('0', 0, None, 'NUMBER'),
            '+.0': ('0', 0, None, 'NUMBER'),
            '+0.0': ('0', 0, None, 'NUMBER'),
            '-0': ('0', 0, None, 'NUMBER'),
            '-00': ('0', 0, None, 'NUMBER'),
            '-.0': ('0', 0, None, 'NUMBER'),
            '-0.0': ('0', 0, None, 'NUMBER'),
            '1': ('1', 1, None, 'NUMBER'),
            '1.0': ('1', 1.0, None, 'NUMBER'),
            '1.1': ('1.1', 1.1, None, 'NUMBER'),
            '+1': ('+1', 1, None, 'NUMBER'),
            '+1.0': ('+1', 1.0, None, 'NUMBER'),
            '+1.1': ('+1.1', 1.1, None, 'NUMBER'),
            '-1': ('-1', -1, None, 'NUMBER'),
            '-1.0': ('-1', -1, None, 'NUMBER'),
            '-1.1': ('-1.1', -1.1, None, 'NUMBER'),
            '0px': ('0', 0, 'px', 'DIMENSION'),
            '1px': ('1px', 1, 'px', 'DIMENSION'),
            '1.0px': ('1px', 1.0, 'px', 'DIMENSION'),
            '1.1px': ('1.1px', 1.1, 'px', 'DIMENSION'),
            '-1px': ('-1px', -1, 'px', 'DIMENSION'),
            '-1.1px': ('-1.1px', -1.1, 'px', 'DIMENSION'),
            '+1px': ('+1px', 1, 'px', 'DIMENSION'),
            '1px1': ('1px1', 1, 'px1', 'DIMENSION'),
            '0%': ('0%', 0, '%', 'PERCENTAGE'),
            '1%': ('1%', 1, '%', 'PERCENTAGE'),
            '1.1%': ('1.1%', 1.1, '%', 'PERCENTAGE'),
            '-1%': ('-1%', -1, '%', 'PERCENTAGE'),
            '-1.1%': ('-1.1%', -1.1, '%', 'PERCENTAGE'),
            '+1%': ('+1%', 1, '%', 'PERCENTAGE'),
        }
        for p, (r, n, d, t) in list(tests.items()):
            v = cssutils.css.DimensionValue(p)
            assert r == v.cssText
            assert t == v.type
            assert n == v.value
            assert d == v.dimension


class CSSFunctionTestCase:
    def test_init(self):
        "CSSFunction.__init__()"
        v = cssutils.css.CSSFunction()
        assert '' == v.cssText
        assert 'FUNCTION' == v.type
        assert v.value == ''

    def test_cssText(self):
        "CSSFunction.cssText"
        tests = {
            'x(x)': ('x(x)', None),
            'X(  X  )': ('x(X)', None),
            'x(1,2)': ('x(1, 2)', None),
            'x(1/**/)': ('x(1 /**/)', 'x(1)'),
            'x(/**/1)': ('x(/**/ 1)', 'x(1)'),
            'x(/**/1/**/)': ('x(/**/ 1 /**/)', 'x(1)'),
            'x(/**/1,x/**/)': ('x(/**/ 1, x /**/)', 'x(1, x)'),
        }
        for f, (cssText, value) in list(tests.items()):
            if value is None:
                value = cssText
            v = cssutils.css.CSSFunction(f)
            assert cssText == v.cssText
            assert 'FUNCTION' == v.type
            assert value == v.value


class CSSVariableTestCase:
    def test_init(self):
        "CSSVariable.__init__()"
        v = cssutils.css.CSSVariable()
        assert '' == v.cssText
        assert 'VARIABLE' == v.type
        assert None is v.name
        assert None is v.value

    def test_cssText(self):
        "CSSVariable.cssText"
        tests = {
            'var(x)': ('var(x)', 'x', None),
            'VAR(  X  )': ('var(X)', 'X', None),
            'var(c1,rgb(14,14,14))': (
                'var(c1, rgb(14, 14, 14))',
                'c1',
                'rgb(14, 14, 14)',
            ),
            'var( L, 1px )': ('var(L, 1px)', 'L', '1px'),
            'var(L,1)': ('var(L, 1)', 'L', '1'),
            'var(T, calc( 2 * 1px ))': ('var(T, calc(2 * 1px))', 'T', 'calc(2 * 1px)'),
            'var(U, url( example.png ) )': (
                'var(U, url(example.png))',
                'U',
                'url(example.png)',
            ),
            'var(C, #f00 )': ('var(C, #f00)', 'C', '#fff'),
        }
        for var, (cssText, name, fallback) in list(tests.items()):
            v = cssutils.css.CSSVariable(var)
            assert cssText == v.cssText
            assert 'VARIABLE' == v.type
            assert name == v.name
            # not resolved so it is None
            assert v.value is None

    @pytest.mark.xfail(reason="not implemented")
    def test_cssValueType(self):
        "CSSValue.cssValueType .cssValueTypeString"
        tests = [
            (['inherit', 'INhe\\rit'], 'CSS_INHERIT', cssutils.css.CSSValue),
            (
                [
                    '1',
                    '1%',
                    '1em',
                    '1ex',
                    '1px',
                    '1cm',
                    '1mm',
                    '1in',
                    '1pt',
                    '1pc',
                    '1deg',
                    '1rad',
                    '1grad',
                    '1ms',
                    '1s',
                    '1hz',
                    '1khz',
                    '1other',
                    '"string"',
                    "'string'",
                    'url(x)',
                    'red',
                    'attr(a)',
                    'counter(x)',
                    'rect(1px, 2px, 3px, 4px)',
                    'rgb(0, 0, 0)',
                    '#000',
                    '#123456',
                    'rgba(0, 0, 0, 0)',
                    'hsl(0, 0, 0)',
                    'hsla(0, 0, 0, 0)',
                ],
                'CSS_PRIMITIVE_VALUE',
                cssutils.css.CSSPrimitiveValue,
            ),
            (
                ['1px 1px', 'red blue green x'],
                'CSS_VALUE_LIST',
                cssutils.css.CSSValueList,
            ),
            # what is a custom value?
            # ([], 'CSS_CUSTOM', cssutils.css.CSSValue)
        ]
        for values, name, cls in tests:
            for value in values:
                v = cssutils.css.CSSValue(cssText=value)
                if value == "'string'":
                    # will be changed to " always
                    value = '"string"'
                assert value == v.cssText
                assert name == v.cssValueTypeString
                assert getattr(v, name) == v.cssValueType
                assert cls == type(v)


@pytest.mark.xfail(reason="not implemented")
class CSSPrimitiveValueTestCase:
    def test_init(self):
        "CSSPrimitiveValue.__init__()"
        v = cssutils.css.CSSPrimitiveValue('1')
        assert '1' == v.cssText

        assert v.CSS_PRIMITIVE_VALUE == v.cssValueType
        assert "CSS_PRIMITIVE_VALUE" == v.cssValueTypeString

        assert v.CSS_NUMBER == v.primitiveType
        assert "CSS_NUMBER" == v.primitiveTypeString

        # DUMMY to be able to test empty constructor call
        # self.assertRaises(xml.dom.SyntaxErr, v.__init__, None)

        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getCounterValue()
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getRGBColorValue()
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getRectValue()
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getStringValue()

    def test_CSS_UNKNOWN(self):
        "CSSPrimitiveValue.CSS_UNKNOWN"
        v = cssutils.css.CSSPrimitiveValue('expression(false)')
        assert v.CSS_UNKNOWN == v.primitiveType
        assert 'CSS_UNKNOWN' == v.primitiveTypeString

    def test_CSS_NUMBER_AND_OTHER_DIMENSIONS(self):
        "CSSPrimitiveValue.CSS_NUMBER .. CSS_DIMENSION"
        defs = [
            ('', 'CSS_NUMBER'),
            ('%', 'CSS_PERCENTAGE'),
            ('em', 'CSS_EMS'),
            ('ex', 'CSS_EXS'),
            ('px', 'CSS_PX'),
            ('cm', 'CSS_CM'),
            ('mm', 'CSS_MM'),
            ('in', 'CSS_IN'),
            ('pt', 'CSS_PT'),
            ('pc', 'CSS_PC'),
            ('deg', 'CSS_DEG'),
            ('rad', 'CSS_RAD'),
            ('grad', 'CSS_GRAD'),
            ('ms', 'CSS_MS'),
            ('s', 'CSS_S'),
            ('hz', 'CSS_HZ'),
            ('khz', 'CSS_KHZ'),
            ('other_dimension', 'CSS_DIMENSION'),
        ]
        for dim, name in defs:
            for n in (0, 1, 1.1, -1, -1.1, -0):
                v = cssutils.css.CSSPrimitiveValue('%i%s' % (n, dim))
                assert name == v.primitiveTypeString
                assert getattr(v, name) == v.primitiveType

    def test_CSS_STRING_AND_OTHER(self):
        "CSSPrimitiveValue.CSS_STRING .. CSS_RGBCOLOR"
        defs = [
            (
                (
                    '""',
                    "''",
                    '"some thing"',
                    "' A\\ND '",
                    # comma separated lists are STRINGS FOR NOW!
                    'a, b',
                    '"a", "b"',
                ),
                'CSS_STRING',
            ),
            (('url(a)', 'url("a b")', "url(' ')"), 'CSS_URI'),
            (('some', 'or_anth-er'), 'CSS_IDENT'),
            (('attr(a)', 'attr(b)'), 'CSS_ATTR'),
            (('counter(1)', 'counter(2)'), 'CSS_COUNTER'),
            (('rect(1,2,3,4)',), 'CSS_RECT'),
            (('rgb(1,2,3)', 'rgb(10%, 20%, 30%)', '#123', '#123456'), 'CSS_RGBCOLOR'),
            (
                (
                    'rgba(1,2,3,4)',
                    'rgba(10%, 20%, 30%, 40%)',
                ),
                'CSS_RGBACOLOR',
            ),
            (('U+0', 'u+ffffff', 'u+000000-f', 'u+0-f, U+ee-ff'), 'CSS_UNICODE_RANGE'),
        ]

        for examples, name in defs:
            for x in examples:
                v = cssutils.css.CSSPrimitiveValue(x)
                assert getattr(v, name) == v.primitiveType
                assert name == v.primitiveTypeString

    def test_getFloat(self):
        "CSSPrimitiveValue.getFloatValue()"
        # NOT TESTED are float values as it seems difficult to
        # compare these. Maybe use decimal.Decimal?

        v = cssutils.css.CSSPrimitiveValue('1px')
        tests = {
            '0': (v.CSS_NUMBER, 0),
            '-1.1': (v.CSS_NUMBER, -1.1),
            '1%': (v.CSS_PERCENTAGE, 1),
            '-1%': (v.CSS_PERCENTAGE, -1),
            '1em': (v.CSS_EMS, 1),
            '-1.1em': (v.CSS_EMS, -1.1),
            '1ex': (v.CSS_EXS, 1),
            '1px': (v.CSS_PX, 1),
            '1cm': (v.CSS_CM, 1),
            # '1cm': (v.CSS_MM, 10),
            '254cm': (v.CSS_IN, 100),
            '1mm': (v.CSS_MM, 1),
            '10mm': (v.CSS_CM, 1),
            '254mm': (v.CSS_IN, 10),
            '1in': (v.CSS_IN, 1),
            '100in': (v.CSS_CM, 254),  # ROUNDED!!!
            '10in': (v.CSS_MM, 254),  # ROUNDED!!!
            '1pt': (v.CSS_PT, 1),
            '1pc': (v.CSS_PC, 1),
            '1deg': (v.CSS_DEG, 1),
            '1rad': (v.CSS_RAD, 1),
            '1grad': (v.CSS_GRAD, 1),
            '1ms': (v.CSS_MS, 1),
            '1000ms': (v.CSS_S, 1),
            '1s': (v.CSS_S, 1),
            # '1s': (v.CSS_MS, 1000),
            '1hz': (v.CSS_HZ, 1),
            '1000hz': (v.CSS_KHZ, 1),
            '1khz': (v.CSS_KHZ, 1),
            # '1khz': (v.CSS_HZ, 1000),
            '1DIMENSION': (v.CSS_DIMENSION, 1),
        }
        for cssText in tests:
            v.cssText = cssText
            unitType, exp = tests[cssText]
            val = v.getFloatValue(unitType)
            if unitType in (v.CSS_IN, v.CSS_CM):
                val = round(val)
            assert val == exp

    def test_setFloat(self):
        "CSSPrimitiveValue.setFloatValue()"
        V = cssutils.css.CSSPrimitiveValue

        tests = {
            # unitType, value
            (V.CSS_NUMBER, 1): [
                # unitType, setvalue,
                #    getvalue or expected exception, msg or cssText
                (V.CSS_NUMBER, 0, 0, '0'),
                (V.CSS_NUMBER, 0.1, 0.1, '0.1'),
                (V.CSS_NUMBER, -0, 0, '0'),
                (V.CSS_NUMBER, 2, 2, '2'),
                (V.CSS_NUMBER, 2.0, 2, '2'),
                (V.CSS_NUMBER, 2.1, 2.1, '2.1'),
                (V.CSS_NUMBER, -2.1, -2.1, '-2.1'),
                # setting with string does work
                (V.CSS_NUMBER, '1', 1, '1'),
                (V.CSS_NUMBER, '1.1', 1.1, '1.1'),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DEG, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_RAD, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_GRAD, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_S, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_MS, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_KHZ, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_HZ, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DIMENSION, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_MM, 2, xml.dom.InvalidAccessErr, None),
                (
                    V.CSS_NUMBER,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: floatValue 'x' is not a float",
                ),
                (
                    V.CSS_NUMBER,
                    '1x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: floatValue '1x' is not a float",
                ),
                (
                    V.CSS_STRING,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_STRING' is not a float type",
                ),
                (
                    V.CSS_URI,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_URI' is not a float type",
                ),
                (
                    V.CSS_ATTR,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_ATTR' is not a float type",
                ),
                (
                    V.CSS_IDENT,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_IDENT' is not a float type",
                ),
                (
                    V.CSS_RGBCOLOR,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_RGBCOLOR' is not a float type",
                ),
                (
                    V.CSS_RGBACOLOR,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_RGBACOLOR' is not a float type",
                ),
                (
                    V.CSS_RECT,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_RECT' is not a float type",
                ),
                (
                    V.CSS_COUNTER,
                    'x',
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: unitType 'CSS_COUNTER' is not a float type",
                ),
                (
                    V.CSS_EMS,
                    1,
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_NUMBER' to 'CSS_EMS'",
                ),
                (
                    V.CSS_EXS,
                    1,
                    xml.dom.InvalidAccessErr,
                    "CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_NUMBER' to 'CSS_EXS'",
                ),
            ],
            (V.CSS_MM, '1mm'): [
                (V.CSS_MM, 2, 2, '2mm'),
                (V.CSS_MM, 0, 0, '0mm'),
                (V.CSS_MM, 0.1, 0.1, '0.1mm'),
                (V.CSS_MM, -0, -0, '0mm'),
                (V.CSS_MM, 3.0, 3, '3mm'),
                (V.CSS_MM, 3.1, 3.1, '3.1mm'),
                (V.CSS_MM, -3.1, -3.1, '-3.1mm'),
                (V.CSS_CM, 1, 10, '10mm'),
                (V.CSS_IN, 10, 254, '254mm'),
                (V.CSS_PT, 1, 1828.8, '1828.8mm'),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_NUMBER, 2, xml.dom.InvalidAccessErr, None),
            ],
            (V.CSS_PT, '1pt'): [
                (V.CSS_PT, 2, 2, '2pt'),
                (V.CSS_PC, 12, 1, '1pt'),
                (V.CSS_NUMBER, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DEG, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
            ],
            (V.CSS_KHZ, '1khz'): [
                (V.CSS_HZ, 2000, 2, '2khz'),
                (V.CSS_NUMBER, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_DEG, 1, xml.dom.InvalidAccessErr, None),
                (V.CSS_PX, 1, xml.dom.InvalidAccessErr, None),
            ],
        }
        for test in tests:
            initialType, initialValue = test
            pv = cssutils.css.CSSPrimitiveValue(initialValue)
            for setType, setValue, exp, cssText in tests[test]:
                if isinstance(exp, types.TypeType):
                    if cssText:
                        self.assertRaisesMsg(
                            exp, cssText, pv.setFloatValue, setType, setValue
                        )
                    else:
                        with pytest.raises(exp):
                            pv.setFloatValue(setType, setValue)
                else:
                    pv.setFloatValue(setType, setValue)
                    assert pv._value[0] == cssText
                    if cssText == '0mm':
                        cssText = '0'
                    assert pv.cssText == cssText
                    assert pv.getFloatValue(initialType) == exp

    def test_getString(self):
        "CSSPrimitiveValue.getStringValue()"
        v = cssutils.css.CSSPrimitiveValue('1px')
        assert v.primitiveType == v.CSS_PX
        with pytest.raises(xml.dom.InvalidAccessErr):
            v.getStringValue()

        pv = cssutils.css.CSSPrimitiveValue
        tests = {
            pv.CSS_STRING: ("'red'", 'red'),
            pv.CSS_STRING: ('"red"', 'red'),
            pv.CSS_URI: ('url(http://example.com)', None),
            pv.CSS_URI: ("url('http://example.com')", "http://example.com"),
            pv.CSS_URI: ('url("http://example.com")', 'http://example.com'),
            pv.CSS_URI: ('url("http://example.com?)")', 'http://example.com?)'),
            pv.CSS_IDENT: ('red', None),
            pv.CSS_ATTR: ('attr(att-name)', 'att-name'),  # the name of the attrr
        }
        for t in tests:
            val, exp = tests[t]
            if not exp:
                exp = val

            v = cssutils.css.CSSPrimitiveValue(val)
            assert v.primitiveType == t
            assert v.getStringValue() == exp

    def test_setString(self):
        "CSSPrimitiveValue.setStringValue()"
        # CSS_STRING
        v = cssutils.css.CSSPrimitiveValue('"a"')
        assert v.CSS_STRING == v.primitiveType
        v.setStringValue(v.CSS_STRING, 'b')
        assert ('b', 'STRING') == v._value
        assert 'b' == v.getStringValue()
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_STRING' to 'CSS_URI'",
        ):
            v.setStringValue(v.CSS_URI, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_STRING' to 'CSS_IDENT'",
        ):
            v.setStringValue(v.CSS_IDENT, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_STRING' to 'CSS_ATTR'",
        ):
            v.setStringValue(v.CSS_ATTR, 'x')

        # CSS_IDENT
        v = cssutils.css.CSSPrimitiveValue('new')
        v.setStringValue(v.CSS_IDENT, 'ident')
        assert v.CSS_IDENT == v.primitiveType
        assert ('ident', 'IDENT') == v._value
        assert 'ident' == v.getStringValue()
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_IDENT' to 'CSS_URI'",
        ):
            v.setStringValue(v.CSS_URI, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_IDENT' to 'CSS_STRING'",
        ):
            v.setStringValue(v.CSS_STRING, '"x"')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_IDENT' to 'CSS_ATTR'",
        ):
            v.setStringValue(v.CSS_ATTR, 'x')

        # CSS_URI
        v = cssutils.css.CSSPrimitiveValue('url(old)')
        v.setStringValue(v.CSS_URI, '(')
        assert ('(', 'URI') == v._value
        assert '(' == v.getStringValue()

        v.setStringValue(v.CSS_URI, ')')
        assert (')', 'URI') == v._value
        assert ')' == v.getStringValue()

        v.setStringValue(v.CSS_URI, '"')
        assert '"' == v.getStringValue()
        assert ('"', 'URI') == v._value

        v.setStringValue(v.CSS_URI, "''")
        assert "''" == v.getStringValue()
        assert ("''", 'URI') == v._value

        v.setStringValue(v.CSS_URI, ',')
        assert ',' == v.getStringValue()
        assert (',', 'URI') == v._value

        v.setStringValue(v.CSS_URI, ' ')
        assert (' ', 'URI') == v._value
        assert ' ' == v.getStringValue()

        v.setStringValue(v.CSS_URI, 'a)')
        assert ('a)', 'URI') == v._value
        assert 'a)' == v.getStringValue()

        v.setStringValue(v.CSS_URI, 'a')
        assert v.CSS_URI == v.primitiveType
        assert ('a', 'URI') == v._value
        assert 'a' == v.getStringValue()

        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_URI' to 'CSS_IDENT'",
        ):
            v.setStringValue(v.CSS_IDENT, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_URI' to 'CSS_STRING'",
        ):
            v.setStringValue(v.CSS_STRING, '"x"')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_URI' to 'CSS_ATTR'",
        ):
            v.setStringValue(v.CSS_ATTR, 'x')

        # CSS_ATTR
        v = cssutils.css.CSSPrimitiveValue('attr(old)')
        v.setStringValue(v.CSS_ATTR, 'a')
        assert v.CSS_ATTR == v.primitiveType
        assert 'a' == v.getStringValue()
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_ATTR' to 'CSS_IDENT'",
        ):
            v.setStringValue(v.CSS_IDENT, 'x')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_ATTR' to 'CSS_STRING'",
        ):
            v.setStringValue(v.CSS_STRING, '"x"')
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: Cannot coerce primitiveType 'CSS_ATTR' to 'CSS_URI'",
        ):
            v.setStringValue(v.CSS_URI, 'x')

        # TypeError as 'x' is no valid type
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: stringType 'x' (UNKNOWN TYPE) is not a string type",
        ):
            v.setStringValue('x', 'brown')
        # IndexError as 111 is no valid type
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: stringType 111 (UNKNOWN TYPE) is not a string type",
        ):
            v.setStringValue(111, 'brown')
        # CSS_PX is no string type
        with pytest.raises(
            xml.dom.InvalidAccessErr,
            match="CSSPrimitiveValue: stringType CSS_PX is not a string type",
        ):
            v.setStringValue(v.CSS_PX, 'brown')

    def test_typeRGBColor(self):
        "RGBColor"
        v = cssutils.css.CSSPrimitiveValue('RGB(1, 5, 10)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        assert 'rgb(1, 5, 10)' == v.cssText

        v = cssutils.css.CSSPrimitiveValue('rgb(1, 5, 10)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        assert 'rgb(1, 5, 10)' == v.cssText

        v = cssutils.css.CSSPrimitiveValue('rgb(1%, 5%, 10%)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        assert 'rgb(1%, 5%, 10%)' == v.cssText

        v = cssutils.css.CSSPrimitiveValue('  rgb(  1 ,5,  10  )')
        assert v.CSS_RGBCOLOR == v.primitiveType
        v = cssutils.css.CSSPrimitiveValue('rgb(1,5,10)')
        assert v.CSS_RGBCOLOR == v.primitiveType
        v = cssutils.css.CSSPrimitiveValue('rgb(1%, .5%, 10.1%)')
        assert v.CSS_RGBCOLOR == v.primitiveType

env_pyrecodes/lib/python3.9/site-packages/cssutils/tests/test_value.py
NikolaBlagojevic/pyrecodes
"""base classes and helper functions for css and stylesheets packages
"""
__all__ = []

from .helper import normalize
from itertools import chain
import cssutils
from . import codec
import codecs
from . import errorhandler
from . import tokenize2
import xml.dom
import re

try:
    from ._fetchgae import _defaultFetcher
except ImportError:
    from ._fetch import _defaultFetcher

log = errorhandler.ErrorHandler()


class _BaseClass:
    """
    Base class for Base, Base2 and _NewBase.

    **Base and Base2 will be removed in the future!**
    """

    _log = errorhandler.ErrorHandler()
    _prods = tokenize2.CSSProductions

    def _checkReadonly(self):
        "Raise xml.dom.NoModificationAllowedErr if rule/... is readonly"
        if hasattr(self, '_readonly') and self._readonly:
            raise xml.dom.NoModificationAllowedErr('%s is readonly.' % self.__class__)
            return True
        return False

    def _valuestr(self, t):
        """
        Return string value of t (t may be a string, a list of token tuples
        or a single tuple in format (type, value, line, col).
        Mainly used to get a string value of t for error messages.
        """
        if not t:
            return ''
        elif isinstance(t, str):
            return t
        else:
            return ''.join([x[1] for x in t])


class _NewBase(_BaseClass):
    """
    New base class for classes using ProdParser.

    **Currently CSSValue and related ones only.**
    """

    def __init__(self):
        self._seq = Seq()

    def _setSeq(self, newseq):
        """Set value of ``seq`` which is readonly."""
        newseq._readonly = True
        self._seq = newseq

    def _clearSeq(self):
        self._seq.clear()

    def _tempSeq(self, readonly=False):
        "Get a writeable Seq() which is used to set ``seq`` later"
        return Seq(readonly=readonly)

    @property
    def seq(self):
        """Internal readonly attribute, **DO NOT USE**!"""
        return self._seq


class _NewListBase(_NewBase):
    """
    (EXPERIMENTAL)
    A base class used for list classes like stylesheets.MediaList

    adds list like behaviour running on inhering class' property ``seq``

    - item in x => bool
    - len(x) => integer
    - get, set and del x[i]
    - for item in x
    - append(item)

    some methods must be overwritten in inheriting class
    """

    def __init__(self):
        self._seq = Seq()

    def __contains__(self, item):
        for it in self._seq:
            if item == it.value:
                return True
        return False

    def __delitem__(self, index):
        del self._seq[index]

    def __getitem__(self, index):
        return self._seq[index].value

    def __iter__(self):
        def gen():
            for x in self._seq:
                yield x.value

        return gen()

    def __len__(self):
        return len(self._seq)

    def __setitem__(self, index, item):
        "must be overwritten"
        raise NotImplementedError

    def append(self, item):
        "must be overwritten"
        raise NotImplementedError


class Base(_BaseClass):
    """
    **Superceded by _NewBase**

    **Superceded by Base2 which is used for new seq handling class.**

    Base class for most CSS and StyleSheets classes

    Contains helper methods for inheriting classes helping parsing

    ``_normalize`` is static as used by Preferences.
    """

    __tokenizer2 = tokenize2.Tokenizer()

    # for more on shorthand properties see
    # http://www.dustindiaz.com/css-shorthand/
    # format: shorthand: [(propname, mandatorycheck?)*]
    _SHORTHANDPROPERTIES = {
        'background': [],
        # u'background-position': [], # list of 2 values!
        'border': [],
        'border-left': [],
        'border-right': [],
        'border-top': [],
        'border-bottom': [],
        # u'border-color': [], # list or single but same values
        # u'border-style': [], # list or single but same values
        # u'border-width': [], # list or single but same values
        'cue': [],
        'font': [],
        'list-style': [],
        # u'margin': [], # list or single but same values
        'outline': [],
        # u'padding': [], # list or single but same values
        'pause': [],
    }

    @staticmethod
    def _normalize(x):
        r"""
        normalizes x, namely:

        - remove any \ before non unicode sequences (0-9a-zA-Z) so for
          x=="c\olor\" return "color" (unicode escape sequences should have
          been resolved by the tokenizer already)
        - lowercase
        """
        return normalize(x)

    def _splitNamespacesOff(self, text_namespaces_tuple):
        """
        returns tuple (text, dict-of-namespaces) or if no namespaces are
        in cssText returns (cssText, {})

        used in Selector, SelectorList, CSSStyleRule, CSSMediaRule and
        CSSStyleSheet
        """
        if isinstance(text_namespaces_tuple, tuple):
            return text_namespaces_tuple[0], _SimpleNamespaces(
                self._log, text_namespaces_tuple[1]
            )
        else:
            return text_namespaces_tuple, _SimpleNamespaces(log=self._log)

    def _tokenize2(self, textortokens):
        """
        returns tokens of textortokens which may already be tokens in which
        case simply returns input
        """
        if not textortokens:
            return None
        elif isinstance(textortokens, str):
            # needs to be tokenized
            return self.__tokenizer2.tokenize(textortokens)
        elif isinstance(textortokens, tuple):
            # a single token (like a comment)
            return [textortokens]
        else:
            # already tokenized but return an iterator
            return iter(textortokens)

    def _nexttoken(self, tokenizer, default=None):
        "returns next token in generator tokenizer or the default value"
        try:
            return next(tokenizer)
        # TypeError for py3
        except (StopIteration, AttributeError, TypeError):
            return default

    def _type(self, token):
        "returns type of Tokenizer token"
        if token:
            return token[0]
        else:
            return None

    def _tokenvalue(self, token, normalize=False):
        "returns value of Tokenizer token"
        if token and normalize:
            return Base._normalize(token[1])
        elif token:
            return token[1]
        else:
            return None

    def _stringtokenvalue(self, token):
        """
        for STRING returns the actual content without surrounding "" or ''
        and without respective escapes, e.g.::

             "with \" char" => with " char
        """
        if token:
            value = token[1]
            return value.replace('\\' + value[0], value[0])[1:-1]
        else:
            return None

    def _uritokenvalue(self, token):
        """
        for URI returns the actual content without surrounding url()
        or url(""), url('') and without respective escapes, e.g.::

             url("\"") => "
        """
        if token:
            value = token[1][4:-1].strip()
            if value and (value[0] in '\'"') and (value[0] == value[-1]):
                # a string "..." or '...'
                value = value.replace('\\' + value[0], value[0])[1:-1]
            return value
        else:
            return None

    def _tokensupto2(  # noqa: C901
        self,
        tokenizer,
        starttoken=None,
        blockstartonly=False,  # {
        blockendonly=False,  # }
        mediaendonly=False,
        importmediaqueryendonly=False,  # ; or STRING
        mediaqueryendonly=False,  # { or STRING
        semicolon=False,  # ;
        propertynameendonly=False,  # :
        propertyvalueendonly=False,  # ! ; }
        propertypriorityendonly=False,  # ; }
        selectorattendonly=False,  # ]
        funcendonly=False,  # )
        listseponly=False,  # ,
        separateEnd=False,  # returns (resulttokens, endtoken)
    ):
        """
        returns tokens upto end of atrule and end index
        end is defined by parameters, might be ; } ) or other

        default looks for ending "}" and ";"
        """
        ends = ';}'
        endtypes = ()
        brace = bracket = parant = 0  # {}, [], ()

        if blockstartonly:  # {
            ends = '{'
            brace = -1  # set to 0 with first {
        elif blockendonly:  # }
            ends = '}'
            brace = 1
        elif mediaendonly:  # }
            ends = '}'
            brace = 1  # rules } and mediarules }
        elif importmediaqueryendonly:
            # end of mediaquery which may be ; or STRING
            ends = ';'
            endtypes = ('STRING',)
        elif mediaqueryendonly:
            # end of mediaquery which may be { or STRING
            # special case, see below
            ends = '{'
            brace = -1  # set to 0 with first {
            endtypes = ('STRING',)
        elif semicolon:
            ends = ';'
        elif propertynameendonly:  # : and ; in case of an error
            ends = ':;'
        elif propertyvalueendonly:  # ; or !important
            ends = ';!'
        elif propertypriorityendonly:  # ;
            ends = ';'
        elif selectorattendonly:  # ]
            ends = ']'
            if starttoken and self._tokenvalue(starttoken) == '[':
                bracket = 1
        elif funcendonly:  # )
            ends = ')'
            parant = 1
        elif listseponly:  # ,
            ends = ','

        resulttokens = []
        if starttoken:
            resulttokens.append(starttoken)
            val = starttoken[1]
            if '[' == val:
                bracket += 1
            elif '{' == val:
                brace += 1
            elif '(' == val:
                parant += 1

        if tokenizer:
            for token in tokenizer:
                typ, val, line, col = token
                if 'EOF' == typ:
                    resulttokens.append(token)
                    break

                if '{' == val:
                    brace += 1
                elif '}' == val:
                    brace -= 1
                elif '[' == val:
                    bracket += 1
                elif ']' == val:
                    bracket -= 1
                # function( or single (
                elif '(' == val or Base._prods.FUNCTION == typ:
                    parant += 1
                elif ')' == val:
                    parant -= 1

                resulttokens.append(token)

                if (brace == bracket == parant == 0) and (
                    val in ends or typ in endtypes
                ):
                    break
                elif (
                    mediaqueryendonly
                    and brace == -1
                    and (bracket == parant == 0)
                    and typ in endtypes
                ):
                    # mediaqueryendonly with STRING
                    break
        if separateEnd:
            # TODO: use this method as generator, then this makes sense
            if resulttokens:
                return resulttokens[:-1], resulttokens[-1]
            else:
                return resulttokens, None
        else:
            return resulttokens

    def _adddefaultproductions(self, productions, new=None):
        """
        adds default productions if not already present, used by
        _parse only

        each production should return the next expected token
        normaly a name like "uri" or "EOF"
        some have no expectation like S or COMMENT, so simply return
        the current value of self.__expected
        """

        def ATKEYWORD(expected, seq, token, tokenizer=None):
            "default impl for unexpected @rule"
            if expected != 'EOF':
                # TODO: parentStyleSheet=self
                rule = cssutils.css.CSSUnknownRule()
                rule.cssText = self._tokensupto2(tokenizer, token)
                if rule.wellformed:
                    seq.append(rule)
                return expected
            else:
                new['wellformed'] = False
                self._log.error('Expected EOF.', token=token)
                return expected

        def COMMENT(expected, seq, token, tokenizer=None):
            "default implementation for COMMENT token adds CSSCommentRule"
            seq.append(cssutils.css.CSSComment([token]))
            return expected

        def S(expected, seq, token, tokenizer=None):
            "default implementation for S token, does nothing"
            return expected

        def EOF(expected=None, seq=None, token=None, tokenizer=None):
            "default implementation for EOF token"
            return 'EOF'

        p = {
            'ATKEYWORD': ATKEYWORD,
            'COMMENT': COMMENT,
            'S': S,
            'EOF': EOF,  # only available if fullsheet
        }
        p.update(productions)
        return p

    def _parse(
        self,
        expected,
        seq,
        tokenizer,
        productions,
        default=None,
        new=None,
        initialtoken=None,
    ):
        """
        puts parsed tokens in seq by calling a production with
            (seq, tokenizer, token)

        expected
            a name what token or value is expected next, e.g. 'uri'
        seq
            to add rules etc to
        tokenizer
            call tokenizer.next() to get next token
        productions
            callbacks {tokentype: callback}
        default
            default callback if tokentype not in productions
        new
            used to init default productions
        initialtoken
            will be used together with tokenizer running 1st this token
            and then all tokens in tokenizer

        returns (wellformed, expected) which the last prod might have set
        """
        wellformed = True

        if initialtoken:
            # add initialtoken to tokenizer
            def tokens():
                "Build new tokenizer including initialtoken"
                yield initialtoken
                yield from tokenizer

            fulltokenizer = chain([initialtoken], tokenizer)
        else:
            fulltokenizer = tokenizer

        if fulltokenizer:
            prods = self._adddefaultproductions(productions, new)
            for token in fulltokenizer:
                p = prods.get(token[0], default)
                if p:
                    expected = p(expected, seq, token, tokenizer)
                else:
                    wellformed = False
                    self._log.error('Unexpected token (%s, %s, %s, %s)' % token)
        return wellformed, expected


class Base2(Base, _NewBase):
    """
    **Superceded by _NewBase.**

    Base class for new seq handling.
    """

    def __init__(self):
        self._seq = Seq()

    def _adddefaultproductions(self, productions, new=None):
        """
        adds default productions if not already present, used by
        _parse only

        each production should return the next expected token
        normaly a name like "uri" or "EOF"
        some have no expectation like S or COMMENT, so simply return
        the current value of self.__expected
        """

        def ATKEYWORD(expected, seq, token, tokenizer=None):
            "default impl for unexpected @rule"
            if expected != 'EOF':
                # TODO: parentStyleSheet=self
                rule = cssutils.css.CSSUnknownRule()
                rule.cssText = self._tokensupto2(tokenizer, token)
                if rule.wellformed:
                    seq.append(
                        rule,
                        cssutils.css.CSSRule.UNKNOWN_RULE,
                        line=token[2],
                        col=token[3],
                    )
                return expected
            else:
                new['wellformed'] = False
                self._log.error('Expected EOF.', token=token)
                return expected

        def COMMENT(expected, seq, token, tokenizer=None):
            "default impl, adds CSSCommentRule if not token == EOF"
            if expected == 'EOF':
                new['wellformed'] = False
                self._log.error('Expected EOF but found comment.', token=token)
            seq.append(cssutils.css.CSSComment([token]), 'COMMENT')
            return expected

        def S(expected, seq, token, tokenizer=None):
            "default impl, does nothing if not token == EOF"
            if expected == 'EOF':
                new['wellformed'] = False
                self._log.error('Expected EOF but found whitespace.', token=token)
            return expected

        def EOF(expected=None, seq=None, token=None, tokenizer=None):
            "default implementation for EOF token"
            return 'EOF'

        defaultproductions = {
            'ATKEYWORD': ATKEYWORD,
            'COMMENT': COMMENT,
            'S': S,
            'EOF': EOF,  # only available if fullsheet
        }
        defaultproductions.update(productions)
        return defaultproductions


class Seq:
    """
    property seq of Base2 inheriting classes, holds a list of Item objects.

    used only by Selector for now

    is normally readonly, only writable during parsing
    """

    def __init__(self, readonly=True):
        """
        only way to write to a Seq is to initialize it with new items
        each itemtuple has (value, type, line) where line is optional
        """
        self._seq = []
        self._readonly = readonly

    def __repr__(self):
        "returns a repr same as a list of tuples of (value, type)"
        return 'cssutils.{}.{}([\n    {}], readonly={!r})'.format(
            self.__module__,
            self.__class__.__name__,
            ',\n    '.join(['%r' % item for item in self._seq]),
            self._readonly,
        )

    def __str__(self):
        vals = []
        for v in self:
            if isinstance(v.value, str):
                vals.append(v.value)
            elif isinstance(v, tuple):
                vals.append(v.value[1])
            else:
                vals.append(repr(v))

        return "<cssutils.{}.{} object length={!r} items={!r} readonly={!r} at 0x{:x}>".format(
            self.__module__,
            self.__class__.__name__,
            len(self),
            vals,
            self._readonly,
            id(self),
        )

    def __delitem__(self, i):
        del self._seq[i]

    def __getitem__(self, i):
        return self._seq[i]

    def __setitem__(self, i, xxx_todo_changeme):
        (val, typ, line, col) = xxx_todo_changeme
        self._seq[i] = Item(val, typ, line, col)

    def __iter__(self):
        return iter(self._seq)

    def __len__(self):
        return len(self._seq)

    def append(self, val, typ=None, line=None, col=None):
        "If not readonly add new Item()"
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            if isinstance(val, Item):
                self._seq.append(val)
            else:
                self._seq.append(Item(val, typ, line, col))

    def appendItem(self, item):
        "if not readonly add item which must be an Item"
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            self._seq.append(item)

    def clear(self):
        del self._seq[:]

    def insert(self, index, val, typ, line=None, col=None):
        "Insert new Item() at index # even if readony!? TODO!"
        self._seq.insert(index, Item(val, typ, line, col))

    def replace(self, index=-1, val=None, typ=None, line=None, col=None):
        """
        if not readonly replace Item at index with new Item or
        simply replace value or type
        """
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            self._seq[index] = Item(val, typ, line, col)

    def rstrip(self):
        "trims S items from end of Seq"
        while self._seq and self._seq[-1].type == tokenize2.CSSProductions.S:
            # TODO: removed S before CSSComment /**/ /**/
            del self._seq[-1]

    def appendToVal(self, val=None, index=-1):
        """
        if not readonly append to Item's value at index
        """
        if self._readonly:
            raise AttributeError('Seq is readonly.')
        else:
            old = self._seq[index]
            self._seq[index] = Item(old.value + val, old.type, old.line, old.col)


class Item:
    """
    an item in the seq list of classes (successor to tuple items in old seq)

    each item has attributes:

    type
        a sematic type like "element", "attribute"
    value
        the actual value which may be a string, number etc or an instance
        of e.g. a CSSComment
    *line*
        **NOT IMPLEMENTED YET, may contain the line in the source later**
    """

    def __init__(self, value, type, line=None, col=None):
        self.__value = value
        self.__type = type
        self.__line = line
        self.__col = col

    type = property(lambda self: self.__type)
    value = property(lambda self: self.__value)
    line = property(lambda self: self.__line)
    col = property(lambda self: self.__col)

    def __repr__(self):
        return "{}.{}(value={!r}, type={!r}, line={!r}, col={!r})".format(
            self.__module__,
            self.__class__.__name__,
            self.__value,
            self.__type,
            self.__line,
            self.__col,
        )


class ListSeq:
    """
    (EXPERIMENTAL)
    A base class used for list classes like cssutils.css.SelectorList or
    stylesheets.MediaList

    adds list like behaviour running on inhering class' property ``seq``

    - item in x => bool
    - len(x) => integer
    - get, set and del x[i]
    - for item in x
    - append(item)

    some methods must be overwritten in inheriting class
    """

    def __init__(self):
        self.seq = []  # does not need to use ``Seq`` as simple list only

    def __contains__(self, item):
        return item in self.seq

    def __delitem__(self, index):
        del self.seq[index]

    def __getitem__(self, index):
        return self.seq[index]

    def __iter__(self):
        def gen():
            yield from self.seq

        return gen()

    def __len__(self):
        return len(self.seq)

    def __setitem__(self, index, item):
        "must be overwritten"
        raise NotImplementedError

    def append(self, item):
        "must be overwritten"
        raise NotImplementedError


class _Namespaces:
    """
    A dictionary like wrapper for @namespace rules used in a CSSStyleSheet.
    Works on effective namespaces, so e.g. if::

        @namespace p1 "uri";
        @namespace p2 "uri";

    only the second rule is effective and kept.

    namespaces
        a dictionary {prefix: namespaceURI} containing the effective namespaces
        only. These are the latest set in the CSSStyleSheet.
    parentStyleSheet
        the parent CSSStyleSheet
    """

    def __init__(self, parentStyleSheet, log=None, *args):
        "no initial values are set, only the relevant sheet is"
        self.parentStyleSheet = parentStyleSheet
        self._log = log

    def __repr__(self):
        return "%r" % self.namespaces

    def __contains__(self, prefix):
        return prefix in self.namespaces

    def __delitem__(self, prefix):
        """deletes CSSNamespaceRule(s) with rule.prefix == prefix

        prefix '' and None are handled the same
        """
        if not prefix:
            prefix = ''
        delrule = self.__findrule(prefix)
        for i, rule in enumerate(
            filter(lambda r: r.type == r.NAMESPACE_RULE, self.parentStyleSheet.cssRules)
        ):
            if rule == delrule:
                self.parentStyleSheet.deleteRule(i)
                return

        self._log.error('Prefix %s not found.' % prefix, error=xml.dom.NamespaceErr)

    def __getitem__(self, prefix):
        try:
            return self.namespaces[prefix]
        except KeyError:
            self._log.error('Prefix %s not found.' % prefix, error=xml.dom.NamespaceErr)

    def __iter__(self):
        return self.namespaces.__iter__()

    def __len__(self):
        return len(self.namespaces)

    def __setitem__(self, prefix, namespaceURI):
        "replaces prefix or sets new rule, may raise NoModificationAllowedErr"
        if not prefix:
            prefix = ''  # None or ''
        rule = self.__findrule(prefix)
        if not rule:
            self.parentStyleSheet.insertRule(
                cssutils.css.CSSNamespaceRule(prefix=prefix, namespaceURI=namespaceURI),
                inOrder=True,
            )
        else:
            if prefix in self.namespaces:
                rule.namespaceURI = namespaceURI  # raises NoModificationAllowedErr
            if namespaceURI in list(self.namespaces.values()):
                rule.prefix = prefix

    def __findrule(self, prefix):
        # returns namespace rule where prefix == key
        for rule in filter(
            lambda r: r.type == r.NAMESPACE_RULE,
            reversed(self.parentStyleSheet.cssRules),
        ):
            if rule.prefix == prefix:
                return rule

    @property
    def namespaces(self):
        """
        A property holding only effective @namespace rules in
        self.parentStyleSheets.
        """
        namespaces = {}
        for rule in filter(
            lambda r: r.type == r.NAMESPACE_RULE,
            reversed(self.parentStyleSheet.cssRules),
        ):
            if rule.namespaceURI not in list(namespaces.values()):
                namespaces[rule.prefix] = rule.namespaceURI
        return namespaces

    def get(self, prefix, default):
        return self.namespaces.get(prefix, default)

    def items(self):
        return list(self.namespaces.items())

    def keys(self):
        return list(self.namespaces.keys())

    def values(self):
        return list(self.namespaces.values())

    def prefixForNamespaceURI(self, namespaceURI):
        """
        returns effective prefix for given namespaceURI or raises IndexError
        if this cannot be found"""
        for prefix, uri in list(self.namespaces.items()):
            if uri == namespaceURI:
                return prefix
        raise IndexError('NamespaceURI %s not found.' % namespaceURI)

    def __str__(self):
        return "<cssutils.util.{} object parentStyleSheet={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            str(self.parentStyleSheet),
            id(self),
        )


class _SimpleNamespaces(_Namespaces):
    """
    namespaces used in objects like Selector as long as they are not connected
    to a CSSStyleSheet
    """

    def __init__(self, log=None, *args):
        """init"""
        super().__init__(parentStyleSheet=None, log=log)
        self.__namespaces = dict(*args)

    def __setitem__(self, prefix, namespaceURI):
        self.__namespaces[prefix] = namespaceURI

    @property
    def namespaces(self):
        """Dict Wrapper for self.sheets @namespace rules."""
        return self.__namespaces

    def __str__(self):
        return "<cssutils.util.{} object namespaces={!r} at 0x{:x}>".format(
            self.__class__.__name__,
            self.namespaces,
            id(self),
        )

    def __repr__(self):
        return f"cssutils.util.{self.__class__.__name__}({self.namespaces!r})"


def _readUrl(  # noqa: C901
    url, fetcher=None, overrideEncoding=None, parentEncoding=None
):
    """
    Read cssText from url and decode it using all relevant methods (HTTP
    header, BOM, @charset). Returns

    - encoding used to decode text (which is needed to set encoding of
      stylesheet properly)
    - type of encoding (how it was retrieved, see list below)
    - decodedCssText

    ``fetcher``
        see cssutils.CSSParser.setFetcher for details
    ``overrideEncoding``
        If given this encoding is used and all other encoding information is
        ignored (HTTP, BOM etc)
    ``parentEncoding``
        Encoding of parent stylesheet (while e.g. reading @import references
        sheets) or document if available.

    Priority or encoding information
    --------------------------------
    **cssutils only**: 0. overrideEncoding

    1. An HTTP "charset" parameter in a "Content-Type" field (or similar
       parameters in other protocols)
    2. BOM and/or @charset (see below)
    3. <link charset=""> or other metadata from the linking mechanism (if any)
    4. charset of referring style sheet or document (if any)
    5. Assume UTF-8

    """
    enctype = None

    if not fetcher:
        fetcher = _defaultFetcher

    r = fetcher(url)
    if r and len(r) == 2 and r[1] is not None:
        httpEncoding, content = r

        if overrideEncoding:
            enctype = 0  # 0. override encoding
            encoding = overrideEncoding
        elif httpEncoding:
            enctype = 1  # 1. HTTP
            encoding = httpEncoding
        else:
            # BOM or @charset
            if isinstance(content, str):
                contentEncoding, explicit = codec.detectencoding_unicode(content)
            else:
                contentEncoding, explicit = codec.detectencoding_str(content)

            if explicit:
                enctype = 2  # 2. BOM/@charset: explicitly
                encoding = contentEncoding

            elif parentEncoding:
                enctype = 4  # 4. parent stylesheet or document
                # may also be None in which case 5. is used in next step anyway
                encoding = parentEncoding

            else:
                enctype = 5  # 5. assume UTF-8
                encoding = 'utf-8'

        if isinstance(content, str):
            decodedCssText = content
        else:
            try:
                # encoding may still be wrong if encoding *is lying*!
                try:
                    decodedCssText = codecs.lookup("css")[1](
                        content, encoding=encoding
                    )[0]
                except AttributeError:
                    # at least in GAE
                    decodedCssText = content.decode(encoding if encoding else 'utf-8')

            except UnicodeDecodeError as e:
                log.warn(e, neverraise=True)
                decodedCssText = None

        return encoding, enctype, decodedCssText
    else:
        return None, None, None


class LazyRegex:
    """A class to represent a lazily compiled regular expression.

    The interface is kept similar to a `re.compile`ed object from the standard
    regex library.

    :ivar pattern: The original regular expression.
    :ivar flags: Flags of the regular expression.
    :ivar matcher: The compiled regular expression, or None if it is not yet
                   compiled.
    :ivar groups: The number of capturing groups in the pattern, or None.
    :ivar groupindex: A dictionary mapping any symbolic group names defined by
                      `(?P<id>)` to group numbers. The dictionary is empty if
                      no symbolic groups were used in the pattern. None if the
                      pattern is not yet compiled.
    """

    __slots__ = ('pattern', 'flags', 'matcher', 'groups', 'groupindex')

    def __init__(self, pattern, flags=0):
        self.pattern = pattern
        self.matcher = None
        self.flags = flags
        self.groups = None
        self.groupindex = None

    def ensure(self):
        """Make sure that the expression is compiled.

        If self.matcher is already a compiled expression, do nothing.
        """
        if self.matcher is not None:
            return
        self.matcher = re.compile(self.pattern, self.flags)
        self.flags = self.matcher.flags
        self.groups = self.matcher.groups
        self.groupindex = self.matcher.groupindex

    def __call__(self, string, pos=None, endpos=None):
        """Shortcut for self.match(string)."""
        return self.match(string, pos, endpos)

    def match(self, string, pos=None, endpos=None):
        """Attempt to do a match of the regular expression.

        This is similar to the `.match` method of `re` objects.
        """
        self.ensure()
        if pos is None:
            pos = 0
        if endpos is None:
            endpos = len(string)
        return self.matcher.match(string, pos, endpos)

    def search(self, string, pos=None, endpos=None):
        """Search the string for the pattern.

        This is similar to the `.search` method of `re` objects.
        """
        self.ensure()
        if pos is None:
            pos = 0
        if endpos is None:
            endpos = len(string)
        return self.matcher.search(string, pos, endpos)

    def split(self, string, maxsplit=0):
        """Split the string at the pattern.

        This is similar to the `.split` method of `re` objects.
        """
        self.ensure()
        return self.matcher.split(string, maxsplit)

    def findall(self, string, pos=None, endpos=None):
        """Find all instances of the pattern in the given string.

        This is similar to the `.findall` method of `re` objects.
        """
        self.ensure()
        if pos is None:
            pos = 0
        if endpos is None:
            endpos = len(string)
        return self.matcher.findall(string, pos, endpos)

    def finditer(self, string, pos=None, endpos=None):
        """Find all instances of the pattern in the given string.

        Returns an iterator.

        This is similar to the `.finditer` method of `re` objects.
        """
        self.ensure()
        if pos is None:
            pos = 0
        if endpos is None:
            endpos = len(string)
        return self.matcher.finditer(string, pos, endpos)

    def sub(self, repl, string, count=0):
        """Replace all occurences of the pattern with the given replacement.

        This is similar to the `.sub` method of `re` objects.
        """
        self.ensure()
        return self.matcher.sub(repl, string, count)

    def subn(self, repl, string, count=0):
        """Replace all occurences of the pattern with the given replacement.

        This is similar to the `.subn` method of `re` objects.
        """
        self.ensure()
        return self.matcher.subn(repl, string, count)

env_pyrecodes/lib/python3.9/site-packages/cssutils/util.py
NikolaBlagojevic/pyrecodes

# This file was generated by 'versioneer.py' (0.23) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.

import json

version_json = '''
{
 "date": "2023-04-03T17:37:14-0700",
 "dirty": false,
 "error": null,
 "full-revisionid": "2f3e0bb7d5984486def3f2e3d246b974cf243b50",
 "version": "1.6.7"
}
'''  # END VERSION_JSON


def get_versions():
    return json.loads(version_json)

env_pyrecodes/lib/python3.9/site-packages/debugpy/_version.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
"""
A μ-library for constructing cascasing style sheets from Python dictionaries.

.. latex:vspace:: 10px
.. seealso:: `css-parser <https://github.com/ebook-utils/css-parser>`_, which this library builds upon.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from io import TextIOBase
from typing import IO, Any, Dict, Mapping, MutableMapping, Sequence, Union, cast

# 3rd party
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.typing import PathLike
from domdf_python_tools.words import TAB

try:
	# 3rd party
	import css_parser  # type: ignore
except ImportError:  # pragma: no cover
	import cssutils as css_parser  # type: ignore

# this package
from dict2css.helpers import em, px, rem
from dict2css.serializer import CSSSerializer

__author__: str = "Dominic Davis-Foster"
__copyright__: str = "2020-2021 Dominic Davis-Foster"
__license__: str = "MIT License"
__version__: str = "0.3.0"
__email__: str = "dominic@davis-foster.co.uk"

__all__ = [
		"IMPORTANT",
		"Style",
		"dumps",
		"dump",
		"loads",
		"load",
		"StyleSheet",
		"make_style",
		]

IMPORTANT = "important"
"""
The string ``'important'``.

.. latex:vspace:: 10px
"""

# Property = Union[Tuple[Union[str, int, None], str], str, int, None]
Property = Union[Sequence, str, int, None]

Style = Mapping[str, Property]
"""
Type annotation representing a style for :func:`~.make_style` and :func:`~.dumps`.

The keys are CSS properties.

The values can be either:

* A :class:`str`, :class:`float` or :py:obj:`None`, giving the value of the property.
* A :class:`tuple` of the property's value (as above) and the priority
  such as :data:`~.IMPORTANT` (which sets ``!important`` on the property).
"""


def dumps(
		styles: Mapping[str, Union[Style, Mapping]],
		*,
		indent: str = TAB,
		trailing_semicolon: bool = False,
		indent_closing_brace: bool = False,
		minify: bool = False,
		) -> str:
	r"""
	Construct a cascading style sheet from a dictionary.

	``styles`` is a mapping of CSS selector strings to styles, which map property names to their values:

	.. code-block:: python

		styles = {".wy-nav-content": {"max-width": (px(1200), IMPORTANT)}}
		print(dumps(styles))

	.. code-block:: css

		.wy-nav-content {
			max-width: 1200px !important
		}

	See the :py:obj:`~.Style` object for more information on the layout.

	The keys can also be media at-rules, with the values mappings of property names to their values:

	.. code-block:: python

		styles = {
			"@media screen and (min-width: 870px)": {
				".wy-nav-content": {"max-width": (px(1200), IMPORTANT)},
				},
			}
		print(dumps(styles))

	.. code-block:: css

		@media screen and (min-width: 870px) {
			.wy-nav-content {
				max-width: 1200px !important
			}
		}

	:param styles: A mapping of CSS selectors to styles.
	:param indent: The indent to use, such as a tab (``\t``), two spaces or four spaces.
	:param trailing_semicolon:  Whether to add a semicolon to the end of the final property.
	:param indent_closing_brace:
	:param minify: Minify the CSS. Overrides all other options.

	:return: The style sheet as a string.

	.. versionchanged:: 0.2.0  Added support for media at-rules.
	"""

	serializer = CSSSerializer(
			indent=indent,
			trailing_semicolon=trailing_semicolon,
			indent_closing_brace=indent_closing_brace,
			minify=minify,
			)

	stylesheet: str = ''

	with serializer.use():
		sheet = StyleSheet()

		for selector, style in styles.items():
			if selector.startswith("@media"):
				sheet.add_media_styles(selector.split("@media")[1].strip(), cast(Mapping[str, Style], style))
			elif selector.startswith('@'):
				raise NotImplementedError("Only @media at-rules are supported at this time.")
			else:
				sheet.add_style(selector, cast(Style, style))

		stylesheet = sheet.tostring()

	if not serializer.minify:
		stylesheet = stylesheet.replace('}', "}\n")

	return stylesheet


def dump(
		styles: Mapping[str, Union[Style, Mapping]],
		fp: Union[PathLike, IO],
		*,
		indent: str = TAB,
		trailing_semicolon: bool = False,
		indent_closing_brace: bool = False,
		minify: bool = False,
		) -> None:
	r"""
	Construct a style sheet from a dictionary and write it to ``fp``.

	.. code-block:: python

		styles = {".wy-nav-content": {"max-width": (px(1200), IMPORTANT)}}
		dump(styles, ...)

	.. code-block:: css

		.wy-nav-content {
			max-width: 1200px !important
		}

	See the :py:obj:`~.Style` object for more information on the layout.

	.. latex:clearpage::

	The keys can also be media at-rules, with the values mappings of property names to their values:

	.. code-block:: python

		styles = {
			"@media screen and (min-width: 870px)": {
				".wy-nav-content": {"max-width": (px(1200), IMPORTANT)},
				},
			}
		dump(styles, ...)

	.. code-block:: css

		@media screen and (min-width: 870px) {
			.wy-nav-content {
				max-width: 1200px !important
			}
		}

	:param styles: A mapping of CSS selectors to styles.
	:param fp: An open file handle, or the filename of a file to write to.
	:param indent: The indent to use, such as a tab (``\t``), two spaces or four spaces.
	:param trailing_semicolon:  Whether to add a semicolon to the end of the final property.
	:param indent_closing_brace:
	:param minify: Minify the CSS. Overrides all other options.

	.. versionchanged:: 0.2.0

		* ``fp`` now accepts :py:obj:`domdf_python_tools.typing.PathLike` objects,
		  representing the path of a file to write to.
		* Added support for media at-rules.
	"""

	css = dumps(
			styles,
			indent=indent,
			trailing_semicolon=trailing_semicolon,
			indent_closing_brace=indent_closing_brace,
			minify=minify,
			)

	if isinstance(fp, TextIOBase):
		fp.write(css)
	else:
		PathPlus(fp).write_clean(css)


def loads(styles: str) -> MutableMapping[str, MutableMapping[str, Any]]:
	r"""
	Parse a style sheet and return its dictionary representation.

	.. versionadded:: 0.2.0

	:param styles:

	:return: The style sheet as a dictionary.
	"""

	parser = css_parser.CSSParser(validate=False)
	stylesheet: css_parser.css.CSSStyleSheet = parser.parseString(styles)

	styles_dict: MutableMapping[str, MutableMapping[str, Any]] = {}

	def parse_style(style: css_parser.css.CSSStyleDeclaration) -> MutableMapping[str, Property]:
		style_dict: Dict[str, Property] = {}

		prop: css_parser.css.Property
		for prop in style.children():
			if prop.priority:
				style_dict[prop.name] = (prop.value, prop.priority)
			else:
				style_dict[prop.name] = prop.value

		return style_dict

	rule: css_parser.css.CSSRule
	for rule in stylesheet.cssRules:
		if isinstance(rule, css_parser.css.CSSStyleRule):
			styles_dict[rule.selectorText] = parse_style(rule.style)

		elif isinstance(rule, css_parser.css.CSSMediaRule):
			styles_dict[f"@media {rule.media.mediaText}"] = {}

			for child in rule.cssRules:
				styles_dict[f"@media {rule.media.mediaText}"][child.selectorText] = parse_style(child.style)

		else:
			raise NotImplementedError(rule)

	return styles_dict


def load(fp: Union[PathLike, IO]) -> MutableMapping[str, MutableMapping[str, Any]]:
	r"""
	Parse a cascading style sheet from the given file and return its dictionary representation.

	.. versionadded:: 0.2.0

	:param fp: An open file handle, or the filename of a file to write to.

	:return: The style sheet as a dictionary.
	"""

	if isinstance(fp, TextIOBase):
		styles = fp.read()
	else:
		styles = PathPlus(fp).read_text()

	return loads(styles)


class StyleSheet(css_parser.css.CSSStyleSheet):
	r"""
	Represents a CSS style sheet.

	.. raw:: latex

		\nopagebreak

	.. autosummary-widths:: 7/16

	"""

	def __init__(self):
		super().__init__(validating=False)

	def add(self, rule: css_parser.css.CSSRule) -> int:
		"""
		Add the ``rule`` to the style sheet.

		:param rule:
		:type rule: :class:`css_parser.css.CSSRule`
		"""

		return super().add(rule)

	def add_style(
			self,
			selector: str,
			styles: Style,
			) -> None:
		"""
		Add a style to the style sheet.

		:param selector:
		:param styles:
		"""

		self.add(make_style(selector, styles))

	def add_media_styles(
			self,
			media_query: str,
			styles: Mapping[str, Style],
			) -> None:
		"""
		Add a set of styles for a media query to the style sheet.

		.. versionadded:: 0.2.0

		:param media_query:
		:param styles:
		"""

		media = css_parser.css.CSSMediaRule(media_query)

		for selector, style in styles.items():
			media.add(make_style(selector, style))

		self.add(media)

	def tostring(self) -> str:
		"""
		Returns the style sheet as a string.
		"""

		return self.cssText.decode("UTF-8")


def make_style(selector: str, styles: Style) -> css_parser.css.CSSStyleRule:
	"""
	Create a CSS Style Rule from a dictionary.

	:param selector:
	:param styles:

	:rtype: :class:`css_parser.css.CSSStyleRule`
	"""

	style = css_parser.css.CSSStyleDeclaration()
	style.validating = False

	for name, properties in styles.items():
		if isinstance(properties, Sequence) and not isinstance(properties, str):
			style[name] = tuple(str(x) for x in properties)
		else:
			style[name] = str(properties)

	return css_parser.css.CSSStyleRule(selectorText=selector, style=style)

env_pyrecodes/lib/python3.9/site-packages/dict2css/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  serializer.py
"""
Serializer for cascading style sheets.

.. versionadded:: 0.2.0
"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from contextlib import contextmanager
from typing import Iterator

# 3rd party
from domdf_python_tools.words import TAB

try:
	# 3rd party
	import css_parser  # type: ignore
except ImportError:  # pragma: no cover
	# 3rd party
	import cssutils as css_parser  # type: ignore

__all__ = ["CSSSerializer"]


class CSSSerializer(css_parser.CSSSerializer):
	r"""
	Serializes a :class:`~.StyleSheet` and its parts.

	This controls the formatting of the style sheet.

	:param indent: The indent to use, such as a tab (``\t``), two spaces or four spaces.
	:param trailing_semicolon:  Whether to add a semicolon to the end of the final property.
	:param indent_closing_brace:
	:param minify: Minify the CSS. Overrides all other options.

	.. autosummary-widths:: 5/16
	"""

	def __init__(
			self,
			*,
			indent: str = TAB,
			trailing_semicolon: bool = False,
			indent_closing_brace: bool = False,
			minify: bool = False,
			):
		super().__init__()
		self.indent = str(indent)
		self.trailing_semicolon = trailing_semicolon
		self.indent_closing_brace = indent_closing_brace
		self.minify = minify

	def reset_style(self) -> None:
		"""
		Reset the serializer to its default style.
		"""

		# Reset CSS Parser to defaults
		self.prefs.useDefaults()

		if self.minify:
			self.prefs.useMinified()

		else:
			# Formatting preferences
			self.prefs.omitLastSemicolon = not self.trailing_semicolon
			self.prefs.indentClosingBrace = self.indent_closing_brace
			self.prefs.indent = self.indent

	@contextmanager
	def use(self) -> Iterator:
		"""
		Contextmanager to use this serializer for the scope of the ``with`` block.
		"""

		# if css_parser.ser is self:
		# 	yield
		# 	return
		#
		current_serializer = css_parser.ser
		self.reset_style()

		try:
			css_parser.ser = self
			yield

		finally:
			css_parser.ser = current_serializer

env_pyrecodes/lib/python3.9/site-packages/dict2css/serializer.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9376 2023-05-09 18:31:42Z grubert $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
This is the Docutils (Python Documentation Utilities) package.

Package Structure
=================

Modules:

- __init__.py: Contains component base classes, exception classes, and
  Docutils version information.

- core.py: Contains the ``Publisher`` class and ``publish_*()`` convenience
  functions.

- frontend.py: Runtime settings (command-line interface, configuration files)
  processing, for Docutils front-ends.

- io.py: Provides a uniform API for low-level input and output.

- nodes.py: Docutils document tree (doctree) node class library.

- statemachine.py: A finite state machine specialized for
  regular-expression-based text filters.

Subpackages:

- languages: Language-specific mappings of terms.

- parsers: Syntax-specific input parser modules or packages.

- readers: Context-specific input handlers which understand the data
  source and manage a parser.

- transforms: Modules used by readers and writers to modify
  the Docutils document tree.

- utils: Contains the ``Reporter`` system warning class and miscellaneous
  utilities used by readers, writers, and transforms.

  utils/urischemes.py: Contains a complete mapping of known URI addressing
  scheme names to descriptions.

- utils/math: Contains functions for conversion of mathematical notation
  between different formats (LaTeX, MathML, text, ...).

- writers: Format-specific output translators.
"""

from collections import namedtuple

__docformat__ = 'reStructuredText'

__version__ = '0.20.1'
"""Docutils version identifier (complies with PEP 440)::

    major.minor[.micro][releaselevel[serial]][.dev]

For version comparison operations, use `__version_info__` (see, below)
rather than parsing the text of `__version__`.

https://docutils.sourceforge.io/docs/dev/policies.html#version-identification
"""

__version_details__ = ''
"""Optional extra version details (e.g. 'snapshot 2005-05-29, r3410').

For development and release status, use `__version__ and `__version_info__`.
"""


class VersionInfo(namedtuple('VersionInfo',
                             'major minor micro releaselevel serial release')):

    def __new__(cls, major=0, minor=0, micro=0,
                releaselevel='final', serial=0, release=True):
        releaselevels = ('alpha', 'beta', 'candidate', 'final')
        if releaselevel not in releaselevels:
            raise ValueError('releaselevel must be one of %r.'
                             % (releaselevels, ))
        if releaselevel == 'final':
            if not release:
                raise ValueError('releaselevel "final" must not be used '
                                 'with development versions (leads to wrong '
                                 'version ordering of the related __version__')
                # cf. https://peps.python.org/pep-0440/#summary-of-permitted-suffixes-and-relative-ordering  # noqa
            if serial != 0:
                raise ValueError('"serial" must be 0 for final releases')

        return super().__new__(cls, major, minor, micro,
                               releaselevel, serial, release)

    def __lt__(self, other):
        if isinstance(other, tuple):
            other = VersionInfo(*other)
        return tuple.__lt__(self, other)

    def __gt__(self, other):
        if isinstance(other, tuple):
            other = VersionInfo(*other)
        return tuple.__gt__(self, other)

    def __le__(self, other):
        if isinstance(other, tuple):
            other = VersionInfo(*other)
        return tuple.__le__(self, other)

    def __ge__(self, other):
        if isinstance(other, tuple):
            other = VersionInfo(*other)
        return tuple.__ge__(self, other)


__version_info__ = VersionInfo(
    major=0,
    minor=20,
    micro=1,
    releaselevel='final',  # one of 'alpha', 'beta', 'candidate', 'final'
    serial=0,  # pre-release number (0 for final releases and snapshots)
    release=True  # True for official releases and pre-releases
    )
"""Comprehensive version information tuple.

https://docutils.sourceforge.io/docs/dev/policies.html#version-identification
"""


class ApplicationError(Exception): pass
class DataError(ApplicationError): pass


class SettingsSpec:

    """
    Runtime setting specification base class.

    SettingsSpec subclass objects used by `docutils.frontend.OptionParser`.
    """

    # TODO: replace settings_specs with a new data structure
    # Backwards compatiblity:
    #   Drop-in components:
    #   Sphinx supplies settings_spec in the current format in some places
    #   Myst parser provides a settings_spec tuple
    #
    #   Sphinx reads a settings_spec in order to set a default value
    #   in writers/html.py:59
    #   https://github.com/sphinx-doc/sphinx/blob/4.x/sphinx/writers/html.py
    #   This should be changed (before retiring the old format)
    #   to use `settings_default_overrides` instead.
    settings_spec = ()
    """Runtime settings specification.  Override in subclasses.

    Defines runtime settings and associated command-line options, as used by
    `docutils.frontend.OptionParser`.  This is a tuple of:

    - Option group title (string or `None` which implies no group, just a list
      of single options).

    - Description (string or `None`).

    - A sequence of option tuples.  Each consists of:

      - Help text (string)

      - List of option strings (e.g. ``['-Q', '--quux']``).

      - Dictionary of keyword arguments sent to the OptionParser/OptionGroup
        ``add_option`` method.

        Runtime setting names are derived implicitly from long option names
        ('--a-setting' becomes ``settings.a_setting``) or explicitly from the
        'dest' keyword argument.

        Most settings will also have a 'validator' keyword & function.  The
        validator function validates setting values (from configuration files
        and command-line option arguments) and converts them to appropriate
        types.  For example, the ``docutils.frontend.validate_boolean``
        function, **required by all boolean settings**, converts true values
        ('1', 'on', 'yes', and 'true') to 1 and false values ('0', 'off',
        'no', 'false', and '') to 0.  Validators need only be set once per
        setting.  See the `docutils.frontend.validate_*` functions.

        See the optparse docs for more details.

    - More triples of group title, description, options, as many times as
      needed.  Thus, `settings_spec` tuples can be simply concatenated.
    """

    settings_defaults = None
    """A dictionary of defaults for settings not in `settings_spec` (internal
    settings, intended to be inaccessible by command-line and config file).
    Override in subclasses."""

    settings_default_overrides = None
    """A dictionary of auxiliary defaults, to override defaults for settings
    defined in other components' `setting_specs`.  Override in subclasses."""

    relative_path_settings = ()
    """Settings containing filesystem paths.  Override in subclasses.
    Settings listed here are to be interpreted relative to the current working
    directory."""

    config_section = None
    """The name of the config file section specific to this component
    (lowercase, no brackets).  Override in subclasses."""

    config_section_dependencies = None
    """A list of names of config file sections that are to be applied before
    `config_section`, in order (from general to specific).  In other words,
    the settings in `config_section` are to be overlaid on top of the settings
    from these sections.  The "general" section is assumed implicitly.
    Override in subclasses."""


class TransformSpec:
    """
    Runtime transform specification base class.

    Provides the interface to register "transforms" and helper functions
    to resolve references with a `docutils.transforms.Transformer`.

    https://docutils.sourceforge.io/docs/ref/transforms.html
    """

    def get_transforms(self):
        """Transforms required by this class.  Override in subclasses."""
        if self.default_transforms != ():
            import warnings
            warnings.warn('TransformSpec: the "default_transforms" attribute '
                          'will be removed in Docutils 2.0.\n'
                          'Use get_transforms() method instead.',
                          DeprecationWarning)
            return list(self.default_transforms)
        return []

    # Deprecated; for compatibility.
    default_transforms = ()

    unknown_reference_resolvers = ()
    """List of functions to try to resolve unknown references.

    Unknown references have a 'refname' attribute which doesn't correspond
    to any target in the document.  Called when the transforms in
    `docutils.transforms.references` are unable to find a correct target.

    The list should contain functions which will try to resolve unknown
    references, with the following signature::

        def reference_resolver(node):
            '''Returns boolean: true if resolved, false if not.'''

    If the function is able to resolve the reference, it should also remove
    the 'refname' attribute and mark the node as resolved::

        del node['refname']
        node.resolved = 1

    Each function must have a "priority" attribute which will affect the order
    the unknown_reference_resolvers are run::

        reference_resolver.priority = 100

    This hook is provided for 3rd party extensions.
    Example use case: the `MoinMoin - ReStructured Text Parser`
    in ``sandbox/mmgilbe/rst.py``.
    """


class Component(SettingsSpec, TransformSpec):

    """Base class for Docutils components."""

    component_type = None
    """Name of the component type ('reader', 'parser', 'writer').  Override in
    subclasses."""

    supported = ()
    """Name and aliases for this component.  Override in subclasses."""

    def supports(self, format):
        """
        Is `format` supported by this component?

        To be used by transforms to ask the dependent component if it supports
        a certain input context or output format.
        """
        return format in self.supported

env_pyrecodes/lib/python3.9/site-packages/docutils/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
# :Copyright: © 2020, 2022 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause
#
# Revision: $Revision: 9107 $
# Date: $Date: 2022-07-06 15:59:57 +0200 (Mi, 06. Jul 2022) $

"""Generic command line interface for the `docutils` package.

See also
https://docs.python.org/3/library/__main__.html#main-py-in-python-packages
"""

import argparse
import locale
import sys

import docutils
from docutils.core import Publisher, publish_cmdline, default_description


class CliSettingsSpec(docutils.SettingsSpec):
    """Runtime settings & command-line options for the generic CLI.

    Configurable reader, parser, and writer components.

    The "--writer" default will change to 'html' in Docutils 2.0
    when 'html' becomes an alias for the current value 'html5'.
    """

    settings_spec = (
        'Docutils Application Options',
        'Reader, writer, and parser settings influence the available options. '
        '  Example: use `--help --writer=latex` to see LaTeX writer options. ',
        # options: ('help text', [<option strings>], {<keyword arguments>})
        (('Reader name (currently: "%default").',
          ['--reader'], {'default': 'standalone', 'metavar': '<reader>'}),
         ('Parser name (currently: "%default").',
          ['--parser'], {'default': 'rst', 'metavar': '<parser>'}),
         ('Writer name (currently: "%default").',
          ['--writer'], {'default': 'html5', 'metavar': '<writer>'}),
         )
    )
    config_section = 'docutils application'
    config_section_dependencies = ('docutils-cli application',  # back-compat
                                   'applications')


def main():
    """Generic command line interface for the Docutils Publisher.
    """
    locale.setlocale(locale.LC_ALL, '')

    description = ('Convert documents into useful formats.  '
                   + default_description)

    # Update component selection from config file(s)
    components = Publisher().get_settings(settings_spec=CliSettingsSpec)

    # Update component selection from command-line
    argparser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
    argparser.add_argument('--reader', default=components.reader)
    argparser.add_argument('--parser', default=components.parser)
    argparser.add_argument('--writer', default=components.writer)
    # other options are parsed in a second pass via `publish_cmdline()`
    (args, remainder) = argparser.parse_known_args()
    # Ensure the current component selections are shown in help:
    CliSettingsSpec.settings_default_overrides = args.__dict__

    try:
        publish_cmdline(reader_name=args.reader,
                        parser_name=args.parser,
                        writer_name=args.writer,
                        settings_spec=CliSettingsSpec,
                        description=description,
                        argv=remainder)
    except ImportError as error:
        print('%s.' % error, file=sys.stderr)
        if '--traceback' in remainder:
            raise
        else:
            print('Use "--traceback" to show details.')


if __name__ == '__main__':
    if sys.argv[0].endswith('__main__.py'):
        # fix "usage" message
        sys.argv[0] = '%s -m docutils' % sys.executable
    main()

env_pyrecodes/lib/python3.9/site-packages/docutils/__main__.py
NikolaBlagojevic/pyrecodes
# $Id: core.py 9369 2023-05-02 23:04:27Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Calling the ``publish_*`` convenience functions (or instantiating a
`Publisher` object) with component names will result in default
behavior.  For custom behavior (setting component options), create
custom component objects first, and pass *them* to
``publish_*``/`Publisher`.  See `The Docutils Publisher`_.

.. _The Docutils Publisher:
    https://docutils.sourceforge.io/docs/api/publisher.html
"""

__docformat__ = 'reStructuredText'

import locale
import pprint
import os
import sys
import warnings

from docutils import (__version__, __version_details__, SettingsSpec,
                      io, utils, readers, writers)
from docutils.frontend import OptionParser
from docutils.readers import doctree


class Publisher:

    """
    A facade encapsulating the high-level logic of a Docutils system.
    """

    def __init__(self, reader=None, parser=None, writer=None,
                 source=None, source_class=io.FileInput,
                 destination=None, destination_class=io.FileOutput,
                 settings=None):
        """
        Initial setup.  If any of `reader`, `parser`, or `writer` are not
        specified, ``set_components()`` or the corresponding ``set_...()``
        method should be called with component names
        (`set_reader` sets the parser as well).
        """

        self.document = None
        """The document tree (`docutils.nodes` objects)."""

        self.reader = reader
        """A `docutils.readers.Reader` instance."""

        self.parser = parser
        """A `docutils.parsers.Parser` instance."""

        self.writer = writer
        """A `docutils.writers.Writer` instance."""

        for component in 'reader', 'parser', 'writer':
            assert not isinstance(getattr(self, component), str), (
                'passed string "%s" as "%s" parameter; pass an instance, '
                'or use the "%s_name" parameter instead (in '
                'docutils.core.publish_* convenience functions).'
                % (getattr(self, component), component, component))

        self.source = source
        """The source of input data, a `docutils.io.Input` instance."""

        self.source_class = source_class
        """The class for dynamically created source objects."""

        self.destination = destination
        """The destination for docutils output, a `docutils.io.Output`
        instance."""

        self.destination_class = destination_class
        """The class for dynamically created destination objects."""

        self.settings = settings
        """An object containing Docutils settings as instance attributes.
        Set by `self.process_command_line()` or `self.get_settings()`."""

        self._stderr = io.ErrorOutput()

    def set_reader(self, reader_name, parser, parser_name):
        """Set `self.reader` by name."""
        reader_class = readers.get_reader_class(reader_name)
        self.reader = reader_class(parser, parser_name)
        self.parser = self.reader.parser

    def set_writer(self, writer_name):
        """Set `self.writer` by name."""
        writer_class = writers.get_writer_class(writer_name)
        self.writer = writer_class()

    def set_components(self, reader_name, parser_name, writer_name):
        if self.reader is None:
            self.set_reader(reader_name, self.parser, parser_name)
        if self.parser is None:
            if self.reader.parser is None:
                self.reader.set_parser(parser_name)
            self.parser = self.reader.parser
        if self.writer is None:
            self.set_writer(writer_name)

    def setup_option_parser(self, usage=None, description=None,
                            settings_spec=None, config_section=None,
                            **defaults):
        warnings.warn('Publisher.setup_option_parser is deprecated, '
                      'and will be removed in Docutils 0.21.',
                      DeprecationWarning, stacklevel=2)
        if config_section:
            if not settings_spec:
                settings_spec = SettingsSpec()
            settings_spec.config_section = config_section
            parts = config_section.split()
            if len(parts) > 1 and parts[-1] == 'application':
                settings_spec.config_section_dependencies = ['applications']
        # @@@ Add self.source & self.destination to components in future?
        return OptionParser(
            components=(self.parser, self.reader, self.writer, settings_spec),
            defaults=defaults, read_config_files=True,
            usage=usage, description=description)

    def _setup_settings_parser(self, *args, **kwargs):
        # Provisional: will change (docutils.frontend.OptionParser will
        # be replaced by a parser based on arparse.ArgumentParser)
        # and may be removed later.
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            return self.setup_option_parser(*args, **kwargs)

    def get_settings(self, usage=None, description=None,
                     settings_spec=None, config_section=None, **defaults):
        """
        Return settings from components and config files.

        Please set components first (`self.set_reader` & `self.set_writer`).
        Use keyword arguments to override component defaults
        (before updating from configuration files).

        Calling this function also sets `self.settings` which makes
        `self.publish()` skip parsing command line options.
        """
        option_parser = self._setup_settings_parser(
            usage, description, settings_spec, config_section, **defaults)
        self.settings = option_parser.get_default_values()
        return self.settings

    def process_programmatic_settings(self, settings_spec,
                                      settings_overrides,
                                      config_section):
        if self.settings is None:
            defaults = settings_overrides.copy() if settings_overrides else {}
            # Propagate exceptions by default when used programmatically:
            defaults.setdefault('traceback', True)
            self.get_settings(settings_spec=settings_spec,
                              config_section=config_section,
                              **defaults)

    def process_command_line(self, argv=None, usage=None, description=None,
                             settings_spec=None, config_section=None,
                             **defaults):
        """
        Parse command line arguments and set ``self.settings``.

        Pass an empty sequence to `argv` to avoid reading `sys.argv`
        (the default behaviour).

        Set components first (`self.set_reader` & `self.set_writer`).
        """
        option_parser = self._setup_settings_parser(
            usage, description, settings_spec, config_section, **defaults)
        if argv is None:
            argv = sys.argv[1:]
        self.settings = option_parser.parse_args(argv)

    def set_io(self, source_path=None, destination_path=None):
        if self.source is None:
            self.set_source(source_path=source_path)
        if self.destination is None:
            self.set_destination(destination_path=destination_path)

    def set_source(self, source=None, source_path=None):
        if source_path is None:
            source_path = self.settings._source
        else:
            self.settings._source = source_path
        self.source = self.source_class(
            source=source, source_path=source_path,
            encoding=self.settings.input_encoding,
            error_handler=self.settings.input_encoding_error_handler)

    def set_destination(self, destination=None, destination_path=None):
        if destination_path is None:
            if (self.settings.output and self.settings._destination
                and self.settings.output != self.settings._destination):
                raise SystemExit('The positional argument <destination> is '
                                 'obsoleted by the --output option.  '
                                 'You cannot use them together.')
            if self.settings.output == '-':      # means stdout
                self.settings.output = None
            destination_path = (self.settings.output
                                or self.settings._destination)
        self.settings._destination = destination_path
        self.destination = self.destination_class(
            destination=destination,
            destination_path=destination_path,
            encoding=self.settings.output_encoding,
            error_handler=self.settings.output_encoding_error_handler)

    def apply_transforms(self):
        self.document.transformer.populate_from_components(
            (self.source, self.reader, self.reader.parser, self.writer,
             self.destination))
        self.document.transformer.apply_transforms()

    def publish(self, argv=None, usage=None, description=None,
                settings_spec=None, settings_overrides=None,
                config_section=None, enable_exit_status=False):
        """
        Process command line options and arguments (if `self.settings` not
        already set), run `self.reader` and then `self.writer`.  Return
        `self.writer`'s output.
        """
        exit = None
        try:
            if self.settings is None:
                self.process_command_line(
                    argv, usage, description, settings_spec, config_section,
                    **(settings_overrides or {}))
            self.set_io()
            self.prompt()
            self.document = self.reader.read(self.source, self.parser,
                                             self.settings)
            self.apply_transforms()
            output = self.writer.write(self.document, self.destination)
            self.writer.assemble_parts()
        except SystemExit as error:
            exit = True
            exit_status = error.code
        except Exception as error:
            if not self.settings:       # exception too early to report nicely
                raise
            if self.settings.traceback:  # Propagate exceptions?
                self.debugging_dumps()
                raise
            self.report_Exception(error)
            exit = True
            exit_status = 1
        self.debugging_dumps()
        if (enable_exit_status and self.document
            and (self.document.reporter.max_level
                 >= self.settings.exit_status_level)):
            sys.exit(self.document.reporter.max_level + 10)
        elif exit:
            sys.exit(exit_status)
        return output

    def debugging_dumps(self):
        if not self.document:
            return
        if self.settings.dump_settings:
            print('\n::: Runtime settings:', file=self._stderr)
            print(pprint.pformat(self.settings.__dict__), file=self._stderr)
        if self.settings.dump_internals:
            print('\n::: Document internals:', file=self._stderr)
            print(pprint.pformat(self.document.__dict__), file=self._stderr)
        if self.settings.dump_transforms:
            print('\n::: Transforms applied:', file=self._stderr)
            print(' (priority, transform class, pending node details, '
                  'keyword args)', file=self._stderr)
            print(pprint.pformat(
                [(priority, '%s.%s' % (xclass.__module__, xclass.__name__),
                  pending and pending.details, kwargs)
                 for priority, xclass, pending, kwargs
                 in self.document.transformer.applied]), file=self._stderr)
        if self.settings.dump_pseudo_xml:
            print('\n::: Pseudo-XML:', file=self._stderr)
            print(self.document.pformat().encode(
                'raw_unicode_escape'), file=self._stderr)

    def prompt(self):
        """Print info and prompt when waiting for input from a terminal."""
        try:
            if not (self.source.isatty() and self._stderr.isatty()):
                return
        except AttributeError:
            return
        eot_key = 'Ctrl+Z' if os.name == 'nt' else 'Ctrl+D'
        in_format = ''
        out_format = 'useful formats'
        try:
            in_format = self.parser.supported[0]
            out_format = self.writer.supported[0]
        except (AttributeError, IndexError):
            pass
        print(f'Docutils {__version__} <https://docutils.sourceforge.io>\n'
              f'converting "{in_format}" into "{out_format}".\n'
              f'Call with option "--help" for more info.\n'
              f'.. Waiting for source text (finish with {eot_key} '
              'on an empty line):',
              file=self._stderr)

    def report_Exception(self, error):
        if isinstance(error, utils.SystemMessage):
            self.report_SystemMessage(error)
        elif isinstance(error, UnicodeEncodeError):
            self.report_UnicodeError(error)
        elif isinstance(error, io.InputError):
            self._stderr.write('Unable to open source file for reading:\n'
                               '  %s\n' % io.error_string(error))
        elif isinstance(error, io.OutputError):
            self._stderr.write(
                'Unable to open destination file for writing:\n'
                '  %s\n' % io.error_string(error))
        else:
            print('%s' % io.error_string(error), file=self._stderr)
            print(f"""\
Exiting due to error.  Use "--traceback" to diagnose.
Please report errors to <docutils-users@lists.sourceforge.net>.
Include "--traceback" output, Docutils version ({__version__}\
{f' [{__version_details__}]' if __version_details__ else ''}),
Python version ({sys.version.split()[0]}), your OS type & version, \
and the command line used.""", file=self._stderr)

    def report_SystemMessage(self, error):
        print('Exiting due to level-%s (%s) system message.' % (
                  error.level, utils.Reporter.levels[error.level]),
              file=self._stderr)

    def report_UnicodeError(self, error):
        data = error.object[error.start:error.end]
        self._stderr.write(
            '%s\n'
            '\n'
            'The specified output encoding (%s) cannot\n'
            'handle all of the output.\n'
            'Try setting "--output-encoding-error-handler" to\n'
            '\n'
            '* "xmlcharrefreplace" (for HTML & XML output);\n'
            '  the output will contain "%s" and should be usable.\n'
            '* "backslashreplace" (for other output formats);\n'
            '  look for "%s" in the output.\n'
            '* "replace"; look for "?" in the output.\n'
            '\n'
            '"--output-encoding-error-handler" is currently set to "%s".\n'
            '\n'
            'Exiting due to error.  Use "--traceback" to diagnose.\n'
            'If the advice above doesn\'t eliminate the error,\n'
            'please report it to <docutils-users@lists.sourceforge.net>.\n'
            'Include "--traceback" output, Docutils version (%s),\n'
            'Python version (%s), your OS type & version, and the\n'
            'command line used.\n'
            % (io.error_string(error),
               self.settings.output_encoding,
               data.encode('ascii', 'xmlcharrefreplace'),
               data.encode('ascii', 'backslashreplace'),
               self.settings.output_encoding_error_handler,
               __version__, sys.version.split()[0]))


default_usage = '%prog [options] [<source> [<destination>]]'
default_description = (
    'Reads from <source> (default is stdin) '
    'and writes to <destination> (default is stdout).  '
    'See https://docutils.sourceforge.io/docs/user/config.html '
    'for a detailed settings reference.')


# TODO: or not to do?  cf. https://clig.dev/#help
#
# Display output on success, but keep it brief.
# Provide a -q option to suppress all non-essential output.
#
# Chain several args as input and use --output or redirection for output:
#   argparser.add_argument('source', nargs='+')
#
def publish_cmdline(reader=None, reader_name='standalone',
                    parser=None, parser_name='restructuredtext',
                    writer=None, writer_name='pseudoxml',
                    settings=None, settings_spec=None,
                    settings_overrides=None, config_section=None,
                    enable_exit_status=True, argv=None,
                    usage=default_usage, description=default_description):
    """
    Set up & run a `Publisher` for command-line-based file I/O (input and
    output file paths taken automatically from the command line).
    Also return the output as `str` or `bytes` (for binary output document
    formats).

    Parameters: see `publish_programmatically()` for the remainder.

    - `argv`: Command-line argument list to use instead of ``sys.argv[1:]``.
    - `usage`: Usage string, output if there's a problem parsing the command
      line.
    - `description`: Program description, output for the "--help" option
      (along with command-line option descriptions).
    """
    publisher = Publisher(reader, parser, writer, settings=settings)
    publisher.set_components(reader_name, parser_name, writer_name)
    output = publisher.publish(
        argv, usage, description, settings_spec, settings_overrides,
        config_section=config_section, enable_exit_status=enable_exit_status)
    return output


def publish_file(source=None, source_path=None,
                 destination=None, destination_path=None,
                 reader=None, reader_name='standalone',
                 parser=None, parser_name='restructuredtext',
                 writer=None, writer_name='pseudoxml',
                 settings=None, settings_spec=None, settings_overrides=None,
                 config_section=None, enable_exit_status=False):
    """
    Set up & run a `Publisher` for programmatic use with file-like I/O.
    Also return the output as `str` or `bytes` (for binary output document
    formats).

    Parameters: see `publish_programmatically()`.
    """
    output, publisher = publish_programmatically(
        source_class=io.FileInput, source=source, source_path=source_path,
        destination_class=io.FileOutput,
        destination=destination, destination_path=destination_path,
        reader=reader, reader_name=reader_name,
        parser=parser, parser_name=parser_name,
        writer=writer, writer_name=writer_name,
        settings=settings, settings_spec=settings_spec,
        settings_overrides=settings_overrides,
        config_section=config_section,
        enable_exit_status=enable_exit_status)
    return output


def publish_string(source, source_path=None, destination_path=None,
                   reader=None, reader_name='standalone',
                   parser=None, parser_name='restructuredtext',
                   writer=None, writer_name='pseudoxml',
                   settings=None, settings_spec=None,
                   settings_overrides=None, config_section=None,
                   enable_exit_status=False):
    """
    Set up & run a `Publisher` for programmatic use with string I/O.

    Accepts a `bytes` or `str` instance as `source`.

    The output is encoded according to the `output_encoding`_ setting;
    the return value is a `bytes` instance (unless `output_encoding`_ is
    "unicode", cf. `docutils.io.StringOutput.write()`).

    Parameters: see `publish_programmatically()`.

    This function is provisional because in Python 3 name and behaviour
    no longer match.

    .. _output_encoding:
        https://docutils.sourceforge.io/docs/user/config.html#output-encoding
    """
    output, publisher = publish_programmatically(
        source_class=io.StringInput, source=source, source_path=source_path,
        destination_class=io.StringOutput,
        destination=None, destination_path=destination_path,
        reader=reader, reader_name=reader_name,
        parser=parser, parser_name=parser_name,
        writer=writer, writer_name=writer_name,
        settings=settings, settings_spec=settings_spec,
        settings_overrides=settings_overrides,
        config_section=config_section,
        enable_exit_status=enable_exit_status)
    return output


def publish_parts(source, source_path=None, source_class=io.StringInput,
                  destination_path=None,
                  reader=None, reader_name='standalone',
                  parser=None, parser_name='restructuredtext',
                  writer=None, writer_name='pseudoxml',
                  settings=None, settings_spec=None,
                  settings_overrides=None, config_section=None,
                  enable_exit_status=False):
    """
    Set up & run a `Publisher`, and return a dictionary of document parts.

    Dictionary keys are the names of parts.
    Dictionary values are `str` instances; encoding is up to the client,
    e.g.::

       parts = publish_parts(...)
       body = parts['body'].encode(parts['encoding'], parts['errors'])

    See the `API documentation`__ for details on the provided parts.

    Parameters: see `publish_programmatically()`.

    __ https://docutils.sourceforge.io/docs/api/publisher.html#publish-parts
    """
    output, publisher = publish_programmatically(
        source=source, source_path=source_path, source_class=source_class,
        destination_class=io.StringOutput,
        destination=None, destination_path=destination_path,
        reader=reader, reader_name=reader_name,
        parser=parser, parser_name=parser_name,
        writer=writer, writer_name=writer_name,
        settings=settings, settings_spec=settings_spec,
        settings_overrides=settings_overrides,
        config_section=config_section,
        enable_exit_status=enable_exit_status)
    return publisher.writer.parts


def publish_doctree(source, source_path=None,
                    source_class=io.StringInput,
                    reader=None, reader_name='standalone',
                    parser=None, parser_name='restructuredtext',
                    settings=None, settings_spec=None,
                    settings_overrides=None, config_section=None,
                    enable_exit_status=False):
    """
    Set up & run a `Publisher` for programmatic use. Return a document tree.

    Parameters: see `publish_programmatically()`.
    """
    _output, publisher = publish_programmatically(
        source=source, source_path=source_path,
        source_class=source_class,
        destination=None, destination_path=None,
        destination_class=io.NullOutput,
        reader=reader, reader_name=reader_name,
        parser=parser, parser_name=parser_name,
        writer=None, writer_name='null',
        settings=settings, settings_spec=settings_spec,
        settings_overrides=settings_overrides, config_section=config_section,
        enable_exit_status=enable_exit_status)
    return publisher.document


def publish_from_doctree(document, destination_path=None,
                         writer=None, writer_name='pseudoxml',
                         settings=None, settings_spec=None,
                         settings_overrides=None, config_section=None,
                         enable_exit_status=False):
    """
    Set up & run a `Publisher` to render from an existing document tree
    data structure. For programmatic use with string output
    (`bytes` or `str`, cf. `publish_string()`).

    Note that ``document.settings`` is overridden; if you want to use the
    settings of the original `document`, pass ``settings=document.settings``.

    Also, new `document.transformer` and `document.reporter` objects are
    generated.

    Parameters: `document` is a `docutils.nodes.document` object, an existing
    document tree.

    Other parameters: see `publish_programmatically()`.

    This function is provisional because in Python 3 name and behaviour
    of the `io.StringOutput` class no longer match.
    """
    reader = doctree.Reader(parser_name='null')
    publisher = Publisher(reader, None, writer,
                          source=io.DocTreeInput(document),
                          destination_class=io.StringOutput,
                          settings=settings)
    if not writer and writer_name:
        publisher.set_writer(writer_name)
    publisher.process_programmatic_settings(
        settings_spec, settings_overrides, config_section)
    publisher.set_destination(None, destination_path)
    return publisher.publish(enable_exit_status=enable_exit_status)


def publish_cmdline_to_binary(reader=None, reader_name='standalone',
                              parser=None, parser_name='restructuredtext',
                              writer=None, writer_name='pseudoxml',
                              settings=None,
                              settings_spec=None,
                              settings_overrides=None,
                              config_section=None,
                              enable_exit_status=True,
                              argv=None,
                              usage=default_usage,
                              description=default_description,
                              destination=None,
                              destination_class=io.BinaryFileOutput):
    """
    Set up & run a `Publisher` for command-line-based file I/O (input and
    output file paths taken automatically from the command line).
    Also return the output as `bytes`.

    This is just like publish_cmdline, except that it uses
    io.BinaryFileOutput instead of io.FileOutput.

    Parameters: see `publish_programmatically()` for the remainder.

    - `argv`: Command-line argument list to use instead of ``sys.argv[1:]``.
    - `usage`: Usage string, output if there's a problem parsing the command
      line.
    - `description`: Program description, output for the "--help" option
      (along with command-line option descriptions).
    """
    publisher = Publisher(reader, parser, writer, settings=settings,
                          destination_class=destination_class)
    publisher.set_components(reader_name, parser_name, writer_name)
    output = publisher.publish(
        argv, usage, description, settings_spec, settings_overrides,
        config_section=config_section, enable_exit_status=enable_exit_status)
    return output


def publish_programmatically(source_class, source, source_path,
                             destination_class, destination, destination_path,
                             reader, reader_name,
                             parser, parser_name,
                             writer, writer_name,
                             settings, settings_spec,
                             settings_overrides, config_section,
                             enable_exit_status):
    """
    Set up & run a `Publisher` for custom programmatic use.

    Return the output (as `str` or `bytes`, depending on `destination_class`,
    writer, and the "output_encoding" setting) and the Publisher object.

    Applications should not need to call this function directly.  If it does
    seem to be necessary to call this function directly, please write to the
    Docutils-develop mailing list
    <https://docutils.sourceforge.io/docs/user/mailing-lists.html#docutils-develop>.

    Parameters:

    * `source_class` **required**: The class for dynamically created source
      objects.  Typically `io.FileInput` or `io.StringInput`.

    * `source`: Type depends on `source_class`:

      - If `source_class` is `io.FileInput`: Either a file-like object
        (must have 'read' and 'close' methods), or ``None``
        (`source_path` is opened).  If neither `source` nor
        `source_path` are supplied, `sys.stdin` is used.

      - If `source_class` is `io.StringInput` **required**:
        The input as either a `bytes` object (ensure the 'input_encoding'
        setting matches its encoding) or a `str` object.

    * `source_path`: Type depends on `source_class`:

      - `io.FileInput`: Path to the input file, opened if no `source`
        supplied.

      - `io.StringInput`: Optional.  Path to the file or name of the
        object that produced `source`.  Only used for diagnostic output.

    * `destination_class` **required**: The class for dynamically created
      destination objects.  Typically `io.FileOutput` or `io.StringOutput`.

    * `destination`: Type depends on `destination_class`:

      - `io.FileOutput`: Either a file-like object (must have 'write' and
        'close' methods), or ``None`` (`destination_path` is opened).  If
        neither `destination` nor `destination_path` are supplied,
        `sys.stdout` is used.

      - `io.StringOutput`: Not used; pass ``None``.

    * `destination_path`: Type depends on `destination_class`:

      - `io.FileOutput`: Path to the output file.  Opened if no `destination`
        supplied.

      - `io.StringOutput`: Path to the file or object which will receive the
        output; optional.  Used for determining relative paths (stylesheets,
        source links, etc.).

    * `reader`: A `docutils.readers.Reader` object.

    * `reader_name`: Name or alias of the Reader class to be instantiated if
      no `reader` supplied.

    * `parser`: A `docutils.parsers.Parser` object.

    * `parser_name`: Name or alias of the Parser class to be instantiated if
      no `parser` supplied.

    * `writer`: A `docutils.writers.Writer` object.

    * `writer_name`: Name or alias of the Writer class to be instantiated if
      no `writer` supplied.

    * `settings`: A runtime settings (`docutils.frontend.Values`) object, for
      dotted-attribute access to runtime settings.  It's the end result of the
      `SettingsSpec`, config file, and option processing.  If `settings` is
      passed, it's assumed to be complete and no further setting/config/option
      processing is done.

    * `settings_spec`: A `docutils.SettingsSpec` subclass or object.  Provides
      extra application-specific settings definitions independently of
      components.  In other words, the application becomes a component, and
      its settings data is processed along with that of the other components.
      Used only if no `settings` specified.

    * `settings_overrides`: A dictionary containing application-specific
      settings defaults that override the defaults of other components.
      Used only if no `settings` specified.

    * `config_section`: A string, the name of the configuration file section
      for this application.  Overrides the ``config_section`` attribute
      defined by `settings_spec`.  Used only if no `settings` specified.

    * `enable_exit_status`: Boolean; enable exit status at end of processing?
    """
    publisher = Publisher(reader, parser, writer, settings=settings,
                          source_class=source_class,
                          destination_class=destination_class)
    publisher.set_components(reader_name, parser_name, writer_name)
    publisher.process_programmatic_settings(
        settings_spec, settings_overrides, config_section)
    publisher.set_source(source, source_path)
    publisher.set_destination(destination, destination_path)
    output = publisher.publish(enable_exit_status=enable_exit_status)
    return output, publisher


# "Entry points" with functionality of the "tools/rst2*.py" scripts
# cf. https://packaging.python.org/en/latest/specifications/entry-points/

def rst2something(writer, documenttype, doc_path=''):
    # Helper function for the common parts of rst2...
    #   writer:       writer name
    #   documenttype: output document type
    #   doc_path:     documentation path (relative to the documentation root)
    description = (
        f'Generate {documenttype} documents '
        'from standalone reStructuredText sources '
        f'<https://docutils.sourceforge.io/docs/{doc_path}>.  '
        + default_description)
    locale.setlocale(locale.LC_ALL, '')
    publish_cmdline(writer_name=writer, description=description)


def rst2html():
    rst2something('html', 'HTML', 'user/html.html#html')


def rst2html4():
    rst2something('html4', 'XHTML 1.1', 'user/html.html#html4css1')


def rst2html5():
    rst2something('html5', 'HTML5', 'user/html.html#html5-polyglot')


def rst2latex():
    rst2something('latex', 'LaTeX', 'user/latex.html')


def rst2man():
    rst2something('manpage', 'Unix manual (troff)', 'user/manpage.html')


def rst2odt():
    rst2something('odt', 'OpenDocument text (ODT)', 'user/odt.html')


def rst2pseudoxml():
    rst2something('pseudoxml', 'pseudo-XML (test)', 'ref/doctree.html')


def rst2s5():
    rst2something('s5', 'S5 HTML slideshow', 'user/slide-shows.html')


def rst2xetex():
    rst2something('xetex', 'LaTeX (XeLaTeX/LuaLaTeX)', 'user/latex.html')


def rst2xml():
    rst2something('xml', 'Docutils-native XML', 'ref/doctree.html')

env_pyrecodes/lib/python3.9/site-packages/docutils/core.py
NikolaBlagojevic/pyrecodes
# $Id: frontend.py 9328 2023-02-07 14:24:37Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Command-line and common processing for Docutils front-end tools.

This module is provisional.
Major changes will happen with the switch from the deprecated
"optparse" module to "arparse".

Applications should use the high-level API provided by `docutils.core`.
See https://docutils.sourceforge.io/docs/api/runtime-settings.html.

Exports the following classes:

* `OptionParser`: Standard Docutils command-line processing.
  Deprecated. Will be replaced by an ArgumentParser.
* `Option`: Customized version of `optparse.Option`; validation support.
  Deprecated. Will be removed.
* `Values`: Runtime settings; objects are simple structs
  (``object.attribute``).  Supports cumulative list settings (attributes).
  Deprecated. Will be removed.
* `ConfigParser`: Standard Docutils config file processing.
  Provisional. Details will change.

Also exports the following functions:

Interface function:
   `get_default_settings()`.  New in 0.19.

Option callbacks:
   `store_multiple()`, `read_config_file()`. Deprecated.

Setting validators:
  `validate_encoding()`, `validate_encoding_error_handler()`,
  `validate_encoding_and_error_handler()`,
  `validate_boolean()`, `validate_ternary()`,
  `validate_nonnegative_int()`, `validate_threshold()`,
  `validate_colon_separated_string_list()`,
  `validate_comma_separated_list()`,
  `validate_url_trailing_slash()`,
  `validate_dependency_file()`,
  `validate_strip_class()`
  `validate_smartquotes_locales()`.

  Provisional.

Misc:
  `make_paths_absolute()`, `filter_settings_spec()`. Provisional.
"""

__docformat__ = 'reStructuredText'


import codecs
import configparser
import optparse
from optparse import SUPPRESS_HELP
import os
import os.path
import sys
import warnings

import docutils
from docutils import io, utils


def store_multiple(option, opt, value, parser, *args, **kwargs):
    """
    Store multiple values in `parser.values`.  (Option callback.)

    Store `None` for each attribute named in `args`, and store the value for
    each key (attribute name) in `kwargs`.
    """
    for attribute in args:
        setattr(parser.values, attribute, None)
    for key, value in kwargs.items():
        setattr(parser.values, key, value)


def read_config_file(option, opt, value, parser):
    """
    Read a configuration file during option processing.  (Option callback.)
    """
    try:
        new_settings = parser.get_config_file_settings(value)
    except ValueError as err:
        parser.error(err)
    parser.values.update(new_settings, parser)


def validate_encoding(setting, value, option_parser,
                      config_parser=None, config_section=None):
    if value == '':
        return None  # allow overwriting a config file value
    try:
        codecs.lookup(value)
    except LookupError:
        raise LookupError('setting "%s": unknown encoding: "%s"'
                          % (setting, value))
    return value


def validate_encoding_error_handler(setting, value, option_parser,
                                    config_parser=None, config_section=None):
    try:
        codecs.lookup_error(value)
    except LookupError:
        raise LookupError(
            'unknown encoding error handler: "%s" (choices: '
            '"strict", "ignore", "replace", "backslashreplace", '
            '"xmlcharrefreplace", and possibly others; see documentation for '
            'the Python ``codecs`` module)' % value)
    return value


def validate_encoding_and_error_handler(
    setting, value, option_parser, config_parser=None, config_section=None):
    """
    Side-effect: if an error handler is included in the value, it is inserted
    into the appropriate place as if it was a separate setting/option.
    """
    if ':' in value:
        encoding, handler = value.split(':')
        validate_encoding_error_handler(
            setting + '_error_handler', handler, option_parser,
            config_parser, config_section)
        if config_parser:
            config_parser.set(config_section, setting + '_error_handler',
                              handler)
        else:
            setattr(option_parser.values, setting + '_error_handler', handler)
    else:
        encoding = value
    validate_encoding(setting, encoding, option_parser,
                      config_parser, config_section)
    return encoding


def validate_boolean(setting, value, option_parser,
                     config_parser=None, config_section=None):
    """Check/normalize boolean settings:
         True:  '1', 'on', 'yes', 'true'
         False: '0', 'off', 'no','false', ''
    """
    if isinstance(value, bool):
        return value
    try:
        return option_parser.booleans[value.strip().lower()]
    except KeyError:
        raise LookupError('unknown boolean value: "%s"' % value)


def validate_ternary(setting, value, option_parser,
                     config_parser=None, config_section=None):
    """Check/normalize three-value settings:
         True:  '1', 'on', 'yes', 'true'
         False: '0', 'off', 'no','false', ''
         any other value: returned as-is.
    """
    if isinstance(value, bool) or value is None:
        return value
    try:
        return option_parser.booleans[value.strip().lower()]
    except KeyError:
        return value


def validate_nonnegative_int(setting, value, option_parser,
                             config_parser=None, config_section=None):
    value = int(value)
    if value < 0:
        raise ValueError('negative value; must be positive or zero')
    return value


def validate_threshold(setting, value, option_parser,
                       config_parser=None, config_section=None):
    try:
        return int(value)
    except ValueError:
        try:
            return option_parser.thresholds[value.lower()]
        except (KeyError, AttributeError):
            raise LookupError('unknown threshold: %r.' % value)


def validate_colon_separated_string_list(
    setting, value, option_parser, config_parser=None, config_section=None):
    if not isinstance(value, list):
        value = value.split(':')
    else:
        last = value.pop()
        value.extend(last.split(':'))
    return value


def validate_comma_separated_list(setting, value, option_parser,
                                  config_parser=None, config_section=None):
    """Check/normalize list arguments (split at "," and strip whitespace).
    """
    # `value` may be ``bytes``, ``str``, or a ``list`` (when  given as
    # command line option and "action" is "append").
    if not isinstance(value, list):
        value = [value]
    # this function is called for every option added to `value`
    # -> split the last item and append the result:
    last = value.pop()
    items = [i.strip(' \t\n') for i in last.split(',') if i.strip(' \t\n')]
    value.extend(items)
    return value


def validate_url_trailing_slash(
    setting, value, option_parser, config_parser=None, config_section=None):
    if not value:
        return './'
    elif value.endswith('/'):
        return value
    else:
        return value + '/'


def validate_dependency_file(setting, value, option_parser,
                             config_parser=None, config_section=None):
    try:
        return utils.DependencyList(value)
    except OSError:
        # TODO: warn/info?
        return utils.DependencyList(None)


def validate_strip_class(setting, value, option_parser,
                         config_parser=None, config_section=None):
    # value is a comma separated string list:
    value = validate_comma_separated_list(setting, value, option_parser,
                                          config_parser, config_section)
    # validate list elements:
    for cls in value:
        normalized = docutils.nodes.make_id(cls)
        if cls != normalized:
            raise ValueError('Invalid class value %r (perhaps %r?)'
                             % (cls, normalized))
    return value


def validate_smartquotes_locales(setting, value, option_parser,
                                 config_parser=None, config_section=None):
    """Check/normalize a comma separated list of smart quote definitions.

    Return a list of (language-tag, quotes) string tuples."""

    # value is a comma separated string list:
    value = validate_comma_separated_list(setting, value, option_parser,
                                          config_parser, config_section)
    # validate list elements
    lc_quotes = []
    for item in value:
        try:
            lang, quotes = item.split(':', 1)
        except AttributeError:
            # this function is called for every option added to `value`
            # -> ignore if already a tuple:
            lc_quotes.append(item)
            continue
        except ValueError:
            raise ValueError('Invalid value "%s".'
                             ' Format is "<language>:<quotes>".'
                             % item.encode('ascii', 'backslashreplace'))
        # parse colon separated string list:
        quotes = quotes.strip()
        multichar_quotes = quotes.split(':')
        if len(multichar_quotes) == 4:
            quotes = multichar_quotes
        elif len(quotes) != 4:
            raise ValueError('Invalid value "%s". Please specify 4 quotes\n'
                             '    (primary open/close; secondary open/close).'
                             % item.encode('ascii', 'backslashreplace'))
        lc_quotes.append((lang, quotes))
    return lc_quotes


def make_paths_absolute(pathdict, keys, base_path=None):
    """
    Interpret filesystem path settings relative to the `base_path` given.

    Paths are values in `pathdict` whose keys are in `keys`.  Get `keys` from
    `OptionParser.relative_path_settings`.
    """
    if base_path is None:
        base_path = os.getcwd()
    for key in keys:
        if key in pathdict:
            value = pathdict[key]
            if isinstance(value, list):
                value = [make_one_path_absolute(base_path, path)
                         for path in value]
            elif value:
                value = make_one_path_absolute(base_path, value)
            pathdict[key] = value


def make_one_path_absolute(base_path, path):
    return os.path.abspath(os.path.join(base_path, path))


def filter_settings_spec(settings_spec, *exclude, **replace):
    """Return a copy of `settings_spec` excluding/replacing some settings.

    `settings_spec` is a tuple of configuration settings
    (cf. `docutils.SettingsSpec.settings_spec`).

    Optional positional arguments are names of to-be-excluded settings.
    Keyword arguments are option specification replacements.
    (See the html4strict writer for an example.)
    """
    settings = list(settings_spec)
    # every third item is a sequence of option tuples
    for i in range(2, len(settings), 3):
        newopts = []
        for opt_spec in settings[i]:
            # opt_spec is ("<help>", [<option strings>], {<keyword args>})
            opt_name = [opt_string[2:].replace('-', '_')
                        for opt_string in opt_spec[1]
                        if opt_string.startswith('--')][0]
            if opt_name in exclude:
                continue
            if opt_name in replace.keys():
                newopts.append(replace[opt_name])
            else:
                newopts.append(opt_spec)
        settings[i] = tuple(newopts)
    return tuple(settings)


class Values(optparse.Values):
    """Storage for option values.

    Updates list attributes by extension rather than by replacement.
    Works in conjunction with the `OptionParser.lists` instance attribute.

    Deprecated. Will be removed.
    """

    def __init__(self, *args, **kwargs):
        warnings.warn('frontend.Values class will be removed '
                      'in Docutils 0.21 or later.',
                      DeprecationWarning, stacklevel=2)
        super().__init__(*args, **kwargs)
        if getattr(self, 'record_dependencies', None) is None:
            # Set up dummy dependency list.
            self.record_dependencies = utils.DependencyList()

    def update(self, other_dict, option_parser):
        if isinstance(other_dict, Values):
            other_dict = other_dict.__dict__
        other_dict = dict(other_dict)  # also works with ConfigParser sections
        for setting in option_parser.lists.keys():
            if hasattr(self, setting) and setting in other_dict:
                value = getattr(self, setting)
                if value:
                    value += other_dict[setting]
                    del other_dict[setting]
        self._update_loose(other_dict)

    def copy(self):
        """Return a shallow copy of `self`."""
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            return self.__class__(defaults=self.__dict__)

    def setdefault(self, name, default):
        """Return ``self.name`` or ``default``.

        If ``self.name`` is unset, set ``self.name = default``.
        """
        if getattr(self, name, None) is None:
            setattr(self, name, default)
        return getattr(self, name)


class Option(optparse.Option):
    """Add validation and override support to `optparse.Option`.

    Deprecated. Will be removed.
    """

    ATTRS = optparse.Option.ATTRS + ['validator', 'overrides']

    def __init__(self, *args, **kwargs):
        warnings.warn('The frontend.Option class will be removed '
                      'in Docutils 0.21 or later.',
                      DeprecationWarning, stacklevel=2)
        super().__init__(*args, **kwargs)

    def process(self, opt, value, values, parser):
        """
        Call the validator function on applicable settings and
        evaluate the 'overrides' option.
        Extends `optparse.Option.process`.
        """
        result = super().process(opt, value, values, parser)
        setting = self.dest
        if setting:
            if self.validator:
                value = getattr(values, setting)
                try:
                    new_value = self.validator(setting, value, parser)
                except Exception as err:
                    raise optparse.OptionValueError(
                        'Error in option "%s":\n    %s'
                        % (opt, io.error_string(err)))
                setattr(values, setting, new_value)
            if self.overrides:
                setattr(values, self.overrides, None)
        return result


class OptionParser(optparse.OptionParser, docutils.SettingsSpec):
    """
    Settings parser for command-line and library use.

    The `settings_spec` specification here and in other Docutils components
    are merged to build the set of command-line options and runtime settings
    for this process.

    Common settings (defined below) and component-specific settings must not
    conflict.  Short options are reserved for common settings, and components
    are restricted to using long options.

    Deprecated.
    Will be replaced by a subclass of `argparse.ArgumentParser`.
    """

    standard_config_files = [
        '/etc/docutils.conf',           # system-wide
        './docutils.conf',              # project-specific
        '~/.docutils']                  # user-specific
    """Docutils configuration files, using ConfigParser syntax.

    Filenames will be tilde-expanded later. Later files override earlier ones.
    """

    threshold_choices = 'info 1 warning 2 error 3 severe 4 none 5'.split()
    """Possible inputs for for --report and --halt threshold values."""

    thresholds = {'info': 1, 'warning': 2, 'error': 3, 'severe': 4, 'none': 5}
    """Lookup table for --report and --halt threshold values."""

    booleans = {'1': True, 'on': True, 'yes': True, 'true': True, '0': False,
                'off': False, 'no': False, 'false': False, '': False}
    """Lookup table for boolean configuration file settings."""

    default_error_encoding = (getattr(sys.stderr, 'encoding', None)
                              or io._locale_encoding  # noqa
                              or 'ascii')

    default_error_encoding_error_handler = 'backslashreplace'

    settings_spec = (
        'General Docutils Options',
        None,
        (('Output destination name. Obsoletes the <destination> '
          'positional argument. Default: None (stdout).',
          ['--output'], {'metavar': '<destination>'}),
         ('Specify the document title as metadata.',
          ['--title'], {'metavar': '<title>'}),
         ('Include a "Generated by Docutils" credit and link.',
          ['--generator', '-g'], {'action': 'store_true',
                                  'validator': validate_boolean}),
         ('Do not include a generator credit.',
          ['--no-generator'], {'action': 'store_false', 'dest': 'generator'}),
         ('Include the date at the end of the document (UTC).',
          ['--date', '-d'], {'action': 'store_const', 'const': '%Y-%m-%d',
                             'dest': 'datestamp'}),
         ('Include the time & date (UTC).',
          ['--time', '-t'], {'action': 'store_const',
                             'const': '%Y-%m-%d %H:%M UTC',
                             'dest': 'datestamp'}),
         ('Do not include a datestamp of any kind.',
          ['--no-datestamp'], {'action': 'store_const', 'const': None,
                               'dest': 'datestamp'}),
         ('Include a "View document source" link.',
          ['--source-link', '-s'], {'action': 'store_true',
                                    'validator': validate_boolean}),
         ('Use <URL> for a source link; implies --source-link.',
          ['--source-url'], {'metavar': '<URL>'}),
         ('Do not include a "View document source" link.',
          ['--no-source-link'],
          {'action': 'callback', 'callback': store_multiple,
           'callback_args': ('source_link', 'source_url')}),
         ('Link from section headers to TOC entries.  (default)',
          ['--toc-entry-backlinks'],
          {'dest': 'toc_backlinks', 'action': 'store_const', 'const': 'entry',
           'default': 'entry'}),
         ('Link from section headers to the top of the TOC.',
          ['--toc-top-backlinks'],
          {'dest': 'toc_backlinks', 'action': 'store_const', 'const': 'top'}),
         ('Disable backlinks to the table of contents.',
          ['--no-toc-backlinks'],
          {'dest': 'toc_backlinks', 'action': 'store_false'}),
         ('Link from footnotes/citations to references. (default)',
          ['--footnote-backlinks'],
          {'action': 'store_true', 'default': 1,
           'validator': validate_boolean}),
         ('Disable backlinks from footnotes and citations.',
          ['--no-footnote-backlinks'],
          {'dest': 'footnote_backlinks', 'action': 'store_false'}),
         ('Enable section numbering by Docutils.  (default)',
          ['--section-numbering'],
          {'action': 'store_true', 'dest': 'sectnum_xform',
           'default': 1, 'validator': validate_boolean}),
         ('Disable section numbering by Docutils.',
          ['--no-section-numbering'],
          {'action': 'store_false', 'dest': 'sectnum_xform'}),
         ('Remove comment elements from the document tree.',
          ['--strip-comments'],
          {'action': 'store_true', 'validator': validate_boolean}),
         ('Leave comment elements in the document tree. (default)',
          ['--leave-comments'],
          {'action': 'store_false', 'dest': 'strip_comments'}),
         ('Remove all elements with classes="<class>" from the document tree. '
          'Warning: potentially dangerous; use with caution. '
          '(Multiple-use option.)',
          ['--strip-elements-with-class'],
          {'action': 'append', 'dest': 'strip_elements_with_classes',
           'metavar': '<class>', 'validator': validate_strip_class}),
         ('Remove all classes="<class>" attributes from elements in the '
          'document tree. Warning: potentially dangerous; use with caution. '
          '(Multiple-use option.)',
          ['--strip-class'],
          {'action': 'append', 'dest': 'strip_classes',
           'metavar': '<class>', 'validator': validate_strip_class}),
         ('Report system messages at or higher than <level>: "info" or "1", '
          '"warning"/"2" (default), "error"/"3", "severe"/"4", "none"/"5"',
          ['--report', '-r'], {'choices': threshold_choices, 'default': 2,
                               'dest': 'report_level', 'metavar': '<level>',
                               'validator': validate_threshold}),
         ('Report all system messages.  (Same as "--report=1".)',
          ['--verbose', '-v'], {'action': 'store_const', 'const': 1,
                                'dest': 'report_level'}),
         ('Report no system messages.  (Same as "--report=5".)',
          ['--quiet', '-q'], {'action': 'store_const', 'const': 5,
                              'dest': 'report_level'}),
         ('Halt execution at system messages at or above <level>.  '
          'Levels as in --report.  Default: 4 (severe).',
          ['--halt'], {'choices': threshold_choices, 'dest': 'halt_level',
                       'default': 4, 'metavar': '<level>',
                       'validator': validate_threshold}),
         ('Halt at the slightest problem.  Same as "--halt=info".',
          ['--strict'], {'action': 'store_const', 'const': 1,
                         'dest': 'halt_level'}),
         ('Enable a non-zero exit status for non-halting system messages at '
          'or above <level>.  Default: 5 (disabled).',
          ['--exit-status'], {'choices': threshold_choices,
                              'dest': 'exit_status_level',
                              'default': 5, 'metavar': '<level>',
                              'validator': validate_threshold}),
         ('Enable debug-level system messages and diagnostics.',
          ['--debug'], {'action': 'store_true',
                        'validator': validate_boolean}),
         ('Disable debug output.  (default)',
          ['--no-debug'], {'action': 'store_false', 'dest': 'debug'}),
         ('Send the output of system messages to <file>.',
          ['--warnings'], {'dest': 'warning_stream', 'metavar': '<file>'}),
         ('Enable Python tracebacks when Docutils is halted.',
          ['--traceback'], {'action': 'store_true', 'default': None,
                            'validator': validate_boolean}),
         ('Disable Python tracebacks.  (default)',
          ['--no-traceback'], {'dest': 'traceback', 'action': 'store_false'}),
         ('Specify the encoding and optionally the '
          'error handler of input text.  Default: <auto-detect>:strict.',
          ['--input-encoding', '-i'],
          {'metavar': '<name[:handler]>',
           'validator': validate_encoding_and_error_handler}),
         ('Specify the error handler for undecodable characters.  '
          'Choices: "strict" (default), "ignore", and "replace".',
          ['--input-encoding-error-handler'],
          {'default': 'strict', 'validator': validate_encoding_error_handler}),
         ('Specify the text encoding and optionally the error handler for '
          'output.  Default: utf-8:strict.',
          ['--output-encoding', '-o'],
          {'metavar': '<name[:handler]>', 'default': 'utf-8',
           'validator': validate_encoding_and_error_handler}),
         ('Specify error handler for unencodable output characters; '
          '"strict" (default), "ignore", "replace", '
          '"xmlcharrefreplace", "backslashreplace".',
          ['--output-encoding-error-handler'],
          {'default': 'strict', 'validator': validate_encoding_error_handler}),
         ('Specify text encoding and optionally error handler '
          'for error output.  Default: %s:%s.'
          % (default_error_encoding, default_error_encoding_error_handler),
          ['--error-encoding', '-e'],
          {'metavar': '<name[:handler]>', 'default': default_error_encoding,
           'validator': validate_encoding_and_error_handler}),
         ('Specify the error handler for unencodable characters in '
          'error output.  Default: %s.'
          % default_error_encoding_error_handler,
          ['--error-encoding-error-handler'],
          {'default': default_error_encoding_error_handler,
           'validator': validate_encoding_error_handler}),
         ('Specify the language (as BCP 47 language tag).  Default: en.',
          ['--language', '-l'], {'dest': 'language_code', 'default': 'en',
                                 'metavar': '<name>'}),
         ('Write output file dependencies to <file>.',
          ['--record-dependencies'],
          {'metavar': '<file>', 'validator': validate_dependency_file,
           'default': None}),           # default set in Values class
         ('Read configuration settings from <file>, if it exists.',
          ['--config'], {'metavar': '<file>', 'type': 'string',
                         'action': 'callback', 'callback': read_config_file}),
         ("Show this program's version number and exit.",
          ['--version', '-V'], {'action': 'version'}),
         ('Show this help message and exit.',
          ['--help', '-h'], {'action': 'help'}),
         # Typically not useful for non-programmatical use:
         (SUPPRESS_HELP, ['--id-prefix'], {'default': ''}),
         (SUPPRESS_HELP, ['--auto-id-prefix'], {'default': '%'}),
         # Hidden options, for development use only:
         (SUPPRESS_HELP, ['--dump-settings'], {'action': 'store_true'}),
         (SUPPRESS_HELP, ['--dump-internals'], {'action': 'store_true'}),
         (SUPPRESS_HELP, ['--dump-transforms'], {'action': 'store_true'}),
         (SUPPRESS_HELP, ['--dump-pseudo-xml'], {'action': 'store_true'}),
         (SUPPRESS_HELP, ['--expose-internal-attribute'],
          {'action': 'append', 'dest': 'expose_internals',
           'validator': validate_colon_separated_string_list}),
         (SUPPRESS_HELP, ['--strict-visitor'], {'action': 'store_true'}),
         ))
    """Runtime settings and command-line options common to all Docutils front
    ends.  Setting specs specific to individual Docutils components are also
    used (see `populate_from_components()`)."""

    settings_defaults = {'_disable_config': None,
                         '_source': None,
                         '_destination': None,
                         '_config_files': None}
    """Defaults for settings without command-line option equivalents.

    See https://docutils.sourceforge.io/docs/user/config.html#internal-settings
    """

    config_section = 'general'

    version_template = ('%%prog (Docutils %s%s, Python %s, on %s)'
                        % (docutils.__version__,
                           docutils.__version_details__
                           and ' [%s]'%docutils.__version_details__ or '',
                           sys.version.split()[0], sys.platform))
    """Default version message."""

    def __init__(self, components=(), defaults=None, read_config_files=False,
                 *args, **kwargs):
        """Set up OptionParser instance.

        `components` is a list of Docutils components each containing a
        ``.settings_spec`` attribute.
        `defaults` is a mapping of setting default overrides.
        """

        self.lists = {}
        """Set of list-type settings."""

        self.config_files = []
        """List of paths of applied configuration files."""

        self.relative_path_settings = ['warning_stream']  # will be modified

        warnings.warn('The frontend.OptionParser class will be replaced '
                      'by a subclass of argparse.ArgumentParser '
                      'in Docutils 0.21 or later.',
                      DeprecationWarning, stacklevel=2)
        super().__init__(option_class=Option, add_help_option=None,
                         formatter=optparse.TitledHelpFormatter(width=78),
                         *args, **kwargs)
        if not self.version:
            self.version = self.version_template
        self.components = (self, *components)
        self.populate_from_components(self.components)
        self.defaults.update(defaults or {})
        if read_config_files and not self.defaults['_disable_config']:
            try:
                config_settings = self.get_standard_config_settings()
            except ValueError as err:
                self.error(err)
            self.defaults.update(config_settings.__dict__)

    def populate_from_components(self, components):
        """Collect settings specification from components.

        For each component, populate from the `SettingsSpec.settings_spec`
        structure, then from the `SettingsSpec.settings_defaults` dictionary.
        After all components have been processed, check for and populate from
        each component's `SettingsSpec.settings_default_overrides` dictionary.
        """
        for component in components:
            if component is None:
                continue
            settings_spec = component.settings_spec
            self.relative_path_settings.extend(
                component.relative_path_settings)
            for i in range(0, len(settings_spec), 3):
                title, description, option_spec = settings_spec[i:i+3]
                if title:
                    group = optparse.OptionGroup(self, title, description)
                    self.add_option_group(group)
                else:
                    group = self        # single options
                for (help_text, option_strings, kwargs) in option_spec:
                    option = group.add_option(help=help_text, *option_strings,
                                              **kwargs)
                    if kwargs.get('action') == 'append':
                        self.lists[option.dest] = True
                if component.settings_defaults:
                    self.defaults.update(component.settings_defaults)
        for component in components:
            if component and component.settings_default_overrides:
                self.defaults.update(component.settings_default_overrides)

    @classmethod
    def get_standard_config_files(cls):
        """Return list of config files, from environment or standard."""
        if 'DOCUTILSCONFIG' in os.environ:
            config_files = os.environ['DOCUTILSCONFIG'].split(os.pathsep)
        else:
            config_files = cls.standard_config_files
        return [os.path.expanduser(f) for f in config_files if f.strip()]

    def get_standard_config_settings(self):
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            settings = Values()
        for filename in self.get_standard_config_files():
            settings.update(self.get_config_file_settings(filename), self)
        return settings

    def get_config_file_settings(self, config_file):
        """Returns a dictionary containing appropriate config file settings."""
        config_parser = ConfigParser()
        # parse config file, add filename if found and successfully read.
        applied = set()
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            self.config_files += config_parser.read(config_file, self)
            settings = Values()
        for component in self.components:
            if not component:
                continue
            for section in (tuple(component.config_section_dependencies or ())
                            + (component.config_section,)):
                if section in applied:
                    continue
                applied.add(section)
                if config_parser.has_section(section):
                    settings.update(config_parser[section], self)
        make_paths_absolute(settings.__dict__,
                            self.relative_path_settings,
                            os.path.dirname(config_file))
        return settings.__dict__

    def check_values(self, values, args):
        """Store positional arguments as runtime settings."""
        values._source, values._destination = self.check_args(args)
        make_paths_absolute(values.__dict__, self.relative_path_settings)
        values._config_files = self.config_files
        return values

    def check_args(self, args):
        source = destination = None
        if args:
            source = args.pop(0)
            if source == '-':           # means stdin
                source = None
        if args:
            destination = args.pop(0)
            if destination == '-':      # means stdout
                destination = None
        if args:
            self.error('Maximum 2 arguments allowed.')
        if source and source == destination:
            self.error('Do not specify the same file for both source and '
                       'destination.  It will clobber the source file.')
        return source, destination

    def set_defaults_from_dict(self, defaults):
        # not used, deprecated, will be removed
        self.defaults.update(defaults)

    def get_default_values(self):
        """Needed to get custom `Values` instances."""
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            defaults = Values(self.defaults)
        defaults._config_files = self.config_files
        return defaults

    def get_option_by_dest(self, dest):
        """
        Get an option by its dest.

        If you're supplying a dest which is shared by several options,
        it is undefined which option of those is returned.

        A KeyError is raised if there is no option with the supplied
        dest.
        """
        for group in self.option_groups + [self]:
            for option in group.option_list:
                if option.dest == dest:
                    return option
        raise KeyError('No option with dest == %r.' % dest)


class ConfigParser(configparser.RawConfigParser):
    """Parser for Docutils configuration files.

    See https://docutils.sourceforge.io/docs/user/config.html.

    Option key normalization includes conversion of '-' to '_'.

    Config file encoding is "utf-8". Encoding errors are reported
    and the affected file(s) skipped.

    This class is provisional and will change in future versions.
    """

    old_settings = {
        'pep_stylesheet': ('pep_html writer', 'stylesheet'),
        'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'),
        'pep_template': ('pep_html writer', 'template')}
    """{old setting: (new section, new setting)} mapping, used by
    `handle_old_config`, to convert settings from the old [options] section.
    """

    old_warning = (
        'The "[option]" section is deprecated.\n'
        'Support for old-format configuration files will be removed in '
        'Docutils 0.21 or later.  Please revise your configuration files.  '
        'See <https://docutils.sourceforge.io/docs/user/config.html>, '
        'section "Old-Format Configuration Files".')

    not_utf8_error = """\
Unable to read configuration file "%s": content not encoded as UTF-8.
Skipping "%s" configuration file.
"""

    def read(self, filenames, option_parser=None):
        # Currently, if a `docutils.frontend.OptionParser` instance is
        # supplied, setting values are validated.
        if option_parser is not None:
            warnings.warn('frontend.ConfigParser.read(): parameter '
                          '"option_parser" will be removed '
                          'in Docutils 0.21 or later.',
                          DeprecationWarning, stacklevel=2)
        read_ok = []
        if isinstance(filenames, str):
            filenames = [filenames]
        for filename in filenames:
            # Config files are UTF-8-encoded:
            try:
                read_ok += super().read(filename, encoding='utf-8')
            except UnicodeDecodeError:
                sys.stderr.write(self.not_utf8_error % (filename, filename))
                continue
            if 'options' in self:
                self.handle_old_config(filename)
            if option_parser is not None:
                self.validate_settings(filename, option_parser)
        return read_ok

    def handle_old_config(self, filename):
        warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning,
                               filename, 0)
        options = self.get_section('options')
        if not self.has_section('general'):
            self.add_section('general')
        for key, value in options.items():
            if key in self.old_settings:
                section, setting = self.old_settings[key]
                if not self.has_section(section):
                    self.add_section(section)
            else:
                section = 'general'
                setting = key
            if not self.has_option(section, setting):
                self.set(section, setting, value)
        self.remove_section('options')

    def validate_settings(self, filename, option_parser):
        """
        Call the validator function and implement overrides on all applicable
        settings.
        """
        for section in self.sections():
            for setting in self.options(section):
                try:
                    option = option_parser.get_option_by_dest(setting)
                except KeyError:
                    continue
                if option.validator:
                    value = self.get(section, setting)
                    try:
                        new_value = option.validator(
                            setting, value, option_parser,
                            config_parser=self, config_section=section)
                    except Exception as err:
                        raise ValueError(f'Error in config file "{filename}", '
                                         f'section "[{section}]":\n'
                                         f'    {io.error_string(err)}\n'
                                         f'        {setting} = {value}')
                    self.set(section, setting, new_value)
                if option.overrides:
                    self.set(section, option.overrides, None)

    def optionxform(self, optionstr):
        """
        Lowercase and transform '-' to '_'.

        So the cmdline form of option names can be used in config files.
        """
        return optionstr.lower().replace('-', '_')

    def get_section(self, section):
        """
        Return a given section as a dictionary.

        Return empty dictionary if the section doesn't exist.

        Deprecated. Use the configparser "Mapping Protocol Access" and
        catch KeyError.
        """
        warnings.warn('frontend.OptionParser.get_section() '
                      'will be removed in Docutils 0.21 or later.',
                      DeprecationWarning, stacklevel=2)
        try:
            return dict(self[section])
        except KeyError:
            return {}


class ConfigDeprecationWarning(FutureWarning):
    """Warning for deprecated configuration file features."""


def get_default_settings(*components):
    """Return default runtime settings for `components`.

    Return a `frontend.Values` instance with defaults for generic Docutils
    settings and settings from the `components` (`SettingsSpec` instances).

    This corresponds to steps 1 and 2 in the `runtime settings priority`__.

    __ https://docutils.sourceforge.io/docs/api/runtime-settings.html
       #settings-priority
    """
    with warnings.catch_warnings():
        warnings.filterwarnings('ignore', category=DeprecationWarning)
        return OptionParser(components).get_default_values()

env_pyrecodes/lib/python3.9/site-packages/docutils/frontend.py
NikolaBlagojevic/pyrecodes
# $Id: io.py 9369 2023-05-02 23:04:27Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
I/O classes provide a uniform API for low-level input and output.  Subclasses
exist for a variety of input/output mechanisms.
"""

__docformat__ = 'reStructuredText'

import codecs
import locale
import os
import re
import sys
import warnings

from docutils import TransformSpec


# Guess the locale's preferred encoding.
# If no valid guess can be made, _locale_encoding is set to `None`:
#
# TODO: check whether this is set correctly with every OS and Python version
#       or whether front-end tools need to call `locale.setlocale()`
#       before importing this module
try:
    # Return locale encoding also in UTF-8 mode
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        _locale_encoding = (locale.getlocale()[1]
                            or locale.getdefaultlocale()[1])
        _locale_encoding = _locale_encoding.lower()
except ValueError as error:  # OS X may set UTF-8 without language code
    # See https://bugs.python.org/issue18378 fixed in 3.8
    # and https://sourceforge.net/p/docutils/bugs/298/.
    # Drop the special case after requiring Python >= 3.8
    if "unknown locale: UTF-8" in error.args:
        _locale_encoding = "utf-8"
    else:
        _locale_encoding = None
except:  # noqa  any other problems determining the locale -> use None
    _locale_encoding = None
try:
    codecs.lookup(_locale_encoding)
except (LookupError, TypeError):
    _locale_encoding = None


class InputError(OSError): pass
class OutputError(OSError): pass


def check_encoding(stream, encoding):
    """Test, whether the encoding of `stream` matches `encoding`.

    Returns

    :None:  if `encoding` or `stream.encoding` are not a valid encoding
            argument (e.g. ``None``) or `stream.encoding is missing.
    :True:  if the encoding argument resolves to the same value as `encoding`,
    :False: if the encodings differ.
    """
    try:
        return codecs.lookup(stream.encoding) == codecs.lookup(encoding)
    except (LookupError, AttributeError, TypeError):
        return None


def error_string(err):
    """Return string representation of Exception `err`.
    """
    return f'{err.__class__.__name__}: {err}'


class Input(TransformSpec):
    """
    Abstract base class for input wrappers.

    Docutils input objects must provide a `read()` method that
    returns the source, typically as `str` instance.

    Inheriting `TransformSpec` allows input objects to add
    "transforms" and "unknown_reference_resolvers" to the "Transformer".
    (Optional for custom input objects since Docutils 0.19.)
    """

    component_type = 'input'

    default_source_path = None

    def __init__(self, source=None, source_path=None, encoding=None,
                 error_handler='strict'):
        self.encoding = encoding
        """Text encoding for the input source."""

        self.error_handler = error_handler
        """Text decoding error handler."""

        self.source = source
        """The source of input data."""

        self.source_path = source_path
        """A text reference to the source."""

        if not source_path:
            self.source_path = self.default_source_path

        self.successful_encoding = None
        """The encoding that successfully decoded the source data."""

    def __repr__(self):
        return '%s: source=%r, source_path=%r' % (self.__class__, self.source,
                                                  self.source_path)

    def read(self):
        """Return input as `str`. Define in subclasses."""
        raise NotImplementedError

    def decode(self, data):
        """
        Decode `data` if required.

        Return Unicode `str` instances unchanged (nothing to decode).

        If `self.encoding` is None, determine encoding from data
        or try UTF-8, locale encoding, and (as last ressort) 'latin-1'.
        The client application should call ``locale.setlocale`` at the
        beginning of processing::

            locale.setlocale(locale.LC_ALL, '')

        Raise UnicodeError if unsuccessful.

        Provisional:
          - Raise UnicodeError (instead of falling back to the locale
            encoding) if decoding the source with the default encoding (UTF-8)
            fails and Python is started in `UTF-8 mode`.

            Raise UnicodeError (instead of falling back to "latin1") if both,
            default and locale encoding, fail.

          - Only remove BOM (U+FEFF ZWNBSP at start of data),
            no other ZWNBSPs.
        """
        if self.encoding and self.encoding.lower() == 'unicode':
            assert isinstance(data, str), ('input encoding is "unicode" '
                                           'but `data` is no `str` instance')
        if isinstance(data, str):
            # nothing to decode
            return data
        if self.encoding:
            # We believe the user/application when the encoding is
            # explicitly given.
            encoding_candidates = [self.encoding]
        else:
            data_encoding = self.determine_encoding_from_data(data)
            if data_encoding:
                # If the data declares its encoding (explicitly or via a BOM),
                # we believe it.
                encoding_candidates = [data_encoding]
            else:
                # Apply heuristics only if no encoding is explicitly given and
                # no BOM found.  Start with UTF-8, because that only matches
                # data that *IS* UTF-8:
                encoding_candidates = ['utf-8']
                # TODO: use `locale.getpreferredlocale(do_setlocale=True)`
                # to respect UTF-8 mode (API change).
                # (Check if it is a valid encoding and not UTF-8)
                if _locale_encoding and _locale_encoding != 'utf-8':
                    encoding_candidates.append(_locale_encoding)
                # TODO: don't fall back to 'latin-1' (API change).
                encoding_candidates.append('latin-1')
        for enc in encoding_candidates:
            try:
                decoded = str(data, enc, self.error_handler)
                self.successful_encoding = enc
                # Return decoded, removing BOM and other ZWNBSPs.
                # TODO: only remove BOM (ZWNBSP at start of data)
                #       and only if 'self.encoding' is None. (API change)
                return decoded.replace('\ufeff', '')
            except (UnicodeError, LookupError) as err:
                # keep exception instance for use outside of the "for" loop.
                error = err
        raise UnicodeError(
            'Unable to decode input data.  Tried the following encodings: '
            f'{", ".join(repr(enc) for enc in encoding_candidates)}.\n'
            f'({error_string(error)})')

    coding_slug = re.compile(br"coding[:=]\s*([-\w.]+)")
    """Encoding declaration pattern."""

    byte_order_marks = ((codecs.BOM_UTF8, 'utf-8'),
                        (codecs.BOM_UTF16_BE, 'utf-16-be'),
                        (codecs.BOM_UTF16_LE, 'utf-16-le'),)
    """Sequence of (start_bytes, encoding) tuples for encoding detection.
    The first bytes of input data are checked against the start_bytes strings.
    A match indicates the given encoding."""

    def determine_encoding_from_data(self, data):
        """
        Try to determine the encoding of `data` by looking *in* `data`.
        Check for a byte order mark (BOM) or an encoding declaration.
        """
        # check for a byte order mark:
        for start_bytes, encoding in self.byte_order_marks:
            if data.startswith(start_bytes):
                return encoding
        # check for an encoding declaration pattern in first 2 lines of file:
        for line in data.splitlines()[:2]:
            match = self.coding_slug.search(line)
            if match:
                return match.group(1).decode('ascii')
        return None

    def isatty(self):
        """Return True, if the input source is connected to a TTY device."""
        try:
            return self.source.isatty()
        except AttributeError:
            return False


class Output(TransformSpec):
    """
    Abstract base class for output wrappers.

    Docutils output objects must provide a `write()` method that
    expects and handles one argument (the output).

    Inheriting `TransformSpec` allows output objects to add
    "transforms" and "unknown_reference_resolvers" to the "Transformer".
    (Optional for custom output objects since Docutils 0.19.)
    """

    component_type = 'output'

    default_destination_path = None

    def __init__(self, destination=None, destination_path=None,
                 encoding=None, error_handler='strict'):
        self.encoding = encoding
        """Text encoding for the output destination."""

        self.error_handler = error_handler or 'strict'
        """Text encoding error handler."""

        self.destination = destination
        """The destination for output data."""

        self.destination_path = destination_path
        """A text reference to the destination."""

        if not destination_path:
            self.destination_path = self.default_destination_path

    def __repr__(self):
        return ('%s: destination=%r, destination_path=%r'
                % (self.__class__, self.destination, self.destination_path))

    def write(self, data):
        """Write `data`. Define in subclasses."""
        raise NotImplementedError

    def encode(self, data):
        """
        Encode and return `data`.

        If `data` is a `bytes` instance, it is returned unchanged.
        Otherwise it is encoded with `self.encoding`.

        Provisional: If `self.encoding` is set to the pseudo encoding name
        "unicode", `data` must be a `str` instance and is returned unchanged.
        """
        if self.encoding and self.encoding.lower() == 'unicode':
            assert isinstance(data, str), ('output encoding is "unicode" '
                                           'but `data` is no `str` instance')
            return data
        if not isinstance(data, str):
            # Non-unicode (e.g. bytes) output.
            return data
        else:
            return data.encode(self.encoding, self.error_handler)


class ErrorOutput:
    """
    Wrapper class for file-like error streams with
    failsafe de- and encoding of `str`, `bytes`, `unicode` and
    `Exception` instances.
    """

    def __init__(self, destination=None, encoding=None,
                 encoding_errors='backslashreplace',
                 decoding_errors='replace'):
        """
        :Parameters:
            - `destination`: a file-like object,
                        a string (path to a file),
                        `None` (write to `sys.stderr`, default), or
                        evaluating to `False` (write() requests are ignored).
            - `encoding`: `destination` text encoding. Guessed if None.
            - `encoding_errors`: how to treat encoding errors.
        """
        if destination is None:
            destination = sys.stderr
        elif not destination:
            destination = False
        # if `destination` is a file name, open it
        elif isinstance(destination, str):
            destination = open(destination, 'w')

        self.destination = destination
        """Where warning output is sent."""

        self.encoding = (encoding or getattr(destination, 'encoding', None)
                         or _locale_encoding or 'ascii')
        """The output character encoding."""

        self.encoding_errors = encoding_errors
        """Encoding error handler."""

        self.decoding_errors = decoding_errors
        """Decoding error handler."""

    def write(self, data):
        """
        Write `data` to self.destination. Ignore, if self.destination is False.

        `data` can be a `bytes`, `str`, or `Exception` instance.
        """
        if not self.destination:
            return
        if isinstance(data, Exception):
            data = str(data)
        try:
            self.destination.write(data)
        except UnicodeEncodeError:
            self.destination.write(data.encode(self.encoding,
                                               self.encoding_errors))
        except TypeError:
            if isinstance(data, str):  # destination may expect bytes
                self.destination.write(data.encode(self.encoding,
                                                   self.encoding_errors))
            elif self.destination in (sys.stderr, sys.stdout):
                # write bytes to raw stream
                self.destination.buffer.write(data)
            else:
                self.destination.write(str(data, self.encoding,
                                           self.decoding_errors))

    def close(self):
        """
        Close the error-output stream.

        Ignored if the destination is` sys.stderr` or `sys.stdout` or has no
        close() method.
        """
        if self.destination in (sys.stdout, sys.stderr):
            return
        try:
            self.destination.close()
        except AttributeError:
            pass

    def isatty(self):
        """Return True, if the destination is connected to a TTY device."""
        try:
            return self.destination.isatty()
        except AttributeError:
            return False


class FileInput(Input):

    """
    Input for single, simple file-like objects.
    """
    def __init__(self, source=None, source_path=None,
                 encoding=None, error_handler='strict',
                 autoclose=True, mode='r'):
        """
        :Parameters:
            - `source`: either a file-like object (which is read directly), or
              `None` (which implies `sys.stdin` if no `source_path` given).
            - `source_path`: a path to a file, which is opened and then read.
            - `encoding`: the expected text encoding of the input file.
            - `error_handler`: the encoding error handler to use.
            - `autoclose`: close automatically after read (except when
              `sys.stdin` is the source).
            - `mode`: how the file is to be opened (see standard function
              `open`). The default is read only ('r').
        """
        Input.__init__(self, source, source_path, encoding, error_handler)
        self.autoclose = autoclose
        self._stderr = ErrorOutput()

        if source is None:
            if source_path:
                try:
                    self.source = open(source_path, mode,
                                       encoding=self.encoding or 'utf-8-sig',
                                       errors=self.error_handler)
                except OSError as error:
                    raise InputError(error.errno, error.strerror, source_path)
            else:
                self.source = sys.stdin
        elif check_encoding(self.source, self.encoding) is False:
            # TODO: re-open, warn or raise error?
            raise UnicodeError('Encoding clash: encoding given is "%s" '
                               'but source is opened with encoding "%s".' %
                               (self.encoding, self.source.encoding))
        if not source_path:
            try:
                self.source_path = self.source.name
            except AttributeError:
                pass

    def read(self):
        """
        Read and decode a single file and return the data (Unicode string).
        """
        try:
            if self.source is sys.stdin:
                # read as binary data to circumvent auto-decoding
                data = self.source.buffer.read()
            else:
                data = self.source.read()
        except (UnicodeError, LookupError):
            if not self.encoding and self.source_path:
                # re-read in binary mode and decode with heuristics
                b_source = open(self.source_path, 'rb')
                data = b_source.read()
                b_source.close()
            else:
                raise
        finally:
            if self.autoclose:
                self.close()
        data = self.decode(data)
        # normalise newlines
        return '\n'.join(data.splitlines()+[''])

    def readlines(self):
        """
        Return lines of a single file as list of Unicode strings.
        """
        return self.read().splitlines(True)

    def close(self):
        if self.source is not sys.stdin:
            self.source.close()


class FileOutput(Output):

    """Output for single, simple file-like objects."""

    default_destination_path = '<file>'

    mode = 'w'
    """The mode argument for `open()`."""
    # 'wb' for binary (e.g. OpenOffice) files (see also `BinaryFileOutput`).
    # (Do not use binary mode ('wb') for text files, as this prevents the
    # conversion of newlines to the system specific default.)

    def __init__(self, destination=None, destination_path=None,
                 encoding=None, error_handler='strict', autoclose=True,
                 handle_io_errors=None, mode=None):
        """
        :Parameters:
            - `destination`: either a file-like object (which is written
              directly) or `None` (which implies `sys.stdout` if no
              `destination_path` given).
            - `destination_path`: a path to a file, which is opened and then
              written.
            - `encoding`: the text encoding of the output file.
            - `error_handler`: the encoding error handler to use.
            - `autoclose`: close automatically after write (except when
              `sys.stdout` or `sys.stderr` is the destination).
            - `handle_io_errors`: ignored, deprecated, will be removed.
            - `mode`: how the file is to be opened (see standard function
              `open`). The default is 'w', providing universal newline
              support for text files.
        """
        Output.__init__(self, destination, destination_path,
                        encoding, error_handler)
        self.opened = True
        self.autoclose = autoclose
        if handle_io_errors is not None:
            warnings.warn('io.FileOutput: init argument "handle_io_errors" '
                          'is ignored and will be removed in '
                          'Docutils 2.0.', DeprecationWarning, stacklevel=2)
        if mode is not None:
            self.mode = mode
        self._stderr = ErrorOutput()
        if destination is None:
            if destination_path:
                self.opened = False
            else:
                self.destination = sys.stdout
        elif (  # destination is file-type object -> check mode:
              mode and hasattr(self.destination, 'mode')
              and mode != self.destination.mode):
            print('Warning: Destination mode "%s" differs from specified '
                  'mode "%s"' % (self.destination.mode, mode),
                  file=self._stderr)
        if not destination_path:
            try:
                self.destination_path = self.destination.name
            except AttributeError:
                pass

    def open(self):
        # Specify encoding
        if 'b' not in self.mode:
            kwargs = {'encoding': self.encoding,
                      'errors': self.error_handler}
        else:
            kwargs = {}
        try:
            self.destination = open(self.destination_path, self.mode, **kwargs)
        except OSError as error:
            raise OutputError(error.errno, error.strerror,
                              self.destination_path)
        self.opened = True

    def write(self, data):
        """Write `data` to a single file, also return it.

        `data` can be a `str` or `bytes` instance.
        If writing `bytes` fails, an attempt is made to write to
        the low-level interface ``self.destination.buffer``.

        If `data` is a `str` instance and `self.encoding` and
        `self.destination.encoding` are  set to different values, `data`
        is encoded to a `bytes` instance using `self.encoding`.

        Provisional: future versions may raise an error if `self.encoding`
        and `self.destination.encoding` are set to different values.
        """
        if not self.opened:
            self.open()
        if (isinstance(data, str)
            and check_encoding(self.destination, self.encoding) is False):
            if os.linesep != '\n':
                data = data.replace('\n', os.linesep)  # fix endings
            data = self.encode(data)

        try:
            self.destination.write(data)
        except TypeError as err:
            if isinstance(data, bytes):
                try:
                    self.destination.buffer.write(data)
                except AttributeError:
                    if check_encoding(self.destination,
                                      self.encoding) is False:
                        raise ValueError(
                            f'Encoding of {self.destination_path} '
                            f'({self.destination.encoding}) differs \n'
                            f'  from specified encoding ({self.encoding})')
                    else:
                        raise err
        except (UnicodeError, LookupError) as err:
            raise UnicodeError(
                'Unable to encode output data. output-encoding is: '
                f'{self.encoding}.\n({error_string(err)})')
        finally:
            if self.autoclose:
                self.close()
        return data

    def close(self):
        if self.destination not in (sys.stdout, sys.stderr):
            self.destination.close()
            self.opened = False


class BinaryFileOutput(FileOutput):
    """
    A version of docutils.io.FileOutput which writes to a binary file.
    """
    # Used by core.publish_cmdline_to_binary() which in turn is used by
    # tools/rst2odt.py but not by core.rst2odt().
    mode = 'wb'


class StringInput(Input):
    """Input from a `str` or `bytes` instance."""

    default_source_path = '<string>'

    def read(self):
        """Return the source as `str` instance.

        Decode, if required (see `Input.decode`).
        """
        return self.decode(self.source)


class StringOutput(Output):
    """Output to a `bytes` or `str` instance.

    Provisional.
    """

    default_destination_path = '<string>'

    def write(self, data):
        """Store `data` in `self.destination`, and return it.

        If `self.encoding` is set to the pseudo encoding name "unicode",
        `data` must be a `str` instance and is stored/returned unchanged
        (cf. `Output.encode`).

        Otherwise, `data` can be a `bytes` or `str` instance and is
        stored/returned as a `bytes` instance
        (`str` data is encoded with `self.encode()`).

        Attention: the `output_encoding`_ setting may affect the content
        of the output (e.g. an encoding declaration in HTML or XML or the
        representation of characters as LaTeX macro vs. literal character).
        """
        self.destination = self.encode(data)
        return self.destination


class NullInput(Input):

    """Degenerate input: read nothing."""

    default_source_path = 'null input'

    def read(self):
        """Return an empty string."""
        return ''


class NullOutput(Output):

    """Degenerate output: write nothing."""

    default_destination_path = 'null output'

    def write(self, data):
        """Do nothing, return None."""
        pass


class DocTreeInput(Input):

    """
    Adapter for document tree input.

    The document tree must be passed in the ``source`` parameter.
    """

    default_source_path = 'doctree input'

    def read(self):
        """Return the document tree."""
        return self.source

env_pyrecodes/lib/python3.9/site-packages/docutils/io.py
NikolaBlagojevic/pyrecodes
# $Id: uk.py 9114 2022-07-28 17:06:10Z milde $
# Author: Dmytro Kazanzhy <dkazanzhy@gmail.com>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <http://docutils.sf.net/docs/howto/i18n.html>.  Two files must be
# translated for each language: one in docutils/languages, the other in
# docutils/parsers/rst/languages.

"""
Ukrainian-language mappings for language-dependent features of Docutils.
"""

__docformat__ = 'reStructuredText'

labels = {
      'abstract': 'Анотація',
      'address': 'Адреса',
      'attention': 'Увага!',
      'author': 'Автор',
      'authors': 'Автори',
      'caution': 'Обережно!',
      'contact': 'Контакт',
      'contents': 'Зміст',
      'copyright': 'Права копіювання',
      'danger': 'НЕБЕЗПЕЧНО!',
      'date': 'Дата',
      'dedication': 'Посвячення',
      'error': 'Помилка',
      'hint': 'Порада',
      'important': 'Важливо',
      'note': 'Примітка',
      'organization': 'Організація',
      'revision': 'Редакція',
      'status': 'Статус',
      'tip': 'Підказка',
      'version': 'Версія',
      'warning': 'Попередження'}
"""Mapping of node class name to label text."""

bibliographic_fields = {
      'анотація': 'abstract',
      'адреса': 'address',
      'автор': 'author',
      'автори': 'authors',
      'контакт': 'contact',
      'права копіювання': 'copyright',
      'дата': 'date',
      'посвячення': 'dedication',
      'організація': 'organization',
      'редакція': 'revision',
      'статус': 'status',
      'версія': 'version'}
"""Ukrainian (lowcased) to canonical name mapping for bibliographic fields."""

author_separators = [';', ',']
"""List of separator strings for the 'Authors' bibliographic field. Tried in
order."""

env_pyrecodes/lib/python3.9/site-packages/docutils/languages/uk.py
NikolaBlagojevic/pyrecodes
# $Id: nodes.py 9310 2022-12-17 12:17:32Z milde $
# Author: David Goodger <goodger@python.org>
# Maintainer: docutils-develop@lists.sourceforge.net
# Copyright: This module has been placed in the public domain.

"""
Docutils document tree element class library.

Classes in CamelCase are abstract base classes or auxiliary classes. The one
exception is `Text`, for a text (PCDATA) node; uppercase is used to
differentiate from element classes.  Classes in lower_case_with_underscores
are element classes, matching the XML element generic identifiers in the DTD_.

The position of each node (the level at which it can occur) is significant and
is represented by abstract base classes (`Root`, `Structural`, `Body`,
`Inline`, etc.).  Certain transformations will be easier because we can use
``isinstance(node, base_class)`` to determine the position of the node in the
hierarchy.

.. _DTD: https://docutils.sourceforge.io/docs/ref/docutils.dtd
"""

__docformat__ = 'reStructuredText'

from collections import Counter
import re
import sys
import warnings
import unicodedata
# import xml.dom.minidom as dom # -> conditional import in Node.asdom()
#                                    and document.asdom()

# import docutils.transforms # -> conditional import in document.__init__()


# ==============================
#  Functional Node Base Classes
# ==============================

class Node:
    """Abstract base class of nodes in a document tree."""

    parent = None
    """Back-reference to the Node immediately containing this Node."""

    source = None
    """Path or description of the input source which generated this Node."""

    line = None
    """The line number (1-based) of the beginning of this Node in `source`."""

    _document = None

    @property
    def document(self):
        """Return the `document` root node of the tree containing this Node.
        """
        try:
            return self._document or self.parent.document
        except AttributeError:
            return None

    @document.setter
    def document(self, value):
        self._document = value

    def __bool__(self):
        """
        Node instances are always true, even if they're empty.  A node is more
        than a simple container.  Its boolean "truth" does not depend on
        having one or more subnodes in the doctree.

        Use `len()` to check node length.
        """
        return True

    def asdom(self, dom=None):
        """Return a DOM **fragment** representation of this Node."""
        if dom is None:
            import xml.dom.minidom as dom
        domroot = dom.Document()
        return self._dom_node(domroot)

    def pformat(self, indent='    ', level=0):
        """
        Return an indented pseudo-XML representation, for test purposes.

        Override in subclasses.
        """
        raise NotImplementedError

    def copy(self):
        """Return a copy of self."""
        raise NotImplementedError

    def deepcopy(self):
        """Return a deep copy of self (also copying children)."""
        raise NotImplementedError

    def astext(self):
        """Return a string representation of this Node."""
        raise NotImplementedError

    def setup_child(self, child):
        child.parent = self
        if self.document:
            child.document = self.document
            if child.source is None:
                child.source = self.document.current_source
            if child.line is None:
                child.line = self.document.current_line

    def walk(self, visitor):
        """
        Traverse a tree of `Node` objects, calling the
        `dispatch_visit()` method of `visitor` when entering each
        node.  (The `walkabout()` method is similar, except it also
        calls the `dispatch_departure()` method before exiting each
        node.)

        This tree traversal supports limited in-place tree
        modifications.  Replacing one node with one or more nodes is
        OK, as is removing an element.  However, if the node removed
        or replaced occurs after the current node, the old node will
        still be traversed, and any new nodes will not.

        Within ``visit`` methods (and ``depart`` methods for
        `walkabout()`), `TreePruningException` subclasses may be raised
        (`SkipChildren`, `SkipSiblings`, `SkipNode`, `SkipDeparture`).

        Parameter `visitor`: A `NodeVisitor` object, containing a
        ``visit`` implementation for each `Node` subclass encountered.

        Return true if we should stop the traversal.
        """
        stop = False
        visitor.document.reporter.debug(
            'docutils.nodes.Node.walk calling dispatch_visit for %s'
            % self.__class__.__name__)
        try:
            try:
                visitor.dispatch_visit(self)
            except (SkipChildren, SkipNode):
                return stop
            except SkipDeparture:           # not applicable; ignore
                pass
            children = self.children
            try:
                for child in children[:]:
                    if child.walk(visitor):
                        stop = True
                        break
            except SkipSiblings:
                pass
        except StopTraversal:
            stop = True
        return stop

    def walkabout(self, visitor):
        """
        Perform a tree traversal similarly to `Node.walk()` (which
        see), except also call the `dispatch_departure()` method
        before exiting each node.

        Parameter `visitor`: A `NodeVisitor` object, containing a
        ``visit`` and ``depart`` implementation for each `Node`
        subclass encountered.

        Return true if we should stop the traversal.
        """
        call_depart = True
        stop = False
        visitor.document.reporter.debug(
            'docutils.nodes.Node.walkabout calling dispatch_visit for %s'
            % self.__class__.__name__)
        try:
            try:
                visitor.dispatch_visit(self)
            except SkipNode:
                return stop
            except SkipDeparture:
                call_depart = False
            children = self.children
            try:
                for child in children[:]:
                    if child.walkabout(visitor):
                        stop = True
                        break
            except SkipSiblings:
                pass
        except SkipChildren:
            pass
        except StopTraversal:
            stop = True
        if call_depart:
            visitor.document.reporter.debug(
                'docutils.nodes.Node.walkabout calling dispatch_departure '
                'for %s' % self.__class__.__name__)
            visitor.dispatch_departure(self)
        return stop

    def _fast_findall(self, cls):
        """Return iterator that only supports instance checks."""
        if isinstance(self, cls):
            yield self
        for child in self.children:
            yield from child._fast_findall(cls)

    def _superfast_findall(self):
        """Return iterator that doesn't check for a condition."""
        # This is different from ``iter(self)`` implemented via
        # __getitem__() and __len__() in the Element subclass,
        # which yields only the direct children.
        yield self
        for child in self.children:
            yield from child._superfast_findall()

    def traverse(self, condition=None, include_self=True, descend=True,
                 siblings=False, ascend=False):
        """Return list of nodes following `self`.

        For looping, Node.findall() is faster and more memory efficient.
        """
        # traverse() may be eventually removed:
        warnings.warn('nodes.Node.traverse() is obsoleted by Node.findall().',
                      PendingDeprecationWarning, stacklevel=2)
        return list(self.findall(condition, include_self, descend,
                                 siblings, ascend))

    def findall(self, condition=None, include_self=True, descend=True,
                siblings=False, ascend=False):
        """
        Return an iterator yielding nodes following `self`:

        * self (if `include_self` is true)
        * all descendants in tree traversal order (if `descend` is true)
        * the following siblings (if `siblings` is true) and their
          descendants (if also `descend` is true)
        * the following siblings of the parent (if `ascend` is true) and
          their descendants (if also `descend` is true), and so on.

        If `condition` is not None, the iterator yields only nodes
        for which ``condition(node)`` is true.  If `condition` is a
        node class ``cls``, it is equivalent to a function consisting
        of ``return isinstance(node, cls)``.

        If `ascend` is true, assume `siblings` to be true as well.

        If the tree structure is modified during iteration, the result
        is undefined.

        For example, given the following tree::

            <paragraph>
                <emphasis>      <--- emphasis.traverse() and
                    <strong>    <--- strong.traverse() are called.
                        Foo
                    Bar
                <reference name="Baz" refid="baz">
                    Baz

        Then tuple(emphasis.traverse()) equals ::

            (<emphasis>, <strong>, <#text: Foo>, <#text: Bar>)

        and list(strong.traverse(ascend=True) equals ::

            [<strong>, <#text: Foo>, <#text: Bar>, <reference>, <#text: Baz>]
        """
        if ascend:
            siblings = True
        # Check for special argument combinations that allow using an
        # optimized version of traverse()
        if include_self and descend and not siblings:
            if condition is None:
                yield from self._superfast_findall()
                return
            elif isinstance(condition, type):
                yield from self._fast_findall(condition)
                return
        # Check if `condition` is a class (check for TypeType for Python
        # implementations that use only new-style classes, like PyPy).
        if isinstance(condition, type):
            node_class = condition

            def condition(node, node_class=node_class):
                return isinstance(node, node_class)

        if include_self and (condition is None or condition(self)):
            yield self
        if descend and len(self.children):
            for child in self:
                yield from child.findall(condition=condition,
                                         include_self=True, descend=True,
                                         siblings=False, ascend=False)
        if siblings or ascend:
            node = self
            while node.parent:
                index = node.parent.index(node)
                # extra check since Text nodes have value-equality
                while node.parent[index] is not node:
                    index = node.parent.index(node, index + 1)
                for sibling in node.parent[index+1:]:
                    yield from sibling.findall(
                        condition=condition,
                        include_self=True, descend=descend,
                        siblings=False, ascend=False)
                if not ascend:
                    break
                else:
                    node = node.parent

    def next_node(self, condition=None, include_self=False, descend=True,
                  siblings=False, ascend=False):
        """
        Return the first node in the iterator returned by findall(),
        or None if the iterable is empty.

        Parameter list is the same as of traverse.  Note that `include_self`
        defaults to False, though.
        """
        try:
            return next(self.findall(condition, include_self,
                                     descend, siblings, ascend))
        except StopIteration:
            return None


class reprunicode(str):
    """
    Deprecated backwards compatibility stub. Use the standard `str` instead.
    """
    def __init__(self, s):
        warnings.warn('nodes.reprunicode() is not required with Python 3'
                      ' and will be removed in Docutils 0.21 or later.',
                      DeprecationWarning, stacklevel=2)
        super().__init__()


def ensure_str(s):
    """
    Deprecated backwards compatibility stub returning `s`.
    """
    warnings.warn('nodes.ensure_str() is not required with Python 3'
                  ' and will be removed in Docutils 0.21 or later.',
                  DeprecationWarning, stacklevel=2)
    return s


# definition moved here from `utils` to avoid circular import dependency
def unescape(text, restore_backslashes=False, respect_whitespace=False):
    """
    Return a string with nulls removed or restored to backslashes.
    Backslash-escaped spaces are also removed.
    """
    # `respect_whitespace` is ignored (since introduction 2016-12-16)
    if restore_backslashes:
        return text.replace('\x00', '\\')
    else:
        for sep in ['\x00 ', '\x00\n', '\x00']:
            text = ''.join(text.split(sep))
        return text


class Text(Node, str):

    """
    Instances are terminal nodes (leaves) containing text only; no child
    nodes or attributes.  Initialize by passing a string to the constructor.

    Access the raw (null-escaped) text with ``str(<instance>)``
    and unescaped text with ``<instance>.astext()``.
    """

    tagname = '#text'

    children = ()
    """Text nodes have no children, and cannot have children."""

    def __new__(cls, data, rawsource=None):
        """Assert that `data` is not an array of bytes
        and warn if the deprecated `rawsource` argument is used.
        """
        if isinstance(data, bytes):
            raise TypeError('expecting str data, not bytes')
        if rawsource is not None:
            warnings.warn('nodes.Text: initialization argument "rawsource" '
                          'is ignored and will be removed in Docutils 2.0.',
                          DeprecationWarning, stacklevel=2)
        return str.__new__(cls, data)

    def shortrepr(self, maxlen=18):
        data = self
        if len(data) > maxlen:
            data = data[:maxlen-4] + ' ...'
        return '<%s: %r>' % (self.tagname, str(data))

    def __repr__(self):
        return self.shortrepr(maxlen=68)

    def _dom_node(self, domroot):
        return domroot.createTextNode(str(self))

    def astext(self):
        return str(unescape(self))

    def copy(self):
        return self.__class__(str(self))

    def deepcopy(self):
        return self.copy()

    def pformat(self, indent='    ', level=0):
        try:
            if self.document.settings.detailed:
                tag = '%s%s' % (indent*level, '<#text>')
                lines = (indent*(level+1) + repr(line)
                         for line in self.splitlines(True))
                return '\n'.join((tag, *lines)) + '\n'
        except AttributeError:
            pass
        indent = indent * level
        lines = [indent+line for line in self.astext().splitlines()]
        if not lines:
            return ''
        return '\n'.join(lines) + '\n'

    # rstrip and lstrip are used by substitution definitions where
    # they are expected to return a Text instance, this was formerly
    # taken care of by UserString.

    def rstrip(self, chars=None):
        return self.__class__(str.rstrip(self, chars))

    def lstrip(self, chars=None):
        return self.__class__(str.lstrip(self, chars))


class Element(Node):

    """
    `Element` is the superclass to all specific elements.

    Elements contain attributes and child nodes.
    They can be described as a cross between a list and a dictionary.

    Elements emulate dictionaries for external [#]_ attributes, indexing by
    attribute name (a string). To set the attribute 'att' to 'value', do::

        element['att'] = 'value'

    .. [#] External attributes correspond to the XML element attributes.
       From its `Node` superclass, Element also inherits "internal"
       class attributes that are accessed using the standard syntax, e.g.
       ``element.parent``.

    There are two special attributes: 'ids' and 'names'.  Both are
    lists of unique identifiers: 'ids' conform to the regular expression
    ``[a-z](-?[a-z0-9]+)*`` (see the make_id() function for rationale and
    details). 'names' serve as user-friendly interfaces to IDs; they are
    case- and whitespace-normalized (see the fully_normalize_name() function).

    Elements emulate lists for child nodes (element nodes and/or text
    nodes), indexing by integer.  To get the first child node, use::

        element[0]

    to iterate over the child nodes (without descending), use::

        for child in element:
            ...

    Elements may be constructed using the ``+=`` operator.  To add one new
    child node to element, do::

        element += node

    This is equivalent to ``element.append(node)``.

    To add a list of multiple child nodes at once, use the same ``+=``
    operator::

        element += [node1, node2]

    This is equivalent to ``element.extend([node1, node2])``.
    """

    basic_attributes = ('ids', 'classes', 'names', 'dupnames')
    """Tuple of attributes which are defined for every Element-derived class
    instance and can be safely transferred to a different node."""

    local_attributes = ('backrefs',)
    """Tuple of class-specific attributes that should not be copied with the
    standard attributes when replacing a node.

    NOTE: Derived classes should override this value to prevent any of its
    attributes being copied by adding to the value in its parent class."""

    list_attributes = basic_attributes + local_attributes
    """Tuple of attributes that are automatically initialized to empty lists
    for all nodes."""

    known_attributes = list_attributes + ('source',)
    """Tuple of attributes that are known to the Element base class."""

    tagname = None
    """The element generic identifier. If None, it is set as an instance
    attribute to the name of the class."""

    child_text_separator = '\n\n'
    """Separator for child nodes, used by `astext()` method."""

    def __init__(self, rawsource='', *children, **attributes):
        self.rawsource = rawsource
        """The raw text from which this element was constructed.

        NOTE: some elements do not set this value (default '').
        """

        self.children = []
        """List of child nodes (elements and/or `Text`)."""

        self.extend(children)           # maintain parent info

        self.attributes = {}
        """Dictionary of attribute {name: value}."""

        # Initialize list attributes.
        for att in self.list_attributes:
            self.attributes[att] = []

        for att, value in attributes.items():
            att = att.lower()
            if att in self.list_attributes:
                # mutable list; make a copy for this node
                self.attributes[att] = value[:]
            else:
                self.attributes[att] = value

        if self.tagname is None:
            self.tagname = self.__class__.__name__

    def _dom_node(self, domroot):
        element = domroot.createElement(self.tagname)
        for attribute, value in self.attlist():
            if isinstance(value, list):
                value = ' '.join(serial_escape('%s' % (v,)) for v in value)
            element.setAttribute(attribute, '%s' % value)
        for child in self.children:
            element.appendChild(child._dom_node(domroot))
        return element

    def __repr__(self):
        data = ''
        for c in self.children:
            data += c.shortrepr()
            if len(data) > 60:
                data = data[:56] + ' ...'
                break
        if self['names']:
            return '<%s "%s": %s>' % (self.__class__.__name__,
                                      '; '.join(self['names']), data)
        else:
            return '<%s: %s>' % (self.__class__.__name__, data)

    def shortrepr(self):
        if self['names']:
            return '<%s "%s"...>' % (self.__class__.__name__,
                                     '; '.join(self['names']))
        else:
            return '<%s...>' % self.tagname

    def __str__(self):
        if self.children:
            return '%s%s%s' % (self.starttag(),
                               ''.join(str(c) for c in self.children),
                               self.endtag())
        else:
            return self.emptytag()

    def starttag(self, quoteattr=None):
        # the optional arg is used by the docutils_xml writer
        if quoteattr is None:
            quoteattr = pseudo_quoteattr
        parts = [self.tagname]
        for name, value in self.attlist():
            if value is None:           # boolean attribute
                parts.append('%s="True"' % name)
                continue
            if isinstance(value, list):
                values = [serial_escape('%s' % (v,)) for v in value]
                value = ' '.join(values)
            else:
                value = str(value)
            value = quoteattr(value)
            parts.append('%s=%s' % (name, value))
        return '<%s>' % ' '.join(parts)

    def endtag(self):
        return '</%s>' % self.tagname

    def emptytag(self):
        attributes = ('%s="%s"' % (n, v) for n, v in self.attlist())
        return '<%s/>' % ' '.join((self.tagname, *attributes))

    def __len__(self):
        return len(self.children)

    def __contains__(self, key):
        # Test for both, children and attributes with operator ``in``.
        if isinstance(key, str):
            return key in self.attributes
        return key in self.children

    def __getitem__(self, key):
        if isinstance(key, str):
            return self.attributes[key]
        elif isinstance(key, int):
            return self.children[key]
        elif isinstance(key, slice):
            assert key.step in (None, 1), 'cannot handle slice with stride'
            return self.children[key.start:key.stop]
        else:
            raise TypeError('element index must be an integer, a slice, or '
                            'an attribute name string')

    def __setitem__(self, key, item):
        if isinstance(key, str):
            self.attributes[str(key)] = item
        elif isinstance(key, int):
            self.setup_child(item)
            self.children[key] = item
        elif isinstance(key, slice):
            assert key.step in (None, 1), 'cannot handle slice with stride'
            for node in item:
                self.setup_child(node)
            self.children[key.start:key.stop] = item
        else:
            raise TypeError('element index must be an integer, a slice, or '
                            'an attribute name string')

    def __delitem__(self, key):
        if isinstance(key, str):
            del self.attributes[key]
        elif isinstance(key, int):
            del self.children[key]
        elif isinstance(key, slice):
            assert key.step in (None, 1), 'cannot handle slice with stride'
            del self.children[key.start:key.stop]
        else:
            raise TypeError('element index must be an integer, a simple '
                            'slice, or an attribute name string')

    def __add__(self, other):
        return self.children + other

    def __radd__(self, other):
        return other + self.children

    def __iadd__(self, other):
        """Append a node or a list of nodes to `self.children`."""
        if isinstance(other, Node):
            self.append(other)
        elif other is not None:
            self.extend(other)
        return self

    def astext(self):
        return self.child_text_separator.join(
                   [child.astext() for child in self.children])

    def non_default_attributes(self):
        atts = {}
        for key, value in self.attributes.items():
            if self.is_not_default(key):
                atts[key] = value
        return atts

    def attlist(self):
        return sorted(self.non_default_attributes().items())

    def get(self, key, failobj=None):
        return self.attributes.get(key, failobj)

    def hasattr(self, attr):
        return attr in self.attributes

    def delattr(self, attr):
        if attr in self.attributes:
            del self.attributes[attr]

    def setdefault(self, key, failobj=None):
        return self.attributes.setdefault(key, failobj)

    has_key = hasattr

    def get_language_code(self, fallback=''):
        """Return node's language tag.

        Look iteratively in self and parents for a class argument
        starting with ``language-`` and return the remainder of it
        (which should be a `BCP49` language tag) or the `fallback`.
        """
        for cls in self.get('classes', []):
            if cls.startswith('language-'):
                return cls[9:]
        try:
            return self.parent.get_language(fallback)
        except AttributeError:
            return fallback

    def append(self, item):
        self.setup_child(item)
        self.children.append(item)

    def extend(self, item):
        for node in item:
            self.append(node)

    def insert(self, index, item):
        if isinstance(item, Node):
            self.setup_child(item)
            self.children.insert(index, item)
        elif item is not None:
            self[index:index] = item

    def pop(self, i=-1):
        return self.children.pop(i)

    def remove(self, item):
        self.children.remove(item)

    def index(self, item, start=0, stop=sys.maxsize):
        return self.children.index(item, start, stop)

    def previous_sibling(self):
        """Return preceding sibling node or ``None``."""
        try:
            i = self.parent.index(self)
        except (AttributeError):
            return None
        return self.parent[i-1] if i > 0 else None

    def is_not_default(self, key):
        if self[key] == [] and key in self.list_attributes:
            return 0
        else:
            return 1

    def update_basic_atts(self, dict_):
        """
        Update basic attributes ('ids', 'names', 'classes',
        'dupnames', but not 'source') from node or dictionary `dict_`.
        """
        if isinstance(dict_, Node):
            dict_ = dict_.attributes
        for att in self.basic_attributes:
            self.append_attr_list(att, dict_.get(att, []))

    def append_attr_list(self, attr, values):
        """
        For each element in values, if it does not exist in self[attr], append
        it.

        NOTE: Requires self[attr] and values to be sequence type and the
        former should specifically be a list.
        """
        # List Concatenation
        for value in values:
            if value not in self[attr]:
                self[attr].append(value)

    def coerce_append_attr_list(self, attr, value):
        """
        First, convert both self[attr] and value to a non-string sequence
        type; if either is not already a sequence, convert it to a list of one
        element.  Then call append_attr_list.

        NOTE: self[attr] and value both must not be None.
        """
        # List Concatenation
        if not isinstance(self.get(attr), list):
            self[attr] = [self[attr]]
        if not isinstance(value, list):
            value = [value]
        self.append_attr_list(attr, value)

    def replace_attr(self, attr, value, force=True):
        """
        If self[attr] does not exist or force is True or omitted, set
        self[attr] to value, otherwise do nothing.
        """
        # One or the other
        if force or self.get(attr) is None:
            self[attr] = value

    def copy_attr_convert(self, attr, value, replace=True):
        """
        If attr is an attribute of self, set self[attr] to
        [self[attr], value], otherwise set self[attr] to value.

        NOTE: replace is not used by this function and is kept only for
              compatibility with the other copy functions.
        """
        if self.get(attr) is not value:
            self.coerce_append_attr_list(attr, value)

    def copy_attr_coerce(self, attr, value, replace):
        """
        If attr is an attribute of self and either self[attr] or value is a
        list, convert all non-sequence values to a sequence of 1 element and
        then concatenate the two sequence, setting the result to self[attr].
        If both self[attr] and value are non-sequences and replace is True or
        self[attr] is None, replace self[attr] with value. Otherwise, do
        nothing.
        """
        if self.get(attr) is not value:
            if isinstance(self.get(attr), list) or \
               isinstance(value, list):
                self.coerce_append_attr_list(attr, value)
            else:
                self.replace_attr(attr, value, replace)

    def copy_attr_concatenate(self, attr, value, replace):
        """
        If attr is an attribute of self and both self[attr] and value are
        lists, concatenate the two sequences, setting the result to
        self[attr].  If either self[attr] or value are non-sequences and
        replace is True or self[attr] is None, replace self[attr] with value.
        Otherwise, do nothing.
        """
        if self.get(attr) is not value:
            if isinstance(self.get(attr), list) and \
               isinstance(value, list):
                self.append_attr_list(attr, value)
            else:
                self.replace_attr(attr, value, replace)

    def copy_attr_consistent(self, attr, value, replace):
        """
        If replace is True or self[attr] is None, replace self[attr] with
        value.  Otherwise, do nothing.
        """
        if self.get(attr) is not value:
            self.replace_attr(attr, value, replace)

    def update_all_atts(self, dict_, update_fun=copy_attr_consistent,
                        replace=True, and_source=False):
        """
        Updates all attributes from node or dictionary `dict_`.

        Appends the basic attributes ('ids', 'names', 'classes',
        'dupnames', but not 'source') and then, for all other attributes in
        dict_, updates the same attribute in self.  When attributes with the
        same identifier appear in both self and dict_, the two values are
        merged based on the value of update_fun.  Generally, when replace is
        True, the values in self are replaced or merged with the values in
        dict_; otherwise, the values in self may be preserved or merged.  When
        and_source is True, the 'source' attribute is included in the copy.

        NOTE: When replace is False, and self contains a 'source' attribute,
              'source' is not replaced even when dict_ has a 'source'
              attribute, though it may still be merged into a list depending
              on the value of update_fun.
        NOTE: It is easier to call the update-specific methods then to pass
              the update_fun method to this function.
        """
        if isinstance(dict_, Node):
            dict_ = dict_.attributes

        # Include the source attribute when copying?
        if and_source:
            filter_fun = self.is_not_list_attribute
        else:
            filter_fun = self.is_not_known_attribute

        # Copy the basic attributes
        self.update_basic_atts(dict_)

        # Grab other attributes in dict_ not in self except the
        # (All basic attributes should be copied already)
        for att in filter(filter_fun, dict_):
            update_fun(self, att, dict_[att], replace)

    def update_all_atts_consistantly(self, dict_, replace=True,
                                     and_source=False):
        """
        Updates all attributes from node or dictionary `dict_`.

        Appends the basic attributes ('ids', 'names', 'classes',
        'dupnames', but not 'source') and then, for all other attributes in
        dict_, updates the same attribute in self.  When attributes with the
        same identifier appear in both self and dict_ and replace is True, the
        values in self are replaced with the values in dict_; otherwise, the
        values in self are preserved.  When and_source is True, the 'source'
        attribute is included in the copy.

        NOTE: When replace is False, and self contains a 'source' attribute,
              'source' is not replaced even when dict_ has a 'source'
              attribute, though it may still be merged into a list depending
              on the value of update_fun.
        """
        self.update_all_atts(dict_, Element.copy_attr_consistent, replace,
                             and_source)

    def update_all_atts_concatenating(self, dict_, replace=True,
                                      and_source=False):
        """
        Updates all attributes from node or dictionary `dict_`.

        Appends the basic attributes ('ids', 'names', 'classes',
        'dupnames', but not 'source') and then, for all other attributes in
        dict_, updates the same attribute in self.  When attributes with the
        same identifier appear in both self and dict_ whose values aren't each
        lists and replace is True, the values in self are replaced with the
        values in dict_; if the values from self and dict_ for the given
        identifier are both of list type, then the two lists are concatenated
        and the result stored in self; otherwise, the values in self are
        preserved.  When and_source is True, the 'source' attribute is
        included in the copy.

        NOTE: When replace is False, and self contains a 'source' attribute,
              'source' is not replaced even when dict_ has a 'source'
              attribute, though it may still be merged into a list depending
              on the value of update_fun.
        """
        self.update_all_atts(dict_, Element.copy_attr_concatenate, replace,
                             and_source)

    def update_all_atts_coercion(self, dict_, replace=True,
                                 and_source=False):
        """
        Updates all attributes from node or dictionary `dict_`.

        Appends the basic attributes ('ids', 'names', 'classes',
        'dupnames', but not 'source') and then, for all other attributes in
        dict_, updates the same attribute in self.  When attributes with the
        same identifier appear in both self and dict_ whose values are both
        not lists and replace is True, the values in self are replaced with
        the values in dict_; if either of the values from self and dict_ for
        the given identifier are of list type, then first any non-lists are
        converted to 1-element lists and then the two lists are concatenated
        and the result stored in self; otherwise, the values in self are
        preserved.  When and_source is True, the 'source' attribute is
        included in the copy.

        NOTE: When replace is False, and self contains a 'source' attribute,
              'source' is not replaced even when dict_ has a 'source'
              attribute, though it may still be merged into a list depending
              on the value of update_fun.
        """
        self.update_all_atts(dict_, Element.copy_attr_coerce, replace,
                             and_source)

    def update_all_atts_convert(self, dict_, and_source=False):
        """
        Updates all attributes from node or dictionary `dict_`.

        Appends the basic attributes ('ids', 'names', 'classes',
        'dupnames', but not 'source') and then, for all other attributes in
        dict_, updates the same attribute in self.  When attributes with the
        same identifier appear in both self and dict_ then first any non-lists
        are converted to 1-element lists and then the two lists are
        concatenated and the result stored in self; otherwise, the values in
        self are preserved.  When and_source is True, the 'source' attribute
        is included in the copy.

        NOTE: When replace is False, and self contains a 'source' attribute,
              'source' is not replaced even when dict_ has a 'source'
              attribute, though it may still be merged into a list depending
              on the value of update_fun.
        """
        self.update_all_atts(dict_, Element.copy_attr_convert,
                             and_source=and_source)

    def clear(self):
        self.children = []

    def replace(self, old, new):
        """Replace one child `Node` with another child or children."""
        index = self.index(old)
        if isinstance(new, Node):
            self.setup_child(new)
            self[index] = new
        elif new is not None:
            self[index:index+1] = new

    def replace_self(self, new):
        """
        Replace `self` node with `new`, where `new` is a node or a
        list of nodes.
        """
        update = new
        if not isinstance(new, Node):
            # `new` is a list; update first child.
            try:
                update = new[0]
            except IndexError:
                update = None
        if isinstance(update, Element):
            update.update_basic_atts(self)
        else:
            # `update` is a Text node or `new` is an empty list.
            # Assert that we aren't losing any attributes.
            for att in self.basic_attributes:
                assert not self[att], \
                       'Losing "%s" attribute: %s' % (att, self[att])
        self.parent.replace(self, new)

    def first_child_matching_class(self, childclass, start=0, end=sys.maxsize):
        """
        Return the index of the first child whose class exactly matches.

        Parameters:

        - `childclass`: A `Node` subclass to search for, or a tuple of `Node`
          classes. If a tuple, any of the classes may match.
        - `start`: Initial index to check.
        - `end`: Initial index to *not* check.
        """
        if not isinstance(childclass, tuple):
            childclass = (childclass,)
        for index in range(start, min(len(self), end)):
            for c in childclass:
                if isinstance(self[index], c):
                    return index
        return None

    def first_child_not_matching_class(self, childclass, start=0,
                                       end=sys.maxsize):
        """
        Return the index of the first child whose class does *not* match.

        Parameters:

        - `childclass`: A `Node` subclass to skip, or a tuple of `Node`
          classes. If a tuple, none of the classes may match.
        - `start`: Initial index to check.
        - `end`: Initial index to *not* check.
        """
        if not isinstance(childclass, tuple):
            childclass = (childclass,)
        for index in range(start, min(len(self), end)):
            for c in childclass:
                if isinstance(self.children[index], c):
                    break
            else:
                return index
        return None

    def pformat(self, indent='    ', level=0):
        tagline = '%s%s\n' % (indent*level, self.starttag())
        childreps = (c.pformat(indent, level+1) for c in self.children)
        return ''.join((tagline, *childreps))

    def copy(self):
        obj = self.__class__(rawsource=self.rawsource, **self.attributes)
        obj._document = self._document
        obj.source = self.source
        obj.line = self.line
        return obj

    def deepcopy(self):
        copy = self.copy()
        copy.extend([child.deepcopy() for child in self.children])
        return copy

    def set_class(self, name):
        """Add a new class to the "classes" attribute."""
        warnings.warn('docutils.nodes.Element.set_class() is deprecated; '
                      ' and will be removed in Docutils 0.21 or later.'
                      "Append to Element['classes'] list attribute directly",
                      DeprecationWarning, stacklevel=2)
        assert ' ' not in name
        self['classes'].append(name.lower())

    def note_referenced_by(self, name=None, id=None):
        """Note that this Element has been referenced by its name
        `name` or id `id`."""
        self.referenced = 1
        # Element.expect_referenced_by_* dictionaries map names or ids
        # to nodes whose ``referenced`` attribute is set to true as
        # soon as this node is referenced by the given name or id.
        # Needed for target propagation.
        by_name = getattr(self, 'expect_referenced_by_name', {}).get(name)
        by_id = getattr(self, 'expect_referenced_by_id', {}).get(id)
        if by_name:
            assert name is not None
            by_name.referenced = 1
        if by_id:
            assert id is not None
            by_id.referenced = 1

    @classmethod
    def is_not_list_attribute(cls, attr):
        """
        Returns True if and only if the given attribute is NOT one of the
        basic list attributes defined for all Elements.
        """
        return attr not in cls.list_attributes

    @classmethod
    def is_not_known_attribute(cls, attr):
        """
        Returns True if and only if the given attribute is NOT recognized by
        this class.
        """
        return attr not in cls.known_attributes


class TextElement(Element):

    """
    An element which directly contains text.

    Its children are all `Text` or `Inline` subclass nodes.  You can
    check whether an element's context is inline simply by checking whether
    its immediate parent is a `TextElement` instance (including subclasses).
    This is handy for nodes like `image` that can appear both inline and as
    standalone body elements.

    If passing children to `__init__()`, make sure to set `text` to
    ``''`` or some other suitable value.
    """

    child_text_separator = ''
    """Separator for child nodes, used by `astext()` method."""

    def __init__(self, rawsource='', text='', *children, **attributes):
        if text != '':
            textnode = Text(text)
            Element.__init__(self, rawsource, textnode, *children,
                             **attributes)
        else:
            Element.__init__(self, rawsource, *children, **attributes)


class FixedTextElement(TextElement):

    """An element which directly contains preformatted text."""

    def __init__(self, rawsource='', text='', *children, **attributes):
        TextElement.__init__(self, rawsource, text, *children, **attributes)
        self.attributes['xml:space'] = 'preserve'


# ========
#  Mixins
# ========

class Resolvable:

    resolved = 0


class BackLinkable:

    def add_backref(self, refid):
        self['backrefs'].append(refid)


# ====================
#  Element Categories
# ====================

class Root:
    pass


class Titular:
    pass


class PreBibliographic:
    """Category of Node which may occur before Bibliographic Nodes."""


class Bibliographic:
    pass


class Decorative(PreBibliographic):
    pass


class Structural:
    pass


class Body:
    pass


class General(Body):
    pass


class Sequential(Body):
    """List-like elements."""


class Admonition(Body): pass


class Special(Body):
    """Special internal body elements."""


class Invisible(PreBibliographic):
    """Internal elements that don't appear in output."""


class Part:
    pass


class Inline:
    pass


class Referential(Resolvable):
    pass


class Targetable(Resolvable):

    referenced = 0

    indirect_reference_name = None
    """Holds the whitespace_normalized_name (contains mixed case) of a target.
    Required for MoinMoin/reST compatibility."""


class Labeled:
    """Contains a `label` as its first element."""


# ==============
#  Root Element
# ==============

class document(Root, Structural, Element):

    """
    The document root element.

    Do not instantiate this class directly; use
    `docutils.utils.new_document()` instead.
    """

    def __init__(self, settings, reporter, *args, **kwargs):
        Element.__init__(self, *args, **kwargs)

        self.current_source = None
        """Path to or description of the input source being processed."""

        self.current_line = None
        """Line number (1-based) of `current_source`."""

        self.settings = settings
        """Runtime settings data record."""

        self.reporter = reporter
        """System message generator."""

        self.indirect_targets = []
        """List of indirect target nodes."""

        self.substitution_defs = {}
        """Mapping of substitution names to substitution_definition nodes."""

        self.substitution_names = {}
        """Mapping of case-normalized substitution names to case-sensitive
        names."""

        self.refnames = {}
        """Mapping of names to lists of referencing nodes."""

        self.refids = {}
        """Mapping of ids to lists of referencing nodes."""

        self.nameids = {}
        """Mapping of names to unique id's."""

        self.nametypes = {}
        """Mapping of names to hyperlink type (boolean: True => explicit,
        False => implicit."""

        self.ids = {}
        """Mapping of ids to nodes."""

        self.footnote_refs = {}
        """Mapping of footnote labels to lists of footnote_reference nodes."""

        self.citation_refs = {}
        """Mapping of citation labels to lists of citation_reference nodes."""

        self.autofootnotes = []
        """List of auto-numbered footnote nodes."""

        self.autofootnote_refs = []
        """List of auto-numbered footnote_reference nodes."""

        self.symbol_footnotes = []
        """List of symbol footnote nodes."""

        self.symbol_footnote_refs = []
        """List of symbol footnote_reference nodes."""

        self.footnotes = []
        """List of manually-numbered footnote nodes."""

        self.citations = []
        """List of citation nodes."""

        self.autofootnote_start = 1
        """Initial auto-numbered footnote number."""

        self.symbol_footnote_start = 0
        """Initial symbol footnote symbol index."""

        self.id_counter = Counter()
        """Numbers added to otherwise identical IDs."""

        self.parse_messages = []
        """System messages generated while parsing."""

        self.transform_messages = []
        """System messages generated while applying transforms."""

        import docutils.transforms
        self.transformer = docutils.transforms.Transformer(self)
        """Storage for transforms to be applied to this document."""

        self.include_log = []
        """The current source's parents (to detect inclusion loops)."""

        self.decoration = None
        """Document's `decoration` node."""

        self._document = self

    def __getstate__(self):
        """
        Return dict with unpicklable references removed.
        """
        state = self.__dict__.copy()
        state['reporter'] = None
        state['transformer'] = None
        return state

    def asdom(self, dom=None):
        """Return a DOM representation of this document."""
        if dom is None:
            import xml.dom.minidom as dom
        domroot = dom.Document()
        domroot.appendChild(self._dom_node(domroot))
        return domroot

    def set_id(self, node, msgnode=None, suggested_prefix=''):
        if node['ids']:
            # register and check for duplicates
            for id in node['ids']:
                self.ids.setdefault(id, node)
                if self.ids[id] is not node:
                    msg = self.reporter.severe('Duplicate ID: "%s".' % id)
                    if msgnode is not None:
                        msgnode += msg
            return id
        # generate and set id
        id_prefix = self.settings.id_prefix
        auto_id_prefix = self.settings.auto_id_prefix
        base_id = ''
        id = ''
        for name in node['names']:
            if id_prefix:
                # allow names starting with numbers if `id_prefix`
                base_id = make_id('x'+name)[1:]
            else:
                base_id = make_id(name)
            # TODO: normalize id-prefix? (would make code simpler)
            id = id_prefix + base_id
            if base_id and id not in self.ids:
                break
        else:
            if base_id and auto_id_prefix.endswith('%'):
                # disambiguate name-derived ID
                # TODO: remove second condition after announcing change
                prefix = id + '-'
            else:
                prefix = id_prefix + auto_id_prefix
                if prefix.endswith('%'):
                    prefix = '%s%s-' % (prefix[:-1],
                                        suggested_prefix
                                        or make_id(node.tagname))
            while True:
                self.id_counter[prefix] += 1
                id = '%s%d' % (prefix, self.id_counter[prefix])
                if id not in self.ids:
                    break
        node['ids'].append(id)
        self.ids[id] = node
        return id

    def set_name_id_map(self, node, id, msgnode=None, explicit=None):
        """
        `self.nameids` maps names to IDs, while `self.nametypes` maps names to
        booleans representing hyperlink type (True==explicit,
        False==implicit).  This method updates the mappings.

        The following state transition table shows how `self.nameids` items
        ("id") and `self.nametypes` items ("type") change with new input
        (a call to this method), and what actions are performed
        ("implicit"-type system messages are INFO/1, and
        "explicit"-type system messages are ERROR/3):

        ====  =====  ========  ========  =======  ====  =====  =====
         Old State    Input          Action        New State   Notes
        -----------  --------  -----------------  -----------  -----
        id    type   new type  sys.msg.  dupname  id    type
        ====  =====  ========  ========  =======  ====  =====  =====
        -     -      explicit  -         -        new   True
        -     -      implicit  -         -        new   False
        -     False  explicit  -         -        new   True
        old   False  explicit  implicit  old      new   True
        -     True   explicit  explicit  new      -     True
        old   True   explicit  explicit  new,old  -     True   [#]_
        -     False  implicit  implicit  new      -     False
        old   False  implicit  implicit  new,old  -     False
        -     True   implicit  implicit  new      -     True
        old   True   implicit  implicit  new      old   True
        ====  =====  ========  ========  =======  ====  =====  =====

        .. [#] Do not clear the name-to-id map or invalidate the old target if
           both old and new targets are external and refer to identical URIs.
           The new target is invalidated regardless.
        """
        for name in tuple(node['names']):
            if name in self.nameids:
                self.set_duplicate_name_id(node, id, name, msgnode, explicit)
                # attention: modifies node['names']
            else:
                self.nameids[name] = id
                self.nametypes[name] = explicit

    def set_duplicate_name_id(self, node, id, name, msgnode, explicit):
        old_id = self.nameids[name]
        old_explicit = self.nametypes[name]
        self.nametypes[name] = old_explicit or explicit
        if explicit:
            if old_explicit:
                level = 2
                if old_id is not None:
                    old_node = self.ids[old_id]
                    if 'refuri' in node:
                        refuri = node['refuri']
                        if (old_node['names']
                            and 'refuri' in old_node
                            and old_node['refuri'] == refuri):
                            level = 1   # just inform if refuri's identical
                    if level > 1:
                        dupname(old_node, name)
                        self.nameids[name] = None
                msg = self.reporter.system_message(
                    level, 'Duplicate explicit target name: "%s".' % name,
                    backrefs=[id], base_node=node)
                if msgnode is not None:
                    msgnode += msg
                dupname(node, name)
            else:
                self.nameids[name] = id
                if old_id is not None:
                    old_node = self.ids[old_id]
                    dupname(old_node, name)
        else:
            if old_id is not None and not old_explicit:
                self.nameids[name] = None
                old_node = self.ids[old_id]
                dupname(old_node, name)
            dupname(node, name)
        if not explicit or (not old_explicit and old_id is not None):
            msg = self.reporter.info(
                'Duplicate implicit target name: "%s".' % name,
                backrefs=[id], base_node=node)
            if msgnode is not None:
                msgnode += msg

    def has_name(self, name):
        return name in self.nameids

    # "note" here is an imperative verb: "take note of".
    def note_implicit_target(self, target, msgnode=None):
        id = self.set_id(target, msgnode)
        self.set_name_id_map(target, id, msgnode, explicit=False)

    def note_explicit_target(self, target, msgnode=None):
        id = self.set_id(target, msgnode)
        self.set_name_id_map(target, id, msgnode, explicit=True)

    def note_refname(self, node):
        self.refnames.setdefault(node['refname'], []).append(node)

    def note_refid(self, node):
        self.refids.setdefault(node['refid'], []).append(node)

    def note_indirect_target(self, target):
        self.indirect_targets.append(target)
        if target['names']:
            self.note_refname(target)

    def note_anonymous_target(self, target):
        self.set_id(target)

    def note_autofootnote(self, footnote):
        self.set_id(footnote)
        self.autofootnotes.append(footnote)

    def note_autofootnote_ref(self, ref):
        self.set_id(ref)
        self.autofootnote_refs.append(ref)

    def note_symbol_footnote(self, footnote):
        self.set_id(footnote)
        self.symbol_footnotes.append(footnote)

    def note_symbol_footnote_ref(self, ref):
        self.set_id(ref)
        self.symbol_footnote_refs.append(ref)

    def note_footnote(self, footnote):
        self.set_id(footnote)
        self.footnotes.append(footnote)

    def note_footnote_ref(self, ref):
        self.set_id(ref)
        self.footnote_refs.setdefault(ref['refname'], []).append(ref)
        self.note_refname(ref)

    def note_citation(self, citation):
        self.citations.append(citation)

    def note_citation_ref(self, ref):
        self.set_id(ref)
        self.citation_refs.setdefault(ref['refname'], []).append(ref)
        self.note_refname(ref)

    def note_substitution_def(self, subdef, def_name, msgnode=None):
        name = whitespace_normalize_name(def_name)
        if name in self.substitution_defs:
            msg = self.reporter.error(
                      'Duplicate substitution definition name: "%s".' % name,
                      base_node=subdef)
            if msgnode is not None:
                msgnode += msg
            oldnode = self.substitution_defs[name]
            dupname(oldnode, name)
        # keep only the last definition:
        self.substitution_defs[name] = subdef
        # case-insensitive mapping:
        self.substitution_names[fully_normalize_name(name)] = name

    def note_substitution_ref(self, subref, refname):
        subref['refname'] = whitespace_normalize_name(refname)

    def note_pending(self, pending, priority=None):
        self.transformer.add_pending(pending, priority)

    def note_parse_message(self, message):
        self.parse_messages.append(message)

    def note_transform_message(self, message):
        self.transform_messages.append(message)

    def note_source(self, source, offset):
        self.current_source = source
        if offset is None:
            self.current_line = offset
        else:
            self.current_line = offset + 1

    def copy(self):
        obj = self.__class__(self.settings, self.reporter,
                             **self.attributes)
        obj.source = self.source
        obj.line = self.line
        return obj

    def get_decoration(self):
        if not self.decoration:
            self.decoration = decoration()
            index = self.first_child_not_matching_class((Titular, meta))
            if index is None:
                self.append(self.decoration)
            else:
                self.insert(index, self.decoration)
        return self.decoration


# ================
#  Title Elements
# ================

class title(Titular, PreBibliographic, TextElement): pass
class subtitle(Titular, PreBibliographic, TextElement): pass
class rubric(Titular, TextElement): pass


# ==================
#  Meta-Data Element
# ==================

class meta(PreBibliographic, Element):
    """Container for "invisible" bibliographic data, or meta-data."""


# ========================
#  Bibliographic Elements
# ========================

class docinfo(Bibliographic, Element): pass
class author(Bibliographic, TextElement): pass
class authors(Bibliographic, Element): pass
class organization(Bibliographic, TextElement): pass
class address(Bibliographic, FixedTextElement): pass
class contact(Bibliographic, TextElement): pass
class version(Bibliographic, TextElement): pass
class revision(Bibliographic, TextElement): pass
class status(Bibliographic, TextElement): pass
class date(Bibliographic, TextElement): pass
class copyright(Bibliographic, TextElement): pass


# =====================
#  Decorative Elements
# =====================

class decoration(Decorative, Element):

    def get_header(self):
        if not len(self.children) or not isinstance(self.children[0], header):
            self.insert(0, header())
        return self.children[0]

    def get_footer(self):
        if not len(self.children) or not isinstance(self.children[-1], footer):
            self.append(footer())
        return self.children[-1]


class header(Decorative, Element): pass
class footer(Decorative, Element): pass


# =====================
#  Structural Elements
# =====================

class section(Structural, Element): pass


class topic(Structural, Element):

    """
    Topics are terminal, "leaf" mini-sections, like block quotes with titles,
    or textual figures.  A topic is just like a section, except that it has no
    subsections, and it doesn't have to conform to section placement rules.

    Topics are allowed wherever body elements (list, table, etc.) are allowed,
    but only at the top level of a section or document.  Topics cannot nest
    inside topics, sidebars, or body elements; you can't have a topic inside a
    table, list, block quote, etc.
    """


class sidebar(Structural, Element):

    """
    Sidebars are like miniature, parallel documents that occur inside other
    documents, providing related or reference material.  A sidebar is
    typically offset by a border and "floats" to the side of the page; the
    document's main text may flow around it.  Sidebars can also be likened to
    super-footnotes; their content is outside of the flow of the document's
    main text.

    Sidebars are allowed wherever body elements (list, table, etc.) are
    allowed, but only at the top level of a section or document.  Sidebars
    cannot nest inside sidebars, topics, or body elements; you can't have a
    sidebar inside a table, list, block quote, etc.
    """


class transition(Structural, Element): pass


# ===============
#  Body Elements
# ===============

class paragraph(General, TextElement): pass
class compound(General, Element): pass
class container(General, Element): pass
class bullet_list(Sequential, Element): pass
class enumerated_list(Sequential, Element): pass
class list_item(Part, Element): pass
class definition_list(Sequential, Element): pass
class definition_list_item(Part, Element): pass
class term(Part, TextElement): pass
class classifier(Part, TextElement): pass
class definition(Part, Element): pass
class field_list(Sequential, Element): pass
class field(Part, Element): pass
class field_name(Part, TextElement): pass
class field_body(Part, Element): pass


class option(Part, Element):

    child_text_separator = ''


class option_argument(Part, TextElement):

    def astext(self):
        return self.get('delimiter', ' ') + TextElement.astext(self)


class option_group(Part, Element):

    child_text_separator = ', '


class option_list(Sequential, Element): pass


class option_list_item(Part, Element):

    child_text_separator = '  '


class option_string(Part, TextElement): pass
class description(Part, Element): pass
class literal_block(General, FixedTextElement): pass
class doctest_block(General, FixedTextElement): pass
class math_block(General, FixedTextElement): pass
class line_block(General, Element): pass


class line(Part, TextElement):

    indent = None


class block_quote(General, Element): pass
class attribution(Part, TextElement): pass
class attention(Admonition, Element): pass
class caution(Admonition, Element): pass
class danger(Admonition, Element): pass
class error(Admonition, Element): pass
class important(Admonition, Element): pass
class note(Admonition, Element): pass
class tip(Admonition, Element): pass
class hint(Admonition, Element): pass
class warning(Admonition, Element): pass
class admonition(Admonition, Element): pass
class comment(Special, Invisible, FixedTextElement): pass
class substitution_definition(Special, Invisible, TextElement): pass
class target(Special, Invisible, Inline, TextElement, Targetable): pass
class footnote(General, BackLinkable, Element, Labeled, Targetable): pass
class citation(General, BackLinkable, Element, Labeled, Targetable): pass
class label(Part, TextElement): pass
class figure(General, Element): pass
class caption(Part, TextElement): pass
class legend(Part, Element): pass
class table(General, Element): pass
class tgroup(Part, Element): pass
class colspec(Part, Element): pass
class thead(Part, Element): pass
class tbody(Part, Element): pass
class row(Part, Element): pass
class entry(Part, Element): pass


class system_message(Special, BackLinkable, PreBibliographic, Element):

    """
    System message element.

    Do not instantiate this class directly; use
    ``document.reporter.info/warning/error/severe()`` instead.
    """

    def __init__(self, message=None, *children, **attributes):
        rawsource = attributes.pop('rawsource', '')
        if message:
            p = paragraph('', message)
            children = (p,) + children
        try:
            Element.__init__(self, rawsource, *children, **attributes)
        except:  # noqa  catchall
            print('system_message: children=%r' % (children,))
            raise

    def astext(self):
        line = self.get('line', '')
        return '%s:%s: (%s/%s) %s' % (self['source'], line, self['type'],
                                      self['level'], Element.astext(self))


class pending(Special, Invisible, Element):

    """
    The "pending" element is used to encapsulate a pending operation: the
    operation (transform), the point at which to apply it, and any data it
    requires.  Only the pending operation's location within the document is
    stored in the public document tree (by the "pending" object itself); the
    operation and its data are stored in the "pending" object's internal
    instance attributes.

    For example, say you want a table of contents in your reStructuredText
    document.  The easiest way to specify where to put it is from within the
    document, with a directive::

        .. contents::

    But the "contents" directive can't do its work until the entire document
    has been parsed and possibly transformed to some extent.  So the directive
    code leaves a placeholder behind that will trigger the second phase of its
    processing, something like this::

        <pending ...public attributes...> + internal attributes

    Use `document.note_pending()` so that the
    `docutils.transforms.Transformer` stage of processing can run all pending
    transforms.
    """

    def __init__(self, transform, details=None,
                 rawsource='', *children, **attributes):
        Element.__init__(self, rawsource, *children, **attributes)

        self.transform = transform
        """The `docutils.transforms.Transform` class implementing the pending
        operation."""

        self.details = details or {}
        """Detail data (dictionary) required by the pending operation."""

    def pformat(self, indent='    ', level=0):
        internals = ['.. internal attributes:',
                     '     .transform: %s.%s' % (self.transform.__module__,
                                                 self.transform.__name__),
                     '     .details:']
        details = sorted(self.details.items())
        for key, value in details:
            if isinstance(value, Node):
                internals.append('%7s%s:' % ('', key))
                internals.extend(['%9s%s' % ('', line)
                                  for line in value.pformat().splitlines()])
            elif (value
                  and isinstance(value, list)
                  and isinstance(value[0], Node)):
                internals.append('%7s%s:' % ('', key))
                for v in value:
                    internals.extend(['%9s%s' % ('', line)
                                      for line in v.pformat().splitlines()])
            else:
                internals.append('%7s%s: %r' % ('', key, value))
        return (Element.pformat(self, indent, level)
                + ''.join(('    %s%s\n' % (indent * level, line))
                          for line in internals))

    def copy(self):
        obj = self.__class__(self.transform, self.details, self.rawsource,
                             **self.attributes)
        obj._document = self._document
        obj.source = self.source
        obj.line = self.line
        return obj


class raw(Special, Inline, PreBibliographic, FixedTextElement):

    """
    Raw data that is to be passed untouched to the Writer.
    """


# =================
#  Inline Elements
# =================

class emphasis(Inline, TextElement): pass
class strong(Inline, TextElement): pass
class literal(Inline, TextElement): pass
class reference(General, Inline, Referential, TextElement): pass
class footnote_reference(Inline, Referential, TextElement): pass
class citation_reference(Inline, Referential, TextElement): pass
class substitution_reference(Inline, TextElement): pass
class title_reference(Inline, TextElement): pass
class abbreviation(Inline, TextElement): pass
class acronym(Inline, TextElement): pass
class superscript(Inline, TextElement): pass
class subscript(Inline, TextElement): pass
class math(Inline, TextElement): pass


class image(General, Inline, Element):

    def astext(self):
        return self.get('alt', '')


class inline(Inline, TextElement): pass
class problematic(Inline, TextElement): pass
class generated(Inline, TextElement): pass


# ========================================
#  Auxiliary Classes, Functions, and Data
# ========================================

node_class_names = """
    Text
    abbreviation acronym address admonition attention attribution author
        authors
    block_quote bullet_list
    caption caution citation citation_reference classifier colspec comment
        compound contact container copyright
    danger date decoration definition definition_list definition_list_item
        description docinfo doctest_block document
    emphasis entry enumerated_list error
    field field_body field_list field_name figure footer
        footnote footnote_reference
    generated
    header hint
    image important inline
    label legend line line_block list_item literal literal_block
    math math_block meta
    note
    option option_argument option_group option_list option_list_item
        option_string organization
    paragraph pending problematic
    raw reference revision row rubric
    section sidebar status strong subscript substitution_definition
        substitution_reference subtitle superscript system_message
    table target tbody term tgroup thead tip title title_reference topic
        transition
    version
    warning""".split()
"""A list of names of all concrete Node subclasses."""


class NodeVisitor:

    """
    "Visitor" pattern [GoF95]_ abstract superclass implementation for
    document tree traversals.

    Each node class has corresponding methods, doing nothing by
    default; override individual methods for specific and useful
    behaviour.  The `dispatch_visit()` method is called by
    `Node.walk()` upon entering a node.  `Node.walkabout()` also calls
    the `dispatch_departure()` method before exiting a node.

    The dispatch methods call "``visit_`` + node class name" or
    "``depart_`` + node class name", resp.

    This is a base class for visitors whose ``visit_...`` & ``depart_...``
    methods must be implemented for *all* compulsory node types encountered
    (such as for `docutils.writers.Writer` subclasses).
    Unimplemented methods will raise exceptions (except for optional nodes).

    For sparse traversals, where only certain node types are of interest, use
    subclass `SparseNodeVisitor` instead.  When (mostly or entirely) uniform
    processing is desired, subclass `GenericNodeVisitor`.

    .. [GoF95] Gamma, Helm, Johnson, Vlissides. *Design Patterns: Elements of
       Reusable Object-Oriented Software*. Addison-Wesley, Reading, MA, USA,
       1995.
    """

    optional = ('meta',)
    """
    Tuple containing node class names (as strings).

    No exception will be raised if writers do not implement visit
    or departure functions for these node classes.

    Used to ensure transitional compatibility with existing 3rd-party writers.
    """

    def __init__(self, document):
        self.document = document

    def dispatch_visit(self, node):
        """
        Call self."``visit_`` + node class name" with `node` as
        parameter.  If the ``visit_...`` method does not exist, call
        self.unknown_visit.
        """
        node_name = node.__class__.__name__
        method = getattr(self, 'visit_' + node_name, self.unknown_visit)
        self.document.reporter.debug(
            'docutils.nodes.NodeVisitor.dispatch_visit calling %s for %s'
            % (method.__name__, node_name))
        return method(node)

    def dispatch_departure(self, node):
        """
        Call self."``depart_`` + node class name" with `node` as
        parameter.  If the ``depart_...`` method does not exist, call
        self.unknown_departure.
        """
        node_name = node.__class__.__name__
        method = getattr(self, 'depart_' + node_name, self.unknown_departure)
        self.document.reporter.debug(
            'docutils.nodes.NodeVisitor.dispatch_departure calling %s for %s'
            % (method.__name__, node_name))
        return method(node)

    def unknown_visit(self, node):
        """
        Called when entering unknown `Node` types.

        Raise an exception unless overridden.
        """
        if (self.document.settings.strict_visitor
            or node.__class__.__name__ not in self.optional):
            raise NotImplementedError(
                '%s visiting unknown node type: %s'
                % (self.__class__, node.__class__.__name__))

    def unknown_departure(self, node):
        """
        Called before exiting unknown `Node` types.

        Raise exception unless overridden.
        """
        if (self.document.settings.strict_visitor
            or node.__class__.__name__ not in self.optional):
            raise NotImplementedError(
                '%s departing unknown node type: %s'
                % (self.__class__, node.__class__.__name__))


class SparseNodeVisitor(NodeVisitor):

    """
    Base class for sparse traversals, where only certain node types are of
    interest.  When ``visit_...`` & ``depart_...`` methods should be
    implemented for *all* node types (such as for `docutils.writers.Writer`
    subclasses), subclass `NodeVisitor` instead.
    """


class GenericNodeVisitor(NodeVisitor):

    """
    Generic "Visitor" abstract superclass, for simple traversals.

    Unless overridden, each ``visit_...`` method calls `default_visit()`, and
    each ``depart_...`` method (when using `Node.walkabout()`) calls
    `default_departure()`. `default_visit()` (and `default_departure()`) must
    be overridden in subclasses.

    Define fully generic visitors by overriding `default_visit()` (and
    `default_departure()`) only. Define semi-generic visitors by overriding
    individual ``visit_...()`` (and ``depart_...()``) methods also.

    `NodeVisitor.unknown_visit()` (`NodeVisitor.unknown_departure()`) should
    be overridden for default behavior.
    """

    def default_visit(self, node):
        """Override for generic, uniform traversals."""
        raise NotImplementedError

    def default_departure(self, node):
        """Override for generic, uniform traversals."""
        raise NotImplementedError


def _call_default_visit(self, node):
    self.default_visit(node)


def _call_default_departure(self, node):
    self.default_departure(node)


def _nop(self, node):
    pass


def _add_node_class_names(names):
    """Save typing with dynamic assignments:"""
    for _name in names:
        setattr(GenericNodeVisitor, "visit_" + _name, _call_default_visit)
        setattr(GenericNodeVisitor, "depart_" + _name, _call_default_departure)
        setattr(SparseNodeVisitor, 'visit_' + _name, _nop)
        setattr(SparseNodeVisitor, 'depart_' + _name, _nop)


_add_node_class_names(node_class_names)


class TreeCopyVisitor(GenericNodeVisitor):

    """
    Make a complete copy of a tree or branch, including element attributes.
    """

    def __init__(self, document):
        GenericNodeVisitor.__init__(self, document)
        self.parent_stack = []
        self.parent = []

    def get_tree_copy(self):
        return self.parent[0]

    def default_visit(self, node):
        """Copy the current node, and make it the new acting parent."""
        newnode = node.copy()
        self.parent.append(newnode)
        self.parent_stack.append(self.parent)
        self.parent = newnode

    def default_departure(self, node):
        """Restore the previous acting parent."""
        self.parent = self.parent_stack.pop()


class TreePruningException(Exception):

    """
    Base class for `NodeVisitor`-related tree pruning exceptions.

    Raise subclasses from within ``visit_...`` or ``depart_...`` methods
    called from `Node.walk()` and `Node.walkabout()` tree traversals to prune
    the tree traversed.
    """


class SkipChildren(TreePruningException):

    """
    Do not visit any children of the current node.  The current node's
    siblings and ``depart_...`` method are not affected.
    """


class SkipSiblings(TreePruningException):

    """
    Do not visit any more siblings (to the right) of the current node.  The
    current node's children and its ``depart_...`` method are not affected.
    """


class SkipNode(TreePruningException):

    """
    Do not visit the current node's children, and do not call the current
    node's ``depart_...`` method.
    """


class SkipDeparture(TreePruningException):

    """
    Do not call the current node's ``depart_...`` method.  The current node's
    children and siblings are not affected.
    """


class NodeFound(TreePruningException):

    """
    Raise to indicate that the target of a search has been found.  This
    exception must be caught by the client; it is not caught by the traversal
    code.
    """


class StopTraversal(TreePruningException):

    """
    Stop the traversal altogether.  The current node's ``depart_...`` method
    is not affected.  The parent nodes ``depart_...`` methods are also called
    as usual.  No other nodes are visited.  This is an alternative to
    NodeFound that does not cause exception handling to trickle up to the
    caller.
    """


def make_id(string):
    """
    Convert `string` into an identifier and return it.

    Docutils identifiers will conform to the regular expression
    ``[a-z](-?[a-z0-9]+)*``.  For CSS compatibility, identifiers (the "class"
    and "id" attributes) should have no underscores, colons, or periods.
    Hyphens may be used.

    - The `HTML 4.01 spec`_ defines identifiers based on SGML tokens:

          ID and NAME tokens must begin with a letter ([A-Za-z]) and may be
          followed by any number of letters, digits ([0-9]), hyphens ("-"),
          underscores ("_"), colons (":"), and periods (".").

    - However the `CSS1 spec`_ defines identifiers based on the "name" token,
      a tighter interpretation ("flex" tokenizer notation; "latin1" and
      "escape" 8-bit characters have been replaced with entities)::

          unicode     \\[0-9a-f]{1,4}
          latin1      [&iexcl;-&yuml;]
          escape      {unicode}|\\[ -~&iexcl;-&yuml;]
          nmchar      [-a-z0-9]|{latin1}|{escape}
          name        {nmchar}+

    The CSS1 "nmchar" rule does not include underscores ("_"), colons (":"),
    or periods ("."), therefore "class" and "id" attributes should not contain
    these characters. They should be replaced with hyphens ("-"). Combined
    with HTML's requirements (the first character must be a letter; no
    "unicode", "latin1", or "escape" characters), this results in the
    ``[a-z](-?[a-z0-9]+)*`` pattern.

    .. _HTML 4.01 spec: https://www.w3.org/TR/html401
    .. _CSS1 spec: https://www.w3.org/TR/REC-CSS1
    """
    id = string.lower()
    id = id.translate(_non_id_translate_digraphs)
    id = id.translate(_non_id_translate)
    # get rid of non-ascii characters.
    # 'ascii' lowercase to prevent problems with turkish locale.
    id = unicodedata.normalize(
            'NFKD', id).encode('ascii', 'ignore').decode('ascii')
    # shrink runs of whitespace and replace by hyphen
    id = _non_id_chars.sub('-', ' '.join(id.split()))
    id = _non_id_at_ends.sub('', id)
    return str(id)


_non_id_chars = re.compile('[^a-z0-9]+')
_non_id_at_ends = re.compile('^[-0-9]+|-+$')
_non_id_translate = {
    0x00f8: 'o',       # o with stroke
    0x0111: 'd',       # d with stroke
    0x0127: 'h',       # h with stroke
    0x0131: 'i',       # dotless i
    0x0142: 'l',       # l with stroke
    0x0167: 't',       # t with stroke
    0x0180: 'b',       # b with stroke
    0x0183: 'b',       # b with topbar
    0x0188: 'c',       # c with hook
    0x018c: 'd',       # d with topbar
    0x0192: 'f',       # f with hook
    0x0199: 'k',       # k with hook
    0x019a: 'l',       # l with bar
    0x019e: 'n',       # n with long right leg
    0x01a5: 'p',       # p with hook
    0x01ab: 't',       # t with palatal hook
    0x01ad: 't',       # t with hook
    0x01b4: 'y',       # y with hook
    0x01b6: 'z',       # z with stroke
    0x01e5: 'g',       # g with stroke
    0x0225: 'z',       # z with hook
    0x0234: 'l',       # l with curl
    0x0235: 'n',       # n with curl
    0x0236: 't',       # t with curl
    0x0237: 'j',       # dotless j
    0x023c: 'c',       # c with stroke
    0x023f: 's',       # s with swash tail
    0x0240: 'z',       # z with swash tail
    0x0247: 'e',       # e with stroke
    0x0249: 'j',       # j with stroke
    0x024b: 'q',       # q with hook tail
    0x024d: 'r',       # r with stroke
    0x024f: 'y',       # y with stroke
}
_non_id_translate_digraphs = {
    0x00df: 'sz',      # ligature sz
    0x00e6: 'ae',      # ae
    0x0153: 'oe',      # ligature oe
    0x0238: 'db',      # db digraph
    0x0239: 'qp',      # qp digraph
}


def dupname(node, name):
    node['dupnames'].append(name)
    node['names'].remove(name)
    # Assume that this method is referenced, even though it isn't; we
    # don't want to throw unnecessary system_messages.
    node.referenced = 1


def fully_normalize_name(name):
    """Return a case- and whitespace-normalized name."""
    return ' '.join(name.lower().split())


def whitespace_normalize_name(name):
    """Return a whitespace-normalized name."""
    return ' '.join(name.split())


def serial_escape(value):
    """Escape string values that are elements of a list, for serialization."""
    return value.replace('\\', r'\\').replace(' ', r'\ ')


def pseudo_quoteattr(value):
    """Quote attributes for pseudo-xml"""
    return '"%s"' % value

env_pyrecodes/lib/python3.9/site-packages/docutils/nodes.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
# :Copyright: © 2020 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause
#
# Revision: $Revision: 9302 $
# Date: $Date: 2022-12-02 18:14:05 +0100 (Fr, 02. Dez 2022) $
"""
A parser for CommonMark Markdown text using `recommonmark`__.

__ https://pypi.org/project/recommonmark/

.. important:: This module is provisional

   * The "recommonmark" package is unmaintained and deprecated.
     This wrapper module will be removed in a future Docutils version.

   * The API is not settled and may change with any minor Docutils version.
"""

from docutils import Component
from docutils import nodes

try:
    # If possible, import Sphinx's 'addnodes'
    from sphinx import addnodes
except ImportError:
    # stub to prevent errors if Sphinx isn't installed
    import sys
    import types

    class pending_xref(nodes.Inline, nodes.Element): ... # NoQA

    sys.modules['sphinx'] = sphinx = types.ModuleType('sphinx')
    sphinx.addnodes = addnodes = types.SimpleNamespace()
    addnodes.pending_xref = pending_xref
try:
    import recommonmark
    from recommonmark.parser import CommonMarkParser
except ImportError as err:
    raise ImportError(
        'Parsing "recommonmark" Markdown flavour requires the\n'
        '  package https://pypi.org/project/recommonmark.'
    ) from err
else:
    if recommonmark.__version__ < '0.6.0':
        raise ImportError('The installed version of "recommonmark" is too old.'
                          ' Update with "pip install -U recommonmark".')


# auxiliary function for `document.findall()`
def is_literal(node):
    return isinstance(node, (nodes.literal, nodes.literal_block))


class Parser(CommonMarkParser):
    """MarkDown parser based on recommonmark.

    This parser is provisional:
    the API is not settled and may change with any minor Docutils version.
    """
    supported = ('recommonmark', 'commonmark', 'markdown', 'md')
    """Formats this parser supports."""

    config_section = 'recommonmark parser'
    config_section_dependencies = ('parsers',)

    def get_transforms(self):
        return Component.get_transforms(self)  # + [AutoStructify]

    def parse(self, inputstring, document):
        """Use the upstream parser and clean up afterwards.
        """
        # check for exorbitantly long lines
        for i, line in enumerate(inputstring.split('\n')):
            if len(line) > document.settings.line_length_limit:
                error = document.reporter.error(
                    'Line %d exceeds the line-length-limit.'%(i+1))
                document.append(error)
                return

        # pass to upstream parser
        try:
            CommonMarkParser.parse(self, inputstring, document)
        except Exception as err:
            if document.settings.traceback:
                raise err
            error = document.reporter.error('Parsing with "recommonmark" '
                                            'returned the error:\n%s'%err)
            document.append(error)

        # Post-Processing
        # ---------------

        # merge adjoining Text nodes:
        for node in document.findall(nodes.TextElement):
            children = node.children
            i = 0
            while i+1 < len(children):
                if (isinstance(children[i], nodes.Text)
                    and isinstance(children[i+1], nodes.Text)):
                    children[i] = nodes.Text(children[i]+children.pop(i+1))
                    children[i].parent = node
                else:
                    i += 1

        # add "code" class argument to literal elements (inline and block)
        for node in document.findall(is_literal):
            if 'code' not in node['classes']:
                node['classes'].append('code')
        # move "language" argument to classes
        for node in document.findall(nodes.literal_block):
            if 'language' in node.attributes:
                node['classes'].append(node['language'])
                del node['language']

        # replace raw nodes if raw is not allowed
        if not document.settings.raw_enabled:
            for node in document.findall(nodes.raw):
                warning = document.reporter.warning('Raw content disabled.')
                node.parent.replace(node, warning)

        # drop pending_xref (Sphinx cross reference extension)
        for node in document.findall(addnodes.pending_xref):
            reference = node.children[0]
            if 'name' not in reference:
                reference['name'] = nodes.fully_normalize_name(
                                                    reference.astext())
            node.parent.replace(node, reference)

    def visit_document(self, node):
        """Dummy function to prevent spurious warnings.

        cf. https://github.com/readthedocs/recommonmark/issues/177
        """
        pass

    # Overwrite parent method with version that
    # doesn't pass deprecated `rawsource` argument to nodes.Text:
    def visit_text(self, mdnode):
        self.current_node.append(nodes.Text(mdnode.literal))

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/recommonmark_wrapper.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9258 2022-11-21 14:51:43Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
This is ``docutils.parsers.rst`` package. It exports a single class, `Parser`,
the reStructuredText parser.


Usage
=====

1. Create a parser::

       parser = docutils.parsers.rst.Parser()

   Several optional arguments may be passed to modify the parser's behavior.
   Please see `Customizing the Parser`_ below for details.

2. Gather input (a multi-line string), by reading a file or the standard
   input::

       input = sys.stdin.read()

3. Create a new empty `docutils.nodes.document` tree::

       document = docutils.utils.new_document(source, settings)

   See `docutils.utils.new_document()` for parameter details.

4. Run the parser, populating the document tree::

       parser.parse(input, document)


Parser Overview
===============

The reStructuredText parser is implemented as a state machine, examining its
input one line at a time. To understand how the parser works, please first
become familiar with the `docutils.statemachine` module, then see the
`states` module.


Customizing the Parser
----------------------

Anything that isn't already customizable is that way simply because that type
of customizability hasn't been implemented yet.  Patches welcome!

When instantiating an object of the `Parser` class, two parameters may be
passed: ``rfc2822`` and ``inliner``.  Pass ``rfc2822=True`` to enable an
initial RFC-2822 style header block, parsed as a "field_list" element (with
"class" attribute set to "rfc2822").  Currently this is the only body-level
element which is customizable without subclassing.  (Tip: subclass `Parser`
and change its "state_classes" and "initial_state" attributes to refer to new
classes. Contact the author if you need more details.)

The ``inliner`` parameter takes an instance of `states.Inliner` or a subclass.
It handles inline markup recognition.  A common extension is the addition of
further implicit hyperlinks, like "RFC 2822".  This can be done by subclassing
`states.Inliner`, adding a new method for the implicit markup, and adding a
``(pattern, method)`` pair to the "implicit_dispatch" attribute of the
subclass.  See `states.Inliner.implicit_inline()` for details.  Explicit
inline markup can be customized in a `states.Inliner` subclass via the
``patterns.initial`` and ``dispatch`` attributes (and new methods as
appropriate).
"""

__docformat__ = 'reStructuredText'


import docutils.parsers
import docutils.statemachine
from docutils.parsers.rst import roles, states
from docutils import frontend, nodes
from docutils.transforms import universal


class Parser(docutils.parsers.Parser):

    """The reStructuredText parser."""

    supported = ('rst', 'restructuredtext', 'rest', 'restx', 'rtxt', 'rstx')
    """Aliases this parser supports."""

    settings_spec = docutils.parsers.Parser.settings_spec + (
        'reStructuredText Parser Options',
        None,
        (('Recognize and link to standalone PEP references (like "PEP 258").',
          ['--pep-references'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Base URL for PEP references '
          '(default "https://peps.python.org/").',
          ['--pep-base-url'],
          {'metavar': '<URL>', 'default': 'https://peps.python.org/',
           'validator': frontend.validate_url_trailing_slash}),
         ('Template for PEP file part of URL. (default "pep-%04d")',
          ['--pep-file-url-template'],
          {'metavar': '<URL>', 'default': 'pep-%04d'}),
         ('Recognize and link to standalone RFC references (like "RFC 822").',
          ['--rfc-references'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Base URL for RFC references '
          '(default "https://tools.ietf.org/html/").',
          ['--rfc-base-url'],
          {'metavar': '<URL>', 'default': 'https://tools.ietf.org/html/',
           'validator': frontend.validate_url_trailing_slash}),
         ('Set number of spaces for tab expansion (default 8).',
          ['--tab-width'],
          {'metavar': '<width>', 'type': 'int', 'default': 8,
           'validator': frontend.validate_nonnegative_int}),
         ('Remove spaces before footnote references.',
          ['--trim-footnote-reference-space'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Leave spaces before footnote references.',
          ['--leave-footnote-reference-space'],
          {'action': 'store_false', 'dest': 'trim_footnote_reference_space'}),
         ('Token name set for parsing code with Pygments: one of '
          '"long", "short", or "none" (no parsing). Default is "long".',
          ['--syntax-highlight'],
          {'choices': ['long', 'short', 'none'],
           'default': 'long', 'metavar': '<format>'}),
         ('Change straight quotation marks to typographic form: '
          'one of "yes", "no", "alt[ernative]" (default "no").',
          ['--smart-quotes'],
          {'default': False, 'metavar': '<yes/no/alt>',
           'validator': frontend.validate_ternary}),
         ('Characters to use as "smart quotes" for <language>. ',
          ['--smartquotes-locales'],
          {'metavar': '<language:quotes[,language:quotes,...]>',
           'action': 'append',
           'validator': frontend.validate_smartquotes_locales}),
         ('Inline markup recognized at word boundaries only '
          '(adjacent to punctuation or whitespace). '
          'Force character-level inline markup recognition with '
          '"\\ " (backslash + space). Default.',
          ['--word-level-inline-markup'],
          {'action': 'store_false', 'dest': 'character_level_inline_markup'}),
         ('Inline markup recognized anywhere, regardless of surrounding '
          'characters. Backslash-escapes must be used to avoid unwanted '
          'markup recognition. Useful for East Asian languages. '
          'Experimental.',
          ['--character-level-inline-markup'],
          {'action': 'store_true', 'default': False,
           'dest': 'character_level_inline_markup'}),
         )
        )

    config_section = 'restructuredtext parser'
    config_section_dependencies = ('parsers',)

    def __init__(self, rfc2822=False, inliner=None):
        if rfc2822:
            self.initial_state = 'RFC2822Body'
        else:
            self.initial_state = 'Body'
        self.state_classes = states.state_classes
        self.inliner = inliner

    def get_transforms(self):
        return super().get_transforms() + [universal.SmartQuotes]

    def parse(self, inputstring, document):
        """Parse `inputstring` and populate `document`, a document tree."""
        self.setup_parse(inputstring, document)
        # provide fallbacks in case the document has only generic settings
        self.document.settings.setdefault('tab_width', 8)
        self.document.settings.setdefault('syntax_highlight', 'long')
        self.statemachine = states.RSTStateMachine(
              state_classes=self.state_classes,
              initial_state=self.initial_state,
              debug=document.reporter.debug_flag)
        inputlines = docutils.statemachine.string2lines(
              inputstring, tab_width=document.settings.tab_width,
              convert_whitespace=True)
        for i, line in enumerate(inputlines):
            if len(line) > self.document.settings.line_length_limit:
                error = self.document.reporter.error(
                            'Line %d exceeds the line-length-limit.'%(i+1))
                self.document.append(error)
                break
        else:
            self.statemachine.run(inputlines, document, inliner=self.inliner)
        # restore the "default" default role after parsing a document
        if '' in roles._roles:
            del roles._roles['']
        self.finish_parse()


class DirectiveError(Exception):

    """
    Store a message and a system message level.

    To be thrown from inside directive code.

    Do not instantiate directly -- use `Directive.directive_error()`
    instead!
    """

    def __init__(self, level, message):
        """Set error `message` and `level`"""
        Exception.__init__(self)
        self.level = level
        self.msg = message


class Directive:

    """
    Base class for reStructuredText directives.

    The following attributes may be set by subclasses.  They are
    interpreted by the directive parser (which runs the directive
    class):

    - `required_arguments`: The number of required arguments (default:
      0).

    - `optional_arguments`: The number of optional arguments (default:
      0).

    - `final_argument_whitespace`: A boolean, indicating if the final
      argument may contain whitespace (default: False).

    - `option_spec`: A dictionary, mapping known option names to
      conversion functions such as `int` or `float` (default: {}, no
      options).  Several conversion functions are defined in the
      directives/__init__.py module.

      Option conversion functions take a single parameter, the option
      argument (a string or ``None``), validate it and/or convert it
      to the appropriate form.  Conversion functions may raise
      `ValueError` and `TypeError` exceptions.

    - `has_content`: A boolean; True if content is allowed.  Client
      code must handle the case where content is required but not
      supplied (an empty content list will be supplied).

    Arguments are normally single whitespace-separated words.  The
    final argument may contain whitespace and/or newlines if
    `final_argument_whitespace` is True.

    If the form of the arguments is more complex, specify only one
    argument (either required or optional) and set
    `final_argument_whitespace` to True; the client code must do any
    context-sensitive parsing.

    When a directive implementation is being run, the directive class
    is instantiated, and the `run()` method is executed.  During
    instantiation, the following instance variables are set:

    - ``name`` is the directive type or name (string).

    - ``arguments`` is the list of positional arguments (strings).

    - ``options`` is a dictionary mapping option names (strings) to
      values (type depends on option conversion functions; see
      `option_spec` above).

    - ``content`` is a list of strings, the directive content line by line.

    - ``lineno`` is the absolute line number of the first line
      of the directive.

    - ``content_offset`` is the line offset of the first line
      of the content from the beginning of the current input.
      Used when initiating a nested parse.

    - ``block_text`` is a string containing the entire directive.

    - ``state`` is the state which called the directive function.

    - ``state_machine`` is the state machine which controls the state
      which called the directive function.

    - ``reporter`` is the state machine's `reporter` instance.

    Directive functions return a list of nodes which will be inserted
    into the document tree at the point where the directive was
    encountered.  This can be an empty list if there is nothing to
    insert.

    For ordinary directives, the list must contain body elements or
    structural elements.  Some directives are intended specifically
    for substitution definitions, and must return a list of `Text`
    nodes and/or inline elements (suitable for inline insertion, in
    place of the substitution reference).  Such directives must verify
    substitution definition context, typically using code like this::

        if not isinstance(state, states.SubstitutionDef):
            error = self.reporter.error(
                'Invalid context: the "%s" directive can only be used '
                'within a substitution definition.' % (name),
                nodes.literal_block(block_text, block_text), line=lineno)
            return [error]
    """

    # There is a "Creating reStructuredText Directives" how-to at
    # <https://docutils.sourceforge.io/docs/howto/rst-directives.html>.  If you
    # update this docstring, please update the how-to as well.

    required_arguments = 0
    """Number of required directive arguments."""

    optional_arguments = 0
    """Number of optional arguments after the required arguments."""

    final_argument_whitespace = False
    """May the final argument contain whitespace?"""

    option_spec = None
    """Mapping of option names to validator functions."""

    has_content = False
    """May the directive have content?"""

    def __init__(self, name, arguments, options, content, lineno,
                 content_offset, block_text, state, state_machine):
        self.name = name
        self.arguments = arguments
        self.options = options
        self.content = content
        self.lineno = lineno
        self.content_offset = content_offset
        self.block_text = block_text
        self.state = state
        self.state_machine = state_machine
        self.reporter = state_machine.reporter

    def run(self):
        raise NotImplementedError('Must override run() in subclass.')

    # Directive errors:

    def directive_error(self, level, message):
        """
        Return a DirectiveError suitable for being thrown as an exception.

        Call "raise self.directive_error(level, message)" from within
        a directive implementation to return one single system message
        at level `level`, which automatically gets the directive block
        and the line number added.

        Preferably use the `debug`, `info`, `warning`, `error`, or `severe`
        wrapper methods, e.g. ``self.error(message)`` to generate an
        ERROR-level directive error.
        """
        return DirectiveError(level, message)

    def debug(self, message):
        return self.directive_error(0, message)

    def info(self, message):
        return self.directive_error(1, message)

    def warning(self, message):
        return self.directive_error(2, message)

    def error(self, message):
        return self.directive_error(3, message)

    def severe(self, message):
        return self.directive_error(4, message)

    # Convenience methods:

    def assert_has_content(self):
        """
        Throw an ERROR-level DirectiveError if the directive doesn't
        have contents.
        """
        if not self.content:
            raise self.error('Content block expected for the "%s" directive; '
                             'none found.' % self.name)

    def add_name(self, node):
        """Append self.options['name'] to node['names'] if it exists.

        Also normalize the name string and register it as explicit target.
        """
        if 'name' in self.options:
            name = nodes.fully_normalize_name(self.options.pop('name'))
            if 'name' in node:
                del node['name']
            node['names'].append(name)
            self.state.document.note_explicit_target(node, node)


def convert_directive_function(directive_fn):
    """
    Define & return a directive class generated from `directive_fn`.

    `directive_fn` uses the old-style, functional interface.
    """

    class FunctionalDirective(Directive):

        option_spec = getattr(directive_fn, 'options', None)
        has_content = getattr(directive_fn, 'content', False)
        _argument_spec = getattr(directive_fn, 'arguments', (0, 0, False))
        required_arguments, optional_arguments, final_argument_whitespace \
            = _argument_spec

        def run(self):
            return directive_fn(
                self.name, self.arguments, self.options, self.content,
                self.lineno, self.content_offset, self.block_text,
                self.state, self.state_machine)

    # Return new-style directive.
    return FunctionalDirective

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9356 2023-04-18 23:50:48Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
This package contains directive implementation modules.
"""

__docformat__ = 'reStructuredText'

import re
import codecs
from importlib import import_module

from docutils import nodes, parsers
from docutils.utils import split_escaped_whitespace, escape2null
from docutils.parsers.rst.languages import en as _fallback_language_module


_directive_registry = {
      'attention': ('admonitions', 'Attention'),
      'caution': ('admonitions', 'Caution'),
      'code': ('body', 'CodeBlock'),
      'danger': ('admonitions', 'Danger'),
      'error': ('admonitions', 'Error'),
      'important': ('admonitions', 'Important'),
      'note': ('admonitions', 'Note'),
      'tip': ('admonitions', 'Tip'),
      'hint': ('admonitions', 'Hint'),
      'warning': ('admonitions', 'Warning'),
      'admonition': ('admonitions', 'Admonition'),
      'sidebar': ('body', 'Sidebar'),
      'topic': ('body', 'Topic'),
      'line-block': ('body', 'LineBlock'),
      'parsed-literal': ('body', 'ParsedLiteral'),
      'math': ('body', 'MathBlock'),
      'rubric': ('body', 'Rubric'),
      'epigraph': ('body', 'Epigraph'),
      'highlights': ('body', 'Highlights'),
      'pull-quote': ('body', 'PullQuote'),
      'compound': ('body', 'Compound'),
      'container': ('body', 'Container'),
      # 'questions': ('body', 'question_list'),
      'table': ('tables', 'RSTTable'),
      'csv-table': ('tables', 'CSVTable'),
      'list-table': ('tables', 'ListTable'),
      'image': ('images', 'Image'),
      'figure': ('images', 'Figure'),
      'contents': ('parts', 'Contents'),
      'sectnum': ('parts', 'Sectnum'),
      'header': ('parts', 'Header'),
      'footer': ('parts', 'Footer'),
      # 'footnotes': ('parts', 'footnotes'),
      # 'citations': ('parts', 'citations'),
      'target-notes': ('references', 'TargetNotes'),
      'meta': ('misc', 'Meta'),
      # 'imagemap': ('html', 'imagemap'),
      'raw': ('misc', 'Raw'),
      'include': ('misc', 'Include'),
      'replace': ('misc', 'Replace'),
      'unicode': ('misc', 'Unicode'),
      'class': ('misc', 'Class'),
      'role': ('misc', 'Role'),
      'default-role': ('misc', 'DefaultRole'),
      'title': ('misc', 'Title'),
      'date': ('misc', 'Date'),
      'restructuredtext-test-directive': ('misc', 'TestDirective'),
      }
"""Mapping of directive name to (module name, class name).  The
directive name is canonical & must be lowercase.  Language-dependent
names are defined in the ``language`` subpackage."""

_directives = {}
"""Cache of imported directives."""


def directive(directive_name, language_module, document):
    """
    Locate and return a directive function from its language-dependent name.
    If not found in the current language, check English.  Return None if the
    named directive cannot be found.
    """
    normname = directive_name.lower()
    messages = []
    msg_text = []
    if normname in _directives:
        return _directives[normname], messages
    canonicalname = None
    try:
        canonicalname = language_module.directives[normname]
    except AttributeError as error:
        msg_text.append('Problem retrieving directive entry from language '
                        'module %r: %s.' % (language_module, error))
    except KeyError:
        msg_text.append('No directive entry for "%s" in module "%s".'
                        % (directive_name, language_module.__name__))
    if not canonicalname:
        try:
            canonicalname = _fallback_language_module.directives[normname]
            msg_text.append('Using English fallback for directive "%s".'
                            % directive_name)
        except KeyError:
            msg_text.append('Trying "%s" as canonical directive name.'
                            % directive_name)
            # The canonical name should be an English name, but just in case:
            canonicalname = normname
    if msg_text:
        message = document.reporter.info(
            '\n'.join(msg_text), line=document.current_line)
        messages.append(message)
    try:
        modulename, classname = _directive_registry[canonicalname]
    except KeyError:
        # Error handling done by caller.
        return None, messages
    try:
        module = import_module('docutils.parsers.rst.directives.'+modulename)
    except ImportError as detail:
        messages.append(document.reporter.error(
            'Error importing directive module "%s" (directive "%s"):\n%s'
            % (modulename, directive_name, detail),
            line=document.current_line))
        return None, messages
    try:
        directive = getattr(module, classname)
        _directives[normname] = directive
    except AttributeError:
        messages.append(document.reporter.error(
            'No directive class "%s" in module "%s" (directive "%s").'
            % (classname, modulename, directive_name),
            line=document.current_line))
        return None, messages
    return directive, messages


def register_directive(name, directive):
    """
    Register a nonstandard application-defined directive function.
    Language lookups are not needed for such functions.
    """
    _directives[name] = directive


def flag(argument):
    """
    Check for a valid flag option (no argument) and return ``None``.
    (Directive option conversion function.)

    Raise ``ValueError`` if an argument is found.
    """
    if argument and argument.strip():
        raise ValueError('no argument is allowed; "%s" supplied' % argument)
    else:
        return None


def unchanged_required(argument):
    """
    Return the argument text, unchanged.
    (Directive option conversion function.)

    Raise ``ValueError`` if no argument is found.
    """
    if argument is None:
        raise ValueError('argument required but none supplied')
    else:
        return argument  # unchanged!


def unchanged(argument):
    """
    Return the argument text, unchanged.
    (Directive option conversion function.)

    No argument implies empty string ("").
    """
    if argument is None:
        return ''
    else:
        return argument  # unchanged!


def path(argument):
    """
    Return the path argument unwrapped (with newlines removed).
    (Directive option conversion function.)

    Raise ``ValueError`` if no argument is found.
    """
    if argument is None:
        raise ValueError('argument required but none supplied')
    else:
        return ''.join(s.strip() for s in argument.splitlines())


def uri(argument):
    """
    Return the URI argument with unescaped whitespace removed.
    (Directive option conversion function.)

    Raise ``ValueError`` if no argument is found.
    """
    if argument is None:
        raise ValueError('argument required but none supplied')
    else:
        parts = split_escaped_whitespace(escape2null(argument))
        return ' '.join(''.join(nodes.unescape(part).split())
                        for part in parts)


def nonnegative_int(argument):
    """
    Check for a nonnegative integer argument; raise ``ValueError`` if not.
    (Directive option conversion function.)
    """
    value = int(argument)
    if value < 0:
        raise ValueError('negative value; must be positive or zero')
    return value


def percentage(argument):
    """
    Check for an integer percentage value with optional percent sign.
    (Directive option conversion function.)
    """
    try:
        argument = argument.rstrip(' %')
    except AttributeError:
        pass
    return nonnegative_int(argument)


length_units = ['em', 'ex', 'px', 'in', 'cm', 'mm', 'pt', 'pc']


def get_measure(argument, units):
    """
    Check for a positive argument of one of the units and return a
    normalized string of the form "<value><unit>" (without space in
    between).
    (Directive option conversion function.)

    To be called from directive option conversion functions.
    """
    match = re.match(r'^([0-9.]+) *(%s)$' % '|'.join(units), argument)
    try:
        float(match.group(1))
    except (AttributeError, ValueError):
        raise ValueError(
            'not a positive measure of one of the following units:\n%s'
            % ' '.join('"%s"' % i for i in units))
    return match.group(1) + match.group(2)


def length_or_unitless(argument):
    return get_measure(argument, length_units + [''])


def length_or_percentage_or_unitless(argument, default=''):
    """
    Return normalized string of a length or percentage unit.
    (Directive option conversion function.)

    Add <default> if there is no unit. Raise ValueError if the argument is not
    a positive measure of one of the valid CSS units (or without unit).

    >>> length_or_percentage_or_unitless('3 pt')
    '3pt'
    >>> length_or_percentage_or_unitless('3%', 'em')
    '3%'
    >>> length_or_percentage_or_unitless('3')
    '3'
    >>> length_or_percentage_or_unitless('3', 'px')
    '3px'
    """
    try:
        return get_measure(argument, length_units + ['%'])
    except ValueError:
        try:
            return get_measure(argument, ['']) + default
        except ValueError:
            # raise ValueError with list of valid units:
            return get_measure(argument, length_units + ['%'])


def class_option(argument):
    """
    Convert the argument into a list of ID-compatible strings and return it.
    (Directive option conversion function.)

    Raise ``ValueError`` if no argument is found.
    """
    if argument is None:
        raise ValueError('argument required but none supplied')
    names = argument.split()
    class_names = []
    for name in names:
        class_name = nodes.make_id(name)
        if not class_name:
            raise ValueError('cannot make "%s" into a class name' % name)
        class_names.append(class_name)
    return class_names


unicode_pattern = re.compile(
    r'(?:0x|x|\\x|U\+?|\\u)([0-9a-f]+)$|&#x([0-9a-f]+);$', re.IGNORECASE)


def unicode_code(code):
    r"""
    Convert a Unicode character code to a Unicode character.
    (Directive option conversion function.)

    Codes may be decimal numbers, hexadecimal numbers (prefixed by ``0x``,
    ``x``, ``\x``, ``U+``, ``u``, or ``\u``; e.g. ``U+262E``), or XML-style
    numeric character entities (e.g. ``&#x262E;``).  Other text remains as-is.

    Raise ValueError for illegal Unicode code values.
    """
    try:
        if code.isdigit():                  # decimal number
            return chr(int(code))
        else:
            match = unicode_pattern.match(code)
            if match:                       # hex number
                value = match.group(1) or match.group(2)
                return chr(int(value, 16))
            else:                           # other text
                return code
    except OverflowError as detail:
        raise ValueError('code too large (%s)' % detail)


def single_char_or_unicode(argument):
    """
    A single character is returned as-is.  Unicode character codes are
    converted as in `unicode_code`.  (Directive option conversion function.)
    """
    char = unicode_code(argument)
    if len(char) > 1:
        raise ValueError('%r invalid; must be a single character or '
                         'a Unicode code' % char)
    return char


def single_char_or_whitespace_or_unicode(argument):
    """
    As with `single_char_or_unicode`, but "tab" and "space" are also supported.
    (Directive option conversion function.)
    """
    if argument == 'tab':
        char = '\t'
    elif argument == 'space':
        char = ' '
    else:
        char = single_char_or_unicode(argument)
    return char


def positive_int(argument):
    """
    Converts the argument into an integer.  Raises ValueError for negative,
    zero, or non-integer values.  (Directive option conversion function.)
    """
    value = int(argument)
    if value < 1:
        raise ValueError('negative or zero value; must be positive')
    return value


def positive_int_list(argument):
    """
    Converts a space- or comma-separated list of values into a Python list
    of integers.
    (Directive option conversion function.)

    Raises ValueError for non-positive-integer values.
    """
    if ',' in argument:
        entries = argument.split(',')
    else:
        entries = argument.split()
    return [positive_int(entry) for entry in entries]


def encoding(argument):
    """
    Verifies the encoding argument by lookup.
    (Directive option conversion function.)

    Raises ValueError for unknown encodings.
    """
    try:
        codecs.lookup(argument)
    except LookupError:
        raise ValueError('unknown encoding: "%s"' % argument)
    return argument


def choice(argument, values):
    """
    Directive option utility function, supplied to enable options whose
    argument must be a member of a finite set of possible values (must be
    lower case).  A custom conversion function must be written to use it.  For
    example::

        from docutils.parsers.rst import directives

        def yesno(argument):
            return directives.choice(argument, ('yes', 'no'))

    Raise ``ValueError`` if no argument is found or if the argument's value is
    not valid (not an entry in the supplied list).
    """
    try:
        value = argument.lower().strip()
    except AttributeError:
        raise ValueError('must supply an argument; choose from %s'
                         % format_values(values))
    if value in values:
        return value
    else:
        raise ValueError('"%s" unknown; choose from %s'
                         % (argument, format_values(values)))


def format_values(values):
    return '%s, or "%s"' % (', '.join('"%s"' % s for s in values[:-1]),
                            values[-1])


def value_or(values, other):
    """
    Directive option conversion function.

    The argument can be any of `values` or `argument_type`.
    """
    def auto_or_other(argument):
        if argument in values:
            return argument
        else:
            return other(argument)
    return auto_or_other


def parser_name(argument):
    """
    Return a docutils parser whose name matches the argument.
    (Directive option conversion function.)

    Return `None`, if the argument evaluates to `False`.
    Raise `ValueError` if importing the parser module fails.
    """
    if not argument:
        return None
    try:
        return parsers.get_parser_class(argument)
    except ImportError as err:
        raise ValueError(str(err))

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/directives/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: misc.py 9358 2023-04-19 23:31:13Z milde $
# Authors: David Goodger <goodger@python.org>; Dethe Elza
# Copyright: This module has been placed in the public domain.

"""Miscellaneous directives."""

__docformat__ = 'reStructuredText'

from pathlib import Path
import re
import time
from urllib.request import urlopen
from urllib.error import URLError

from docutils import io, nodes, statemachine, utils
from docutils.parsers.rst import Directive, convert_directive_function
from docutils.parsers.rst import directives, roles, states
from docutils.parsers.rst.directives.body import CodeBlock, NumberLines
from docutils.transforms import misc


class Include(Directive):

    """
    Include content read from a separate source file.

    Content may be parsed by the parser, or included as a literal
    block.  The encoding of the included file can be specified.  Only
    a part of the given file argument may be included by specifying
    start and end line or text to match before and/or after the text
    to be used.

    https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment
    """

    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec = {'literal': directives.flag,
                   'code': directives.unchanged,
                   'encoding': directives.encoding,
                   'parser': directives.parser_name,
                   'tab-width': int,
                   'start-line': int,
                   'end-line': int,
                   'start-after': directives.unchanged_required,
                   'end-before': directives.unchanged_required,
                   # ignored except for 'literal' or 'code':
                   'number-lines': directives.unchanged,  # integer or None
                   'class': directives.class_option,
                   'name': directives.unchanged}

    standard_include_path = Path(states.__file__).parent / 'include'

    def run(self):
        """Include a file as part of the content of this reST file.

        Depending on the options, the file (or a clipping) is
        converted to nodes and returned or inserted into the input stream.
        """
        if not self.state.document.settings.file_insertion_enabled:
            raise self.warning('"%s" directive disabled.' % self.name)
        current_source = self.state.document.current_source
        path = directives.path(self.arguments[0])
        if path.startswith('<') and path.endswith('>'):
            _base = self.standard_include_path
            path = path[1:-1]
        else:
            _base = Path(current_source).parent
        path = utils.relative_path(None, _base/path)
        encoding = self.options.get(
            'encoding', self.state.document.settings.input_encoding)
        e_handler = self.state.document.settings.input_encoding_error_handler
        tab_width = self.options.get(
            'tab-width', self.state.document.settings.tab_width)
        try:
            include_file = io.FileInput(source_path=path,
                                        encoding=encoding,
                                        error_handler=e_handler)
        except UnicodeEncodeError:
            raise self.severe(f'Problems with "{self.name}" directive path:\n'
                              f'Cannot encode input file path "{path}" '
                              '(wrong locale?).')
        except OSError as error:
            raise self.severe(f'Problems with "{self.name}" directive '
                              f'path:\n{io.error_string(error)}.')
        else:
            self.state.document.settings.record_dependencies.add(path)

        # Get to-be-included content
        startline = self.options.get('start-line', None)
        endline = self.options.get('end-line', None)
        try:
            if startline or (endline is not None):
                lines = include_file.readlines()
                rawtext = ''.join(lines[startline:endline])
            else:
                rawtext = include_file.read()
        except UnicodeError as error:
            raise self.severe(f'Problem with "{self.name}" directive:\n'
                              + io.error_string(error))
        # start-after/end-before: no restrictions on newlines in match-text,
        # and no restrictions on matching inside lines vs. line boundaries
        after_text = self.options.get('start-after', None)
        if after_text:
            # skip content in rawtext before *and incl.* a matching text
            after_index = rawtext.find(after_text)
            if after_index < 0:
                raise self.severe('Problem with "start-after" option of "%s" '
                                  'directive:\nText not found.' % self.name)
            rawtext = rawtext[after_index + len(after_text):]
        before_text = self.options.get('end-before', None)
        if before_text:
            # skip content in rawtext after *and incl.* a matching text
            before_index = rawtext.find(before_text)
            if before_index < 0:
                raise self.severe('Problem with "end-before" option of "%s" '
                                  'directive:\nText not found.' % self.name)
            rawtext = rawtext[:before_index]

        include_lines = statemachine.string2lines(rawtext, tab_width,
                                                  convert_whitespace=True)
        for i, line in enumerate(include_lines):
            if len(line) > self.state.document.settings.line_length_limit:
                raise self.warning('"%s": line %d exceeds the'
                                   ' line-length-limit.' % (path, i+1))

        if 'literal' in self.options:
            # Don't convert tabs to spaces, if `tab_width` is negative.
            if tab_width >= 0:
                text = rawtext.expandtabs(tab_width)
            else:
                text = rawtext
            literal_block = nodes.literal_block(
                                rawtext, source=path,
                                classes=self.options.get('class', []))
            literal_block.line = 1
            self.add_name(literal_block)
            if 'number-lines' in self.options:
                try:
                    startline = int(self.options['number-lines'] or 1)
                except ValueError:
                    raise self.error(':number-lines: with non-integer '
                                     'start value')
                endline = startline + len(include_lines)
                if text.endswith('\n'):
                    text = text[:-1]
                tokens = NumberLines([([], text)], startline, endline)
                for classes, value in tokens:
                    if classes:
                        literal_block += nodes.inline(value, value,
                                                      classes=classes)
                    else:
                        literal_block += nodes.Text(value)
            else:
                literal_block += nodes.Text(text)
            return [literal_block]

        if 'code' in self.options:
            self.options['source'] = path
            # Don't convert tabs to spaces, if `tab_width` is negative:
            if tab_width < 0:
                include_lines = rawtext.splitlines()
            codeblock = CodeBlock(self.name,
                                  [self.options.pop('code')],  # arguments
                                  self.options,
                                  include_lines,  # content
                                  self.lineno,
                                  self.content_offset,
                                  self.block_text,
                                  self.state,
                                  self.state_machine)
            return codeblock.run()

        # Prevent circular inclusion:
        clip_options = (startline, endline, before_text, after_text)
        include_log = self.state.document.include_log
        # log entries are tuples (<source>, <clip-options>)
        if not include_log:  # new document, initialize with document source
            include_log.append((utils.relative_path(None, current_source),
                                (None, None, None, None)))
        if (path, clip_options) in include_log:
            master_paths = (pth for (pth, opt) in reversed(include_log))
            inclusion_chain = '\n> '.join((path, *master_paths))
            raise self.warning('circular inclusion in "%s" directive:\n%s'
                               % (self.name, inclusion_chain))

        if 'parser' in self.options:
            # parse into a dummy document and return created nodes
            document = utils.new_document(path, self.state.document.settings)
            document.include_log = include_log + [(path, clip_options)]
            parser = self.options['parser']()
            parser.parse('\n'.join(include_lines), document)
            # clean up doctree and complete parsing
            document.transformer.populate_from_components((parser,))
            document.transformer.apply_transforms()
            return document.children

        # Include as rST source:
        #
        # mark end (cf. parsers.rst.states.Body.comment())
        include_lines += ['', '.. end of inclusion from "%s"' % path]
        self.state_machine.insert_input(include_lines, path)
        # update include-log
        include_log.append((path, clip_options))
        return []


class Raw(Directive):

    """
    Pass through content unchanged

    Content is included in output based on type argument

    Content may be included inline (content section of directive) or
    imported from a file or url.
    """

    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec = {'file': directives.path,
                   'url': directives.uri,
                   'encoding': directives.encoding,
                   'class': directives.class_option}
    has_content = True

    def run(self):
        if (not self.state.document.settings.raw_enabled
            or (not self.state.document.settings.file_insertion_enabled
                and ('file' in self.options
                     or 'url' in self.options))):
            raise self.warning('"%s" directive disabled.' % self.name)
        attributes = {'format': ' '.join(self.arguments[0].lower().split())}
        encoding = self.options.get(
            'encoding', self.state.document.settings.input_encoding)
        e_handler = self.state.document.settings.input_encoding_error_handler
        if self.content:
            if 'file' in self.options or 'url' in self.options:
                raise self.error(
                    '"%s" directive may not both specify an external file '
                    'and have content.' % self.name)
            text = '\n'.join(self.content)
        elif 'file' in self.options:
            if 'url' in self.options:
                raise self.error(
                    'The "file" and "url" options may not be simultaneously '
                    'specified for the "%s" directive.' % self.name)
            path = self.options['file']
            _base = Path(self.state.document.current_source).parent
            path = utils.relative_path(None, _base/path)
            try:
                raw_file = io.FileInput(source_path=path,
                                        encoding=encoding,
                                        error_handler=e_handler)
            except OSError as error:
                raise self.severe(f'Problems with "{self.name}" directive '
                                  f'path:\n{io.error_string(error)}.')
            else:
                # TODO: currently, raw input files are recorded as
                # dependencies even if not used for the chosen output format.
                self.state.document.settings.record_dependencies.add(path)
            try:
                text = raw_file.read()
            except UnicodeError as error:
                raise self.severe(f'Problem with "{self.name}" directive:\n'
                                  + io.error_string(error))
            attributes['source'] = path
        elif 'url' in self.options:
            source = self.options['url']
            try:
                raw_text = urlopen(source).read()
            except (URLError, OSError) as error:
                raise self.severe(f'Problems with "{self.name}" directive URL '
                                  f'"{self.options["url"]}":\n'
                                  f'{io.error_string(error)}.')
            raw_file = io.StringInput(source=raw_text, source_path=source,
                                      encoding=encoding,
                                      error_handler=e_handler)
            try:
                text = raw_file.read()
            except UnicodeError as error:
                raise self.severe(f'Problem with "{self.name}" directive:\n'
                                  + io.error_string(error))
            attributes['source'] = source
        else:
            # This will always fail because there is no content.
            self.assert_has_content()
        raw_node = nodes.raw('', text, classes=self.options.get('class', []),
                             **attributes)
        (raw_node.source,
         raw_node.line) = self.state_machine.get_source_and_line(self.lineno)
        return [raw_node]


class Replace(Directive):

    has_content = True

    def run(self):
        if not isinstance(self.state, states.SubstitutionDef):
            raise self.error(
                'Invalid context: the "%s" directive can only be used within '
                'a substitution definition.' % self.name)
        self.assert_has_content()
        text = '\n'.join(self.content)
        element = nodes.Element(text)
        self.state.nested_parse(self.content, self.content_offset,
                                element)
        # element might contain [paragraph] + system_message(s)
        node = None
        messages = []
        for elem in element:
            if not node and isinstance(elem, nodes.paragraph):
                node = elem
            elif isinstance(elem, nodes.system_message):
                elem['backrefs'] = []
                messages.append(elem)
            else:
                return [
                    self.reporter.error(
                        f'Error in "{self.name}" directive: may contain '
                        'a single paragraph only.', line=self.lineno)]
        if node:
            return messages + node.children
        return messages


class Unicode(Directive):

    r"""
    Convert Unicode character codes (numbers) to characters.  Codes may be
    decimal numbers, hexadecimal numbers (prefixed by ``0x``, ``x``, ``\x``,
    ``U+``, ``u``, or ``\u``; e.g. ``U+262E``), or XML-style numeric character
    entities (e.g. ``&#x262E;``).  Text following ".." is a comment and is
    ignored.  Spaces are ignored, and any other text remains as-is.
    """

    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec = {'trim': directives.flag,
                   'ltrim': directives.flag,
                   'rtrim': directives.flag}

    comment_pattern = re.compile(r'( |\n|^)\.\. ')

    def run(self):
        if not isinstance(self.state, states.SubstitutionDef):
            raise self.error(
                'Invalid context: the "%s" directive can only be used within '
                'a substitution definition.' % self.name)
        substitution_definition = self.state_machine.node
        if 'trim' in self.options:
            substitution_definition.attributes['ltrim'] = 1
            substitution_definition.attributes['rtrim'] = 1
        if 'ltrim' in self.options:
            substitution_definition.attributes['ltrim'] = 1
        if 'rtrim' in self.options:
            substitution_definition.attributes['rtrim'] = 1
        codes = self.comment_pattern.split(self.arguments[0])[0].split()
        element = nodes.Element()
        for code in codes:
            try:
                decoded = directives.unicode_code(code)
            except ValueError as error:
                raise self.error('Invalid character code: %s\n%s'
                                 % (code, io.error_string(error)))
            element += nodes.Text(decoded)
        return element.children


class Class(Directive):

    """
    Set a "class" attribute on the directive content or the next element.
    When applied to the next element, a "pending" element is inserted, and a
    transform does the work later.
    """

    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = True

    def run(self):
        try:
            class_value = directives.class_option(self.arguments[0])
        except ValueError:
            raise self.error(
                'Invalid class attribute value for "%s" directive: "%s".'
                % (self.name, self.arguments[0]))
        node_list = []
        if self.content:
            container = nodes.Element()
            self.state.nested_parse(self.content, self.content_offset,
                                    container)
            for node in container:
                node['classes'].extend(class_value)
            node_list.extend(container.children)
        else:
            pending = nodes.pending(
                misc.ClassAttribute,
                {'class': class_value, 'directive': self.name},
                self.block_text)
            self.state_machine.document.note_pending(pending)
            node_list.append(pending)
        return node_list


class Role(Directive):

    has_content = True

    argument_pattern = re.compile(r'(%s)\s*(\(\s*(%s)\s*\)\s*)?$'
                                  % ((states.Inliner.simplename,) * 2))

    def run(self):
        """Dynamically create and register a custom interpreted text role."""
        if self.content_offset > self.lineno or not self.content:
            raise self.error('"%s" directive requires arguments on the first '
                             'line.' % self.name)
        args = self.content[0]
        match = self.argument_pattern.match(args)
        if not match:
            raise self.error('"%s" directive arguments not valid role names: '
                             '"%s".' % (self.name, args))
        new_role_name = match.group(1)
        base_role_name = match.group(3)
        messages = []
        if base_role_name:
            base_role, messages = roles.role(
                base_role_name, self.state_machine.language, self.lineno,
                self.state.reporter)
            if base_role is None:
                error = self.state.reporter.error(
                    'Unknown interpreted text role "%s".' % base_role_name,
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                return messages + [error]
        else:
            base_role = roles.generic_custom_role
        assert not hasattr(base_role, 'arguments'), (
            'Supplemental directive arguments for "%s" directive not '
            'supported (specified by "%r" role).' % (self.name, base_role))
        try:
            converted_role = convert_directive_function(base_role)
            (arguments, options, content, content_offset
             ) = self.state.parse_directive_block(
                    self.content[1:], self.content_offset,
                    converted_role, option_presets={})
        except states.MarkupError as detail:
            error = self.reporter.error(
                'Error in "%s" directive:\n%s.' % (self.name, detail),
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            return messages + [error]
        if 'class' not in options:
            try:
                options['class'] = directives.class_option(new_role_name)
            except ValueError as detail:
                error = self.reporter.error(
                    'Invalid argument for "%s" directive:\n%s.'
                    % (self.name, detail),
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                return messages + [error]
        role = roles.CustomRole(new_role_name, base_role, options, content)
        roles.register_local_role(new_role_name, role)
        return messages


class DefaultRole(Directive):

    """Set the default interpreted text role."""

    optional_arguments = 1
    final_argument_whitespace = False

    def run(self):
        if not self.arguments:
            if '' in roles._roles:
                # restore the "default" default role
                del roles._roles['']
            return []
        role_name = self.arguments[0]
        role, messages = roles.role(role_name, self.state_machine.language,
                                    self.lineno, self.state.reporter)
        if role is None:
            error = self.state.reporter.error(
                'Unknown interpreted text role "%s".' % role_name,
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            return messages + [error]
        roles._roles[''] = role
        return messages


class Title(Directive):

    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True

    def run(self):
        self.state_machine.document['title'] = self.arguments[0]
        return []


class MetaBody(states.SpecializedBody):

    def field_marker(self, match, context, next_state):
        """Meta element."""
        node, blank_finish = self.parsemeta(match)
        self.parent += node
        return [], next_state, []

    def parsemeta(self, match):
        name = self.parse_field_marker(match)
        name = nodes.unescape(utils.escape2null(name))
        (indented, indent, line_offset, blank_finish
         ) = self.state_machine.get_first_known_indented(match.end())
        node = nodes.meta()
        node['content'] = nodes.unescape(utils.escape2null(
                                            ' '.join(indented)))
        if not indented:
            line = self.state_machine.line
            msg = self.reporter.info(
                  'No content for meta tag "%s".' % name,
                  nodes.literal_block(line, line))
            return msg, blank_finish
        tokens = name.split()
        try:
            attname, val = utils.extract_name_value(tokens[0])[0]
            node[attname.lower()] = val
        except utils.NameValueError:
            node['name'] = tokens[0]
        for token in tokens[1:]:
            try:
                attname, val = utils.extract_name_value(token)[0]
                node[attname.lower()] = val
            except utils.NameValueError as detail:
                line = self.state_machine.line
                msg = self.reporter.error(
                      'Error parsing meta tag attribute "%s": %s.'
                      % (token, detail), nodes.literal_block(line, line))
                return msg, blank_finish
        return node, blank_finish


class Meta(Directive):

    has_content = True

    SMkwargs = {'state_classes': (MetaBody,)}

    def run(self):
        self.assert_has_content()
        node = nodes.Element()
        new_line_offset, blank_finish = self.state.nested_list_parse(
            self.content, self.content_offset, node,
            initial_state='MetaBody', blank_finish=True,
            state_machine_kwargs=self.SMkwargs)
        if (new_line_offset - self.content_offset) != len(self.content):
            # incomplete parse of block?
            error = self.reporter.error(
                'Invalid meta directive.',
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            node += error
        # insert at begin of document
        index = self.state.document.first_child_not_matching_class(
                                        (nodes.Titular, nodes.meta)) or 0
        self.state.document[index:index] = node.children
        return []


class Date(Directive):

    has_content = True

    def run(self):
        if not isinstance(self.state, states.SubstitutionDef):
            raise self.error(
                'Invalid context: the "%s" directive can only be used within '
                'a substitution definition.' % self.name)
        format_str = '\n'.join(self.content) or '%Y-%m-%d'
        # @@@
        # Use timestamp from the `SOURCE_DATE_EPOCH`_ environment variable?
        # Pro: Docutils-generated documentation
        #      can easily be part of `reproducible software builds`__
        #
        #      __ https://reproducible-builds.org/
        #
        # Con: Changes the specs, hard to predict behaviour,
        #
        # See also the discussion about \date \time \year in TeX
        # http://tug.org/pipermail/tex-k/2016-May/002704.html
        # source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH')
        # if (source_date_epoch):
        #     text = time.strftime(format_str,
        #                          time.gmtime(int(source_date_epoch)))
        # else:
        text = time.strftime(format_str)
        return [nodes.Text(text)]


class TestDirective(Directive):

    """This directive is useful only for testing purposes."""

    optional_arguments = 1
    final_argument_whitespace = True
    option_spec = {'option': directives.unchanged_required}
    has_content = True

    def run(self):
        if self.content:
            text = '\n'.join(self.content)
            info = self.reporter.info(
                'Directive processed. Type="%s", arguments=%r, options=%r, '
                'content:' % (self.name, self.arguments, self.options),
                nodes.literal_block(text, text), line=self.lineno)
        else:
            info = self.reporter.info(
                'Directive processed. Type="%s", arguments=%r, options=%r, '
                'content: None' % (self.name, self.arguments, self.options),
                line=self.lineno)
        return [info]

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/directives/misc.py
NikolaBlagojevic/pyrecodes
# $Id: tables.py 9363 2023-04-24 16:37:37Z aa-turner $
# Authors: David Goodger <goodger@python.org>; David Priest
# Copyright: This module has been placed in the public domain.

"""
Directives for table elements.
"""

__docformat__ = 'reStructuredText'


import csv
from pathlib import Path
import warnings

from docutils import io, nodes, statemachine, utils
from docutils.utils import SystemMessagePropagation
from docutils.parsers.rst import Directive
from docutils.parsers.rst import directives
from urllib.request import urlopen
from urllib.error import URLError


def align(argument):
    return directives.choice(argument, ('left', 'center', 'right'))


class Table(Directive):

    """
    Generic table base class.
    """

    optional_arguments = 1
    final_argument_whitespace = True
    option_spec = {'class': directives.class_option,
                   'name': directives.unchanged,
                   'align': align,
                   'width': directives.length_or_percentage_or_unitless,
                   'widths': directives.value_or(('auto', 'grid'),
                                                 directives.positive_int_list)}
    has_content = True

    def make_title(self):
        if self.arguments:
            title_text = self.arguments[0]
            text_nodes, messages = self.state.inline_text(title_text,
                                                          self.lineno)
            title = nodes.title(title_text, '', *text_nodes)
            (title.source,
             title.line) = self.state_machine.get_source_and_line(self.lineno)
        else:
            title = None
            messages = []
        return title, messages

    def process_header_option(self):
        # Provisional
        # * Will move to CSVTable in Docutils 0.21
        #   as it calls `self.HeaderDialect()` only defined in CSVTable.
        # * Will change to use the same CSV dialect as the body to get in line
        #   with the specification in ref/rst/directives.txt in Docutils 0.21.
        source = self.state_machine.get_source(self.lineno - 1)
        table_head = []
        max_header_cols = 0
        if 'header' in self.options:   # separate table header in option
            with warnings.catch_warnings():
                warnings.simplefilter('ignore')
                header_dialect = self.HeaderDialect()
            rows, max_header_cols = self.parse_csv_data_into_rows(
                self.options['header'].split('\n'), header_dialect,
                source)
            table_head.extend(rows)
        return table_head, max_header_cols

    def check_table_dimensions(self, rows, header_rows, stub_columns):
        if len(rows) < header_rows:
            error = self.reporter.error('%s header row(s) specified but '
                'only %s row(s) of data supplied ("%s" directive).'
                % (header_rows, len(rows), self.name),
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            raise SystemMessagePropagation(error)
        if len(rows) == header_rows > 0:
            error = self.reporter.error(
                f'Insufficient data supplied ({len(rows)} row(s)); '
                'no data remaining for table body, '
                f'required by "{self.name}" directive.',
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            raise SystemMessagePropagation(error)
        for row in rows:
            if len(row) < stub_columns:
                error = self.reporter.error(
                    f'{stub_columns} stub column(s) specified '
                    f'but only {len(row)} columns(s) of data supplied '
                    f'("{self.name}" directive).',
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(error)
            if len(row) == stub_columns > 0:
                error = self.reporter.error(
                    'Insufficient data supplied (%s columns(s)); '
                    'no data remaining for table body, required '
                    'by "%s" directive.' % (len(row), self.name),
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(error)

    def set_table_width(self, table_node):
        if 'width' in self.options:
            table_node['width'] = self.options.get('width')

    @property
    def widths(self):
        return self.options.get('widths', '')

    def get_column_widths(self, n_cols):
        if isinstance(self.widths, list):
            if len(self.widths) != n_cols:
                # TODO: use last value for missing columns?
                error = self.reporter.error('"%s" widths do not match the '
                    'number of columns in table (%s).' % (self.name, n_cols),
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(error)
            col_widths = self.widths
        elif n_cols:
            col_widths = [100 // n_cols] * n_cols
        else:
            error = self.reporter.error('No table data detected in CSV file.',
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            raise SystemMessagePropagation(error)
        return col_widths

    def extend_short_rows_with_empty_cells(self, columns, parts):
        for part in parts:
            for row in part:
                if len(row) < columns:
                    row.extend([(0, 0, 0, [])] * (columns - len(row)))


class RSTTable(Table):
    """
    Class for the `"table" directive`__ for formal tables using rST syntax.

    __ https://docutils.sourceforge.io/docs/ref/rst/directives.html
    """

    def run(self):
        if not self.content:
            warning = self.reporter.warning('Content block expected '
                'for the "%s" directive; none found.' % self.name,
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            return [warning]
        title, messages = self.make_title()
        node = nodes.Element()          # anonymous container for parsing
        self.state.nested_parse(self.content, self.content_offset, node)
        if len(node) != 1 or not isinstance(node[0], nodes.table):
            error = self.reporter.error('Error parsing content block for the '
                '"%s" directive: exactly one table expected.' % self.name,
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            return [error]
        table_node = node[0]
        table_node['classes'] += self.options.get('class', [])
        self.set_table_width(table_node)
        if 'align' in self.options:
            table_node['align'] = self.options.get('align')
        if isinstance(self.widths, list):
            tgroup = table_node[0]
            try:
                col_widths = self.get_column_widths(tgroup["cols"])
            except SystemMessagePropagation as detail:
                return [detail.args[0]]
            colspecs = [child for child in tgroup.children
                        if child.tagname == 'colspec']
            for colspec, col_width in zip(colspecs, col_widths):
                colspec['colwidth'] = col_width
        if self.widths == 'auto':
            table_node['classes'] += ['colwidths-auto']
        elif self.widths:  # "grid" or list of integers
            table_node['classes'] += ['colwidths-given']
        self.add_name(table_node)
        if title:
            table_node.insert(0, title)
        return [table_node] + messages


class CSVTable(Table):

    option_spec = {'header-rows': directives.nonnegative_int,
                   'stub-columns': directives.nonnegative_int,
                   'header': directives.unchanged,
                   'width': directives.length_or_percentage_or_unitless,
                   'widths': directives.value_or(('auto', ),
                                                 directives.positive_int_list),
                   'file': directives.path,
                   'url': directives.uri,
                   'encoding': directives.encoding,
                   'class': directives.class_option,
                   'name': directives.unchanged,
                   'align': align,
                   # field delimiter char
                   'delim': directives.single_char_or_whitespace_or_unicode,
                   # treat whitespace after delimiter as significant
                   'keepspace': directives.flag,
                   # text field quote/unquote char:
                   'quote': directives.single_char_or_unicode,
                   # char used to escape delim & quote as-needed:
                   'escape': directives.single_char_or_unicode}

    class DocutilsDialect(csv.Dialect):

        """CSV dialect for `csv_table` directive."""

        delimiter = ','
        quotechar = '"'
        doublequote = True
        skipinitialspace = True
        strict = True
        lineterminator = '\n'
        quoting = csv.QUOTE_MINIMAL

        def __init__(self, options):
            if 'delim' in options:
                self.delimiter = options['delim']
            if 'keepspace' in options:
                self.skipinitialspace = False
            if 'quote' in options:
                self.quotechar = options['quote']
            if 'escape' in options:
                self.doublequote = False
                self.escapechar = options['escape']
            super().__init__()

    class HeaderDialect(csv.Dialect):
        """
        CSV dialect used for the "header" option data.

        Deprecated. Will be removed in Docutils 0.22.
        """
        # The separate HeaderDialect was introduced in revision 2294
        # (2004-06-17) in the sandbox before the "csv-table" directive moved
        # to the trunk in r2309. Discussion in docutils-devel around this time
        # did not mention a rationale (part of the discussion was in private
        # mail).
        # This is in conflict with the documentation, which always said:
        # ""
        # and did not change in this aspect.
        #
        # Maybe it was intended to have similar escape rules for rST and CSV,
        # however with the current implementation this means we need
        # `\\` for rST markup and ``\\\\`` for a literal backslash
        # in the "option" header but ``\`` and ``\\`` in the header-lines and
        # table cells of the main CSV data.
        delimiter = ','
        quotechar = '"'
        escapechar = '\\'
        doublequote = False
        skipinitialspace = True
        strict = True
        lineterminator = '\n'
        quoting = csv.QUOTE_MINIMAL

        def __init__(self):
            warnings.warn('CSVTable.HeaderDialect will be removed '
                          'in Docutils 0.22.',
                          PendingDeprecationWarning, stacklevel=2)
            super().__init__()

    @staticmethod
    def check_requirements():
        warnings.warn('CSVTable.check_requirements()'
                      ' is not required with Python 3'
                      ' and will be removed in Docutils 0.22.',
                      DeprecationWarning, stacklevel=2)

    def run(self):
        try:
            if (not self.state.document.settings.file_insertion_enabled
                and ('file' in self.options
                     or 'url' in self.options)):
                warning = self.reporter.warning('File and URL access '
                    'deactivated; ignoring "%s" directive.' % self.name,
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                return [warning]
            title, messages = self.make_title()
            csv_data, source = self.get_csv_data()
            table_head, max_header_cols = self.process_header_option()
            rows, max_cols = self.parse_csv_data_into_rows(
                csv_data, self.DocutilsDialect(self.options), source)
            max_cols = max(max_cols, max_header_cols)
            header_rows = self.options.get('header-rows', 0)
            stub_columns = self.options.get('stub-columns', 0)
            self.check_table_dimensions(rows, header_rows, stub_columns)
            table_head.extend(rows[:header_rows])
            table_body = rows[header_rows:]
            col_widths = self.get_column_widths(max_cols)
            self.extend_short_rows_with_empty_cells(max_cols,
                                                    (table_head, table_body))
        except SystemMessagePropagation as detail:
            return [detail.args[0]]
        except csv.Error as detail:
            message = str(detail)
            error = self.reporter.error('Error with CSV data'
                ' in "%s" directive:\n%s' % (self.name, message),
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            return [error]
        table = (col_widths, table_head, table_body)
        table_node = self.state.build_table(table, self.content_offset,
                                            stub_columns, widths=self.widths)
        table_node['classes'] += self.options.get('class', [])
        if 'align' in self.options:
            table_node['align'] = self.options.get('align')
        self.set_table_width(table_node)
        self.add_name(table_node)
        if title:
            table_node.insert(0, title)
        return [table_node] + messages

    def get_csv_data(self):
        """
        Get CSV data from the directive content, from an external
        file, or from a URL reference.
        """
        encoding = self.options.get(
            'encoding', self.state.document.settings.input_encoding)
        error_handler = self.state.document.settings.input_encoding_error_handler  # noqa:E501
        if self.content:
            # CSV data is from directive content.
            if 'file' in self.options or 'url' in self.options:
                error = self.reporter.error('"%s" directive may not both '
                    'specify an external file and have content.' % self.name,
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(error)
            source = self.content.source(0)
            csv_data = self.content
        elif 'file' in self.options:
            # CSV data is from an external file.
            if 'url' in self.options:
                error = self.reporter.error('The "file" and "url" options '
                    'may not be simultaneously specified '
                    'for the "%s" directive.' % self.name,
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(error)
            source = self.options['file']
            # resolve path to external file
            _base = Path(self.state.document.current_source).parent
            source = utils.relative_path(None, _base/source)
            try:
                csv_file = io.FileInput(source_path=source,
                                        encoding=encoding,
                                        error_handler=error_handler)
                csv_data = csv_file.read().splitlines()
            except OSError as error:
                severe = self.reporter.severe(
                    'Problems with "%s" directive path:\n%s.'
                    % (self.name, error),
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(severe)
            else:
                self.state.document.settings.record_dependencies.add(source)
        elif 'url' in self.options:
            source = self.options['url']
            try:
                with urlopen(source) as response:
                    csv_text = response.read()
            except (URLError, OSError, ValueError) as error:
                severe = self.reporter.severe(
                      'Problems with "%s" directive URL "%s":\n%s.'
                      % (self.name, self.options['url'], error),
                      nodes.literal_block(self.block_text, self.block_text),
                      line=self.lineno)
                raise SystemMessagePropagation(severe)
            csv_file = io.StringInput(
                source=csv_text, source_path=source, encoding=encoding,
                error_handler=(self.state.document.settings.
                               input_encoding_error_handler))
            csv_data = csv_file.read().splitlines()
        else:
            error = self.reporter.warning(
                'The "%s" directive requires content; none supplied.'
                % self.name,
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            raise SystemMessagePropagation(error)
        return csv_data, source

    @staticmethod
    def decode_from_csv(s):
        warnings.warn('CSVTable.decode_from_csv()'
                  ' is not required with Python 3'
                  ' and will be removed in Docutils 0.21 or later.',
                  DeprecationWarning, stacklevel=2)
        return s

    @staticmethod
    def encode_for_csv(s):
        warnings.warn('CSVTable.encode_from_csv()'
                  ' is not required with Python 3'
                  ' and will be removed in Docutils 0.21 or later.',
                  DeprecationWarning, stacklevel=2)
        return s

    def parse_csv_data_into_rows(self, csv_data, dialect, source):
        csv_reader = csv.reader((line + '\n' for line in csv_data),
                                dialect=dialect)
        rows = []
        max_cols = 0
        for row in csv_reader:
            row_data = []
            for cell in row:
                cell_data = (0, 0, 0, statemachine.StringList(
                    cell.splitlines(), source=source))
                row_data.append(cell_data)
            rows.append(row_data)
            max_cols = max(max_cols, len(row))
        return rows, max_cols


class ListTable(Table):

    """
    Implement tables whose data is encoded as a uniform two-level bullet list.
    For further ideas, see
    https://docutils.sourceforge.io/docs/dev/rst/alternatives.html#list-driven-tables
    """

    option_spec = {'header-rows': directives.nonnegative_int,
                   'stub-columns': directives.nonnegative_int,
                   'width': directives.length_or_percentage_or_unitless,
                   'widths': directives.value_or(('auto', ),
                                                 directives.positive_int_list),
                   'class': directives.class_option,
                   'name': directives.unchanged,
                   'align': align}

    def run(self):
        if not self.content:
            error = self.reporter.error('The "%s" directive is empty; '
                'content required.' % self.name,
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            return [error]
        title, messages = self.make_title()
        node = nodes.Element()          # anonymous container for parsing
        self.state.nested_parse(self.content, self.content_offset, node)
        try:
            num_cols, col_widths = self.check_list_content(node)
            table_data = [[item.children for item in row_list[0]]
                          for row_list in node[0]]
            header_rows = self.options.get('header-rows', 0)
            stub_columns = self.options.get('stub-columns', 0)
            self.check_table_dimensions(table_data, header_rows, stub_columns)
        except SystemMessagePropagation as detail:
            return [detail.args[0]]
        table_node = self.build_table_from_list(table_data, col_widths,
                                                header_rows, stub_columns)
        if 'align' in self.options:
            table_node['align'] = self.options.get('align')
        table_node['classes'] += self.options.get('class', [])
        self.set_table_width(table_node)
        self.add_name(table_node)
        if title:
            table_node.insert(0, title)
        return [table_node] + messages

    def check_list_content(self, node):
        if len(node) != 1 or not isinstance(node[0], nodes.bullet_list):
            error = self.reporter.error(
                'Error parsing content block for the "%s" directive: '
                'exactly one bullet list expected.' % self.name,
                nodes.literal_block(self.block_text, self.block_text),
                line=self.lineno)
            raise SystemMessagePropagation(error)
        list_node = node[0]
        num_cols = 0
        # Check for a uniform two-level bullet list:
        for item_index in range(len(list_node)):
            item = list_node[item_index]
            if len(item) != 1 or not isinstance(item[0], nodes.bullet_list):
                error = self.reporter.error(
                    'Error parsing content block for the "%s" directive: '
                    'two-level bullet list expected, but row %s does not '
                    'contain a second-level bullet list.'
                    % (self.name, item_index + 1),
                    nodes.literal_block(self.block_text, self.block_text),
                    line=self.lineno)
                raise SystemMessagePropagation(error)
            elif item_index:
                if len(item[0]) != num_cols:
                    error = self.reporter.error(
                        'Error parsing content block for the "%s" directive: '
                        'uniform two-level bullet list expected, but row %s '
                        'does not contain the same number of items as row 1 '
                        '(%s vs %s).'
                        % (self.name, item_index + 1, len(item[0]), num_cols),
                        nodes.literal_block(self.block_text, self.block_text),
                        line=self.lineno)
                    raise SystemMessagePropagation(error)
            else:
                num_cols = len(item[0])
        col_widths = self.get_column_widths(num_cols)
        return num_cols, col_widths

    def build_table_from_list(self, table_data,
                              col_widths, header_rows, stub_columns):
        table = nodes.table()
        if self.widths == 'auto':
            table['classes'] += ['colwidths-auto']
        elif self.widths:  # explicitly set column widths
            table['classes'] += ['colwidths-given']
        tgroup = nodes.tgroup(cols=len(col_widths))
        table += tgroup
        for col_width in col_widths:
            colspec = nodes.colspec()
            if col_width is not None:
                colspec.attributes['colwidth'] = col_width
            if stub_columns:
                colspec.attributes['stub'] = 1
                stub_columns -= 1
            tgroup += colspec
        rows = []
        for row in table_data:
            row_node = nodes.row()
            for cell in row:
                entry = nodes.entry()
                entry += cell
                row_node += entry
            rows.append(row_node)
        if header_rows:
            thead = nodes.thead()
            thead.extend(rows[:header_rows])
            tgroup += thead
        tbody = nodes.tbody()
        tbody.extend(rows[header_rows:])
        tgroup += tbody
        return table

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/directives/tables.py
NikolaBlagojevic/pyrecodes
# $Id: af.py 9116 2022-07-28 17:06:51Z milde $
# Author: Jannie Hofmeyr <jhsh@sun.ac.za>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
Afrikaans-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      'aandag': 'attention',
      'versigtig': 'caution',
      'code (translation required)': 'code',
      'gevaar': 'danger',
      'fout': 'error',
      'wenk': 'hint',
      'belangrik': 'important',
      'nota': 'note',
      'tip': 'tip',  # hint and tip both have the same translation: wenk
      'waarskuwing': 'warning',
      'vermaning': 'admonition',  # sic! Not used in this sense in rST.
      'kantstreep': 'sidebar',
      'onderwerp': 'topic',
      'lynblok': 'line-block',
      'math (translation required)': 'math',
      'parsed-literal (translation required)': 'parsed-literal',
      'rubriek': 'rubric',
      'epigraaf': 'epigraph',
      'hoogtepunte': 'highlights',
      'pull-quote (translation required)': 'pull-quote',
      'compound (translation required)': 'compound',
      'container (translation required)': 'container',
      # 'vrae': 'questions',
      # 'qa': 'questions',
      # 'faq': 'questions',
      'table (translation required)': 'table',
      'csv-table (translation required)': 'csv-table',
      'list-table (translation required)': 'list-table',
      'meta': 'meta',
      # 'beeldkaart': 'imagemap',
      'beeld': 'image',
      'figuur': 'figure',
      'insluiting': 'include',
      'rou': 'raw',
      'vervang': 'replace',
      'unicode': 'unicode',  # should this be translated? unikode
      'datum': 'date',
      'klas': 'class',
      'role (translation required)': 'role',
      'default-role (translation required)': 'default-role',
      'title (translation required)': 'title',
      'inhoud': 'contents',
      'sectnum': 'sectnum',
      'section-numbering': 'sectnum',
      'header (translation required)': 'header',
      'footer (translation required)': 'footer',
      # 'voetnote': 'footnotes',
      # 'aanhalings': 'citations',
      'teikennotas': 'target-notes',
      'restructuredtext-test-directive': 'restructuredtext-test-directive'}
"""Afrikaans name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
    'afkorting': 'abbreviation',
    'ab': 'abbreviation',
    'akroniem': 'acronym',
    'ac': 'acronym',
    'code (translation required)': 'code',
    'indeks': 'index',
    'i': 'index',
    'voetskrif': 'subscript',
    'sub': 'subscript',
    'boskrif': 'superscript',
    'sup': 'superscript',
    'titelverwysing': 'title-reference',
    'titel': 'title-reference',
    't': 'title-reference',
    'pep-verwysing': 'pep-reference',
    'pep': 'pep-reference',
    'rfc-verwysing': 'rfc-reference',
    'rfc': 'rfc-reference',
    'nadruk': 'emphasis',
    'sterk': 'strong',
    'literal (translation required)': 'literal',
    'math (translation required)': 'math',
    'benoemde verwysing': 'named-reference',
    'anonieme verwysing': 'anonymous-reference',
    'voetnootverwysing': 'footnote-reference',
    'aanhalingverwysing': 'citation-reference',
    'vervangingsverwysing': 'substitution-reference',
    'teiken': 'target',
    'uri-verwysing': 'uri-reference',
    'uri': 'uri-reference',
    'url': 'uri-reference',
    'rou': 'raw',
    }
"""Mapping of Afrikaans role names to canonical names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/af.py
NikolaBlagojevic/pyrecodes
# $Id: da.py 9116 2022-07-28 17:06:51Z milde $
# Author: E D
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
Danish-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      # language-dependent: fixed
      'giv agt': 'attention',
      'pas på': 'caution',
      'kode': 'code',
      'kode-blok': 'code',
      'kildekode': 'code',
      'fare': 'danger',
      'fejl': 'error',
      'vink': 'hint',
      'vigtigt': 'important',
      'bemærk': 'note',
      'tips': 'tip',
      'advarsel': 'warning',
      'formaning': 'admonition',  # sic! Not used in this sense in rST.
      'sidebjælke': 'sidebar',
      'emne': 'topic',
      'linje-blok': 'line-block',
      'linie-blok': 'line-block',
      'parset-literal': 'parsed-literal',
      'rubrik': 'rubric',
      'epigraf': 'epigraph',
      'fremhævninger': 'highlights',
      'pull-quote (translation required)': 'pull-quote',
      'compound (translation required)': 'compound',
      'container (translation required)': 'container',
      # 'questions': 'questions',
      'tabel': 'table',
      'csv-tabel': 'csv-table',
      'liste-tabel': 'list-table',
      # 'qa': 'questions',
      # 'faq': 'questions',
      'meta': 'meta',
      'math (translation required)': 'math',
      # 'imagemap': 'imagemap',
      'billede': 'image',
      'figur': 'figure',
      'inkludér': 'include',
      'inkluder': 'include',
      'rå': 'raw',
      'erstat': 'replace',
      'unicode': 'unicode',
      'dato': 'date',
      'klasse': 'class',
      'rolle': 'role',
      'forvalgt-rolle': 'default-role',
      'titel': 'title',
      'indhold': 'contents',
      'sektnum': 'sectnum',
      'sektions-nummerering': 'sectnum',
      'sidehovede': 'header',
      'sidefod': 'footer',
      # 'footnotes': 'footnotes',
      # 'citations': 'citations',
      'target-notes (translation required)': 'target-notes',
      'restructuredtext-test-direktiv': 'restructuredtext-test-directive'}
"""Danish name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
    # language-dependent: fixed
    'forkortelse': 'abbreviation',
    'fork': 'abbreviation',
    'akronym': 'acronym',
    'ac (translation required)': 'acronym',
    'kode': 'code',
    'indeks': 'index',
    'i': 'index',
    'subscript (translation required)': 'subscript',
    'sub (translation required)': 'subscript',
    'superscript (translation required)': 'superscript',
    'sup (translation required)': 'superscript',
    'titel-reference': 'title-reference',
    'titel': 'title-reference',
    't': 'title-reference',
    'pep-reference': 'pep-reference',
    'pep': 'pep-reference',
    'rfc-reference': 'rfc-reference',
    'rfc': 'rfc-reference',
    'emfase': 'emphasis',
    'kraftig': 'strong',
    'literal': 'literal',
    'math (translation required)': 'math',
    'navngivet-reference': 'named-reference',
    'anonym-reference': 'anonymous-reference',
    'fodnote-reference': 'footnote-reference',
    'citation-reference (translation required)': 'citation-reference',
    'substitutions-reference': 'substitution-reference',
    'target (translation required)': 'target',
    'uri-reference': 'uri-reference',
    'uri': 'uri-reference',
    'url': 'uri-reference',
    'rå': 'raw',
    }
"""Mapping of Danish role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/da.py
NikolaBlagojevic/pyrecodes
# $Id: de.py 9120 2022-09-13 10:24:30Z milde $
# Authors: Engelbert Gruber <grubert@users.sourceforge.net>;
#          Lea Wiemann <LeWiemann@gmail.com>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
German-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      'achtung': 'attention',
      'vorsicht': 'caution',
      'code': 'code',
      'gefahr': 'danger',
      'fehler': 'error',
      'hinweis': 'hint',  # Wink
      'wichtig': 'important',
      'notiz': 'note',
      'tipp': 'tip',
      'warnung': 'warning',
      'ermahnung': 'admonition',  # sic! Not used in this sense in rST.
      # TODO: Rat(schlag), Empfehlung, Warnhinweis, ...?
      'kasten': 'sidebar',
      'seitenkasten': 'sidebar',  # kept for backwards compatibiltity
      'seitenleiste': 'sidebar',
      'thema': 'topic',
      'zeilenblock': 'line-block',
      'parsed-literal (translation required)': 'parsed-literal',
      'rubrik': 'rubric',
      'epigraph': 'epigraph',
      'highlights': 'highlights',
      'pull-quote': 'pull-quote',  # commonly used in German too
      'seitenansprache': 'pull-quote',
      # cf. http://www.typografie.info/2/wiki.php?title=Seitenansprache
      'zusammengesetzt': 'compound',
      'verbund': 'compound',
      'container': 'container',
      # 'fragen': 'questions',
      'tabelle': 'table',
      'csv-tabelle': 'csv-table',
      'listentabelle': 'list-table',
      'mathe': 'math',
      'formel': 'math',
      'meta': 'meta',
      # 'imagemap': 'imagemap',
      'bild': 'image',
      'abbildung': 'figure',
      'unverändert': 'raw',
      'roh': 'raw',
      'einfügen': 'include',
      'ersetzung': 'replace',
      'ersetzen': 'replace',
      'ersetze': 'replace',
      'unicode': 'unicode',
      'datum': 'date',
      'klasse': 'class',
      'rolle': 'role',
      'standardrolle': 'default-role',
      'titel': 'title',
      'inhalt': 'contents',
      'kapitelnummerierung': 'sectnum',
      'abschnittsnummerierung': 'sectnum',
      'linkziel-fußnoten': 'target-notes',
      'kopfzeilen': 'header',
      'fußzeilen': 'footer',
      # 'fußnoten': 'footnotes',
      # 'zitate': 'citations',
      }
"""German name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
      'abkürzung': 'abbreviation',
      'akronym': 'acronym',
      'code': 'code',
      'index': 'index',
      'tiefgestellt': 'subscript',
      'hochgestellt': 'superscript',
      'titel-referenz': 'title-reference',
      'pep-referenz': 'pep-reference',
      'rfc-referenz': 'rfc-reference',
      'betonung': 'emphasis',  # for backwards compatibility
      'betont': 'emphasis',
      'fett': 'strong',
      'wörtlich': 'literal',
      'mathe': 'math',
      'benannte-referenz': 'named-reference',
      'unbenannte-referenz': 'anonymous-reference',
      'fußnoten-referenz': 'footnote-reference',
      'zitat-referenz': 'citation-reference',
      'ersetzungs-referenz': 'substitution-reference',
      'ziel': 'target',
      'uri-referenz': 'uri-reference',
      'unverändert': 'raw',
      'roh': 'raw',
      }
"""Mapping of German role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/de.py
NikolaBlagojevic/pyrecodes
# $Id: en.py 9120 2022-09-13 10:24:30Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
English-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      # language-dependent: fixed
      'attention': 'attention',
      'caution': 'caution',
      'danger': 'danger',
      'error': 'error',
      'hint': 'hint',
      'important': 'important',
      'note': 'note',
      'tip': 'tip',
      'warning': 'warning',
      'admonition': 'admonition',  # advice/caveat/remark, not reprimand
      'sidebar': 'sidebar',
      'topic': 'topic',
      'line-block': 'line-block',
      'parsed-literal': 'parsed-literal',
      'code': 'code',
      'code-block': 'code',
      'sourcecode': 'code',
      'math': 'math',
      'rubric': 'rubric',
      'epigraph': 'epigraph',
      'highlights': 'highlights',
      'pull-quote': 'pull-quote',
      'compound': 'compound',
      'container': 'container',
      'table': 'table',
      'csv-table': 'csv-table',
      'list-table': 'list-table',
      'meta': 'meta',
      # 'imagemap': 'imagemap',
      'image': 'image',
      'figure': 'figure',
      'include': 'include',
      'raw': 'raw',
      'replace': 'replace',
      'unicode': 'unicode',
      'date': 'date',
      'class': 'class',
      'role': 'role',
      'default-role': 'default-role',
      'title': 'title',
      'contents': 'contents',
      'sectnum': 'sectnum',
      'section-numbering': 'sectnum',
      'header': 'header',
      'footer': 'footer',
      # 'footnotes': 'footnotes',
      # 'citations': 'citations',
      'target-notes': 'target-notes',
      'restructuredtext-test-directive': 'restructuredtext-test-directive'}
"""Mapping of English directive name to registered directive names

Cf. https://docutils.sourceforge.io/docs/ref/rst/directives.html
and `_directive_registry` in ``directives/__init__.py``.
"""

roles = {
    # language-dependent: fixed
    'abbreviation': 'abbreviation',
    'ab': 'abbreviation',
    'acronym': 'acronym',
    'ac': 'acronym',
    'code': 'code',
    'emphasis': 'emphasis',
    'literal': 'literal',
    'math': 'math',
    'pep-reference': 'pep-reference',
    'pep': 'pep-reference',
    'rfc-reference': 'rfc-reference',
    'rfc': 'rfc-reference',
    'strong': 'strong',
    'subscript': 'subscript',
    'sub': 'subscript',
    'superscript': 'superscript',
    'sup': 'superscript',
    'title-reference': 'title-reference',
    'title': 'title-reference',
    't': 'title-reference',
    'raw': 'raw',
    # the following roles are not implemented in Docutils
    'index': 'index',
    'i': 'index',
    'anonymous-reference': 'anonymous-reference',
    'citation-reference': 'citation-reference',
    'footnote-reference': 'footnote-reference',
    'named-reference': 'named-reference',
    'substitution-reference': 'substitution-reference',
    'uri-reference': 'uri-reference',
    'uri': 'uri-reference',
    'url': 'uri-reference',
    'target': 'target',
    }
"""Mapping of English role names to canonical role names for interpreted text.

Cf. https://docutils.sourceforge.io/docs/ref/rst/roles.html
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/en.py
NikolaBlagojevic/pyrecodes
# $Id: es.py 9116 2022-07-28 17:06:51Z milde $
# Author: Marcelo Huerta San Martín <richieadler@users.sourceforge.net>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
Spanish-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'

directives = {
      'atenci\u00f3n': 'attention',
      'atencion': 'attention',
      'precauci\u00f3n': 'caution',
      'code (translation required)': 'code',
      'precaucion': 'caution',
      'peligro': 'danger',
      'error': 'error',
      'sugerencia': 'hint',
      'importante': 'important',
      'nota': 'note',
      'consejo': 'tip',
      'advertencia': 'warning',
      'exhortacion': 'admonition',  # sic! Not used in this sense in rST.
      'exhortaci\u00f3n': 'admonition',
      'nota-al-margen': 'sidebar',
      'tema': 'topic',
      'bloque-de-lineas': 'line-block',
      'bloque-de-l\u00edneas': 'line-block',
      'literal-evaluado': 'parsed-literal',
      'firma': 'rubric',
      'ep\u00edgrafe': 'epigraph',
      'epigrafe': 'epigraph',
      'destacado': 'highlights',
      'cita-destacada': 'pull-quote',
      'combinacion': 'compound',
      'combinaci\u00f3n': 'compound',
      'contenedor': 'container',
      # 'questions': 'questions',
      # 'qa': 'questions',
      # 'faq': 'questions',
      'tabla': 'table',
      'tabla-vsc': 'csv-table',
      'tabla-csv': 'csv-table',
      'tabla-lista': 'list-table',
      'meta': 'meta',
      'math (translation required)': 'math',
      # 'imagemap': 'imagemap',
      'imagen': 'image',
      'figura': 'figure',
      'incluir': 'include',
      'sin-analisis': 'raw',
      'sin-an\u00e1lisis': 'raw',
      'reemplazar': 'replace',
      'unicode': 'unicode',
      'fecha': 'date',
      'clase': 'class',
      'rol': 'role',
      'rol-por-omision': 'default-role',
      'rol-por-omisi\u00f3n': 'default-role',
      'titulo': 'title',
      't\u00edtulo': 'title',
      'contenido': 'contents',
      'numseccion': 'sectnum',
      'numsecci\u00f3n': 'sectnum',
      'numeracion-seccion': 'sectnum',
      'numeraci\u00f3n-secci\u00f3n': 'sectnum',
      'notas-destino': 'target-notes',
      'cabecera': 'header',
      'pie': 'footer',
      # 'footnotes': 'footnotes',
      # 'citations': 'citations',
      'restructuredtext-test-directive': 'restructuredtext-test-directive'}
"""Spanish name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
    'abreviatura': 'abbreviation',
    'ab': 'abbreviation',
    'acronimo': 'acronym',
    'ac': 'acronym',
    'code (translation required)': 'code',
    'indice': 'index',
    'i': 'index',
    'subindice': 'subscript',
    'sub\u00edndice': 'subscript',
    'superindice': 'superscript',
    'super\u00edndice': 'superscript',
    'referencia-titulo': 'title-reference',
    'titulo': 'title-reference',
    't': 'title-reference',
    'referencia-pep': 'pep-reference',
    'pep': 'pep-reference',
    'referencia-rfc': 'rfc-reference',
    'rfc': 'rfc-reference',
    'enfasis': 'emphasis',
    '\u00e9nfasis': 'emphasis',
    'destacado': 'strong',
    'literal': 'literal',  # "literal" is also a word in Spanish :-)
    'math (translation required)': 'math',
    'referencia-con-nombre': 'named-reference',
    'referencia-anonima': 'anonymous-reference',
    'referencia-an\u00f3nima': 'anonymous-reference',
    'referencia-nota-al-pie': 'footnote-reference',
    'referencia-cita': 'citation-reference',
    'referencia-sustitucion': 'substitution-reference',
    'referencia-sustituci\u00f3n': 'substitution-reference',
    'destino': 'target',
    'referencia-uri': 'uri-reference',
    'uri': 'uri-reference',
    'url': 'uri-reference',
    'sin-analisis': 'raw',
    'sin-an\u00e1lisis': 'raw',
}
"""Mapping of Spanish role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/es.py
NikolaBlagojevic/pyrecodes
# $Id: fr.py 9383 2023-05-11 15:01:20Z milde $
# Authors: David Goodger <goodger@python.org>; William Dode
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
French-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      'attention': 'attention',
      'précaution': 'caution',
      'danger': 'danger',
      'erreur': 'error',
      'conseil': 'hint',
      'important': 'important',
      'note': 'note',
      'astuce': 'tip',
      'avertissement': 'warning',
      'admonition': 'admonition',  # sic! Not used in this sense in rST.
      # suggestions: annonce, avis, indication, remarque, renseignement
      # see also https://sourceforge.net/p/docutils/bugs/453/
      'encadré': 'sidebar',
      'sujet': 'topic',
      'bloc-textuel': 'line-block',
      'bloc-interprété': 'parsed-literal',
      'code-interprété': 'parsed-literal',
      'code': 'code',
      'math (translation required)': 'math',
      'intertitre': 'rubric',
      'exergue': 'epigraph',
      'épigraphe': 'epigraph',
      'chapeau': 'highlights',
      'accroche': 'pull-quote',
      'compound (translation required)': 'compound',
      'container (translation required)': 'container',
      'tableau': 'table',
      'csv-table (translation required)': 'csv-table',
      'list-table (translation required)': 'list-table',
      'méta': 'meta',
      # 'imagemap (translation required)': 'imagemap',
      'image': 'image',
      'figure': 'figure',
      'inclure': 'include',
      'brut': 'raw',
      'remplacer': 'replace',
      'remplace': 'replace',
      'unicode': 'unicode',
      'date': 'date',
      'classe': 'class',
      'role (translation required)': 'role',
      'default-role (translation required)': 'default-role',
      'titre (translation required)': 'title',
      'sommaire': 'contents',
      'table-des-matières': 'contents',
      'sectnum': 'sectnum',
      'section-numérotée': 'sectnum',
      'liens': 'target-notes',
      'header (translation required)': 'header',
      'footer (translation required)': 'footer',
      # 'footnotes (translation required)': 'footnotes',
      # 'citations (translation required)': 'citations',
      }
"""Mapping of French directive names to registered directive names

Cf. https://docutils.sourceforge.io/docs/ref/rst/directives.html
and `_directive_registry` in ``directives/__init__.py``.
"""

roles = {
      'abréviation': 'abbreviation',
      'acronyme': 'acronym',
      'sigle': 'acronym',
      'code': 'code',
      'emphase': 'emphasis',
      'littéral': 'literal',
      'math (translation required)': 'math',
      'pep-référence': 'pep-reference',
      'rfc-référence': 'rfc-reference',
      'fort': 'strong',
      'indice': 'subscript',
      'ind': 'subscript',
      'exposant': 'superscript',
      'exp': 'superscript',
      'titre-référence': 'title-reference',
      'titre': 'title-reference',
      'brut': 'raw',
      # the following roles are not implemented in Docutils
      'index': 'index',
      'nommée-référence': 'named-reference',
      'anonyme-référence': 'anonymous-reference',
      'note-référence': 'footnote-reference',
      'citation-référence': 'citation-reference',
      'substitution-référence': 'substitution-reference',
      'lien': 'target',
      'uri-référence': 'uri-reference',
      }
"""Mapping of French role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/fr.py
NikolaBlagojevic/pyrecodes
# $Id: it.py 9116 2022-07-28 17:06:51Z milde $
# Authors: Nicola Larosa <docutils@tekNico.net>;
#          Lele Gaifax <lele@seldati.it>
# Copyright: This module has been placed in the public domain.

# Beware: the italian translation of the reStructuredText documentation
# at http://docit.bice.dyndns.org/static/ReST, in particular
# http://docit.bice.dyndns.org/static/ReST/ref/rst/directives.html, needs
# to be synced with the content of this file.

"""
Italian-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      'attenzione': 'attention',
      'cautela': 'caution',
      'code (translation required)': 'code',
      'pericolo': 'danger',
      'errore': 'error',
      'suggerimento': 'hint',
      'importante': 'important',
      'nota': 'note',
      'consiglio': 'tip',
      'avvertenza': 'warning',
      'ammonizione': 'admonition',  # sic! Not used in this sense in rST.
      'riquadro': 'sidebar',
      'argomento': 'topic',
      'blocco-di-righe': 'line-block',
      'blocco-interpretato': 'parsed-literal',
      'rubrica': 'rubric',
      'epigrafe': 'epigraph',
      'punti-salienti': 'highlights',
      'estratto-evidenziato': 'pull-quote',
      'composito': 'compound',
      'container (translation required)': 'container',
      # 'questions': 'questions',
      # 'qa': 'questions',
      # 'faq': 'questions',
      'tabella': 'table',
      'tabella-csv': 'csv-table',
      'tabella-elenco': 'list-table',
      'meta': 'meta',
      'math (translation required)': 'math',
      # 'imagemap': 'imagemap',
      'immagine': 'image',
      'figura': 'figure',
      'includi': 'include',
      'grezzo': 'raw',
      'sostituisci': 'replace',
      'unicode': 'unicode',
      'data': 'date',
      'classe': 'class',
      'ruolo': 'role',
      'ruolo-predefinito': 'default-role',
      'titolo': 'title',
      'indice': 'contents',
      'contenuti': 'contents',
      'seznum': 'sectnum',
      'sezioni-autonumerate': 'sectnum',
      'annota-riferimenti-esterni': 'target-notes',
      'intestazione': 'header',
      'piede-pagina': 'footer',
      # 'footnotes': 'footnotes',
      # 'citations': 'citations',
      'restructuredtext-test-directive': 'restructuredtext-test-directive'}
"""Italian name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
      'abbreviazione': 'abbreviation',
      'acronimo': 'acronym',
      'code (translation required)': 'code',
      'indice': 'index',
      'deponente': 'subscript',
      'esponente': 'superscript',
      'riferimento-titolo': 'title-reference',
      'riferimento-pep': 'pep-reference',
      'riferimento-rfc': 'rfc-reference',
      'enfasi': 'emphasis',
      'forte': 'strong',
      'letterale': 'literal',
      'math (translation required)': 'math',
      'riferimento-con-nome': 'named-reference',
      'riferimento-anonimo': 'anonymous-reference',
      'riferimento-nota': 'footnote-reference',
      'riferimento-citazione': 'citation-reference',
      'riferimento-sostituzione': 'substitution-reference',
      'destinazione': 'target',
      'riferimento-uri': 'uri-reference',
      'grezzo': 'raw',
      }
"""Mapping of Italian role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/it.py
NikolaBlagojevic/pyrecodes
# $Id: nl.py 9116 2022-07-28 17:06:51Z milde $
# Author: Martijn Pieters <mjpieters@users.sourceforge.net>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
Dutch-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      # language-dependent: fixed
      'attentie': 'attention',
      'let-op': 'caution',
      'code (translation required)': 'code',
      'gevaar': 'danger',
      'fout': 'error',
      'hint': 'hint',
      'belangrijk': 'important',
      'opmerking': 'note',
      'tip': 'tip',
      'waarschuwing': 'warning',
      'aanmaning': 'admonition',  # sic! Not used in this sense in rST.
      'katern': 'sidebar',
      'onderwerp': 'topic',
      'lijn-blok': 'line-block',
      'letterlijk-ontleed': 'parsed-literal',
      'rubriek': 'rubric',
      'opschrift': 'epigraph',
      'hoogtepunten': 'highlights',
      'pull-quote': 'pull-quote',  # Dutch printers use the english term
      'samenstelling': 'compound',
      'verbinding': 'compound',
      'container (translation required)': 'container',
      # 'vragen': 'questions',
      'tabel': 'table',
      'csv-tabel': 'csv-table',
      'lijst-tabel': 'list-table',
      # 'veelgestelde-vragen': 'questions',
      'meta': 'meta',
      'math (translation required)': 'math',
      # 'imagemap': 'imagemap',
      'beeld': 'image',
      'figuur': 'figure',
      'opnemen': 'include',
      'onbewerkt': 'raw',
      'vervang': 'replace',
      'vervanging': 'replace',
      'unicode': 'unicode',
      'datum': 'date',
      'klasse': 'class',
      'rol': 'role',
      'default-role (translation required)': 'default-role',
      'title (translation required)': 'title',
      'inhoud': 'contents',
      'sectnum': 'sectnum',
      'sectie-nummering': 'sectnum',
      'hoofdstuk-nummering': 'sectnum',
      'header (translation required)': 'header',
      'footer (translation required)': 'footer',
      # 'voetnoten': 'footnotes',
      # 'citaten': 'citations',
      'verwijzing-voetnoten': 'target-notes',
      'restructuredtext-test-instructie': 'restructuredtext-test-directive'}
"""Dutch name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
    # language-dependent: fixed
    'afkorting': 'abbreviation',
    # 'ab': 'abbreviation',
    'acroniem': 'acronym',
    'ac': 'acronym',
    'code (translation required)': 'code',
    'index': 'index',
    'i': 'index',
    'inferieur': 'subscript',
    'inf': 'subscript',
    'superieur': 'superscript',
    'sup': 'superscript',
    'titel-referentie': 'title-reference',
    'titel': 'title-reference',
    't': 'title-reference',
    'pep-referentie': 'pep-reference',
    'pep': 'pep-reference',
    'rfc-referentie': 'rfc-reference',
    'rfc': 'rfc-reference',
    'nadruk': 'emphasis',
    'extra': 'strong',
    'extra-nadruk': 'strong',
    'vet': 'strong',
    'letterlijk': 'literal',
    'math (translation required)': 'math',
    'benoemde-referentie': 'named-reference',
    'anonieme-referentie': 'anonymous-reference',
    'voetnoot-referentie': 'footnote-reference',
    'citaat-referentie': 'citation-reference',
    'substitie-reference': 'substitution-reference',
    'verwijzing': 'target',
    'uri-referentie': 'uri-reference',
    'uri': 'uri-reference',
    'url': 'uri-reference',
    'onbewerkt': 'raw',
    }
"""Mapping of Dutch role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/nl.py
NikolaBlagojevic/pyrecodes
# $Id$
# Author: Robert Wojciechowicz <rw@smsnet.pl>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <https://docutils.sourceforge.io/docs/howto/i18n.html>.
# Two files must be translated for each language: one in docutils/languages,
# the other in docutils/parsers/rst/languages.

"""
Polish-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'


directives = {
      # language-dependent: fixed
      'uwaga': 'attention',
      'ostro\u017cnie': 'caution',
      'code (translation required)': 'code',
      'niebezpiecze\u0144stwo': 'danger',
      'b\u0142\u0105d': 'error',
      'wskaz\u00f3wka': 'hint',
      'wa\u017cne': 'important',
      'przypis': 'note',
      'rada': 'tip',
      'ostrze\u017cenie': 'warning',
      'upomnienie': 'admonition',  # sic! Not used in this sense in rST.
      # 'zauważenie': 'admonition',  # remark
      'ramka': 'sidebar',
      'temat': 'topic',
      'blok-linii': 'line-block',
      'sparsowany-litera\u0142': 'parsed-literal',
      'rubryka': 'rubric',
      'epigraf': 'epigraph',
      'highlights': 'highlights',  # FIXME no polish equivalent?
      'pull-quote': 'pull-quote',  # FIXME no polish equivalent?
      'z\u0142o\u017cony': 'compound',
      'kontener': 'container',
      # 'questions': 'questions',
      'tabela': 'table',
      'tabela-csv': 'csv-table',
      'tabela-listowa': 'list-table',
      # 'qa': 'questions',
      # 'faq': 'questions',
      'meta': 'meta',
      'math (translation required)': 'math',
      # 'imagemap': 'imagemap',
      'obraz': 'image',
      'rycina': 'figure',
      'do\u0142\u0105cz': 'include',
      'surowe': 'raw',
      'zast\u0105p': 'replace',
      'unikod': 'unicode',
      'data': 'date',
      'klasa': 'class',
      'rola': 'role',
      'rola-domy\u015blna': 'default-role',
      'tytu\u0142': 'title',
      'tre\u015b\u0107': 'contents',
      'sectnum': 'sectnum',
      'numeracja-sekcji': 'sectnum',
      'nag\u0142\u00f3wek': 'header',
      'stopka': 'footer',
      # 'footnotes': 'footnotes',
      # 'citations': 'citations',
      'target-notes': 'target-notes',  # FIXME no polish equivalent?
      'restructuredtext-test-directive': 'restructuredtext-test-directive'}
"""Polish name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
    # language-dependent: fixed
    'skr\u00f3t': 'abbreviation',
    'akronim': 'acronym',
    'code (translation required)': 'code',
    'indeks': 'index',
    'indeks-dolny': 'subscript',
    'indeks-g\u00f3rny': 'superscript',
    'referencja-tytu\u0142': 'title-reference',
    'referencja-pep': 'pep-reference',
    'referencja-rfc': 'rfc-reference',
    'podkre\u015blenie': 'emphasis',
    'wyt\u0142uszczenie': 'strong',
    'dos\u0142ownie': 'literal',
    'math (translation required)': 'math',
    'referencja-nazwana': 'named-reference',
    'referencja-anonimowa': 'anonymous-reference',
    'referencja-przypis': 'footnote-reference',
    'referencja-cytat': 'citation-reference',
    'referencja-podstawienie': 'substitution-reference',
    'cel': 'target',
    'referencja-uri': 'uri-reference',
    'uri': 'uri-reference',
    'url': 'uri-reference',
    'surowe': 'raw',
    }
"""Mapping of Polish role names to canonical role names for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/pl.py
NikolaBlagojevic/pyrecodes
# $Id: uk.py 9114 2022-07-28 17:06:10Z milde $
# Author: Dmytro Kazanzhy <dkazanzhy@gmail.com>
# Copyright: This module has been placed in the public domain.

# New language mappings are welcome.  Before doing a new translation, please
# read <http://docutils.sf.net/docs/howto/i18n.html>.  Two files must be
# translated for each language: one in docutils/languages, the other in
# docutils/parsers/rst/languages.

"""
Ukrainian-language mappings for language-dependent features of
reStructuredText.
"""

__docformat__ = 'reStructuredText'

directives = {
    'блок-строк': 'line-block',
    'мета': 'meta',
    'математика': 'math',
    'оброблений-літерал': 'parsed-literal',
    'виділена-цитата': 'pull-quote',
    'код': 'code',
    'складений абзац': 'compound',
    'контейнер': 'container',
    'таблиця': 'table',
    'таблиця-csv': 'csv-table',
    'таблиця-списків': 'list-table',
    'сирий': 'raw',
    'заміна': 'replace',
    'тестова-директива-restructuredtext': 'restructuredtext-test-directive',
    'цільові-виноски': 'target-notes',
    'юнікод': 'unicode',
    'дата': 'date',
    'бічна-панель': 'sidebar',
    'важливо': 'important',
    'включати': 'include',
    'увага': 'attention',
    'виділення': 'highlights',
    'зауваження': 'admonition',
    'зображення': 'image',
    'клас': 'class',
    'роль': 'role',
    'роль-за-замовчуванням': 'default-role',
    'заголовок': 'title',
    'номер-розділу': 'sectnum',
    'нумерація-розділів': 'sectnum',
    'небезпечно': 'danger',
    'обережно': 'caution',
    'помилка': 'error',
    'підказка': 'tip',
    'попередження': 'warning',
    'примітка': 'note',
    'малюнок': 'figure',
    'рубрика': 'rubric',
    'порада': 'hint',
    'зміст': 'contents',
    'тема': 'topic',
    'епіграф': 'epigraph',
    'верхній колонтитул': 'header',
    'нижній колонтитул': 'footer',
    }
"""Ukrainian name to registered (in directives/__init__.py) directive name
mapping."""

roles = {
    'акронім': 'acronym',
    'код': 'code',
    'анонімне-посилання': 'anonymous-reference',
    'буквально': 'literal',
    'математика': 'math',
    'верхній-індекс': 'superscript',
    'наголос': 'emphasis',
    'іменоване-посилання': 'named-reference',
    'індекс': 'index',
    'нижній-індекс': 'subscript',
    'жирне-накреслення': 'strong',
    'скорочення': 'abbreviation',
    'посилання-заміна': 'substitution-reference',
    'посилання-на-pep': 'pep-reference',
    'посилання-на-rfc': 'rfc-reference',
    'посилання-на-uri': 'uri-reference',
    'посилання-на-заголовок': 'title-reference',
    'посилання-на-зноску': 'footnote-reference',
    'посилання-на-цитату': 'citation-reference',
    'ціль': 'target',
    'сирий': 'raw',
    }
"""Mapping of Ukrainian role names to canonical role names
for interpreted text.
"""

env_pyrecodes/lib/python3.9/site-packages/docutils/parsers/rst/languages/uk.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9258 2022-11-21 14:51:43Z milde $
# Authors: David Goodger <goodger@python.org>; Ueli Schlaepfer
# Copyright: This module has been placed in the public domain.

"""
This package contains Docutils Reader modules.
"""

__docformat__ = 'reStructuredText'

from importlib import import_module

from docutils import utils, parsers, Component
from docutils.transforms import universal


class Reader(Component):

    """
    Abstract base class for docutils Readers.

    Each reader module or package must export a subclass also called 'Reader'.

    The two steps of a Reader's responsibility are to read data from the
    source Input object and parse the data with the Parser object.
    Call `read()` to process a document.
    """

    component_type = 'reader'
    config_section = 'readers'

    def get_transforms(self):
        return super().get_transforms() + [universal.Decorations,
                                           universal.ExposeInternals,
                                           universal.StripComments]

    def __init__(self, parser=None, parser_name=None):
        """
        Initialize the Reader instance.

        Several instance attributes are defined with dummy initial values.
        Subclasses may use these attributes as they wish.
        """

        self.parser = parser
        """A `parsers.Parser` instance shared by all doctrees.  May be left
        unspecified if the document source determines the parser."""

        if parser is None and parser_name:
            self.set_parser(parser_name)

        self.source = None
        """`docutils.io` IO object, source of input data."""

        self.input = None
        """Raw text input; either a single string or, for more complex cases,
        a collection of strings."""

    def set_parser(self, parser_name):
        """Set `self.parser` by name."""
        parser_class = parsers.get_parser_class(parser_name)
        self.parser = parser_class()

    def read(self, source, parser, settings):
        self.source = source
        if not self.parser:
            self.parser = parser
        self.settings = settings
        self.input = self.source.read()
        self.parse()
        return self.document

    def parse(self):
        """Parse `self.input` into a document tree."""
        self.document = document = self.new_document()
        self.parser.parse(self.input, document)
        document.current_source = document.current_line = None

    def new_document(self):
        """Create and return a new empty document tree (root node)."""
        return utils.new_document(self.source.source_path, self.settings)


class ReReader(Reader):

    """
    A reader which rereads an existing document tree (e.g. a
    deserializer).

    Often used in conjunction with `writers.UnfilteredWriter`.
    """

    def get_transforms(self):
        # Do not add any transforms.  They have already been applied
        # by the reader which originally created the document.
        return Component.get_transforms(self)


_reader_aliases = {}


def get_reader_class(reader_name):
    """Return the Reader class from the `reader_name` module."""
    name = reader_name.lower()
    name = _reader_aliases.get(name, name)
    try:
        module = import_module('docutils.readers.'+name)
    except ImportError:
        try:
            module = import_module(name)
        except ImportError as err:
            raise ImportError(f'Reader "{reader_name}" not found. {err}')
    return module.Reader

env_pyrecodes/lib/python3.9/site-packages/docutils/readers/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: pep.py 9258 2022-11-21 14:51:43Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Python Enhancement Proposal (PEP) Reader.
"""

__docformat__ = 'reStructuredText'


from docutils.readers import standalone
from docutils.transforms import peps, frontmatter
from docutils.parsers import rst


class Reader(standalone.Reader):

    supported = ('pep',)
    """Contexts this reader supports."""

    settings_spec = (
        'PEP Reader Option Defaults',
        'The --pep-references and --rfc-references options (for the '
        'reStructuredText parser) are on by default.',
        ())

    config_section = 'pep reader'
    config_section_dependencies = ('readers', 'standalone reader')

    def get_transforms(self):
        transforms = super().get_transforms()
        # We have PEP-specific frontmatter handling.
        transforms.remove(frontmatter.DocTitle)
        transforms.remove(frontmatter.SectionSubTitle)
        transforms.remove(frontmatter.DocInfo)
        transforms.extend([peps.Headers, peps.Contents, peps.TargetNotes])
        return transforms

    settings_default_overrides = {'pep_references': 1, 'rfc_references': 1}

    inliner_class = rst.states.Inliner

    def __init__(self, parser=None, parser_name=None):
        """`parser` should be ``None``."""
        if parser is None:
            parser = rst.Parser(rfc2822=True, inliner=self.inliner_class())
        standalone.Reader.__init__(self, parser, '')

env_pyrecodes/lib/python3.9/site-packages/docutils/readers/pep.py
NikolaBlagojevic/pyrecodes
# $Id: standalone.py 9258 2022-11-21 14:51:43Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Standalone file Reader for the reStructuredText markup syntax.
"""

__docformat__ = 'reStructuredText'


from docutils import frontend, readers
from docutils.transforms import frontmatter, references, misc


class Reader(readers.Reader):

    supported = ('standalone',)
    """Contexts this reader supports."""

    document = None
    """A single document tree."""

    settings_spec = (
        'Standalone Reader Options',
        None,
        (('Disable the promotion of a lone top-level section title to '
          'document title (and subsequent section title to document '
          'subtitle promotion; enabled by default).',
          ['--no-doc-title'],
          {'dest': 'doctitle_xform', 'action': 'store_false', 'default': 1,
           'validator': frontend.validate_boolean}),
         ('Disable the bibliographic field list transform (enabled by '
          'default).',
          ['--no-doc-info'],
          {'dest': 'docinfo_xform', 'action': 'store_false', 'default': 1,
           'validator': frontend.validate_boolean}),
         ('Activate the promotion of lone subsection titles to '
          'section subtitles (disabled by default).',
          ['--section-subtitles'],
          {'dest': 'sectsubtitle_xform', 'action': 'store_true', 'default': 0,
           'validator': frontend.validate_boolean}),
         ('Deactivate the promotion of lone subsection titles.',
          ['--no-section-subtitles'],
          {'dest': 'sectsubtitle_xform', 'action': 'store_false'}),
         ))

    config_section = 'standalone reader'
    config_section_dependencies = ('readers',)

    def get_transforms(self):
        return super().get_transforms() + [
            references.Substitutions,
            references.PropagateTargets,
            frontmatter.DocTitle,
            frontmatter.SectionSubTitle,
            frontmatter.DocInfo,
            references.AnonymousHyperlinks,
            references.IndirectHyperlinks,
            references.Footnotes,
            references.ExternalTargets,
            references.InternalTargets,
            references.DanglingReferences,
            misc.Transitions,
            ]

env_pyrecodes/lib/python3.9/site-packages/docutils/readers/standalone.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9320 2023-01-17 15:40:15Z milde $
# Authors: David Goodger <goodger@python.org>; Ueli Schlaepfer
# Copyright: This module has been placed in the public domain.

"""
This package contains modules for standard tree transforms available
to Docutils components. Tree transforms serve a variety of purposes:

- To tie up certain syntax-specific "loose ends" that remain after the
  initial parsing of the input plaintext. These transforms are used to
  supplement a limited syntax.

- To automate the internal linking of the document tree (hyperlink
  references, footnote references, etc.).

- To extract useful information from the document tree. These
  transforms may be used to construct (for example) indexes and tables
  of contents.

Each transform is an optional step that a Docutils component may
choose to perform on the parsed document.
"""

__docformat__ = 'reStructuredText'


from docutils import languages, ApplicationError, TransformSpec


class TransformError(ApplicationError):
    pass


class Transform:
    """Docutils transform component abstract base class."""

    default_priority = None
    """Numerical priority of this transform, 0 through 999 (override)."""

    def __init__(self, document, startnode=None):
        """
        Initial setup for in-place document transforms.
        """

        self.document = document
        """The document tree to transform."""

        self.startnode = startnode
        """Node from which to begin the transform.  For many transforms which
        apply to the document as a whole, `startnode` is not set (i.e. its
        value is `None`)."""

        self.language = languages.get_language(
            document.settings.language_code, document.reporter)
        """Language module local to this document."""

    def apply(self, **kwargs):
        """Override to apply the transform to the document tree."""
        raise NotImplementedError('subclass must override this method')


class Transformer(TransformSpec):
    """
    Store "transforms" and apply them to the document tree.

    Collect lists of `Transform` instances and "unknown_reference_resolvers"
    from Docutils components (`TransformSpec` instances).
    Apply collected "transforms" to the document tree.

    Also keeps track of components by component type name.

    https://docutils.sourceforge.io/docs/peps/pep-0258.html#transformer
    """

    def __init__(self, document):
        self.transforms = []
        """List of transforms to apply.  Each item is a 4-tuple:
        ``(priority string, transform class, pending node or None, kwargs)``.
        """

        self.unknown_reference_resolvers = []
        """List of hook functions which assist in resolving references."""

        self.document = document
        """The `nodes.document` object this Transformer is attached to."""

        self.applied = []
        """Transforms already applied, in order."""

        self.sorted = False
        """Boolean: is `self.tranforms` sorted?"""

        self.components = {}
        """Mapping of component type name to component object.

        Set by `self.populate_from_components()`.
        """

        self.serialno = 0
        """Internal serial number to keep track of the add order of
        transforms."""

    def add_transform(self, transform_class, priority=None, **kwargs):
        """
        Store a single transform.  Use `priority` to override the default.
        `kwargs` is a dictionary whose contents are passed as keyword
        arguments to the `apply` method of the transform.  This can be used to
        pass application-specific data to the transform instance.
        """
        if priority is None:
            priority = transform_class.default_priority
        priority_string = self.get_priority_string(priority)
        self.transforms.append(
            (priority_string, transform_class, None, kwargs))
        self.sorted = False

    def add_transforms(self, transform_list):
        """Store multiple transforms, with default priorities."""
        for transform_class in transform_list:
            priority_string = self.get_priority_string(
                transform_class.default_priority)
            self.transforms.append(
                (priority_string, transform_class, None, {}))
        self.sorted = False

    def add_pending(self, pending, priority=None):
        """Store a transform with an associated `pending` node."""
        transform_class = pending.transform
        if priority is None:
            priority = transform_class.default_priority
        priority_string = self.get_priority_string(priority)
        self.transforms.append(
            (priority_string, transform_class, pending, {}))
        self.sorted = False

    def get_priority_string(self, priority):
        """
        Return a string, `priority` combined with `self.serialno`.

        This ensures FIFO order on transforms with identical priority.
        """
        self.serialno += 1
        return '%03d-%03d' % (priority, self.serialno)

    def populate_from_components(self, components):
        """
        Store each component's default transforms and reference resolvers

        Transforms are stored with default priorities for later sorting.
        "Unknown reference resolvers" are sorted and stored.
        Components that don't inherit from `TransformSpec` are ignored.

        Also, store components by type name in a mapping for later lookup.
        """
        resolvers = []
        for component in components:
            if not isinstance(component, TransformSpec):
                continue
            self.add_transforms(component.get_transforms())
            self.components[component.component_type] = component
            resolvers.extend(component.unknown_reference_resolvers)
        self.sorted = False  # sort transform list in self.apply_transforms()

        # Sort and add helper functions to help resolve unknown references.
        def keyfun(f):
            return f.priority
        resolvers.sort(key=keyfun)
        self.unknown_reference_resolvers += resolvers

    def apply_transforms(self):
        """Apply all of the stored transforms, in priority order."""
        self.document.reporter.attach_observer(
            self.document.note_transform_message)
        while self.transforms:
            if not self.sorted:
                # Unsorted initially, and whenever a transform is added
                # (transforms may add other transforms).
                self.transforms.sort(reverse=True)
                self.sorted = True
            priority, transform_class, pending, kwargs = self.transforms.pop()
            transform = transform_class(self.document, startnode=pending)
            transform.apply(**kwargs)
            self.applied.append((priority, transform_class, pending, kwargs))

env_pyrecodes/lib/python3.9/site-packages/docutils/transforms/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: frontmatter.py 9351 2023-04-17 20:26:33Z milde $
# Author: David Goodger, Ueli Schlaepfer <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Transforms related to the front matter of a document or a section
(information found before the main text):

- `DocTitle`: Used to transform a lone top level section's title to
  the document title, promote a remaining lone top-level section's
  title to the document subtitle, and determine the document's title
  metadata (document['title']) based on the document title and/or the
  "title" setting.

- `SectionSubTitle`: Used to transform a lone subsection into a
  subtitle.

- `DocInfo`: Used to transform a bibliographic field list into docinfo
  elements.
"""

__docformat__ = 'reStructuredText'

import re

from docutils import nodes, parsers, utils
from docutils.transforms import TransformError, Transform


class TitlePromoter(Transform):

    """
    Abstract base class for DocTitle and SectionSubTitle transforms.
    """

    def promote_title(self, node):
        """
        Transform the following tree::

            <node>
                <section>
                    <title>
                    ...

        into ::

            <node>
                <title>
                ...

        `node` is normally a document.
        """
        # Type check
        if not isinstance(node, nodes.Element):
            raise TypeError('node must be of Element-derived type.')

        # `node` must not have a title yet.
        assert not (len(node) and isinstance(node[0], nodes.title))
        section, index = self.candidate_index(node)
        if index is None:
            return False

        # Transfer the section's attributes to the node:
        # NOTE: Change `replace` to False to NOT replace attributes that
        #       already exist in node with those in section.
        # NOTE: Remove `and_source` to NOT copy the 'source'
        #       attribute from section
        node.update_all_atts_concatenating(section, replace=True,
                                           and_source=True)

        # setup_child is called automatically for all nodes.
        node[:] = (section[:1]        # section title
                   + node[:index]     # everything that was in the
                                      # node before the section
                   + section[1:])     # everything that was in the section
        assert isinstance(node[0], nodes.title)
        return True

    def promote_subtitle(self, node):
        """
        Transform the following node tree::

            <node>
                <title>
                <section>
                    <title>
                    ...

        into ::

            <node>
                <title>
                <subtitle>
                ...
        """
        # Type check
        if not isinstance(node, nodes.Element):
            raise TypeError('node must be of Element-derived type.')

        subsection, index = self.candidate_index(node)
        if index is None:
            return False
        subtitle = nodes.subtitle()

        # Transfer the subsection's attributes to the new subtitle
        # NOTE: Change `replace` to False to NOT replace attributes
        #       that already exist in node with those in section.
        # NOTE: Remove `and_source` to NOT copy the 'source'
        #       attribute from section.
        subtitle.update_all_atts_concatenating(subsection, replace=True,
                                               and_source=True)

        # Transfer the contents of the subsection's title to the
        # subtitle:
        subtitle[:] = subsection[0][:]
        node[:] = (node[:1]       # title
                   + [subtitle]
                   # everything that was before the section:
                   + node[1:index]
                   # everything that was in the subsection:
                   + subsection[1:])
        return True

    def candidate_index(self, node):
        """
        Find and return the promotion candidate and its index.

        Return (None, None) if no valid candidate was found.
        """
        index = node.first_child_not_matching_class(
            nodes.PreBibliographic)
        if (index is None or len(node) > (index + 1)
            or not isinstance(node[index], nodes.section)):
            return None, None
        else:
            return node[index], index


class DocTitle(TitlePromoter):

    """
    In reStructuredText_, there is no way to specify a document title
    and subtitle explicitly. Instead, we can supply the document title
    (and possibly the subtitle as well) implicitly, and use this
    two-step transform to "raise" or "promote" the title(s) (and their
    corresponding section contents) to the document level.

    1. If the document contains a single top-level section as its
       first non-comment element, the top-level section's title
       becomes the document's title, and the top-level section's
       contents become the document's immediate contents. The lone
       top-level section header must be the first non-comment element
       in the document.

       For example, take this input text::

           =================
            Top-Level Title
           =================

           A paragraph.

       Once parsed, it looks like this::

           <document>
               <section names="top-level title">
                   <title>
                       Top-Level Title
                   <paragraph>
                       A paragraph.

       After running the DocTitle transform, we have::

           <document names="top-level title">
               <title>
                   Top-Level Title
               <paragraph>
                   A paragraph.

    2. If step 1 successfully determines the document title, we
       continue by checking for a subtitle.

       If the lone top-level section itself contains a single
       second-level section as its first non-comment element, that
       section's title is promoted to the document's subtitle, and
       that section's contents become the document's immediate
       contents. Given this input text::

           =================
            Top-Level Title
           =================

           Second-Level Title
           ~~~~~~~~~~~~~~~~~~

           A paragraph.

       After parsing and running the Section Promotion transform, the
       result is::

           <document names="top-level title">
               <title>
                   Top-Level Title
               <subtitle names="second-level title">
                   Second-Level Title
               <paragraph>
                   A paragraph.

       (Note that the implicit hyperlink target generated by the
       "Second-Level Title" is preserved on the "subtitle" element
       itself.)

    Any comment elements occurring before the document title or
    subtitle are accumulated and inserted as the first body elements
    after the title(s).

    This transform also sets the document's metadata title
    (document['title']).

    .. _reStructuredText: https://docutils.sourceforge.io/rst.html
    """

    default_priority = 320

    def set_metadata(self):
        """
        Set document['title'] metadata title from the following
        sources, listed in order of priority:

        * Existing document['title'] attribute.
        * "title" setting.
        * Document title node (as promoted by promote_title).
        """
        if not self.document.hasattr('title'):
            if self.document.settings.title is not None:
                self.document['title'] = self.document.settings.title
            elif len(self.document) and isinstance(self.document[0],
                                                   nodes.title):
                self.document['title'] = self.document[0].astext()

    def apply(self):
        if self.document.settings.setdefault('doctitle_xform', True):
            # promote_(sub)title defined in TitlePromoter base class.
            if self.promote_title(self.document):
                # If a title has been promoted, also try to promote a
                # subtitle.
                self.promote_subtitle(self.document)
        # Set document['title'].
        self.set_metadata()


class SectionSubTitle(TitlePromoter):

    """
    This works like document subtitles, but for sections.  For example, ::

        <section>
            <title>
                Title
            <section>
                <title>
                    Subtitle
                ...

    is transformed into ::

        <section>
            <title>
                Title
            <subtitle>
                Subtitle
            ...

    For details refer to the docstring of DocTitle.
    """

    default_priority = 350

    def apply(self):
        if not self.document.settings.setdefault('sectsubtitle_xform', True):
            return
        for section in self.document.findall(nodes.section):
            # On our way through the node tree, we are modifying it
            # but only the not-yet-visited part, so that the iterator
            # returned by findall() is not corrupted.
            self.promote_subtitle(section)


class DocInfo(Transform):

    """
    This transform is specific to the reStructuredText_ markup syntax;
    see "Bibliographic Fields" in the `reStructuredText Markup
    Specification`_ for a high-level description. This transform
    should be run *after* the `DocTitle` transform.

    Given a field list as the first non-comment element after the
    document title and subtitle (if present), registered bibliographic
    field names are transformed to the corresponding DTD elements,
    becoming child elements of the "docinfo" element (except for a
    dedication and/or an abstract, which become "topic" elements after
    "docinfo").

    For example, given this document fragment after parsing::

        <document>
            <title>
                Document Title
            <field_list>
                <field>
                    <field_name>
                        Author
                    <field_body>
                        <paragraph>
                            A. Name
                <field>
                    <field_name>
                        Status
                    <field_body>
                        <paragraph>
                            $RCSfile$
            ...

    After running the bibliographic field list transform, the
    resulting document tree would look like this::

        <document>
            <title>
                Document Title
            <docinfo>
                <author>
                    A. Name
                <status>
                    frontmatter.py
            ...

    The "Status" field contained an expanded RCS keyword, which is
    normally (but optionally) cleaned up by the transform. The sole
    contents of the field body must be a paragraph containing an
    expanded RCS keyword of the form "$keyword: expansion text $". Any
    RCS keyword can be processed in any bibliographic field. The
    dollar signs and leading RCS keyword name are removed. Extra
    processing is done for the following RCS keywords:

    - "RCSfile" expands to the name of the file in the RCS or CVS
      repository, which is the name of the source file with a ",v"
      suffix appended. The transform will remove the ",v" suffix.

    - "Date" expands to the format "YYYY/MM/DD hh:mm:ss" (in the UTC
      time zone). The RCS Keywords transform will extract just the
      date itself and transform it to an ISO 8601 format date, as in
      "2000-12-31".

      (Since the source file for this text is itself stored under CVS,
      we can't show an example of the "Date" RCS keyword because we
      can't prevent any RCS keywords used in this explanation from
      being expanded. Only the "RCSfile" keyword is stable; its
      expansion text changes only if the file name changes.)

    .. _reStructuredText: https://docutils.sourceforge.io/rst.html
    .. _reStructuredText Markup Specification:
       https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html
    """

    default_priority = 340

    biblio_nodes = {
          'author': nodes.author,
          'authors': nodes.authors,
          'organization': nodes.organization,
          'address': nodes.address,
          'contact': nodes.contact,
          'version': nodes.version,
          'revision': nodes.revision,
          'status': nodes.status,
          'date': nodes.date,
          'copyright': nodes.copyright,
          'dedication': nodes.topic,
          'abstract': nodes.topic}
    """Canonical field name (lowcased) to node class name mapping for
    bibliographic fields (field_list)."""

    def apply(self):
        if not self.document.settings.setdefault('docinfo_xform', True):
            return
        document = self.document
        index = document.first_child_not_matching_class(
              nodes.PreBibliographic)
        if index is None:
            return
        candidate = document[index]
        if isinstance(candidate, nodes.field_list):
            biblioindex = document.first_child_not_matching_class(
                  (nodes.Titular, nodes.Decorative, nodes.meta))
            nodelist = self.extract_bibliographic(candidate)
            del document[index]         # untransformed field list (candidate)
            document[biblioindex:biblioindex] = nodelist

    def extract_bibliographic(self, field_list):
        docinfo = nodes.docinfo()
        bibliofields = self.language.bibliographic_fields
        labels = self.language.labels
        topics = {'dedication': None, 'abstract': None}
        for field in field_list:
            try:
                name = field[0][0].astext()
                normedname = nodes.fully_normalize_name(name)
                if not (len(field) == 2 and normedname in bibliofields
                        and self.check_empty_biblio_field(field, name)):
                    raise TransformError
                canonical = bibliofields[normedname]
                biblioclass = self.biblio_nodes[canonical]
                if issubclass(biblioclass, nodes.TextElement):
                    if not self.check_compound_biblio_field(field, name):
                        raise TransformError
                    utils.clean_rcs_keywords(
                          field[1][0], self.rcs_keyword_substitutions)
                    docinfo.append(biblioclass('', '', *field[1][0]))
                elif issubclass(biblioclass, nodes.authors):
                    self.extract_authors(field, name, docinfo)
                elif issubclass(biblioclass, nodes.topic):
                    if topics[canonical]:
                        field[-1] += self.document.reporter.warning(
                            'There can only be one "%s" field.' % name,
                            base_node=field)
                        raise TransformError
                    title = nodes.title(name, labels[canonical])
                    title[0].rawsource = labels[canonical]
                    topics[canonical] = biblioclass(
                        '', title, classes=[canonical], *field[1].children)
                else:
                    docinfo.append(biblioclass('', *field[1].children))
            except TransformError:
                if len(field[-1]) == 1 \
                       and isinstance(field[-1][0], nodes.paragraph):
                    utils.clean_rcs_keywords(
                        field[-1][0], self.rcs_keyword_substitutions)
                # if normedname not in bibliofields:
                classvalue = nodes.make_id(normedname)
                if classvalue:
                    field['classes'].append(classvalue)
                docinfo.append(field)
        nodelist = []
        if len(docinfo) != 0:
            nodelist.append(docinfo)
        for name in ('dedication', 'abstract'):
            if topics[name]:
                nodelist.append(topics[name])
        return nodelist

    def check_empty_biblio_field(self, field, name):
        if len(field[-1]) < 1:
            field[-1] += self.document.reporter.warning(
                  f'Cannot extract empty bibliographic field "{name}".',
                  base_node=field)
            return False
        return True

    def check_compound_biblio_field(self, field, name):
        # Check that the `field` body contains a single paragraph
        # (i.e. it must *not* be a compound element).
        f_body = field[-1]
        if len(f_body) == 1 and isinstance(f_body[0], nodes.paragraph):
            return True
        # Restore single author name with initial (E. Xampl) parsed as
        # enumerated list
        # https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#enumerated-lists
        if (isinstance(f_body[0], nodes.enumerated_list)
            and '\n' not in f_body.rawsource.strip()):
            # parse into a dummy document and use created nodes
            _document = utils.new_document('*DocInfo transform*',
                                           field.document.settings)
            parser = parsers.rst.Parser()
            parser.parse('\\'+f_body.rawsource, _document)
            if (len(_document.children) == 1
                and isinstance(_document.children[0], nodes.paragraph)):
                f_body.children = _document.children
                return True
        # Check failed, add a warning
        content = [f'<{e.tagname}>' for e in f_body.children]
        if len(content) > 1:
            content = '[' + ', '.join(content) + ']'
        else:
            content = 'a ' + content[0]
        f_body += self.document.reporter.warning(
                      f'Bibliographic field "{name}"\nmust contain '
                      f'a single <paragraph>, not {content}.',
                      base_node=field)
        return False

    rcs_keyword_substitutions = [
          (re.compile(r'\$' r'Date: (\d\d\d\d)[-/](\d\d)[-/](\d\d)[ T][\d:]+'
                      r'[^$]* \$', re.IGNORECASE), r'\1-\2-\3'),
          (re.compile(r'\$' r'RCSfile: (.+),v \$', re.IGNORECASE), r'\1'),
          (re.compile(r'\$[a-zA-Z]+: (.+) \$'), r'\1')]

    def extract_authors(self, field, name, docinfo):
        try:
            if len(field[1]) == 1:
                if isinstance(field[1][0], nodes.paragraph):
                    authors = self.authors_from_one_paragraph(field)
                elif isinstance(field[1][0], nodes.bullet_list):
                    authors = self.authors_from_bullet_list(field)
                else:
                    raise TransformError
            else:
                authors = self.authors_from_paragraphs(field)
            authornodes = [nodes.author('', '', *author)
                           for author in authors if author]
            if len(authornodes) >= 1:
                docinfo.append(nodes.authors('', *authornodes))
            else:
                raise TransformError
        except TransformError:
            field[-1] += self.document.reporter.warning(
                f'Cannot extract "{name}" from bibliographic field:\n'
                f'Bibliographic field "{name}" must contain either\n'
                ' a single paragraph (with author names separated by one of '
                f'"{"".join(self.language.author_separators)}"),\n'
                ' multiple paragraphs (one per author),\n'
                ' or a bullet list with one author name per item.\n'
                'Note: Leading initials can cause (mis)recognizing names '
                'as enumerated list.',
                base_node=field)
            raise

    def authors_from_one_paragraph(self, field):
        """Return list of Text nodes with author names in `field`.

        Author names must be separated by one of the "autor separators"
        defined for the document language (default: ";" or ",").
        """
        # @@ keep original formatting? (e.g. ``:authors: A. Test, *et-al*``)
        text = ''.join(str(node)
                       for node in field[1].findall(nodes.Text))
        if not text:
            raise TransformError
        for authorsep in self.language.author_separators:
            # don't split at escaped `authorsep`:
            pattern = '(?<!\x00)%s' % authorsep
            authornames = re.split(pattern, text)
            if len(authornames) > 1:
                break
        authornames = (name.strip() for name in authornames)
        return [[nodes.Text(name)] for name in authornames if name]

    def authors_from_bullet_list(self, field):
        authors = []
        for item in field[1][0]:
            if isinstance(item, nodes.comment):
                continue
            if len(item) != 1 or not isinstance(item[0], nodes.paragraph):
                raise TransformError
            authors.append(item[0].children)
        if not authors:
            raise TransformError
        return authors

    def authors_from_paragraphs(self, field):
        for item in field[1]:
            if not isinstance(item, (nodes.paragraph, nodes.comment)):
                raise TransformError
        authors = [item.children for item in field[1]
                   if not isinstance(item, nodes.comment)]
        return authors

env_pyrecodes/lib/python3.9/site-packages/docutils/transforms/frontmatter.py
NikolaBlagojevic/pyrecodes
# $Id: references.py 9312 2022-12-19 20:43:08Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Transforms for resolving references.
"""

__docformat__ = 'reStructuredText'

from docutils import nodes, utils
from docutils.transforms import Transform


class PropagateTargets(Transform):

    """
    Propagate empty internal targets to the next element.

    Given the following nodes::

        <target ids="internal1" names="internal1">
        <target anonymous="1" ids="id1">
        <target ids="internal2" names="internal2">
        <paragraph>
            This is a test.

    PropagateTargets propagates the ids and names of the internal
    targets preceding the paragraph to the paragraph itself::

        <target refid="internal1">
        <target anonymous="1" refid="id1">
        <target refid="internal2">
        <paragraph ids="internal2 id1 internal1" names="internal2 internal1">
            This is a test.
    """

    default_priority = 260

    def apply(self):
        for target in self.document.findall(nodes.target):
            # Only block-level targets without reference (like ".. _target:"):
            if (isinstance(target.parent, nodes.TextElement)
                or (target.hasattr('refid') or target.hasattr('refuri')
                    or target.hasattr('refname'))):
                continue
            assert len(target) == 0, 'error: block-level target has children'
            next_node = target.next_node(ascend=True)
            # skip system messages (may be removed by universal.FilterMessages)
            while isinstance(next_node, nodes.system_message):
                next_node = next_node.next_node(ascend=True, descend=False)
            # Do not move names and ids into Invisibles (we'd lose the
            # attributes) or different Targetables (e.g. footnotes).
            if (next_node is None
                or isinstance(next_node, (nodes.Invisible, nodes.Targetable))
                and not isinstance(next_node, nodes.target)):
                continue
            next_node['ids'].extend(target['ids'])
            next_node['names'].extend(target['names'])
            # Set defaults for next_node.expect_referenced_by_name/id.
            if not hasattr(next_node, 'expect_referenced_by_name'):
                next_node.expect_referenced_by_name = {}
            if not hasattr(next_node, 'expect_referenced_by_id'):
                next_node.expect_referenced_by_id = {}
            for id in target['ids']:
                # Update IDs to node mapping.
                self.document.ids[id] = next_node
                # If next_node is referenced by id ``id``, this
                # target shall be marked as referenced.
                next_node.expect_referenced_by_id[id] = target
            for name in target['names']:
                next_node.expect_referenced_by_name[name] = target
            # If there are any expect_referenced_by_... attributes
            # in target set, copy them to next_node.
            next_node.expect_referenced_by_name.update(
                getattr(target, 'expect_referenced_by_name', {}))
            next_node.expect_referenced_by_id.update(
                getattr(target, 'expect_referenced_by_id', {}))
            # Set refid to point to the first former ID of target
            # which is now an ID of next_node.
            target['refid'] = target['ids'][0]
            # Clear ids and names; they have been moved to
            # next_node.
            target['ids'] = []
            target['names'] = []
            self.document.note_refid(target)


class AnonymousHyperlinks(Transform):

    """
    Link anonymous references to targets.  Given::

        <paragraph>
            <reference anonymous="1">
                internal
            <reference anonymous="1">
                external
        <target anonymous="1" ids="id1">
        <target anonymous="1" ids="id2" refuri="http://external">

    Corresponding references are linked via "refid" or resolved via "refuri"::

        <paragraph>
            <reference anonymous="1" refid="id1">
                text
            <reference anonymous="1" refuri="http://external">
                external
        <target anonymous="1" ids="id1">
        <target anonymous="1" ids="id2" refuri="http://external">
    """

    default_priority = 440

    def apply(self):
        anonymous_refs = []
        anonymous_targets = []
        for node in self.document.findall(nodes.reference):
            if node.get('anonymous'):
                anonymous_refs.append(node)
        for node in self.document.findall(nodes.target):
            if node.get('anonymous'):
                anonymous_targets.append(node)
        if len(anonymous_refs) != len(anonymous_targets):
            msg = self.document.reporter.error(
                  'Anonymous hyperlink mismatch: %s references but %s '
                  'targets.\nSee "backrefs" attribute for IDs.'
                  % (len(anonymous_refs), len(anonymous_targets)))
            msgid = self.document.set_id(msg)
            for ref in anonymous_refs:
                prb = nodes.problematic(
                      ref.rawsource, ref.rawsource, refid=msgid)
                prbid = self.document.set_id(prb)
                msg.add_backref(prbid)
                ref.replace_self(prb)
            return
        for ref, target in zip(anonymous_refs, anonymous_targets):
            target.referenced = 1
            while True:
                if target.hasattr('refuri'):
                    ref['refuri'] = target['refuri']
                    ref.resolved = 1
                    break
                else:
                    if not target['ids']:
                        # Propagated target.
                        target = self.document.ids[target['refid']]
                        continue
                    ref['refid'] = target['ids'][0]
                    self.document.note_refid(ref)
                    break


class IndirectHyperlinks(Transform):

    """
    a) Indirect external references::

           <paragraph>
               <reference refname="indirect external">
                   indirect external
           <target id="id1" name="direct external"
               refuri="http://indirect">
           <target id="id2" name="indirect external"
               refname="direct external">

       The "refuri" attribute is migrated back to all indirect targets
       from the final direct target (i.e. a target not referring to
       another indirect target)::

           <paragraph>
               <reference refname="indirect external">
                   indirect external
           <target id="id1" name="direct external"
               refuri="http://indirect">
           <target id="id2" name="indirect external"
               refuri="http://indirect">

       Once the attribute is migrated, the preexisting "refname" attribute
       is dropped.

    b) Indirect internal references::

           <target id="id1" name="final target">
           <paragraph>
               <reference refname="indirect internal">
                   indirect internal
           <target id="id2" name="indirect internal 2"
               refname="final target">
           <target id="id3" name="indirect internal"
               refname="indirect internal 2">

       Targets which indirectly refer to an internal target become one-hop
       indirect (their "refid" attributes are directly set to the internal
       target's "id"). References which indirectly refer to an internal
       target become direct internal references::

           <target id="id1" name="final target">
           <paragraph>
               <reference refid="id1">
                   indirect internal
           <target id="id2" name="indirect internal 2" refid="id1">
           <target id="id3" name="indirect internal" refid="id1">
    """

    default_priority = 460

    def apply(self):
        for target in self.document.indirect_targets:
            if not target.resolved:
                self.resolve_indirect_target(target)
            self.resolve_indirect_references(target)

    def resolve_indirect_target(self, target):
        refname = target.get('refname')
        if refname is None:
            reftarget_id = target['refid']
        else:
            reftarget_id = self.document.nameids.get(refname)
            if not reftarget_id:
                # Check the unknown_reference_resolvers
                for resolver_function in \
                        self.document.transformer.unknown_reference_resolvers:
                    if resolver_function(target):
                        break
                else:
                    self.nonexistent_indirect_target(target)
                return
        reftarget = self.document.ids[reftarget_id]
        reftarget.note_referenced_by(id=reftarget_id)
        if (isinstance(reftarget, nodes.target)
            and not reftarget.resolved
            and reftarget.hasattr('refname')):
            if hasattr(target, 'multiply_indirect'):
                self.circular_indirect_reference(target)
                return
            target.multiply_indirect = 1
            self.resolve_indirect_target(reftarget)  # multiply indirect
            del target.multiply_indirect
        if reftarget.hasattr('refuri'):
            target['refuri'] = reftarget['refuri']
            if 'refid' in target:
                del target['refid']
        elif reftarget.hasattr('refid'):
            target['refid'] = reftarget['refid']
            self.document.note_refid(target)
        else:
            if reftarget['ids']:
                target['refid'] = reftarget_id
                self.document.note_refid(target)
            else:
                self.nonexistent_indirect_target(target)
                return
        if refname is not None:
            del target['refname']
        target.resolved = 1

    def nonexistent_indirect_target(self, target):
        if target['refname'] in self.document.nameids:
            self.indirect_target_error(target, 'which is a duplicate, and '
                                       'cannot be used as a unique reference')
        else:
            self.indirect_target_error(target, 'which does not exist')

    def circular_indirect_reference(self, target):
        self.indirect_target_error(target, 'forming a circular reference')

    def indirect_target_error(self, target, explanation):
        naming = ''
        reflist = []
        if target['names']:
            naming = '"%s" ' % target['names'][0]
        for name in target['names']:
            reflist.extend(self.document.refnames.get(name, []))
        for id in target['ids']:
            reflist.extend(self.document.refids.get(id, []))
        if target['ids']:
            naming += '(id="%s")' % target['ids'][0]
        msg = self.document.reporter.error(
              'Indirect hyperlink target %s refers to target "%s", %s.'
              % (naming, target['refname'], explanation), base_node=target)
        msgid = self.document.set_id(msg)
        for ref in utils.uniq(reflist):
            prb = nodes.problematic(
                  ref.rawsource, ref.rawsource, refid=msgid)
            prbid = self.document.set_id(prb)
            msg.add_backref(prbid)
            ref.replace_self(prb)
        target.resolved = 1

    def resolve_indirect_references(self, target):
        if target.hasattr('refid'):
            attname = 'refid'
            call_method = self.document.note_refid
        elif target.hasattr('refuri'):
            attname = 'refuri'
            call_method = None
        else:
            return
        attval = target[attname]
        for name in target['names']:
            reflist = self.document.refnames.get(name, [])
            if reflist:
                target.note_referenced_by(name=name)
            for ref in reflist:
                if ref.resolved:
                    continue
                del ref['refname']
                ref[attname] = attval
                if call_method:
                    call_method(ref)
                ref.resolved = 1
                if isinstance(ref, nodes.target):
                    self.resolve_indirect_references(ref)
        for id in target['ids']:
            reflist = self.document.refids.get(id, [])
            if reflist:
                target.note_referenced_by(id=id)
            for ref in reflist:
                if ref.resolved:
                    continue
                del ref['refid']
                ref[attname] = attval
                if call_method:
                    call_method(ref)
                ref.resolved = 1
                if isinstance(ref, nodes.target):
                    self.resolve_indirect_references(ref)


class ExternalTargets(Transform):

    """
    Given::

        <paragraph>
            <reference refname="direct external">
                direct external
        <target id="id1" name="direct external" refuri="http://direct">

    The "refname" attribute is replaced by the direct "refuri" attribute::

        <paragraph>
            <reference refuri="http://direct">
                direct external
        <target id="id1" name="direct external" refuri="http://direct">
    """

    default_priority = 640

    def apply(self):
        for target in self.document.findall(nodes.target):
            if target.hasattr('refuri'):
                refuri = target['refuri']
                for name in target['names']:
                    reflist = self.document.refnames.get(name, [])
                    if reflist:
                        target.note_referenced_by(name=name)
                    for ref in reflist:
                        if ref.resolved:
                            continue
                        del ref['refname']
                        ref['refuri'] = refuri
                        ref.resolved = 1


class InternalTargets(Transform):

    default_priority = 660

    def apply(self):
        for target in self.document.findall(nodes.target):
            if not target.hasattr('refuri') and not target.hasattr('refid'):
                self.resolve_reference_ids(target)

    def resolve_reference_ids(self, target):
        """
        Given::

            <paragraph>
                <reference refname="direct internal">
                    direct internal
            <target id="id1" name="direct internal">

        The "refname" attribute is replaced by "refid" linking to the target's
        "id"::

            <paragraph>
                <reference refid="id1">
                    direct internal
            <target id="id1" name="direct internal">
        """
        for name in target['names']:
            refid = self.document.nameids.get(name)
            reflist = self.document.refnames.get(name, [])
            if reflist:
                target.note_referenced_by(name=name)
            for ref in reflist:
                if ref.resolved:
                    continue
                if refid:
                    del ref['refname']
                    ref['refid'] = refid
                ref.resolved = 1


class Footnotes(Transform):

    """
    Assign numbers to autonumbered footnotes, and resolve links to footnotes,
    citations, and their references.

    Given the following ``document`` as input::

        <document>
            <paragraph>
                A labeled autonumbered footnote reference:
                <footnote_reference auto="1" id="id1" refname="footnote">
            <paragraph>
                An unlabeled autonumbered footnote reference:
                <footnote_reference auto="1" id="id2">
            <footnote auto="1" id="id3">
                <paragraph>
                    Unlabeled autonumbered footnote.
            <footnote auto="1" id="footnote" name="footnote">
                <paragraph>
                    Labeled autonumbered footnote.

    Auto-numbered footnotes have attribute ``auto="1"`` and no label.
    Auto-numbered footnote_references have no reference text (they're
    empty elements). When resolving the numbering, a ``label`` element
    is added to the beginning of the ``footnote``, and reference text
    to the ``footnote_reference``.

    The transformed result will be::

        <document>
            <paragraph>
                A labeled autonumbered footnote reference:
                <footnote_reference auto="1" id="id1" refid="footnote">
                    2
            <paragraph>
                An unlabeled autonumbered footnote reference:
                <footnote_reference auto="1" id="id2" refid="id3">
                    1
            <footnote auto="1" id="id3" backrefs="id2">
                <label>
                    1
                <paragraph>
                    Unlabeled autonumbered footnote.
            <footnote auto="1" id="footnote" name="footnote" backrefs="id1">
                <label>
                    2
                <paragraph>
                    Labeled autonumbered footnote.

    Note that the footnotes are not in the same order as the references.

    The labels and reference text are added to the auto-numbered ``footnote``
    and ``footnote_reference`` elements.  Footnote elements are backlinked to
    their references via "refids" attributes.  References are assigned "id"
    and "refid" attributes.

    After adding labels and reference text, the "auto" attributes can be
    ignored.
    """

    default_priority = 620

    autofootnote_labels = None
    """Keep track of unlabeled autonumbered footnotes."""

    symbols = [
          # Entries 1-4 and 6 below are from section 12.51 of
          # The Chicago Manual of Style, 14th edition.
          '*',                          # asterisk/star
          '\u2020',                    # † &dagger; dagger
          '\u2021',                    # ‡ &Dagger; double dagger
          '\u00A7',                    # § &sect; section mark
          '\u00B6',                    # ¶ &para; paragraph mark (pilcrow)
                                        # (parallels ['||'] in CMoS)
          '#',                          # number sign
          # The entries below were chosen arbitrarily.
          '\u2660',                    # ♠ &spades; spade suit
          '\u2665',                    # ♡ &hearts; heart suit
          '\u2666',                    # ♢ &diams; diamond suit
          '\u2663',                    # ♣ &clubs; club suit
          ]

    def apply(self):
        self.autofootnote_labels = []
        startnum = self.document.autofootnote_start
        self.document.autofootnote_start = self.number_footnotes(startnum)
        self.number_footnote_references(startnum)
        self.symbolize_footnotes()
        self.resolve_footnotes_and_citations()

    def number_footnotes(self, startnum):
        """
        Assign numbers to autonumbered footnotes.

        For labeled autonumbered footnotes, copy the number over to
        corresponding footnote references.
        """
        for footnote in self.document.autofootnotes:
            while True:
                label = str(startnum)
                startnum += 1
                if label not in self.document.nameids:
                    break
            footnote.insert(0, nodes.label('', label))
            for name in footnote['names']:
                for ref in self.document.footnote_refs.get(name, []):
                    ref += nodes.Text(label)
                    ref.delattr('refname')
                    assert len(footnote['ids']) == len(ref['ids']) == 1
                    ref['refid'] = footnote['ids'][0]
                    footnote.add_backref(ref['ids'][0])
                    self.document.note_refid(ref)
                    ref.resolved = 1
            if not footnote['names'] and not footnote['dupnames']:
                footnote['names'].append(label)
                self.document.note_explicit_target(footnote, footnote)
                self.autofootnote_labels.append(label)
        return startnum

    def number_footnote_references(self, startnum):
        """Assign numbers to autonumbered footnote references."""
        i = 0
        for ref in self.document.autofootnote_refs:
            if ref.resolved or ref.hasattr('refid'):
                continue
            try:
                label = self.autofootnote_labels[i]
            except IndexError:
                msg = self.document.reporter.error(
                      'Too many autonumbered footnote references: only %s '
                      'corresponding footnotes available.'
                      % len(self.autofootnote_labels), base_node=ref)
                msgid = self.document.set_id(msg)
                for ref in self.document.autofootnote_refs[i:]:
                    if ref.resolved or ref.hasattr('refname'):
                        continue
                    prb = nodes.problematic(
                          ref.rawsource, ref.rawsource, refid=msgid)
                    prbid = self.document.set_id(prb)
                    msg.add_backref(prbid)
                    ref.replace_self(prb)
                break
            ref += nodes.Text(label)
            id = self.document.nameids[label]
            footnote = self.document.ids[id]
            ref['refid'] = id
            self.document.note_refid(ref)
            assert len(ref['ids']) == 1
            footnote.add_backref(ref['ids'][0])
            ref.resolved = 1
            i += 1

    def symbolize_footnotes(self):
        """Add symbols indexes to "[*]"-style footnotes and references."""
        labels = []
        for footnote in self.document.symbol_footnotes:
            reps, index = divmod(self.document.symbol_footnote_start,
                                 len(self.symbols))
            labeltext = self.symbols[index] * (reps + 1)
            labels.append(labeltext)
            footnote.insert(0, nodes.label('', labeltext))
            self.document.symbol_footnote_start += 1
            self.document.set_id(footnote)
        i = 0
        for ref in self.document.symbol_footnote_refs:
            try:
                ref += nodes.Text(labels[i])
            except IndexError:
                msg = self.document.reporter.error(
                      'Too many symbol footnote references: only %s '
                      'corresponding footnotes available.' % len(labels),
                      base_node=ref)
                msgid = self.document.set_id(msg)
                for ref in self.document.symbol_footnote_refs[i:]:
                    if ref.resolved or ref.hasattr('refid'):
                        continue
                    prb = nodes.problematic(
                          ref.rawsource, ref.rawsource, refid=msgid)
                    prbid = self.document.set_id(prb)
                    msg.add_backref(prbid)
                    ref.replace_self(prb)
                break
            footnote = self.document.symbol_footnotes[i]
            assert len(footnote['ids']) == 1
            ref['refid'] = footnote['ids'][0]
            self.document.note_refid(ref)
            footnote.add_backref(ref['ids'][0])
            i += 1

    def resolve_footnotes_and_citations(self):
        """
        Link manually-labeled footnotes and citations to/from their
        references.
        """
        for footnote in self.document.footnotes:
            for label in footnote['names']:
                if label in self.document.footnote_refs:
                    reflist = self.document.footnote_refs[label]
                    self.resolve_references(footnote, reflist)
        for citation in self.document.citations:
            for label in citation['names']:
                if label in self.document.citation_refs:
                    reflist = self.document.citation_refs[label]
                    self.resolve_references(citation, reflist)

    def resolve_references(self, note, reflist):
        assert len(note['ids']) == 1
        id = note['ids'][0]
        for ref in reflist:
            if ref.resolved:
                continue
            ref.delattr('refname')
            ref['refid'] = id
            assert len(ref['ids']) == 1
            note.add_backref(ref['ids'][0])
            ref.resolved = 1
        note.resolved = 1


class CircularSubstitutionDefinitionError(Exception):
    pass


class Substitutions(Transform):

    """
    Given the following ``document`` as input::

        <document>
            <paragraph>
                The
                <substitution_reference refname="biohazard">
                    biohazard
                 symbol is deservedly scary-looking.
            <substitution_definition name="biohazard">
                <image alt="biohazard" uri="biohazard.png">

    The ``substitution_reference`` will simply be replaced by the
    contents of the corresponding ``substitution_definition``.

    The transformed result will be::

        <document>
            <paragraph>
                The
                <image alt="biohazard" uri="biohazard.png">
                 symbol is deservedly scary-looking.
            <substitution_definition name="biohazard">
                <image alt="biohazard" uri="biohazard.png">
    """

    default_priority = 220
    """The Substitutions transform has to be applied very early, before
    `docutils.transforms.frontmatter.DocTitle` and others."""

    def apply(self):
        defs = self.document.substitution_defs
        normed = self.document.substitution_names
        nested = {}
        line_length_limit = getattr(self.document.settings,
                                    "line_length_limit", 10000)

        subreflist = list(self.document.findall(nodes.substitution_reference))
        for ref in subreflist:
            msg = ''
            refname = ref['refname']
            if refname in defs:
                key = refname
            else:
                normed_name = refname.lower()
                key = normed.get(normed_name, None)
            if key is None:
                msg = self.document.reporter.error(
                      'Undefined substitution referenced: "%s".'
                      % refname, base_node=ref)
            else:
                subdef = defs[key]
                if len(subdef.astext()) > line_length_limit:
                    msg = self.document.reporter.error(
                            'Substitution definition "%s" exceeds the'
                            ' line-length-limit.' % key)
            if msg:
                msgid = self.document.set_id(msg)
                prb = nodes.problematic(
                      ref.rawsource, ref.rawsource, refid=msgid)
                prbid = self.document.set_id(prb)
                msg.add_backref(prbid)
                ref.replace_self(prb)
                continue

            parent = ref.parent
            index = parent.index(ref)
            if ('ltrim' in subdef.attributes
                or 'trim' in subdef.attributes):
                if index > 0 and isinstance(parent[index - 1],
                                            nodes.Text):
                    parent[index - 1] = parent[index - 1].rstrip()
            if ('rtrim' in subdef.attributes
                or 'trim' in subdef.attributes):
                if (len(parent) > index + 1
                    and isinstance(parent[index + 1], nodes.Text)):
                    parent[index + 1] = parent[index + 1].lstrip()
            subdef_copy = subdef.deepcopy()
            try:
                # Take care of nested substitution references:
                for nested_ref in subdef_copy.findall(
                        nodes.substitution_reference):
                    nested_name = normed[nested_ref['refname'].lower()]
                    if nested_name in nested.setdefault(nested_name, []):
                        raise CircularSubstitutionDefinitionError
                    nested[nested_name].append(key)
                    nested_ref['ref-origin'] = ref
                    subreflist.append(nested_ref)
            except CircularSubstitutionDefinitionError:
                parent = ref.parent
                if isinstance(parent, nodes.substitution_definition):
                    msg = self.document.reporter.error(
                        'Circular substitution definition detected:',
                        nodes.literal_block(parent.rawsource,
                                            parent.rawsource),
                        line=parent.line, base_node=parent)
                    parent.replace_self(msg)
                else:
                    # find original ref substitution which caused this error
                    ref_origin = ref
                    while ref_origin.hasattr('ref-origin'):
                        ref_origin = ref_origin['ref-origin']
                    msg = self.document.reporter.error(
                        'Circular substitution definition referenced: '
                        '"%s".' % refname, base_node=ref_origin)
                    msgid = self.document.set_id(msg)
                    prb = nodes.problematic(
                        ref.rawsource, ref.rawsource, refid=msgid)
                    prbid = self.document.set_id(prb)
                    msg.add_backref(prbid)
                    ref.replace_self(prb)
                continue
            ref.replace_self(subdef_copy.children)
            # register refname of the replacement node(s)
            # (needed for resolution of references)
            for node in subdef_copy.children:
                if isinstance(node, nodes.Referential):
                    # HACK: verify refname attribute exists.
                    # Test with docs/dev/todo.txt, see. |donate|
                    if 'refname' in node:
                        self.document.note_refname(node)


class TargetNotes(Transform):

    """
    Creates a footnote for each external target in the text, and corresponding
    footnote references after each reference.
    """

    default_priority = 540
    """The TargetNotes transform has to be applied after `IndirectHyperlinks`
    but before `Footnotes`."""

    def __init__(self, document, startnode):
        Transform.__init__(self, document, startnode=startnode)

        self.classes = startnode.details.get('class', [])

    def apply(self):
        notes = {}
        nodelist = []
        for target in self.document.findall(nodes.target):
            # Only external targets.
            if not target.hasattr('refuri'):
                continue
            names = target['names']
            refs = []
            for name in names:
                refs.extend(self.document.refnames.get(name, []))
            if not refs:
                continue
            footnote = self.make_target_footnote(target['refuri'], refs,
                                                 notes)
            if target['refuri'] not in notes:
                notes[target['refuri']] = footnote
                nodelist.append(footnote)
        # Take care of anonymous references.
        for ref in self.document.findall(nodes.reference):
            if not ref.get('anonymous'):
                continue
            if ref.hasattr('refuri'):
                footnote = self.make_target_footnote(ref['refuri'], [ref],
                                                     notes)
                if ref['refuri'] not in notes:
                    notes[ref['refuri']] = footnote
                    nodelist.append(footnote)
        self.startnode.replace_self(nodelist)

    def make_target_footnote(self, refuri, refs, notes):
        if refuri in notes:  # duplicate?
            footnote = notes[refuri]
            assert len(footnote['names']) == 1
            footnote_name = footnote['names'][0]
        else:                           # original
            footnote = nodes.footnote()
            footnote_id = self.document.set_id(footnote)
            # Use uppercase letters and a colon; they can't be
            # produced inside names by the parser.
            footnote_name = 'TARGET_NOTE: ' + footnote_id
            footnote['auto'] = 1
            footnote['names'] = [footnote_name]
            footnote_paragraph = nodes.paragraph()
            footnote_paragraph += nodes.reference('', refuri, refuri=refuri)
            footnote += footnote_paragraph
            self.document.note_autofootnote(footnote)
            self.document.note_explicit_target(footnote, footnote)
        for ref in refs:
            if isinstance(ref, nodes.target):
                continue
            refnode = nodes.footnote_reference(refname=footnote_name, auto=1)
            refnode['classes'] += self.classes
            self.document.note_autofootnote_ref(refnode)
            self.document.note_footnote_ref(refnode)
            index = ref.parent.index(ref) + 1
            reflist = [refnode]
            if not utils.get_trim_footnote_ref_space(self.document.settings):
                if self.classes:
                    reflist.insert(
                        0, nodes.inline(text=' ', Classes=self.classes))
                else:
                    reflist.insert(0, nodes.Text(' '))
            ref.parent.insert(index, reflist)
        return footnote


class DanglingReferences(Transform):

    """
    Check for dangling references (incl. footnote & citation) and for
    unreferenced targets.
    """

    default_priority = 850

    def apply(self):
        visitor = DanglingReferencesVisitor(
            self.document,
            self.document.transformer.unknown_reference_resolvers)
        self.document.walk(visitor)
        # *After* resolving all references, check for unreferenced
        # targets:
        for target in self.document.findall(nodes.target):
            if not target.referenced:
                if target.get('anonymous'):
                    # If we have unreferenced anonymous targets, there
                    # is already an error message about anonymous
                    # hyperlink mismatch; no need to generate another
                    # message.
                    continue
                if target['names']:
                    naming = target['names'][0]
                elif target['ids']:
                    naming = target['ids'][0]
                else:
                    # Hack: Propagated targets always have their refid
                    # attribute set.
                    naming = target['refid']
                self.document.reporter.info(
                    'Hyperlink target "%s" is not referenced.'
                    % naming, base_node=target)


class DanglingReferencesVisitor(nodes.SparseNodeVisitor):

    def __init__(self, document, unknown_reference_resolvers):
        nodes.SparseNodeVisitor.__init__(self, document)
        self.document = document
        self.unknown_reference_resolvers = unknown_reference_resolvers

    def unknown_visit(self, node):
        pass

    def visit_reference(self, node):
        if node.resolved or not node.hasattr('refname'):
            return
        refname = node['refname']
        id = self.document.nameids.get(refname)
        if id is None:
            for resolver_function in self.unknown_reference_resolvers:
                if resolver_function(node):
                    break
            else:
                if (getattr(self.document.settings, 'use_bibtex', False)
                    and isinstance(node, nodes.citation_reference)):
                    # targets added from BibTeX database by LaTeX
                    node.resolved = True
                    return
                if refname in self.document.nameids:
                    msg = self.document.reporter.error(
                        'Duplicate target name, cannot be used as a unique '
                        'reference: "%s".' % (node['refname']), base_node=node)
                else:
                    msg = self.document.reporter.error(
                        f'Unknown target name: "{node["refname"]}".',
                        base_node=node)
                msgid = self.document.set_id(msg)
                prb = nodes.problematic(
                      node.rawsource, node.rawsource, refid=msgid)
                try:
                    prbid = node['ids'][0]
                except IndexError:
                    prbid = self.document.set_id(prb)
                msg.add_backref(prbid)
                node.replace_self(prb)
        else:
            del node['refname']
            node['refid'] = id
            self.document.ids[id].note_referenced_by(id=id)
            node.resolved = True

    visit_footnote_reference = visit_citation_reference = visit_reference

env_pyrecodes/lib/python3.9/site-packages/docutils/transforms/references.py
NikolaBlagojevic/pyrecodes
# $Id: universal.py 9320 2023-01-17 15:40:15Z milde $
# Authors: David Goodger <goodger@python.org>; Ueli Schlaepfer; Günter Milde
# Maintainer: docutils-develop@lists.sourceforge.net
# Copyright: This module has been placed in the public domain.

"""
Transforms needed by most or all documents:

- `Decorations`: Generate a document's header & footer.
- `ExposeInternals`: Expose internal attributes.
- `Messages`: Placement of system messages generated after parsing.
- `FilterMessages`: Remove system messages below verbosity threshold.
- `TestMessages`: Like `Messages`, used on test runs.
- `StripComments`: Remove comment elements from the document tree.
- `StripClassesAndElements`: Remove elements with classes
  in `self.document.settings.strip_elements_with_classes`
  and class values in `self.document.settings.strip_classes`.
- `SmartQuotes`: Replace ASCII quotation marks with typographic form.
"""

__docformat__ = 'reStructuredText'

import re
import time
from docutils import nodes, utils
from docutils.transforms import Transform
from docutils.utils import smartquotes


class Decorations(Transform):

    """
    Populate a document's decoration element (header, footer).
    """

    default_priority = 820

    def apply(self):
        header_nodes = self.generate_header()
        if header_nodes:
            decoration = self.document.get_decoration()
            header = decoration.get_header()
            header.extend(header_nodes)
        footer_nodes = self.generate_footer()
        if footer_nodes:
            decoration = self.document.get_decoration()
            footer = decoration.get_footer()
            footer.extend(footer_nodes)

    def generate_header(self):
        return None

    def generate_footer(self):
        # @@@ Text is hard-coded for now.
        # Should be made dynamic (language-dependent).
        # @@@ Use timestamp from the `SOURCE_DATE_EPOCH`_ environment variable
        # for the datestamp?
        # See https://sourceforge.net/p/docutils/patches/132/
        # and https://reproducible-builds.org/specs/source-date-epoch/
        settings = self.document.settings
        if (settings.generator or settings.datestamp
            or settings.source_link or settings.source_url):
            text = []
            if (settings.source_link and settings._source
                or settings.source_url):
                if settings.source_url:
                    source = settings.source_url
                else:
                    source = utils.relative_path(settings._destination,
                                                 settings._source)
                text.extend([
                    nodes.reference('', 'View document source',
                                    refuri=source),
                    nodes.Text('.\n')])
            if settings.datestamp:
                datestamp = time.strftime(settings.datestamp, time.gmtime())
                text.append(nodes.Text('Generated on: ' + datestamp + '.\n'))
            if settings.generator:
                text.extend([
                    nodes.Text('Generated by '),
                    nodes.reference('', 'Docutils',
                                    refuri='https://docutils.sourceforge.io/'),
                    nodes.Text(' from '),
                    nodes.reference('', 'reStructuredText',
                                    refuri='https://docutils.sourceforge.io/'
                                    'rst.html'),
                    nodes.Text(' source.\n')])
            return [nodes.paragraph('', '', *text)]
        else:
            return None


class ExposeInternals(Transform):

    """
    Expose internal attributes if ``expose_internals`` setting is set.
    """

    default_priority = 840

    def not_Text(self, node):
        return not isinstance(node, nodes.Text)

    def apply(self):
        if self.document.settings.expose_internals:
            for node in self.document.findall(self.not_Text):
                for att in self.document.settings.expose_internals:
                    value = getattr(node, att, None)
                    if value is not None:
                        node['internal:' + att] = value


class Messages(Transform):

    """
    Place any system messages generated after parsing into a dedicated section
    of the document.
    """

    default_priority = 860

    def apply(self):
        messages = self.document.transform_messages
        loose_messages = [msg for msg in messages if not msg.parent]
        if loose_messages:
            section = nodes.section(classes=['system-messages'])
            # @@@ get this from the language module?
            section += nodes.title('', 'Docutils System Messages')
            section += loose_messages
            self.document.transform_messages[:] = []
            self.document += section


# TODO: fix bug #435:

# Messages are filtered at a very late stage
# This breaks the link from inline error messages to the corresponding
# system message at the end of document.

class FilterMessages(Transform):

    """
    Remove system messages below verbosity threshold.

    Convert <problematic> nodes referencing removed messages to <Text>.
    Remove "System Messages" section if empty.
    """

    default_priority = 870

    def apply(self):
        for node in tuple(self.document.findall(nodes.system_message)):
            if node['level'] < self.document.reporter.report_level:
                node.parent.remove(node)
                try:  # also remove id-entry
                    del self.document.ids[node['ids'][0]]
                except (IndexError):
                    pass
        for node in tuple(self.document.findall(nodes.problematic)):
            if node['refid'] not in self.document.ids:
                node.parent.replace(node, nodes.Text(node.astext()))
        for node in self.document.findall(nodes.section):
            if "system-messages" in node['classes'] and len(node) == 1:
                node.parent.remove(node)


class TestMessages(Transform):

    """
    Append all post-parse system messages to the end of the document.

    Used for testing purposes.
    """

    # marker for pytest to ignore this class during test discovery
    __test__ = False

    default_priority = 880

    def apply(self):
        for msg in self.document.transform_messages:
            if not msg.parent:
                self.document += msg


class StripComments(Transform):

    """
    Remove comment elements from the document tree (only if the
    ``strip_comments`` setting is enabled).
    """

    default_priority = 740

    def apply(self):
        if self.document.settings.strip_comments:
            for node in tuple(self.document.findall(nodes.comment)):
                node.parent.remove(node)


class StripClassesAndElements(Transform):

    """
    Remove from the document tree all elements with classes in
    `self.document.settings.strip_elements_with_classes` and all "classes"
    attribute values in `self.document.settings.strip_classes`.
    """

    default_priority = 420

    def apply(self):
        if self.document.settings.strip_elements_with_classes:
            self.strip_elements = {*self.document.settings
                                   .strip_elements_with_classes}
            # Iterate over a tuple as removing the current node
            # corrupts the iterator returned by `iter`:
            for node in tuple(self.document.findall(self.check_classes)):
                node.parent.remove(node)

        if not self.document.settings.strip_classes:
            return
        strip_classes = self.document.settings.strip_classes
        for node in self.document.findall(nodes.Element):
            for class_value in strip_classes:
                try:
                    node['classes'].remove(class_value)
                except ValueError:
                    pass

    def check_classes(self, node):
        if not isinstance(node, nodes.Element):
            return False
        for class_value in node['classes'][:]:
            if class_value in self.strip_elements:
                return True
        return False


class SmartQuotes(Transform):

    """
    Replace ASCII quotation marks with typographic form.

    Also replace multiple dashes with em-dash/en-dash characters.
    """

    default_priority = 855

    nodes_to_skip = (nodes.FixedTextElement, nodes.Special)
    """Do not apply "smartquotes" to instances of these block-level nodes."""

    literal_nodes = (nodes.FixedTextElement, nodes.Special,
                     nodes.image, nodes.literal, nodes.math,
                     nodes.raw, nodes.problematic)
    """Do not apply smartquotes to instances of these inline nodes."""

    smartquotes_action = 'qDe'
    """Setting to select smartquote transformations.

    The default 'qDe' educates normal quote characters: (", '),
    em- and en-dashes (---, --) and ellipses (...).
    """

    def __init__(self, document, startnode):
        Transform.__init__(self, document, startnode=startnode)
        self.unsupported_languages = set()

    def get_tokens(self, txtnodes):
        # A generator that yields ``(texttype, nodetext)`` tuples for a list
        # of "Text" nodes (interface to ``smartquotes.educate_tokens()``).
        for node in txtnodes:
            if (isinstance(node.parent, self.literal_nodes)
                or isinstance(node.parent.parent, self.literal_nodes)):
                yield 'literal', str(node)
            else:
                # SmartQuotes uses backslash escapes instead of null-escapes
                # Insert backslashes before escaped "active" characters.
                txt = re.sub('(?<=\x00)([-\\\'".`])', r'\\\1', str(node))
                yield 'plain', txt

    def apply(self):
        smart_quotes = self.document.settings.setdefault('smart_quotes',
                                                         False)
        if not smart_quotes:
            return
        try:
            alternative = smart_quotes.startswith('alt')
        except AttributeError:
            alternative = False

        document_language = self.document.settings.language_code
        lc_smartquotes = self.document.settings.smartquotes_locales
        if lc_smartquotes:
            smartquotes.smartchars.quotes.update(dict(lc_smartquotes))

        # "Educate" quotes in normal text. Handle each block of text
        # (TextElement node) as a unit to keep context around inline nodes:
        for node in self.document.findall(nodes.TextElement):
            # skip preformatted text blocks and special elements:
            if isinstance(node, self.nodes_to_skip):
                continue
            # nested TextElements are not "block-level" elements:
            if isinstance(node.parent, nodes.TextElement):
                continue

            # list of text nodes in the "text block":
            txtnodes = [txtnode for txtnode in node.findall(nodes.Text)
                        if not isinstance(txtnode.parent,
                                          nodes.option_string)]

            # language: use typographical quotes for language "lang"
            lang = node.get_language_code(document_language)
            # use alternative form if `smart-quotes` setting starts with "alt":
            if alternative:
                if '-x-altquot' in lang:
                    lang = lang.replace('-x-altquot', '')
                else:
                    lang += '-x-altquot'
            # drop unsupported subtags:
            for tag in utils.normalize_language_tag(lang):
                if tag in smartquotes.smartchars.quotes:
                    lang = tag
                    break
            else:  # language not supported -- keep ASCII quotes
                if lang not in self.unsupported_languages:
                    self.document.reporter.warning(
                        'No smart quotes defined for language "%s".' % lang,
                        base_node=node)
                self.unsupported_languages.add(lang)
                lang = ''

            # Iterator educating quotes in plain text:
            # (see "utils/smartquotes.py" for the attribute setting)
            teacher = smartquotes.educate_tokens(
                self.get_tokens(txtnodes),
                attr=self.smartquotes_action, language=lang)

            for txtnode, newtext in zip(txtnodes, teacher):
                txtnode.parent.replace(txtnode, nodes.Text(newtext))

        self.unsupported_languages.clear()

env_pyrecodes/lib/python3.9/site-packages/docutils/transforms/universal.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9283 2022-11-28 23:55:57Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
Miscellaneous utilities for the documentation utilities.
"""

__docformat__ = 'reStructuredText'

import sys
import os
import os.path
from pathlib import PurePath, Path
import re
import itertools
import warnings
import unicodedata

from docutils import ApplicationError, DataError, __version_info__
from docutils import io, nodes
# for backwards compatibility
from docutils.nodes import unescape  # noqa: F401


class SystemMessage(ApplicationError):

    def __init__(self, system_message, level):
        Exception.__init__(self, system_message.astext())
        self.level = level


class SystemMessagePropagation(ApplicationError):
    pass


class Reporter:

    """
    Info/warning/error reporter and ``system_message`` element generator.

    Five levels of system messages are defined, along with corresponding
    methods: `debug()`, `info()`, `warning()`, `error()`, and `severe()`.

    There is typically one Reporter object per process.  A Reporter object is
    instantiated with thresholds for reporting (generating warnings) and
    halting processing (raising exceptions), a switch to turn debug output on
    or off, and an I/O stream for warnings.  These are stored as instance
    attributes.

    When a system message is generated, its level is compared to the stored
    thresholds, and a warning or error is generated as appropriate.  Debug
    messages are produced if the stored debug switch is on, independently of
    other thresholds.  Message output is sent to the stored warning stream if
    not set to ''.

    The Reporter class also employs a modified form of the "Observer" pattern
    [GoF95]_ to track system messages generated.  The `attach_observer` method
    should be called before parsing, with a bound method or function which
    accepts system messages.  The observer can be removed with
    `detach_observer`, and another added in its place.

    .. [GoF95] Gamma, Helm, Johnson, Vlissides. *Design Patterns: Elements of
       Reusable Object-Oriented Software*. Addison-Wesley, Reading, MA, USA,
       1995.
    """

    levels = 'DEBUG INFO WARNING ERROR SEVERE'.split()
    """List of names for system message levels, indexed by level."""

    # system message level constants:
    (DEBUG_LEVEL,
     INFO_LEVEL,
     WARNING_LEVEL,
     ERROR_LEVEL,
     SEVERE_LEVEL) = range(5)

    def __init__(self, source, report_level, halt_level, stream=None,
                 debug=False, encoding=None, error_handler='backslashreplace'):
        """
        :Parameters:
            - `source`: The path to or description of the source data.
            - `report_level`: The level at or above which warning output will
              be sent to `stream`.
            - `halt_level`: The level at or above which `SystemMessage`
              exceptions will be raised, halting execution.
            - `debug`: Show debug (level=0) system messages?
            - `stream`: Where warning output is sent.  Can be file-like (has a
              ``.write`` method), a string (file name, opened for writing),
              '' (empty string) or `False` (for discarding all stream messages)
              or `None` (implies `sys.stderr`; default).
            - `encoding`: The output encoding.
            - `error_handler`: The error handler for stderr output encoding.
        """

        self.source = source
        """The path to or description of the source data."""

        self.error_handler = error_handler
        """The character encoding error handler."""

        self.debug_flag = debug
        """Show debug (level=0) system messages?"""

        self.report_level = report_level
        """The level at or above which warning output will be sent
        to `self.stream`."""

        self.halt_level = halt_level
        """The level at or above which `SystemMessage` exceptions
        will be raised, halting execution."""

        if not isinstance(stream, io.ErrorOutput):
            stream = io.ErrorOutput(stream, encoding, error_handler)

        self.stream = stream
        """Where warning output is sent."""

        self.encoding = encoding or getattr(stream, 'encoding', 'ascii')
        """The output character encoding."""

        self.observers = []
        """List of bound methods or functions to call with each system_message
        created."""

        self.max_level = -1
        """The highest level system message generated so far."""

    def set_conditions(self, category, report_level, halt_level,
                       stream=None, debug=False):
        warnings.warn('docutils.utils.Reporter.set_conditions() deprecated; '
                      'Will be removed in Docutils 0.21 or later. '
                      'Set attributes via configuration settings or directly.',
                      DeprecationWarning, stacklevel=2)
        self.report_level = report_level
        self.halt_level = halt_level
        if not isinstance(stream, io.ErrorOutput):
            stream = io.ErrorOutput(stream, self.encoding, self.error_handler)
        self.stream = stream
        self.debug_flag = debug

    def attach_observer(self, observer):
        """
        The `observer` parameter is a function or bound method which takes one
        argument, a `nodes.system_message` instance.
        """
        self.observers.append(observer)

    def detach_observer(self, observer):
        self.observers.remove(observer)

    def notify_observers(self, message):
        for observer in self.observers:
            observer(message)

    def system_message(self, level, message, *children, **kwargs):
        """
        Return a system_message object.

        Raise an exception or generate a warning if appropriate.
        """
        # `message` can be a `str` or `Exception` instance.
        if isinstance(message, Exception):
            message = str(message)

        attributes = kwargs.copy()
        if 'base_node' in kwargs:
            source, line = get_source_line(kwargs['base_node'])
            del attributes['base_node']
            if source is not None:
                attributes.setdefault('source', source)
            if line is not None:
                attributes.setdefault('line', line)
                # assert source is not None, "line- but no source-argument"
        if 'source' not in attributes:
            # 'line' is absolute line number
            try:
                source, line = self.get_source_and_line(attributes.get('line'))
            except AttributeError:
                source, line = None, None
            if source is not None:
                attributes['source'] = source
            if line is not None:
                attributes['line'] = line
        # assert attributes['line'] is not None, (message, kwargs)
        # assert attributes['source'] is not None, (message, kwargs)
        attributes.setdefault('source', self.source)

        msg = nodes.system_message(message, level=level,
                                   type=self.levels[level],
                                   *children, **attributes)
        if self.stream and (level >= self.report_level
                            or self.debug_flag and level == self.DEBUG_LEVEL
                            or level >= self.halt_level):
            self.stream.write(msg.astext() + '\n')
        if level >= self.halt_level:
            raise SystemMessage(msg, level)
        if level > self.DEBUG_LEVEL or self.debug_flag:
            self.notify_observers(msg)
        self.max_level = max(level, self.max_level)
        return msg

    def debug(self, *args, **kwargs):
        """
        Level-0, "DEBUG": an internal reporting issue. Typically, there is no
        effect on the processing. Level-0 system messages are handled
        separately from the others.
        """
        if self.debug_flag:
            return self.system_message(self.DEBUG_LEVEL, *args, **kwargs)

    def info(self, *args, **kwargs):
        """
        Level-1, "INFO": a minor issue that can be ignored. Typically there is
        no effect on processing, and level-1 system messages are not reported.
        """
        return self.system_message(self.INFO_LEVEL, *args, **kwargs)

    def warning(self, *args, **kwargs):
        """
        Level-2, "WARNING": an issue that should be addressed. If ignored,
        there may be unpredictable problems with the output.
        """
        return self.system_message(self.WARNING_LEVEL, *args, **kwargs)

    def error(self, *args, **kwargs):
        """
        Level-3, "ERROR": an error that should be addressed. If ignored, the
        output will contain errors.
        """
        return self.system_message(self.ERROR_LEVEL, *args, **kwargs)

    def severe(self, *args, **kwargs):
        """
        Level-4, "SEVERE": a severe error that must be addressed. If ignored,
        the output will contain severe errors. Typically level-4 system
        messages are turned into exceptions which halt processing.
        """
        return self.system_message(self.SEVERE_LEVEL, *args, **kwargs)


class ExtensionOptionError(DataError): pass
class BadOptionError(ExtensionOptionError): pass
class BadOptionDataError(ExtensionOptionError): pass
class DuplicateOptionError(ExtensionOptionError): pass


def extract_extension_options(field_list, options_spec):
    """
    Return a dictionary mapping extension option names to converted values.

    :Parameters:
        - `field_list`: A flat field list without field arguments, where each
          field body consists of a single paragraph only.
        - `options_spec`: Dictionary mapping known option names to a
          conversion function such as `int` or `float`.

    :Exceptions:
        - `KeyError` for unknown option names.
        - `ValueError` for invalid option values (raised by the conversion
           function).
        - `TypeError` for invalid option value types (raised by conversion
           function).
        - `DuplicateOptionError` for duplicate options.
        - `BadOptionError` for invalid fields.
        - `BadOptionDataError` for invalid option data (missing name,
          missing data, bad quotes, etc.).
    """
    option_list = extract_options(field_list)
    return assemble_option_dict(option_list, options_spec)


def extract_options(field_list):
    """
    Return a list of option (name, value) pairs from field names & bodies.

    :Parameter:
        `field_list`: A flat field list, where each field name is a single
        word and each field body consists of a single paragraph only.

    :Exceptions:
        - `BadOptionError` for invalid fields.
        - `BadOptionDataError` for invalid option data (missing name,
          missing data, bad quotes, etc.).
    """
    option_list = []
    for field in field_list:
        if len(field[0].astext().split()) != 1:
            raise BadOptionError(
                'extension option field name may not contain multiple words')
        name = str(field[0].astext().lower())
        body = field[1]
        if len(body) == 0:
            data = None
        elif (len(body) > 1
              or not isinstance(body[0], nodes.paragraph)
              or len(body[0]) != 1
              or not isinstance(body[0][0], nodes.Text)):
            raise BadOptionDataError(
                  'extension option field body may contain\n'
                  'a single paragraph only (option "%s")' % name)
        else:
            data = body[0][0].astext()
        option_list.append((name, data))
    return option_list


def assemble_option_dict(option_list, options_spec):
    """
    Return a mapping of option names to values.

    :Parameters:
        - `option_list`: A list of (name, value) pairs (the output of
          `extract_options()`).
        - `options_spec`: Dictionary mapping known option names to a
          conversion function such as `int` or `float`.

    :Exceptions:
        - `KeyError` for unknown option names.
        - `DuplicateOptionError` for duplicate options.
        - `ValueError` for invalid option values (raised by conversion
           function).
        - `TypeError` for invalid option value types (raised by conversion
           function).
    """
    options = {}
    for name, value in option_list:
        convertor = options_spec[name]  # raises KeyError if unknown
        if convertor is None:
            raise KeyError(name)        # or if explicitly disabled
        if name in options:
            raise DuplicateOptionError('duplicate option "%s"' % name)
        try:
            options[name] = convertor(value)
        except (ValueError, TypeError) as detail:
            raise detail.__class__('(option: "%s"; value: %r)\n%s'
                                   % (name, value, ' '.join(detail.args)))
    return options


class NameValueError(DataError): pass


def decode_path(path):
    """
    Ensure `path` is Unicode. Return `str` instance.

    Decode file/path string in a failsafe manner if not already done.
    """
    # TODO: is this still required with Python 3?
    if isinstance(path, str):
        return path
    try:
        path = path.decode(sys.getfilesystemencoding(), 'strict')
    except AttributeError:  # default value None has no decode method
        if not path:
            return ''
        raise ValueError('`path` value must be a String or ``None``, '
                         f'not {path!r}')
    except UnicodeDecodeError:
        try:
            path = path.decode('utf-8', 'strict')
        except UnicodeDecodeError:
            path = path.decode('ascii', 'replace')
    return path


def extract_name_value(line):
    """
    Return a list of (name, value) from a line of the form "name=value ...".

    :Exception:
        `NameValueError` for invalid input (missing name, missing data, bad
        quotes, etc.).
    """
    attlist = []
    while line:
        equals = line.find('=')
        if equals == -1:
            raise NameValueError('missing "="')
        attname = line[:equals].strip()
        if equals == 0 or not attname:
            raise NameValueError(
                  'missing attribute name before "="')
        line = line[equals+1:].lstrip()
        if not line:
            raise NameValueError(
                  'missing value after "%s="' % attname)
        if line[0] in '\'"':
            endquote = line.find(line[0], 1)
            if endquote == -1:
                raise NameValueError(
                      'attribute "%s" missing end quote (%s)'
                      % (attname, line[0]))
            if len(line) > endquote + 1 and line[endquote + 1].strip():
                raise NameValueError(
                      'attribute "%s" end quote (%s) not followed by '
                      'whitespace' % (attname, line[0]))
            data = line[1:endquote]
            line = line[endquote+1:].lstrip()
        else:
            space = line.find(' ')
            if space == -1:
                data = line
                line = ''
            else:
                data = line[:space]
                line = line[space+1:].lstrip()
        attlist.append((attname.lower(), data))
    return attlist


def new_reporter(source_path, settings):
    """
    Return a new Reporter object.

    :Parameters:
        `source` : string
            The path to or description of the source text of the document.
        `settings` : optparse.Values object
            Runtime settings.
    """
    reporter = Reporter(
        source_path, settings.report_level, settings.halt_level,
        stream=settings.warning_stream, debug=settings.debug,
        encoding=settings.error_encoding,
        error_handler=settings.error_encoding_error_handler)
    return reporter


def new_document(source_path, settings=None):
    """
    Return a new empty document object.

    :Parameters:
        `source_path` : string
            The path to or description of the source text of the document.
        `settings` : optparse.Values object
            Runtime settings.  If none are provided, a default core set will
            be used.  If you will use the document object with any Docutils
            components, you must provide their default settings as well.

            For example, if parsing rST, at least provide the rst-parser
            settings, obtainable as follows:

            Defaults for parser component::

                settings = docutils.frontend.get_default_settings(
                               docutils.parsers.rst.Parser)

            Defaults and configuration file customizations::

                settings = docutils.core.Publisher(
                    parser=docutils.parsers.rst.Parser).get_settings()

    """
    # Import at top of module would lead to circular dependency!
    from docutils import frontend
    if settings is None:
        settings = frontend.get_default_settings()
    source_path = decode_path(source_path)
    reporter = new_reporter(source_path, settings)
    document = nodes.document(settings, reporter, source=source_path)
    document.note_source(source_path, -1)
    return document


def clean_rcs_keywords(paragraph, keyword_substitutions):
    if len(paragraph) == 1 and isinstance(paragraph[0], nodes.Text):
        textnode = paragraph[0]
        for pattern, substitution in keyword_substitutions:
            match = pattern.search(textnode)
            if match:
                paragraph[0] = nodes.Text(pattern.sub(substitution, textnode))
                return


def relative_path(source, target):
    """
    Build and return a path to `target`, relative to `source` (both files).

    Differences to `os.relpath()`:

    * Inverse argument order.
    * `source` expects path to a FILE (while os.relpath expects a dir)!
      (Add a "dummy" file name if `source` points to a directory.)
    * Always use Posix path separator ("/") for the output.
    * Use `os.sep` for parsing the input (ignored by `os.relpath()`).
    * If there is no common prefix, return the absolute path to `target`.

    """
    source_parts = os.path.abspath(source or type(target)('dummy_file')
                                   ).split(os.sep)
    target_parts = os.path.abspath(target).split(os.sep)
    # Check first 2 parts because '/dir'.split('/') == ['', 'dir']:
    if source_parts[:2] != target_parts[:2]:
        # Nothing in common between paths.
        # Return absolute path, using '/' for URLs:
        return '/'.join(target_parts)
    source_parts.reverse()
    target_parts.reverse()
    while (source_parts and target_parts
           and source_parts[-1] == target_parts[-1]):
        # Remove path components in common:
        source_parts.pop()
        target_parts.pop()
    target_parts.reverse()
    parts = ['..'] * (len(source_parts) - 1) + target_parts
    return '/'.join(parts)


def get_stylesheet_reference(settings, relative_to=None):
    """
    Retrieve a stylesheet reference from the settings object.

    Deprecated. Use get_stylesheet_list() instead to
    enable specification of multiple stylesheets as a comma-separated
    list.
    """
    warnings.warn('utils.get_stylesheet_reference()'
                  ' is obsoleted by utils.get_stylesheet_list()'
                  ' and will be removed in Docutils 2.0.',
                  DeprecationWarning, stacklevel=2)
    if settings.stylesheet_path:
        assert not settings.stylesheet, (
            'stylesheet and stylesheet_path are mutually exclusive.')
        if relative_to is None:
            relative_to = settings._destination
        return relative_path(relative_to, settings.stylesheet_path)
    else:
        return settings.stylesheet


# Return 'stylesheet' or 'stylesheet_path' arguments as list.
#
# The original settings arguments are kept unchanged: you can test
# with e.g. ``if settings.stylesheet_path: ...``.
#
# Differences to the depracated `get_stylesheet_reference()`:
# * return value is a list
# * no re-writing of the path (and therefore no optional argument)
#   (if required, use ``utils.relative_path(source, target)``
#   in the calling script)
def get_stylesheet_list(settings):
    """
    Retrieve list of stylesheet references from the settings object.
    """
    assert not (settings.stylesheet and settings.stylesheet_path), (
            'stylesheet and stylesheet_path are mutually exclusive.')
    stylesheets = settings.stylesheet_path or settings.stylesheet or []
    # programmatically set default may be string with comma separated list:
    if not isinstance(stylesheets, list):
        stylesheets = [path.strip() for path in stylesheets.split(',')]
    if settings.stylesheet_path:
        # expand relative paths if found in stylesheet-dirs:
        stylesheets = [find_file_in_dirs(path, settings.stylesheet_dirs)
                       for path in stylesheets]
    return stylesheets


def find_file_in_dirs(path, dirs):
    """
    Search for `path` in the list of directories `dirs`.

    Return the first expansion that matches an existing file.
    """
    path = Path(path)
    if path.is_absolute():
        return path.as_posix()
    for d in dirs:
        f = Path(d).expanduser() / path
        if f.exists():
            return f.as_posix()
    return path.as_posix()


def get_trim_footnote_ref_space(settings):
    """
    Return whether or not to trim footnote space.

    If trim_footnote_reference_space is not None, return it.

    If trim_footnote_reference_space is None, return False unless the
    footnote reference style is 'superscript'.
    """
    if settings.setdefault('trim_footnote_reference_space', None) is None:
        return getattr(settings, 'footnote_references', None) == 'superscript'
    else:
        return settings.trim_footnote_reference_space


def get_source_line(node):
    """
    Return the "source" and "line" attributes from the `node` given or from
    its closest ancestor.
    """
    while node:
        if node.source or node.line:
            return node.source, node.line
        node = node.parent
    return None, None


def escape2null(text):
    """Return a string with escape-backslashes converted to nulls."""
    parts = []
    start = 0
    while True:
        found = text.find('\\', start)
        if found == -1:
            parts.append(text[start:])
            return ''.join(parts)
        parts.append(text[start:found])
        parts.append('\x00' + text[found+1:found+2])
        start = found + 2               # skip character after escape


def split_escaped_whitespace(text):
    """
    Split `text` on escaped whitespace (null+space or null+newline).
    Return a list of strings.
    """
    strings = text.split('\x00 ')
    strings = [string.split('\x00\n') for string in strings]
    # flatten list of lists of strings to list of strings:
    return list(itertools.chain(*strings))


def strip_combining_chars(text):
    return ''.join(c for c in text if not unicodedata.combining(c))


def find_combining_chars(text):
    """Return indices of all combining chars in  Unicode string `text`.

    >>> from docutils.utils import find_combining_chars
    >>> find_combining_chars('A t̆ab̆lĕ')
    [3, 6, 9]

    """
    return [i for i, c in enumerate(text) if unicodedata.combining(c)]


def column_indices(text):
    """Indices of Unicode string `text` when skipping combining characters.

    >>> from docutils.utils import column_indices
    >>> column_indices('A t̆ab̆lĕ')
    [0, 1, 2, 4, 5, 7, 8]

    """
    # TODO: account for asian wide chars here instead of using dummy
    # replacements in the tableparser?
    string_indices = list(range(len(text)))
    for index in find_combining_chars(text):
        string_indices[index] = None
    return [i for i in string_indices if i is not None]


east_asian_widths = {'W': 2,   # Wide
                     'F': 2,   # Full-width (wide)
                     'Na': 1,  # Narrow
                     'H': 1,   # Half-width (narrow)
                     'N': 1,   # Neutral (not East Asian, treated as narrow)
                     'A': 1,   # Ambiguous (s/b wide in East Asian context,
                     }         # narrow otherwise, but that doesn't work)
"""Mapping of result codes from `unicodedata.east_asian_widt()` to character
column widths."""


def column_width(text):
    """Return the column width of text.

    Correct ``len(text)`` for wide East Asian and combining Unicode chars.
    """
    width = sum(east_asian_widths[unicodedata.east_asian_width(c)]
                for c in text)
    # correction for combining chars:
    width -= len(find_combining_chars(text))
    return width


def uniq(L):
    r = []
    for item in L:
        if item not in r:
            r.append(item)
    return r


def normalize_language_tag(tag):
    """Return a list of normalized combinations for a `BCP 47` language tag.

    Example:

    >>> from docutils.utils import normalize_language_tag
    >>> normalize_language_tag('de_AT-1901')
    ['de-at-1901', 'de-at', 'de-1901', 'de']
    >>> normalize_language_tag('de-CH-x_altquot')
    ['de-ch-x-altquot', 'de-ch', 'de-x-altquot', 'de']

    """
    # normalize:
    tag = tag.lower().replace('-', '_')
    # split (except singletons, which mark the following tag as non-standard):
    tag = re.sub(r'_([a-zA-Z0-9])_', r'_\1-', tag)
    subtags = [subtag for subtag in tag.split('_')]
    base_tag = (subtags.pop(0),)
    # find all combinations of subtags
    taglist = []
    for n in range(len(subtags), 0, -1):
        for tags in itertools.combinations(subtags, n):
            taglist.append('-'.join(base_tag+tags))
    taglist += base_tag
    return taglist


def xml_declaration(encoding=None):
    """Return an XML text declaration.

    Include an encoding declaration, if `encoding`
    is not 'unicode', '', or None.
    """
    if encoding and encoding.lower() != 'unicode':
        encoding_declaration = f' encoding="{encoding}"'
    else:
        encoding_declaration = ''
    return f'<?xml version="1.0"{encoding_declaration}?>\n'


class DependencyList:

    """
    List of dependencies, with file recording support.

    Note that the output file is not automatically closed.  You have
    to explicitly call the close() method.
    """

    def __init__(self, output_file=None, dependencies=()):
        """
        Initialize the dependency list, automatically setting the
        output file to `output_file` (see `set_output()`) and adding
        all supplied dependencies.

        If output_file is None, no file output is done when calling add().
        """
        self.list = []
        self.file = None
        if output_file:
            self.set_output(output_file)
        self.add(*dependencies)

    def set_output(self, output_file):
        """
        Set the output file and clear the list of already added
        dependencies.

        `output_file` must be a string.  The specified file is
        immediately overwritten.

        If output_file is '-', the output will be written to stdout.
        """
        if output_file:
            if output_file == '-':
                self.file = sys.stdout
            else:
                self.file = open(output_file, 'w', encoding='utf-8')

    def add(self, *paths):
        """
        Append `path` to `self.list` unless it is already there.

        Also append to `self.file` unless it is already there
        or `self.file is `None`.
        """
        for path in paths:
            if isinstance(path, PurePath):
                path = path.as_posix()  # use '/' as separator
            if path not in self.list:
                self.list.append(path)
                if self.file is not None:
                    self.file.write(path+'\n')

    def close(self):
        """
        Close the output file.
        """
        if self.file is not sys.stdout:
            self.file.close()
        self.file = None

    def __repr__(self):
        try:
            output_file = self.file.name
        except AttributeError:
            output_file = None
        return '%s(%r, %s)' % (self.__class__.__name__, output_file, self.list)


release_level_abbreviations = {
    'alpha': 'a',
    'beta': 'b',
    'candidate': 'rc',
    'final': ''}


def version_identifier(version_info=None):
    """
    Return a version identifier string built from `version_info`, a
    `docutils.VersionInfo` namedtuple instance or compatible tuple. If
    `version_info` is not provided, by default return a version identifier
    string based on `docutils.__version_info__` (i.e. the current Docutils
    version).
    """
    if version_info is None:
        version_info = __version_info__
    if version_info.micro:
        micro = '.%s' % version_info.micro
    else:
        # 0 is omitted:
        micro = ''
    releaselevel = release_level_abbreviations[version_info.releaselevel]
    if version_info.serial:
        serial = version_info.serial
    else:
        # 0 is omitted:
        serial = ''
    if version_info.release:
        dev = ''
    else:
        dev = '.dev'
    version = '%s.%s%s%s%s%s' % (
        version_info.major,
        version_info.minor,
        micro,
        releaselevel,
        serial,
        dev)
    return version

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/__init__.py
NikolaBlagojevic/pyrecodes
# :Author: Georg Brandl; Lea Wiemann; Günter Milde
# :Date: $Date: 2022-11-16 15:01:31 +0100 (Mi, 16. Nov 2022) $
# :Copyright: This module has been placed in the public domain.

"""Lexical analysis of formal languages (i.e. code) using Pygments."""

from docutils import ApplicationError
try:
    import pygments
    from pygments.lexers import get_lexer_by_name
    from pygments.formatters.html import _get_ttype_class
    with_pygments = True
except ImportError:
    with_pygments = False

# Filter the following token types from the list of class arguments:
unstyled_tokens = ['token',  # Token (base token type)
                   'text',   # Token.Text
                   '']       # short name for Token and Text
# (Add, e.g., Token.Punctuation with ``unstyled_tokens += 'punctuation'``.)


class LexerError(ApplicationError):
    pass


class Lexer:
    """Parse `code` lines and yield "classified" tokens.

    Arguments

      code       -- string of source code to parse,
      language   -- formal language the code is written in,
      tokennames -- either 'long', 'short', or 'none' (see below).

    Merge subsequent tokens of the same token-type.

    Iterating over an instance yields the tokens as ``(tokentype, value)``
    tuples. The value of `tokennames` configures the naming of the tokentype:

      'long':  downcased full token type name,
      'short': short name defined by pygments.token.STANDARD_TYPES
               (= class argument used in pygments html output),
      'none':  skip lexical analysis.
    """

    def __init__(self, code, language, tokennames='short'):
        """
        Set up a lexical analyzer for `code` in `language`.
        """
        self.code = code
        self.language = language
        self.tokennames = tokennames
        self.lexer = None
        # get lexical analyzer for `language`:
        if language in ('', 'text') or tokennames == 'none':
            return
        if not with_pygments:
            raise LexerError('Cannot analyze code. '
                             'Pygments package not found.')
        try:
            self.lexer = get_lexer_by_name(self.language)
        except pygments.util.ClassNotFound:
            raise LexerError('Cannot analyze code. '
                             'No Pygments lexer found for "%s".' % language)
        # self.lexer.add_filter('tokenmerge')
        # Since version 1.2. (released Jan 01, 2010) Pygments has a
        # TokenMergeFilter. # ``self.merge(tokens)`` in __iter__ could
        # be replaced by ``self.lexer.add_filter('tokenmerge')`` in __init__.
        # However, `merge` below also strips a final newline added by pygments.
        #
        # self.lexer.add_filter('tokenmerge')

    def merge(self, tokens):
        """Merge subsequent tokens of same token-type.

           Also strip the final newline (added by pygments).
        """
        tokens = iter(tokens)
        (lasttype, lastval) = next(tokens)
        for ttype, value in tokens:
            if ttype is lasttype:
                lastval += value
            else:
                yield lasttype, lastval
                (lasttype, lastval) = (ttype, value)
        if lastval.endswith('\n'):
            lastval = lastval[:-1]
        if lastval:
            yield lasttype, lastval

    def __iter__(self):
        """Parse self.code and yield "classified" tokens.
        """
        if self.lexer is None:
            yield [], self.code
            return
        tokens = pygments.lex(self.code, self.lexer)
        for tokentype, value in self.merge(tokens):
            if self.tokennames == 'long':  # long CSS class args
                classes = str(tokentype).lower().split('.')
            else:  # short CSS class args
                classes = [_get_ttype_class(tokentype)]
            classes = [cls for cls in classes if cls not in unstyled_tokens]
            yield classes, value


class NumberLines:
    """Insert linenumber-tokens at the start of every code line.

    Arguments

       tokens    -- iterable of ``(classes, value)`` tuples
       startline -- first line number
       endline   -- last line number

    Iterating over an instance yields the tokens with a
    ``(['ln'], '<the line number>')`` token added for every code line.
    Multi-line tokens are split."""

    def __init__(self, tokens, startline, endline):
        self.tokens = tokens
        self.startline = startline
        # pad linenumbers, e.g. endline == 100 -> fmt_str = '%3d '
        self.fmt_str = '%%%dd ' % len(str(endline))

    def __iter__(self):
        lineno = self.startline
        yield ['ln'], self.fmt_str % lineno
        for ttype, value in self.tokens:
            lines = value.split('\n')
            for line in lines[:-1]:
                yield ttype, line + '\n'
                lineno += 1
                yield ['ln'], self.fmt_str % lineno
            yield ttype, lines[-1]

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/code_analyzer.py
NikolaBlagojevic/pyrecodes
# :Id: $Id: latex2mathml.py 9338 2023-04-08 21:08:47Z milde $
# :Copyright: © 2005 Jens Jørgen Mortensen [1]_
#             © 2010, 2021 Günter Milde.
#
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause
#
# .. [1] the original `rst2mathml.py` in `sandbox/jensj/latex_math`

"""Convert LaTex maths code into presentational MathML.

This module is provisional:
the API is not settled and may change with any minor Docutils version.
"""

# Usage:
#
# >>> from latex2mathml import *

import re
import unicodedata

from docutils.utils.math import tex2unichar, toplevel_code


# Character data
# --------------

# LaTeX math macro to Unicode mappings.
# Character categories.

# identifiers -> <mi>

letters = tex2unichar.mathalpha
letters['hbar'] = '\u210F'  # compatibility mapping to ℏ (\hslash).
# (ħ LATIN SMALL LETTER H WITH STROKE is upright)

# special case: Capital Greek letters: (upright in TeX style)
greek_capitals = {
    'Phi': '\u03a6', 'Xi': '\u039e', 'Sigma': '\u03a3',
    'Psi': '\u03a8', 'Delta': '\u0394', 'Theta': '\u0398',
    'Upsilon': '\u03d2', 'Pi': '\u03a0', 'Omega': '\u03a9',
    'Gamma': '\u0393', 'Lambda': '\u039b'}

# functions -> <mi>
functions = {
    # functions with a space in the name
    'liminf': 'lim\u202finf',
    'limsup': 'lim\u202fsup',
    'injlim': 'inj\u202flim',
    'projlim': 'proj\u202flim',
    # embellished function names (see handle_cmd() below)
    'varlimsup': 'lim',
    'varliminf': 'lim',
    'varprojlim': 'lim',
    'varinjlim': 'lim',
    # custom function name
    'operatorname': None,
}
functions.update((name, name) for name in
                 ('arccos', 'arcsin', 'arctan', 'arg',  'cos',
                  'cosh',   'cot',    'coth',   'csc',  'deg',
                  'det',    'dim',    'exp',    'gcd',  'hom',
                  'ker',    'lg',     'ln',     'log',  'Pr',
                  'sec',    'sin',    'sinh',   'tan',  'tanh'))
# Function with limits: 'lim', 'sup', 'inf', 'max', 'min':
# use <mo> to allow "movablelimits" attribute (see below).

# modulo operator/arithmetic
modulo_functions = {
    # cmdname: (binary, named, parentheses, padding)
    'bmod': (True,  True,  False, '0.278em'),  # a mod n
    'pmod': (False, True,  True,  '0.444em'),  # a  (mod n)
    'mod':  (False, True,  False, '0.667em'),  # a  mod n
    'pod':  (False, False, True,  '0.444em'),  # a  (n)
    }


# math font selection -> <mi mathvariant=...> or <mstyle mathvariant=...>
math_alphabets = {
    # 'cmdname':  'mathvariant value'        # package
    'boldsymbol': 'bold',
    'mathbf':     'bold',
    'mathit':     'italic',
    'mathtt':     'monospace',
    'mathrm':     'normal',
    'mathsf':     'sans-serif',
    'mathcal':    'script',
    'mathbfit':   'bold-italic',             # isomath
    'mathbb':     'double-struck',           # amssymb
    'mathfrak':   'fraktur',                 # amssymb
    'mathsfit':   'sans-serif-italic',       # isomath
    'mathsfbfit': 'sans-serif-bold-italic',  # isomath
    'mathscr':    'script',                  # mathrsfs
    # unsupported: bold-fraktur
    #              bold-script
    #              bold-sans-serif
}

# operator, fence, or separator -> <mo>


stretchables = {
    # extensible delimiters allowed in left/right cmds
    'backslash':   '\\',
    'uparrow':     '\u2191',  # ↑ UPWARDS ARROW
    'downarrow':   '\u2193',  # ↓ DOWNWARDS ARROW
    'updownarrow': '\u2195',  # ↕ UP DOWN ARROW
    'Uparrow':     '\u21d1',  # ⇑ UPWARDS DOUBLE ARROW
    'Downarrow':   '\u21d3',  # ⇓ DOWNWARDS DOUBLE ARROW
    'Updownarrow': '\u21d5',  # ⇕ UP DOWN DOUBLE ARROW
    'lmoustache':  '\u23b0',  # ⎰ … CURLY BRACKET SECTION
    'rmoustache':  '\u23b1',  # ⎱ … LEFT CURLY BRACKET SECTION
    'arrowvert':   '\u23d0',  # ⏐ VERTICAL LINE EXTENSION
    'bracevert':   '\u23aa',  # ⎪ CURLY BRACKET EXTENSION
    'lvert':      '|',        # left  |
    'lVert':      '\u2016',   # left  ‖
    'rvert':      '|',        # right |
    'rVert':      '\u2016',   # right ‖
    'Arrowvert':  '\u2016',   # ‖
}
stretchables.update(tex2unichar.mathfence)
stretchables.update(tex2unichar.mathopen)   # Braces
stretchables.update(tex2unichar.mathclose)  # Braces

# >>> print(' '.join(sorted(set(stretchables.values()))))
# [ \ ] { | } ‖ ↑ ↓ ↕ ⇑ ⇓ ⇕ ⌈ ⌉ ⌊ ⌋ ⌜ ⌝ ⌞ ⌟ ⎪ ⎰ ⎱ ⏐ ⟅ ⟆ ⟦ ⟧ ⟨ ⟩ ⟮ ⟯ ⦇ ⦈

operators = {
    # negated symbols without pre-composed Unicode character
    'nleqq':      '\u2266\u0338',  # ≦̸
    'ngeqq':      '\u2267\u0338',  # ≧̸
    'nleqslant':  '\u2a7d\u0338',  # ⩽̸
    'ngeqslant':  '\u2a7e\u0338',  # ⩾̸
    'ngtrless':   '\u2277\u0338',  # txfonts
    'nlessgtr':   '\u2276\u0338',  # txfonts
    'nsubseteqq': '\u2AC5\u0338',  # ⫅̸
    'nsupseteqq': '\u2AC6\u0338',  # ⫆̸
    # compatibility definitions:
    'centerdot': '\u2B1D',  # BLACK VERY SMALL SQUARE | mathbin
    'varnothing': '\u2300',  # ⌀ DIAMETER SIGN | empty set
    'varpropto': '\u221d',  # ∝ PROPORTIONAL TO | sans serif
    'triangle': '\u25B3',  # WHITE UP-POINTING TRIANGLE | mathord
    'triangledown': '\u25BD',  # WHITE DOWN-POINTING TRIANGLE | mathord
    # alias commands:
    'dotsb': '\u22ef',  # ⋯ with binary operators/relations
    'dotsc': '\u2026',  # … with commas
    'dotsi': '\u22ef',  # ⋯ with integrals
    'dotsm': '\u22ef',  # ⋯ multiplication dots
    'dotso': '\u2026',  # … other dots
    # functions with movable limits (requires <mo>)
    'lim': 'lim',
    'sup': 'sup',
    'inf': 'inf',
    'max': 'max',
    'min': 'min',
}
operators.update(tex2unichar.mathbin)    # Binary symbols
operators.update(tex2unichar.mathrel)    # Relation symbols, arrow symbols
operators.update(tex2unichar.mathord)    # Miscellaneous symbols
operators.update(tex2unichar.mathpunct)  # Punctuation
operators.update(tex2unichar.mathop)     # Variable-sized symbols
operators.update(stretchables)


# special cases

thick_operators = {
    # style='font-weight: bold;'
    'thicksim':       '\u223C',  # ∼
    'thickapprox':    '\u2248',  # ≈
}

small_operators = {
    # mathsize='75%'
    'shortmid':       '\u2223',  # ∣
    'shortparallel':  '\u2225',  # ∥
    'nshortmid':      '\u2224',  # ∤
    'nshortparallel': '\u2226',  # ∦
    'smallfrown':     '\u2322',  # ⌢ FROWN
    'smallsmile':     '\u2323',  # ⌣ SMILE
    'smallint':       '\u222b',  # ∫ INTEGRAL
}

# Operators and functions with limits above/below in display formulas
# and in index position inline (movablelimits=True)
movablelimits = ('bigcap', 'bigcup', 'bigodot', 'bigoplus', 'bigotimes',
                 'bigsqcup', 'biguplus', 'bigvee', 'bigwedge',
                 'coprod', 'intop', 'ointop', 'prod', 'sum',
                 'lim', 'max', 'min', 'sup', 'inf')
# Depending on settings, integrals may also be in this category.
# (e.g. if "amsmath" is loaded with option "intlimits", see
#  http://mirror.ctan.org/macros/latex/required/amsmath/amsldoc.pdf)
# movablelimits.extend(('fint', 'iiiint', 'iiint', 'iint', 'int', 'oiint',
#                       'oint', 'ointctrclockwise', 'sqint',
#                       'varointclockwise',))

# horizontal space -> <mspace>

spaces = {'qquad':         '2em',        # two \quad
          'quad':          '1em',        # 18 mu
          'thickspace':    '0.2778em',   # 5mu = 5/18em
          ';':             '0.2778em',   # 5mu thickspace
          ' ':             '0.25em',     # inter word space
          'medspace':      '0.2222em',   # 4mu = 2/9em
          ':':             '0.2222em',   # 4mu medspace
          'thinspace':     '0.1667em',   # 3mu = 1/6em
          ',':             '0.1667em',   # 3mu thinspace
          'negthinspace':  '-0.1667em',  # -3mu = -1/6em
          '!':             '-0.1667em',  # negthinspace
          'negmedspace':   '-0.2222em',  # -4mu = -2/9em
          'negthickspace': '-0.2778em',  # -5mu = -5/18em
          }

# accents -> <mover stretchy="false">
accents = {
    # TeX:    (spacing, combining)
    'acute':    ('´', '\u0301'),
    'bar':      ('ˉ', '\u0304'),
    'breve':    ('˘', '\u0306'),
    'check':    ('ˇ', '\u030C'),
    'dot':      ('˙', '\u0307'),
    'ddot':     ('¨', '\u0308'),
    'dddot':    ('⋯', '\u20DB'),
    'grave':    ('`', '\u0300'),
    'hat':      ('ˆ', '\u0302'),
    'mathring': ('˚', '\u030A'),
    'tilde':    ('˜', '\u0303'),  # tilde ~ or small tilde ˜?
    'vec':      ('→', '\u20d7'),  # → too heavy, accents="false"
    # TODO: ddddot
}

# limits etc. -> <mover> or <munder>
over = {
    # TeX:                  (char,     offset-correction/em)
    'overbrace':            ('\u23DE', -0.2),  # DejaVu Math -0.6
    'overleftarrow':        ('\u2190', -0.2),
    'overleftrightarrow':   ('\u2194', -0.2),
    'overline':             ('_',      -0.2),  # \u2012 does not stretch
    'overrightarrow':       ('\u2192', -0.2),
    'widehat':              ('^',      -0.5),
    'widetilde':            ('~',      -0.3),
}
under = {'underbrace':          ('\u23DF',  0.1),  # DejaVu Math -0.7
         'underleftarrow':      ('\u2190', -0.2),
         'underleftrightarrow': ('\u2194', -0.2),
         'underline':           ('_',      -0.8),
         'underrightarrow':     ('\u2192', -0.2),
         }

# Character translations
# ----------------------
# characters with preferred alternative in mathematical use
# cf. https://www.w3.org/TR/MathML3/chapter7.html#chars.anomalous
anomalous_chars = {'-': '\u2212',  # HYPHEN-MINUS -> MINUS SIGN
                   ':': '\u2236',  # COLON -> RATIO
                   '~': '\u00a0',  # NO-BREAK SPACE
                   }

# blackboard bold (Greek characters not working with "mathvariant" (Firefox 78)
mathbb = {'Γ': '\u213E',    # ℾ
          'Π': '\u213F',    # ℿ
          'Σ': '\u2140',    # ⅀
          'γ': '\u213D',    # ℽ
          'π': '\u213C',    # ℼ
          }

# Matrix environments
matrices = {
    # name:    fences
    'matrix':  ('', ''),
    'smallmatrix':  ('', ''),  # smaller, see begin_environment()!
    'pmatrix': ('(', ')'),
    'bmatrix': ('[', ']'),
    'Bmatrix': ('{', '}'),
    'vmatrix': ('|', '|'),
    'Vmatrix': ('\u2016', '\u2016'),  # ‖
    'cases':   ('{', ''),
}

layout_styles = {
    'displaystyle':      {'displaystyle': True,  'scriptlevel': 0},
    'textstyle':         {'displaystyle': False, 'scriptlevel': 0},
    'scriptstyle':       {'displaystyle': False, 'scriptlevel': 1},
    'scriptscriptstyle': {'displaystyle': False, 'scriptlevel': 2},
    }
# See also https://www.w3.org/TR/MathML3/chapter3.html#presm.scriptlevel

fractions = {
    # name:   style_attrs, frac_attrs
    'frac':   ({}, {}),
    'cfrac':  ({'displaystyle': True,  'scriptlevel': 0,
                'CLASS': 'cfrac'}, {}),  # in LaTeX with padding
    'dfrac':  (layout_styles['displaystyle'], {}),
    'tfrac':  (layout_styles['textstyle'], {}),
    'binom':  ({}, {'linethickness': 0}),
    'dbinom': (layout_styles['displaystyle'], {'linethickness': 0}),
    'tbinom': (layout_styles['textstyle'], {'linethickness': 0}),
}

delimiter_sizes = ['', '1.2em', '1.623em', '2.047em', '2.470em']
bigdelimiters = {'left':  0,
                 'right': 0,
                 'bigl':  1,
                 'bigr':  1,
                 'Bigl':  2,
                 'Bigr':  2,
                 'biggl': 3,
                 'biggr': 3,
                 'Biggl': 4,
                 'Biggr': 4,
                 }


# MathML element classes
# ----------------------

class math:
    """Base class for MathML elements and root of MathML trees."""

    nchildren = None
    """Expected number of children or None"""
    # cf. https://www.w3.org/TR/MathML3/chapter3.html#id.3.1.3.2
    parent = None
    """Parent node in MathML DOM tree."""
    _level = 0  # indentation level (static class variable)
    xml_entities = {
        # for invalid and invisible characters
        ord('<'): '&lt;',
        ord('>'): '&gt;',
        ord('&'): '&amp;',
        0x2061:   '&ApplyFunction;',
    }
    _boolstrings = {True: 'true', False: 'false'}
    """String representation of boolean MathML attribute values."""

    html_tagname = 'span'
    """Tag name for HTML representation."""

    def __init__(self, *children, **attributes):
        """Set up node with `children` and `attributes`.

        Attributes are downcased: Use CLASS to set "class" value.
        >>> math(mn(3), CLASS='test')
        math(mn(3), class='test')
        >>> math(CLASS='test').toprettyxml()
        '<math class="test">\n</math>'

        """
        self.children = []
        self.extend(children)

        self.attributes = {}
        for key in attributes.keys():
            # Use .lower() to allow argument `CLASS` for attribute `class`
            # (Python keyword). MathML uses only lowercase attributes.
            self.attributes[key.lower()] = attributes[key]

    def __repr__(self):
        content = [repr(item) for item in getattr(self, 'children', [])]
        if hasattr(self, 'data'):
            content.append(repr(self.data))
        if isinstance(self, MathSchema) and self.switch:
            content.append('switch=True')
        content += ["%s=%r"%(k, v) for k, v in self.attributes.items()
                    if v is not None]

        return self.__class__.__name__ + '(%s)' % ', '.join(content)

    def __len__(self):
        return len(self.children)

    # emulate dictionary-like access to attributes
    # see `docutils.nodes.Element` for dict/list interface
    def __getitem__(self, key):
        return self.attributes[key]

    def __setitem__(self, key, item):
        self.attributes[key] = item

    def get(self, *args, **kwargs):
        return self.attributes.get(*args, **kwargs)

    def full(self):
        """Return boolean indicating whether children may be appended."""
        return (self.nchildren is not None
                and len(self) >= self.nchildren)

    def append(self, child):
        """Append child and return self or first non-full parent.

        If self is full, go up the tree and return first non-full node or
        `None`.
        """
        if self.full():
            raise SyntaxError('Node %s already full!' % self)
        self.children.append(child)
        child.parent = self
        if self.full():
            return self.close()
        return self

    def extend(self, children):
        for child in children:
            self.append(child)
        return self

    def close(self):
        """Close element and return first non-full parent or None."""
        parent = self.parent
        while parent is not None and parent.full():
            parent = parent.parent
        return parent

    def toprettyxml(self):
        """Return XML representation of self as string."""
        return ''.join(self._xml())

    def _xml(self, level=0):
        return ([self.xml_starttag()]
                + self._xml_body(level)
                + ['</%s>' % self.__class__.__name__])

    def xml_starttag(self):
        attrs = ('%s="%s"' % (k, str(v).replace('True', 'true').replace('False', 'false'))
                 for k, v in self.attributes.items()
                 if v is not None)
        return '<%s>' % ' '.join((self.__class__.__name__, *attrs))

    def _xml_body(self, level=0):
        xml = []
        for child in self.children:
            xml.extend(['\n', '  ' * (level+1)])
            xml.extend(child._xml(level+1))
        xml.extend(['\n', '  ' * level])
        return xml

    def is_block(self):
        """Return true, if `self` or a parent has ``display='block'``."""
        try:
            return self['display'] == 'block'
        except KeyError:
            try:
                return self.parent.is_block()
            except AttributeError:
                return False

# >>> n2 = math(mn(2))
# >>> n2
# math(mn(2))
# >>> n2.toprettyxml()
# '<math>\n  <mn>2</mn>\n</math>'
# >>> len(n2)
# 1
# >>> eq3 = math(id='eq3', display='block')
# >>> eq3
# math(id='eq3', display='block')
# >>> eq3.toprettyxml()
# '<math id="eq3" display="block">\n</math>'
# >>> len(eq3)
# 0
# >>> math(CLASS='bold').xml_starttag()
# '<math class="bold">'
# >>> n2.is_block()
# False
# >>> node = n2.append(mrow())
# >>> node.is_block()
# False
# >>> eq3.is_block()
# True
# >>> node = eq3.append(mrow())
# >>> node.is_block()
# True


class mtable(math): pass


# >>> mt = mtable(displaystyle=True)
# >>> mt
# mtable(displaystyle=True)
# >>> math(mt).toprettyxml()
# '<math>\n  <mtable displaystyle="true">\n  </mtable>\n</math>'

class mrow(math):
    """Group sub-expressions as a horizontal row."""

    def close(self):
        """Close element and return first non-full parent or None.

        Remove <mrow>, if it is single child and the parent infers an mrow
        or if it has only one child element.
        """
        parent = self.parent
        if isinstance(parent, MathRowSchema) and parent.nchildren == 1:
            parent.nchildren = len(parent.children)
            parent.children = self.children
            for child in self.children:
                child.parent = parent
            return parent.close()
        if len(self) == 1:
            try:
                parent.children[parent.children.index(self)] = self.children[0]
                self.children[0].parent = parent
            except (AttributeError, ValueError):
                return self.children[0]
        return super().close()

# >>> mrow(displaystyle=False)
# mrow(displaystyle=False)


# The elements <msqrt>, <mstyle>, <merror>, <mpadded>, <mphantom>, <menclose>,
# <mtd>, <mscarry>, and <math> treat their contents as a single inferred mrow
# formed from all their children.
class MathRowSchema(math):
    """Base class for elements treating content as a single inferred mrow."""


class mtr(MathRowSchema): pass
class mtd(MathRowSchema): pass


class menclose(MathRowSchema):
    nchildren = 1  # \boxed expects one argument or a group


class mphantom(MathRowSchema):
    nchildren = 1  # \phantom expects one argument or a group


class msqrt(MathRowSchema):
    nchildren = 1  # \sqrt expects one argument or a group


class mstyle(MathRowSchema):
    nchildren = 1  # \mathrm, ... expect one argument or a group


class MathToken(math):
    """Token Element: contains textual data instead of children.

    Base class for mo, mi, and mn.
    """
    nchildren = 0

    def __init__(self, data, **attributes):
        self.data = data
        super().__init__(**attributes)

    def _xml_body(self, level=0):
        return [str(self.data).translate(self.xml_entities)]


class mtext(MathToken): pass
class mi(MathToken): pass
class mo(MathToken): pass
class mn(MathToken): pass


# >>> mo('<')
# mo('<')
# >>> mo('<')._xml()
# ['<mo>', '&lt;', '</mo>']

class MathSchema(math):
    """Base class for schemata expecting 2 or more children.

    The special attribute `switch` indicates that the last two child
    elements are in reversed order and must be switched before XML-export.
    """

    nchildren = 2

    def __init__(self, *children, **kwargs):
        self.switch = kwargs.pop('switch', False)
        math.__init__(self, *children, **kwargs)

    def append(self, child):
        current_node = super().append(child)
        # normalize order if full
        if self.switch and self.full():
            self.children[-1], self.children[-2] = self.children[-2], self.children[-1]
            self.switch = False
        return current_node


class msub(MathSchema): pass
class msup(MathSchema): pass


class msubsup(MathSchema):
    nchildren = 3


# >>> msub(mi('x'), mo('-'))
# msub(mi('x'), mo('-'))
# >>> msubsup(mi('base'), mi('sub'), mi('super'))
# msubsup(mi('base'), mi('sub'), mi('super'))
# >>> msubsup(mi('base'), mi('super'), mi('sub'), switch=True)
# msubsup(mi('base'), mi('sub'), mi('super'))

class munder(msub): pass
class mover(msup): pass


# >>> munder(mi('lim'), mo('-'), accent=False)
# munder(mi('lim'), mo('-'), accent=False)
# >>> mu = munder(mo('-'), accent=False, switch=True)
# >>> mu
# munder(mo('-'), switch=True, accent=False)
# >>> mu.append(mi('lim'))
# >>> mu
# munder(mi('lim'), mo('-'), accent=False)
# >>> mu.append(mi('lim'))
# Traceback (most recent call last):
# SyntaxError: Node munder(mi('lim'), mo('-'), accent=False) already full!
# >>> munder(mo('-'), mi('lim'), accent=False, switch=True).toprettyxml()
# '<munder accent="false">\n  <mi>lim</mi>\n  <mo>-</mo>\n</munder>'

class munderover(msubsup): pass


class mroot(MathSchema):
    nchildren = 2


class mfrac(math):
    nchildren = 2


class mspace(math):
    nchildren = 0


# LaTeX to MathML translation
# ---------------------------

# auxiliary functions
# ~~~~~~~~~~~~~~~~~~~

def tex_cmdname(string):
    """Return leading TeX command name and remainder of `string`.

    >>> tex_cmdname('mymacro2') # up to first non-letter
    ('mymacro', '2')
    >>> tex_cmdname('name 2') # strip trailing whitespace
    ('name', '2')
    >>> tex_cmdname('_2') # single non-letter character
    ('_', '2')

    """
    m = re.match(r'([a-zA-Z]+) *(.*)', string)
    if m is None:
        m = re.match(r'(.?)(.*)', string)
    return m.group(1), m.group(2)


# Test:
#
# >>> tex_cmdname('name_2') # first non-letter terminates
# ('name', '_2')
# >>> tex_cmdname(' next') # leading whitespace is returned
# (' ', 'next')
# >>> tex_cmdname('1 2') # whitespace after non-letter is kept
# ('1', ' 2')
# >>> tex_cmdname('') # empty string
# ('', '')


def tex_number(string):
    """Return leading number literal and remainder of `string`.

    >>> tex_number('123.4')
    ('123.4', '')

    """
    m = re.match(r'([0-9.,]*[0-9]+)(.*)', string)
    if m is None:
        return '', string
    return m.group(1), m.group(2)


# Test:
#
# >>> tex_number(' 23.4b') # leading whitespace -> no number
# ('', ' 23.4b')
# >>> tex_number('23,400/2') # comma separator included
# ('23,400', '/2')
# >>> tex_number('23. 4/2') # trailing separator not included
# ('23', '. 4/2')
# >>> tex_number('4, 2') # trailing separator not included
# ('4', ', 2')
# >>> tex_number('1 000.4')
# ('1', ' 000.4')


def tex_token(string):
    """Return first simple TeX token and remainder of `string`.

    >>> tex_token('\\command{without argument}')
    ('\\command', '{without argument}')
    >>> tex_token('or first character')
    ('o', 'r first character')

    """
    m = re.match(r"""((?P<cmd>\\[a-zA-Z]+)\s* # TeX command, skip whitespace
                      |(?P<chcmd>\\.)          # one-character TeX command
                      |(?P<ch>.?))            # first character (or empty)
                     (?P<remainder>.*$)    # remaining part of string
                 """, string, re.VERBOSE)
    cmd, chcmd, ch, remainder = m.group('cmd', 'chcmd', 'ch', 'remainder')
    return cmd or chcmd or ch, remainder

# Test:
#
# >>> tex_token('{opening bracket of group}')
# ('{', 'opening bracket of group}')
# >>> tex_token('\\skip whitespace after macro name')
# ('\\skip', 'whitespace after macro name')
# >>> tex_token('. but not after single char')
# ('.', ' but not after single char')
# >>> tex_token('') # empty string.
# ('', '')
# >>> tex_token('\{escaped bracket')
# ('\\{', 'escaped bracket')


def tex_group(string):
    """Return first TeX group or token and remainder of `string`.

    >>> tex_group('{first group} returned without brackets')
    ('first group', ' returned without brackets')

    """
    split_index = 0
    nest_level = 0   # level of {{nested} groups}
    escape = False   # the next character is escaped (\)

    if not string.startswith('{'):
        # special case: there is no group, return first token and remainder
        return string[:1], string[1:]
    for c in string:
        split_index += 1
        if escape:
            escape = False
        elif c == '\\':
            escape = True
        elif c == '{':
            nest_level += 1
        elif c == '}':
            nest_level -= 1
        if nest_level == 0:
            break
    else:
        raise SyntaxError('Group without closing bracket')
    return string[1:split_index-1], string[split_index:]


# >>> tex_group('{} empty group')
# ('', ' empty group')
# >>> tex_group('{group with {nested} group} ')
# ('group with {nested} group', ' ')
# >>> tex_group('{group with {nested group}} at the end')
# ('group with {nested group}', ' at the end')
# >>> tex_group('{{group} {with {{complex }nesting}} constructs}')
# ('{group} {with {{complex }nesting}} constructs', '')
# >>> tex_group('{group with \\{escaped\\} brackets}')
# ('group with \\{escaped\\} brackets', '')
# >>> tex_group('{group followed by closing bracket}} from outer group')
# ('group followed by closing bracket', '} from outer group')
# >>> tex_group('No group? Return first character.')
# ('N', 'o group? Return first character.')
# >>> tex_group(' {also whitespace}')
# (' ', '{also whitespace}')


def tex_token_or_group(string):
    """Return first TeX group or token and remainder of `string`.

    >>> tex_token_or_group('\\command{without argument}')
    ('\\command', '{without argument}')
    >>> tex_token_or_group('first character')
    ('f', 'irst character')
    >>> tex_token_or_group(' also whitespace')
    (' ', 'also whitespace')
    >>> tex_token_or_group('{first group} keep rest')
    ('first group', ' keep rest')

    """
    arg, remainder = tex_token(string)
    if arg == '{':
        arg, remainder = tex_group(string.lstrip())
    return arg, remainder

# >>> tex_token_or_group('\{no group but left bracket')
# ('\\{', 'no group but left bracket')


def tex_optarg(string):
    """Return optional argument and remainder.

    >>> tex_optarg('[optional argument] returned without brackets')
    ('optional argument', ' returned without brackets')
    >>> tex_optarg('{empty string, if there is no optional arg}')
    ('', '{empty string, if there is no optional arg}')

    """
    m = re.match(r"""\s*                            # leading whitespace
                 \[(?P<optarg>(\\]|[^\[\]]|\\])*)\] # [group] without nested groups
                 (?P<remainder>.*$)
                 """, string, re.VERBOSE)
    if m is None and not string.startswith('['):
        return '', string
    try:
        return m.group('optarg'), m.group('remainder')
    except AttributeError:
        raise SyntaxError('Could not extract optional argument from %r' % string)

# Test:
# >>> tex_optarg(' [optional argument] after whitespace')
# ('optional argument', ' after whitespace')
# >>> tex_optarg('[missing right bracket')
# Traceback (most recent call last):
# SyntaxError: Could not extract optional argument from '[missing right bracket'
# >>> tex_optarg('[group with [nested group]]')
# Traceback (most recent call last):
# SyntaxError: Could not extract optional argument from '[group with [nested group]]'


def parse_latex_math(node, string):
    """Append MathML conversion of `string` to `node` and return it.

    >>> parse_latex_math(math(), r'\alpha')
    math(mi('α'))
    >>> parse_latex_math(mrow(), r'x_{n}')
    mrow(msub(mi('x'), mi('n')))

    """
    # Normalize white-space:
    string = ' '.join(string.split())
    tree = node

    while len(string) > 0:
        # Take off first character:
        c, string = string[0], string[1:]

        if c == ' ':
            continue  # whitespace is ignored in LaTeX math mode
        if c == '\\':  # start of a LaTeX macro
            cmdname, string = tex_cmdname(string)
            node, string = handle_cmd(cmdname, node, string)
        elif c in "_^":
            node = handle_script_or_limit(node, c)
        elif c == '{':
            new_node = mrow()
            node.append(new_node)
            node = new_node
        elif c == '}':
            node = node.close()
        elif c == '&':
            new_node = mtd()
            node.close().append(new_node)
            node = new_node
        elif c.isalpha():
            node = node.append(mi(c))
        elif c.isdigit():
            number, string = tex_number(string)
            node = node.append(mn(c+number))
        elif c in anomalous_chars:
            # characters with a special meaning in LaTeX math mode
            # fix spacing before "unary" minus.
            attributes = {}
            if c == '-' and node.children:
                previous_node = node.children[-1]
                if (getattr(previous_node, 'data', '-') in '([='
                    or previous_node.get('class') == 'mathopen'):
                    attributes['form'] = 'prefix'
            node = node.append(mo(anomalous_chars[c], **attributes))
        elif c in "/()[]|":
            node = node.append(mo(c, stretchy=False))
        elif c in "+*=<>,.!?`';@":
            node = node.append(mo(c))
        else:
            raise SyntaxError('Unsupported character: "%s"' % c)
    return tree

# Test:

# >>> print(parse_latex_math(math(), ''))
# math()
# >>> parse_latex_math(math(), ' \\sqrt{ \\alpha}')
# math(msqrt(mi('α')))
# >>> parse_latex_math(math(), '23.4x')
# math(mn('23.4'), mi('x'))
# >>> parse_latex_math(math(), '\\sqrt 2 \\ne 3')
# math(msqrt(mn('2')), mo('≠'), mn('3'))
# >>> parse_latex_math(math(), '\\sqrt{2 + 3} < 3')
# math(msqrt(mn('2'), mo('+'), mn('3')), mo('<'), mn('3'))
# >>> parse_latex_math(math(), '\\sqrt[3]{2 + 3}')
# math(mroot(mrow(mn('2'), mo('+'), mn('3')), mn('3')))
# >>> parse_latex_math(math(), '\max_x') # function takes limits
# math(munder(mo('max', movablelimits=True), mi('x')))
# >>> parse_latex_math(math(), 'x^j_i') # ensure correct order: base, sub, sup
# math(msubsup(mi('x'), mi('i'), mi('j')))
# >>> parse_latex_math(math(), '\int^j_i') # ensure correct order
# math(msubsup(mo('∫'), mi('i'), mi('j')))
# >>> parse_latex_math(math(), 'x_{\\alpha}')
# math(msub(mi('x'), mi('α')))
# >>> parse_latex_math(math(), 'x_\\text{in}')
# math(msub(mi('x'), mtext('in')))


def handle_cmd(name, node, string):  # noqa: C901 TODO make this less complex
    """Process LaTeX command `name` followed by `string`.

    Append result to `node`.
    If needed, parse `string` for command argument.
    Return new current node and remainder of `string`:

    >>> handle_cmd('hbar', math(), r' \frac')
    (math(mi('ℏ')), ' \\frac')
    >>> handle_cmd('hspace', math(), r'{1ex} (x)')
    (math(mspace(width='1ex')), ' (x)')

    """

    # Token elements
    # ==============

    # identifier  ->  <mi>

    if name in letters:
        new_node = mi(letters[name])
        if name in greek_capitals:
            # upright in "TeX style" but MathML sets them italic ("ISO style").
            # CSS styling does not change the font style in Firefox 78.
            # Use 'mathvariant="normal"'?
            new_node['class'] = 'capital-greek'
        node = node.append(new_node)
        return node, string

    if name in functions:
        # use <mi> followed by invisible function applicator character
        # (see https://www.w3.org/TR/MathML3/chapter3.html#presm.mi)
        if name == 'operatorname':
            # custom function name, e.g. ``\operatorname{abs}(x)``
            # TODO: \operatorname* -> with limits
            arg, string = tex_token_or_group(string)
            new_node = mi(arg, mathvariant='normal')
        else:
            new_node = mi(functions[name])
        # embellished function names:
        if name == 'varliminf':    # \underline\lim
            new_node = munder(new_node, mo('_'))
        elif name == 'varlimsup':  # \overline\lim
            new_node = mover(new_node, mo('¯'), accent=False)
        elif name == 'varprojlim':  # \underleftarrow\lim
            new_node = munder(new_node, mo('\u2190'))
        elif name == 'varinjlim':  # \underrightarrow\lim
            new_node = munder(new_node, mo('\u2192'))

        node = node.append(new_node)
        # add ApplyFunction when appropriate (not \sin^2(x), say)
        # cf. https://www.w3.org/TR/MathML3/chapter3.html#presm.mi
        if string and string[0] not in ('^', '_'):
            node = node.append(mo('\u2061'))  # &ApplyFunction;
        return node, string

    if name in modulo_functions:
        (binary, named, parentheses, padding) = modulo_functions[name]
        if binary:
            node = node.append(mo('mod', lspace=padding, rspace=padding))
            return node, string
        # left padding
        if node.is_block():
            padding = '1em'
        node = node.append(mspace(width=padding))
        if parentheses:
            node = node.append(mo('(', stretchy=False))
        if named:
            node = node.append(mi('mod'))
            node = node.append(mspace(width='0.333em'))
        arg, string = tex_token_or_group(string)
        node = parse_latex_math(node, arg)
        if parentheses:
            node = node.append(mo(')', stretchy=False))
        return node, string

    if name in math_alphabets:
        if name == 'boldsymbol':
            attributes = {'class': 'boldsymbol'}
        else:
            attributes = {'mathvariant': math_alphabets[name]}
        if name == 'mathscr':
            attributes['class'] = 'mathscr'
        # Check for single symbol (letter, name, or ⅀)
        arg, remainder = tex_token_or_group(string)
        if arg.startswith('\\'):
            # convert single letters (so the isalpha() test below works).
            # TODO: convert all LICRs in a group (\matrm{\mu\Omega})
            arg = letters.get(arg[1:], arg)
        if name == 'mathbb':
            # mathvariant="double-struck" is ignored for Greek letters
            # (tested in Firefox 78). Use literal Unicode characters.
            arg = mathbb.get(arg, arg)
        if arg.isalpha() or arg == '\u2140':
            node = node.append(mi(arg, **attributes))
            return node, remainder
        # Wrap in <style>
        style = mstyle(**attributes)
        node.append(style)
        return style, string

    # operator, fence, or separator  ->  <mo>

    if name == 'colon':  # trailing punctuation, not binary relation
        node = node.append(mo(':', form='postfix', lspace='0', rspace='0.28em'))
        return node, string

    if name == 'idotsint':
        node = parse_latex_math(node, r'\int\dotsi\int')
        return node, string

    if name in thick_operators:
        node = node.append(mo(thick_operators[name], style='font-weight: bold'))
        return node, string

    if name in small_operators:
        node = node.append(mo(small_operators[name], mathsize='75%'))
        return node, string

    if name in operators:
        attributes = {}
        if name in movablelimits and string and string[0] in ' _^':
            attributes['movablelimits'] = True
        elif name in ('lvert', 'lVert'):
            attributes['class'] = 'mathopen'
        node = node.append(mo(operators[name], **attributes))
        return node, string

    if name in bigdelimiters:
        delimiter_attributes = {}
        size = delimiter_sizes[bigdelimiters[name]]
        delimiter, string = tex_token_or_group(string)
        if delimiter not in '()[]/|.':
            try:
                delimiter = stretchables[delimiter.lstrip('\\')]
            except KeyError:
                raise SyntaxError('Unsupported "\\%s" delimiter "%s"!'
                                  % (name, delimiter))
        if size:
            delimiter_attributes['maxsize'] = size
            delimiter_attributes['minsize'] = size
            delimiter_attributes['symmetric'] = True
        if name == 'left' or name.endswith('l'):
            row = mrow()
            node.append(row)
            node = row
        if delimiter != '.':  # '.' stands for "empty delimiter"
            node.append(mo(delimiter, **delimiter_attributes))
        if name == 'right' or name.endswith('r'):
            node = node.close()
        return node, string

    if name == 'not':
        arg, string = tex_token(string)
        if arg == '{':
            return node, '{\\not ' + string
        if arg.startswith('\\'):  # LaTeX macro
            try:
                arg = operators[arg[1:]]
            except KeyError:
                raise SyntaxError('\\not: Cannot negate: "%s"!'%arg)
        arg = unicodedata.normalize('NFC', arg+'\u0338')
        node = node.append(mo(arg))
        return node, string

    # arbitrary text (usually comments)  ->  <mtext>
    if name in ('text', 'mbox', 'textrm'):
        arg, string = tex_token_or_group(string)
        parts = arg.split('$')  # extract inline math
        for i, part in enumerate(parts):
            if i % 2 == 0:  # i is even
                part = re.sub('(^ | $)', '\u00a0', part)
                node = node.append(mtext(part))
            else:
                parse_latex_math(node, part)
        return node, string

    # horizontal space -> <mspace>
    if name in spaces:
        node = node.append(mspace(width='%s'%spaces[name]))
        return node, string

    if name in ('hspace', 'mspace'):
        arg, string = tex_group(string)
        if arg.endswith('mu'):
            # unit "mu" (1mu=1/18em) not supported by MathML
            arg = '%sem' % (float(arg[:-2])/18)
        node = node.append(mspace(width='%s'%arg))
        return node, string

    if name == 'phantom':
        new_node = mphantom()
        node.append(new_node)
        return new_node, string

    if name == 'boxed':
        new_node = menclose(notation='box')
        node.append(new_node)
        return new_node, string

    # Complex elements (Layout schemata)
    # ==================================

    if name == 'sqrt':
        radix, string = tex_optarg(string)
        if radix:
            indexnode = mrow()
            new_node = mroot(indexnode, switch=True)
            parse_latex_math(indexnode, radix)
            indexnode.close()
        else:
            new_node = msqrt()
        node.append(new_node)
        return new_node, string

    if name in fractions:
        (style_atts, frac_atts) = fractions[name]
        if name == 'cfrac':
            optarg, string = tex_optarg(string)
            optargs = {'l': 'left', 'r': 'right'}
            if optarg in optargs:
                frac_atts = frac_atts.copy()
                frac_atts['numalign'] = optargs[optarg]  # "numalign" is deprecated
                frac_atts['class'] = 'numalign-' + optargs[optarg]
        new_node = frac = mfrac(**frac_atts)
        if name.endswith('binom'):
            new_node = mrow(mo('('), new_node, mo(')'), CLASS='binom')
            new_node.nchildren = 3
        if style_atts:
            new_node = mstyle(new_node, **style_atts)
        node.append(new_node)
        return frac, string

    if name == '\\':  # end of a row
        entry = mtd()
        new_node = mtr(entry)
        node.close().close().append(new_node)
        return entry, string

    if name in accents:
        new_node = mover(mo(accents[name][0], stretchy=False), switch=True)
        if name == 'vec':
            new_node.children[0]['accent'] = False  # scale down arrow but drop i-dot
        new_node.tex_cmd = name  # for HTML export
        node.append(new_node)
        return new_node, string

    if name in over:
        # set "accent" to False (otherwise dots on i and j are dropped)
        # but to True on accent node get "textstyle" (full size) symbols on top
        new_node = mover(mo(over[name][0], accent=True),
                         switch=True, accent=False)
        new_node.tex_cmd = name  # for HTML export
        node.append(new_node)
        return new_node, string

    if name == 'overset':
        new_node = mover(switch=True)
        node.append(new_node)
        return new_node, string

    if name in under:
        new_node = munder(mo(under[name][0]), switch=True)
        new_node.tex_cmd = name  # for HTML export
        node.append(new_node)
        return new_node, string

    if name == 'underset':
        new_node = munder(switch=True)
        node.append(new_node)
        return new_node, string

    if name in ('xleftarrow', 'xrightarrow'):
        subscript, string = tex_optarg(string)
        base = mo(operators['long'+name[1:]])
        if subscript:
            new_node = munderover(base)
            sub_node = parse_latex_math(mrow(), subscript)
            if len(sub_node) == 1:
                sub_node = sub_node.children[0]
            new_node.append(sub_node)
        else:
            new_node = mover(base)
        node.append(new_node)
        return new_node, string

    if name in layout_styles:  # 'displaystyle', 'textstyle', ...
        new_node = mstyle(**layout_styles[name])
        new_node.nchildren = None
        if isinstance(node, mrow) and len(node) == 0:
            # replace node with new_node
            node.parent.children[node.parent.children.index(node)] = new_node
            new_node.parent = node.parent
        elif node.__class__.__name__ == 'math':
            node.append(new_node)
        else:
            raise SyntaxError('Declaration "\\%s" must be first command '
                              'in a group.' % name)
        return new_node, string

    if name.endswith('limits'):
        arg, remainder = tex_token(string)
        if arg in '_^':  # else ignore
            string = remainder
            node = handle_script_or_limit(node, arg, limits=name)
        return node, string

    # Environments

    if name == 'begin':
        return begin_environment(node, string)

    if name == 'end':
        return end_environment(node, string)

    raise SyntaxError('Unknown LaTeX command: \\' + name)

# >>> handle_cmd('left', math(), '[a\\right]')
# (mrow(mo('[')), 'a\\right]')
# >>> handle_cmd('left', math(), '. a)') # empty \left
# (mrow(), ' a)')
# >>> handle_cmd('left', math(), '\\uparrow a)') # cmd
# (mrow(mo('↑')), 'a)')
# >>> handle_cmd('not', math(), '\\equiv \\alpha)') # cmd
# (math(mo('≢')), '\\alpha)')
# >>> handle_cmd('text', math(), '{ for } i>0') # group
# (math(mtext('\xa0for\xa0')), ' i>0')
# >>> handle_cmd('text', math(), '{B}T') # group
# (math(mtext('B')), 'T')
# >>> handle_cmd('text', math(), '{number of apples}}') # group
# (math(mtext('number of apples')), '}')
# >>> handle_cmd('text', math(), 'i \\sin(x)') # single char
# (math(mtext('i')), ' \\sin(x)')
# >>> handle_cmd('sin', math(), '(\\alpha)')
# (math(mi('sin'), mo('\u2061')), '(\\alpha)')
# >>> handle_cmd('sin', math(), ' \\alpha')
# (math(mi('sin'), mo('\u2061')), ' \\alpha')
# >>> handle_cmd('operatorname', math(), '{abs}(x)')
# (math(mi('abs', mathvariant='normal'), mo('\u2061')), '(x)')
# >>> handle_cmd('mathrm', math(), '\\alpha')
# (math(mi('α', mathvariant='normal')), '')
# >>> handle_cmd('mathrm', math(), '{out} = 3')
# (math(mi('out', mathvariant='normal')), ' = 3')
# >>> handle_cmd('overline', math(), '{981}')
# (mover(mo('_', accent=True), switch=True, accent=False), '{981}')
# >>> handle_cmd('bar', math(), '{x}')
# (mover(mo('ˉ', stretchy=False), switch=True), '{x}')
# >>> handle_cmd('xleftarrow', math(), r'[\alpha]{10}')
# (munderover(mo('⟵'), mi('α')), '{10}')
# >>> handle_cmd('xleftarrow', math(), r'[\alpha=5]{10}')
# (munderover(mo('⟵'), mrow(mi('α'), mo('='), mn('5'))), '{10}')


def handle_script_or_limit(node, c, limits=''):
    """Append script or limit element to `node`."""
    child = node.children.pop()
    if limits == 'limits':
        child['movablelimits'] = False
    elif (limits == 'movablelimits'
          or getattr(child, 'data', '') in movablelimits):
        child['movablelimits'] = True

    if c == '_':
        if isinstance(child, mover):
            new_node = munderover(*child.children, switch=True)
        elif isinstance(child, msup):
            new_node = msubsup(*child.children, switch=True)
        elif (limits in ('limits', 'movablelimits')
              or limits == '' and child.get('movablelimits', None)):
            new_node = munder(child)
        else:
            new_node = msub(child)
    elif c == '^':
        if isinstance(child, munder):
            new_node = munderover(*child.children)
        elif isinstance(child, msub):
            new_node = msubsup(*child.children)
        elif (limits in ('limits', 'movablelimits')
              or limits == '' and child.get('movablelimits', None)):
            new_node = mover(child)
        else:
            new_node = msup(child)
    node.append(new_node)
    return new_node


def begin_environment(node, string):
    name, string = tex_group(string)
    if name in matrices:
        left_delimiter = matrices[name][0]
        attributes = {}
        if left_delimiter:
            wrapper = mrow(mo(left_delimiter))
            if name == 'cases':
                wrapper = mrow(mo(left_delimiter, rspace='0.17em'))
                attributes['columnalign'] = 'left'
            node.append(wrapper)
            node = wrapper
        elif name == 'smallmatrix':
            attributes['rowspacing'] = '0.02em'
            attributes['columnspacing'] = '0.333em'
            wrapper = mstyle(scriptlevel='1')
            node.append(wrapper)
            node = wrapper
        # TODO: aligned, alignedat
        # take an optional [t], [b] or the default [c]
        entry = mtd()
        node.append(mtable(mtr(entry), **attributes))
        node = entry
    else:
        raise SyntaxError('Environment not supported!')
    return node, string


def end_environment(node, string):
    name, string = tex_group(string)
    if name in matrices:
        node = node.close().close().close()  # close: mtd, mdr, mtable
        right_delimiter = matrices[name][1]
        if right_delimiter:
            node = node.append(mo(right_delimiter))
            node = node.close()
        elif name == 'cases':
            node = node.close()
    else:
        raise SyntaxError('Environment not supported!')
    return node, string


# Return the number of "equation_columns" in `code_lines`. cf. "alignat"
# in http://mirror.ctan.org/macros/latex/required/amsmath/amsldoc.pdf
def tex_equation_columns(rows):
    tabs = max(row.count('&') - row.count(r'\&') for row in rows)
    if tabs == 0:
        return 0
    return int(tabs/2 + 1)

# >>> tex_equation_columns(['a = b'])
# 0
# >>> tex_equation_columns(['a &= b'])
# 1
# >>> tex_equation_columns(['a &= b & a \in S'])
# 2
# >>> tex_equation_columns(['a &= b & c &= d'])
# 2


# Return dictionary with attributes to style an <mtable> as align environment:
def align_attributes(rows):
    atts = {'class': 'align',
            'displaystyle': True}
    tabs = max(row.count('&') - row.count(r'\&') for row in rows)
    if tabs:
        aligns = ['right', 'left'] * tabs
        spacing = ['0', '2em'] * tabs
        atts['columnalign'] = ' '.join(aligns[:tabs+1])
        atts['columnspacing'] = ' '.join(spacing[:tabs])
    return atts

# >>> align_attributes(['a = b'])
# {'class': 'align', 'displaystyle': True}
# >>> align_attributes(['a &= b'])
# {'class': 'align', 'displaystyle': True, 'columnalign': 'right left', 'columnspacing': '0'}
# >>> align_attributes(['a &= b & a \in S'])
# {'class': 'align', 'displaystyle': True, 'columnalign': 'right left right', 'columnspacing': '0 2em'}
# >>> align_attributes(['a &= b & c &= d'])
# {'class': 'align', 'displaystyle': True, 'columnalign': 'right left right left', 'columnspacing': '0 2em 0'}


def tex2mathml(tex_math, inline=True):
    """Return string with MathML code corresponding to `tex_math`.

    Set `inline` to False for displayed math.
    """
    # Set up tree
    math_tree = math(xmlns='http://www.w3.org/1998/Math/MathML')
    node = math_tree
    if not inline:
        math_tree['display'] = 'block'
        rows = toplevel_code(tex_math).split(r'\\')
        if len(rows) > 1:
            # emulate align* environment with a math table
            node = mtd()
            math_tree.append(mtable(mtr(node),
                                    **align_attributes(rows)))
    parse_latex_math(node, tex_math)
    return math_tree.toprettyxml()

# >>> print(tex2mathml('3'))
# <math xmlns="http://www.w3.org/1998/Math/MathML">
#   <mn>3</mn>
# </math>
# >>> print(tex2mathml('3', inline=False))
# <math xmlns="http://www.w3.org/1998/Math/MathML" display="block">
#   <mn>3</mn>
# </math>
# >>> print(tex2mathml(r'a & b \\ c & d', inline=False))
# <math xmlns="http://www.w3.org/1998/Math/MathML" display="block">
#   <mtable class="align" displaystyle="true" columnalign="right left" columnspacing="0">
#     <mtr>
#       <mtd>
#         <mi>a</mi>
#       </mtd>
#       <mtd>
#         <mi>b</mi>
#       </mtd>
#     </mtr>
#     <mtr>
#       <mtd>
#         <mi>c</mi>
#       </mtd>
#       <mtd>
#         <mi>d</mi>
#       </mtd>
#     </mtr>
#   </mtable>
# </math>
# >>> print(tex2mathml(r'a \\ b', inline=False))
# <math xmlns="http://www.w3.org/1998/Math/MathML" display="block">
#   <mtable class="align" displaystyle="true">
#     <mtr>
#       <mtd>
#         <mi>a</mi>
#       </mtd>
#     </mtr>
#     <mtr>
#       <mtd>
#         <mi>b</mi>
#       </mtd>
#     </mtr>
#   </mtable>
# </math>


# TODO: look up more symbols from tr25, e.g.
#
#
# Table 2.8 Using Vertical Line or Solidus Overlay
#   some of the negated forms of mathematical relations that can only be
#   encoded by using either U+0338 COMBINING LONG SOLIDUS OVERLAY or U+20D2
#   COMBINING LONG VERTICAL LINE OVERLAY . (For issues with using 0338 in
#   MathML, see Section 3.2.7, Combining Marks.
#
# Table 2.9 Variants of Mathematical Symbols using VS1?
#
# Sequence      Description
# 0030 + VS1    DIGIT ZERO - short diagonal stroke form
# 2205 + VS1    EMPTY SET - zero with long diagonal stroke overlay form
# 2229 + VS1    INTERSECTION - with serifs
# 222A + VS1    UNION - with serifs
# 2268 + VS1    LESS-THAN BUT NOT EQUAL TO - with vertical stroke
# 2269 + VS1    GREATER-THAN BUT NOT EQUAL TO - with vertical stroke
# 2272 + VS1    LESS-THAN OR EQUIVALENT TO - following the slant of the lower leg
# 2273 + VS1    GREATER-THAN OR EQUIVALENT TO - following the slant of the lower leg
# 228A + VS1    SUBSET OF WITH NOT EQUAL TO - variant with stroke through bottom members
# 228B + VS1    SUPERSET OF WITH NOT EQUAL TO - variant with stroke through bottom members
# 2293 + VS1    SQUARE CAP - with serifs
# 2294 + VS1    SQUARE CUP - with serifs
# 2295 + VS1    CIRCLED PLUS - with white rim
# 2297 + VS1    CIRCLED TIMES - with white rim
# 229C + VS1    CIRCLED EQUALS - equal sign inside and touching the circle
# 22DA + VS1    LESS-THAN slanted EQUAL TO OR GREATER-THAN
# 22DB + VS1    GREATER-THAN slanted EQUAL TO OR LESS-THAN
# 2A3C + VS1    INTERIOR PRODUCT - tall variant with narrow foot
# 2A3D + VS1    RIGHTHAND INTERIOR PRODUCT - tall variant with narrow foot
# 2A9D + VS1    SIMILAR OR LESS-THAN - following the slant of the upper leg
# 2A9E + VS1    SIMILAR OR GREATER-THAN - following the slant of the upper leg
# 2AAC + VS1    SMALLER THAN OR slanted EQUAL
# 2AAD + VS1    LARGER THAN OR slanted EQUAL
# 2ACB + VS1    SUBSET OF ABOVE NOT EQUAL TO - variant with stroke through bottom members
# 2ACC + VS1    SUPERSET OF ABOVE NOT EQUAL TO - variant with stroke through bottom members

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/math/latex2mathml.py
NikolaBlagojevic/pyrecodes
#! /usr/bin/env python3
#   math2html: convert LaTeX equations to HTML output.
#
#   Copyright (C) 2009-2011 Alex Fernández, 2021 Günter Milde
#
#   Released under the terms of the `2-Clause BSD license'_, in short:
#   Copying and distribution of this file, with or without modification,
#   are permitted in any medium without royalty provided the copyright
#   notice and this notice are preserved.
#   This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause

#   Based on eLyXer: convert LyX source files to HTML output.
#   http://alexfernandez.github.io/elyxer/

# Versions:
# 1.2.5  2015-02-26  eLyXer standalone formula conversion to HTML.
# 1.3    2021-06-02  Removed code for conversion of LyX files not
#                    required for LaTeX math.
#                    Support for more math commands from the AMS "math-guide".
# 2.0    2021-12-31  Drop 2.7 compatibility code.

import pathlib
import sys
import unicodedata

from docutils.utils.math import tex2unichar


__version__ = '1.3 (2021-06-02)'


class Trace:
    "A tracing class"

    debugmode = False
    quietmode = False
    showlinesmode = False

    prefix = None

    def debug(cls, message):
        "Show a debug message"
        if not Trace.debugmode or Trace.quietmode:
            return
        Trace.show(message, sys.stdout)

    def message(cls, message):
        "Show a trace message"
        if Trace.quietmode:
            return
        if Trace.prefix and Trace.showlinesmode:
            message = Trace.prefix + message
        Trace.show(message, sys.stdout)

    def error(cls, message):
        "Show an error message"
        message = '* ' + message
        if Trace.prefix and Trace.showlinesmode:
            message = Trace.prefix + message
        Trace.show(message, sys.stderr)

    def fatal(cls, message):
        "Show an error message and terminate"
        Trace.error('FATAL: ' + message)
        exit(-1)

    def show(cls, message, channel):
        "Show a message out of a channel"
        channel.write(message + '\n')

    debug = classmethod(debug)
    message = classmethod(message)
    error = classmethod(error)
    fatal = classmethod(fatal)
    show = classmethod(show)


class ContainerConfig:
    "Configuration class from elyxer.config file"

    extracttext = {
        'allowed': ['FormulaConstant'],
        'extracted': ['AlphaCommand',
                      'Bracket',
                      'BracketCommand',
                      'CombiningFunction',
                      'EmptyCommand',
                      'FontFunction',
                      'Formula',
                      'FormulaNumber',
                      'FormulaSymbol',
                      'OneParamFunction',
                      'OversetFunction',
                      'RawText',
                      'SpacedCommand',
                      'SymbolFunction',
                      'TextFunction',
                      'UndersetFunction',
                      ],
    }


class EscapeConfig:
    "Configuration class from elyxer.config file"

    chars = {
        '\n': '',
        "'": '’',
        '`': '‘',
    }

    entities = {
        '&': '&amp;',
        '<': '&lt;',
        '>': '&gt;',
    }


class FormulaConfig:
    "Configuration class from elyxer.config file"

    alphacommands = {
        '\\AmS': '<span class="textsc">AmS</span>',
        '\\AA':        'Å',
        '\\AE':        'Æ',
        '\\DH':        'Ð',
        '\\L':         'Ł',
        '\\O':         'Ø',
        '\\OE':        'Œ',
        '\\TH':        'Þ',
        '\\aa':        'å',
        '\\ae':        'æ',
        '\\dh':        'ð',
        '\\i':         'ı',
        '\\j':         'ȷ',
        '\\l':         'ł',
        '\\o':         'ø',
        '\\oe':        'œ',
        '\\ss':        'ß',
        '\\th':        'þ',
    }
    for key, value in tex2unichar.mathalpha.items():
        alphacommands['\\'+key] = value

    array = {
        'begin': r'\begin',
        'cellseparator': '&',
        'end': r'\end',
        'rowseparator': r'\\',
    }

    bigbrackets = {'(': ['⎛', '⎜', '⎝'],
                   ')': ['⎞', '⎟', '⎠'],
                   '[': ['⎡', '⎢', '⎣'],
                   ']': ['⎤', '⎥', '⎦'],
                   '{': ['⎧', '⎪', '⎨', '⎩'],
                   '}': ['⎫', '⎪', '⎬', '⎭'],
                   # TODO: 2-row brackets with ⎰⎱ (\lmoustache \rmoustache)
                   '|': ['|'],  # 007C VERTICAL LINE
                   # '|': ['⎮'],  # 23AE INTEGRAL EXTENSION
                   # '|': ['⎪'],  # 23AA CURLY BRACKET EXTENSION
                   '‖': ['‖'],  # 2016 DOUBLE VERTICAL LINE
                   # '∥': ['∥'],  # 2225 PARALLEL TO
                   }

    bracketcommands = {
        '\\left': 'span class="stretchy"',
        '\\left.': '<span class="leftdot"></span>',
        '\\middle': 'span class="stretchy"',
        '\\right': 'span class="stretchy"',
        '\\right.': '<span class="rightdot"></span>',
    }

    combiningfunctions = {
        "\\'":           '\u0301',  # x́
        '\\"':           '\u0308',  # ẍ
        '\\^':           '\u0302',  # x̂
        '\\`':           '\u0300',  # x̀
        '\\~':           '\u0303',  # x̃
        '\\c':           '\u0327',  # x̧
        '\\r':           '\u030a',  # x̊
        '\\s':           '\u0329',  # x̩
        '\\textcircled': '\u20dd',  # x⃝
        '\\textsubring': '\u0325',  # x̥
        '\\v':           '\u030c',  # x̌
    }
    for key, value in tex2unichar.mathaccent.items():
        combiningfunctions['\\'+key] = value

    commands = {
        '\\\\': '<br/>',
        '\\\n': ' ',  # escaped whitespace
        '\\\t': ' ',  # escaped whitespace
        '\\centerdot': '\u2B1D',  # BLACK VERY SMALL SQUARE, mathbin
        '\\colon': ': ',
        '\\copyright': '©',
        '\\dotminus': '∸',
        '\\dots': '…',
        '\\dotsb': '⋯',
        '\\dotsc': '…',
        '\\dotsi': '⋯',
        '\\dotsm': '⋯',
        '\\dotso': '…',
        '\\euro': '€',
        '\\guillemotleft': '«',
        '\\guillemotright': '»',
        '\\hbar': '<i>\u0127</i>',  # ħ LATIN SMALL LETTER H WITH STROKE
        '\\lVert': '‖',
        '\\Arrowvert':  '\u2016',  # ‖
        '\\lvert': '|',
        '\\newline': '<br/>',
        '\\nobreakspace': ' ',
        '\\nolimits': '',
        '\\nonumber': '',
        '\\qquad': '  ',
        '\\rVert': '‖',
        '\\rvert': '|',
        '\\textasciicircum': '^',
        '\\textasciitilde': '~',
        '\\textbackslash': '\\',
        '\\textcopyright': '©',
        '\\textdegree': '°',
        '\\textellipsis': '…',
        '\\textemdash': '—',
        '\\textendash': '—',
        '\\texteuro': '€',
        '\\textgreater': '>',
        '\\textless': '<',
        '\\textordfeminine': 'ª',
        '\\textordmasculine': 'º',
        '\\textquotedblleft': '“',
        '\\textquotedblright': '”',
        '\\textquoteright': '’',
        '\\textregistered': '®',
        '\\textrightarrow': '→',
        '\\textsection': '§',
        '\\texttrademark': '™',
        '\\texttwosuperior': '²',
        '\\textvisiblespace': ' ',
        '\\thickspace': '<span class="thickspace"> </span>',  # 5/13 em
        '\\;': '<span class="thickspace"> </span>',  # 5/13 em
        '\\triangle': '\u25B3',  # WHITE UP-POINTING TRIANGLE, mathord
        '\\triangledown': '\u25BD',  # WHITE DOWN-POINTING TRIANGLE, mathord
        '\\varnothing': '\u2300',  # ⌀ DIAMETER SIGN
        # functions
        '\\Pr': 'Pr',
        '\\arccos': 'arccos',
        '\\arcsin': 'arcsin',
        '\\arctan': 'arctan',
        '\\arg': 'arg',
        '\\cos': 'cos',
        '\\cosh': 'cosh',
        '\\cot': 'cot',
        '\\coth': 'coth',
        '\\csc': 'csc',
        '\\deg': 'deg',
        '\\det': 'det',
        '\\dim': 'dim',
        '\\exp': 'exp',
        '\\gcd': 'gcd',
        '\\hom': 'hom',
        '\\injlim': 'inj lim',
        '\\ker': 'ker',
        '\\lg': 'lg',
        '\\liminf': 'lim inf',
        '\\limsup': 'lim sup',
        '\\ln': 'ln',
        '\\log': 'log',
        '\\projlim': 'proj lim',
        '\\sec': 'sec',
        '\\sin': 'sin',
        '\\sinh': 'sinh',
        '\\tan': 'tan',
        '\\tanh': 'tanh',
    }
    cmddict = {}
    cmddict.update(tex2unichar.mathbin)  # TODO: spacing around binary operators
    cmddict.update(tex2unichar.mathopen)
    cmddict.update(tex2unichar.mathclose)
    cmddict.update(tex2unichar.mathfence)
    cmddict.update(tex2unichar.mathord)
    cmddict.update(tex2unichar.mathpunct)
    cmddict.update(tex2unichar.space)
    commands.update(('\\' + key, value) for key, value in cmddict.items())

    oversetfunctions = {
        # math accents (cf. combiningfunctions)
        # '\\acute':    '´',
        '\\bar':      '‒',  # FIGURE DASH
        # '\\breve':    '˘',
        # '\\check':    'ˇ',
        '\\dddot':    '<span class="smallsymbol">⋯</span>',
        # '\\ddot':     '··', # ¨ too high
        # '\\dot':      '·',
        # '\\grave':    '`',
        # '\\hat':      '^',
        # '\\mathring': '˚',
        # '\\tilde':    '~',
        '\\vec':      '<span class="smallsymbol">→</span>',
        # embellishments
        '\\overleftarrow': '⟵',
        '\\overleftrightarrow': '⟷',
        '\\overrightarrow': '⟶',
        '\\widehat': '^',
        '\\widetilde': '～',
    }

    undersetfunctions = {
        '\\underleftarrow': '⟵',
        '\\underleftrightarrow': '⟷',
        '\\underrightarrow': '⟶',
    }

    endings = {
        'bracket': '}',
        'complex': '\\]',
        'endafter': '}',
        'endbefore': '\\end{',
        'squarebracket': ']',
    }

    environments = {
        'align': ['r', 'l'],
        'eqnarray': ['r', 'c', 'l'],
        'gathered': ['l', 'l'],
        'smallmatrix': ['c', 'c'],
    }

    fontfunctions = {
        '\\boldsymbol': 'b', '\\mathbb': 'span class="blackboard"',
        '\\mathbb{A}': '𝔸', '\\mathbb{B}': '𝔹', '\\mathbb{C}': 'ℂ',
        '\\mathbb{D}': '𝔻', '\\mathbb{E}': '𝔼', '\\mathbb{F}': '𝔽',
        '\\mathbb{G}': '𝔾', '\\mathbb{H}': 'ℍ', '\\mathbb{J}': '𝕁',
        '\\mathbb{K}': '𝕂', '\\mathbb{L}': '𝕃', '\\mathbb{N}': 'ℕ',
        '\\mathbb{O}': '𝕆', '\\mathbb{P}': 'ℙ', '\\mathbb{Q}': 'ℚ',
        '\\mathbb{R}': 'ℝ', '\\mathbb{S}': '𝕊', '\\mathbb{T}': '𝕋',
        '\\mathbb{W}': '𝕎', '\\mathbb{Z}': 'ℤ', '\\mathbf': 'b',
        '\\mathcal': 'span class="scriptfont"',
        '\\mathcal{B}': 'ℬ', '\\mathcal{E}': 'ℰ', '\\mathcal{F}':
        'ℱ', '\\mathcal{H}': 'ℋ', '\\mathcal{I}': 'ℐ',
        '\\mathcal{L}': 'ℒ', '\\mathcal{M}': 'ℳ', '\\mathcal{R}': 'ℛ',
        '\\mathfrak': 'span class="fraktur"',
        '\\mathfrak{C}': 'ℭ', '\\mathfrak{F}': '𝔉', '\\mathfrak{H}': 'ℌ',
        '\\mathfrak{I}': 'ℑ', '\\mathfrak{R}': 'ℜ', '\\mathfrak{Z}': 'ℨ',
        '\\mathit': 'i',
        '\\mathring{A}': 'Å', '\\mathring{U}': 'Ů',
        '\\mathring{a}': 'å', '\\mathring{u}': 'ů', '\\mathring{w}': 'ẘ',
        '\\mathring{y}': 'ẙ',
        '\\mathrm': 'span class="mathrm"',
        '\\mathscr': 'span class="mathscr"',
        '\\mathscr{B}': 'ℬ', '\\mathscr{E}': 'ℰ', '\\mathscr{F}': 'ℱ',
        '\\mathscr{H}': 'ℋ', '\\mathscr{I}': 'ℐ', '\\mathscr{L}': 'ℒ',
        '\\mathscr{M}': 'ℳ', '\\mathscr{R}': 'ℛ',
        '\\mathsf': 'span class="mathsf"',
        '\\mathtt': 'span class="mathtt"',
        '\\operatorname': 'span class="mathrm"',
    }

    hybridfunctions = {
        '\\addcontentsline': ['{$p!}{$q!}{$r!}', 'f0{}', 'ignored'],
        '\\addtocontents': ['{$p!}{$q!}', 'f0{}', 'ignored'],
        '\\backmatter': ['', 'f0{}', 'ignored'],
        '\\binom': ['{$1}{$2}', 'f2{(}f0{f1{$1}f1{$2}}f2{)}', 'span class="binom"', 'span class="binomstack"', 'span class="bigdelimiter size2"'],
        '\\boxed': ['{$1}', 'f0{$1}', 'span class="boxed"'],
        '\\cfrac': ['[$p!]{$1}{$2}', 'f0{f3{(}f1{$1}f3{)/(}f2{$2}f3{)}}', 'span class="fullfraction"', 'span class="numerator align-$p"', 'span class="denominator"', 'span class="ignored"'],
        '\\color': ['{$p!}{$1}', 'f0{$1}', 'span style="color: $p;"'],
        '\\colorbox': ['{$p!}{$1}', 'f0{$1}', 'span class="colorbox" style="background: $p;"'],
        '\\dbinom': ['{$1}{$2}', '(f0{f1{f2{$1}}f1{f2{ }}f1{f2{$2}}})', 'span class="binomial"', 'span class="binomrow"', 'span class="binomcell"'],
        '\\dfrac': ['{$1}{$2}', 'f0{f3{(}f1{$1}f3{)/(}f2{$2}f3{)}}', 'span class="fullfraction"', 'span class="numerator"', 'span class="denominator"', 'span class="ignored"'],
        '\\displaystyle': ['{$1}', 'f0{$1}', 'span class="displaystyle"'],
        '\\fancyfoot': ['[$p!]{$q!}', 'f0{}', 'ignored'],
        '\\fancyhead': ['[$p!]{$q!}', 'f0{}', 'ignored'],
        '\\fbox': ['{$1}', 'f0{$1}', 'span class="fbox"'],
        '\\fboxrule': ['{$p!}', 'f0{}', 'ignored'],
        '\\fboxsep': ['{$p!}', 'f0{}', 'ignored'],
        '\\fcolorbox': ['{$p!}{$q!}{$1}', 'f0{$1}', 'span class="boxed" style="border-color: $p; background: $q;"'],
        '\\frac': ['{$1}{$2}', 'f0{f3{(}f1{$1}f3{)/(}f2{$2}f3{)}}', 'span class="fraction"', 'span class="numerator"', 'span class="denominator"', 'span class="ignored"'],
        '\\framebox': ['[$p!][$q!]{$1}', 'f0{$1}', 'span class="framebox align-$q" style="width: $p;"'],
        '\\frontmatter': ['', 'f0{}', 'ignored'],
        '\\href': ['[$o]{$u!}{$t!}', 'f0{$t}', 'a href="$u"'],
        '\\hspace': ['{$p!}', 'f0{ }', 'span class="hspace" style="width: $p;"'],
        '\\leftroot': ['{$p!}', 'f0{ }', 'span class="leftroot" style="width: $p;px"'],
        # TODO: convert 1 mu to 1/18 em
        # '\\mspace': ['{$p!}', 'f0{ }', 'span class="hspace" style="width: $p;"'],
        '\\nicefrac': ['{$1}{$2}', 'f0{f1{$1}⁄f2{$2}}', 'span class="fraction"', 'sup class="numerator"', 'sub class="denominator"', 'span class="ignored"'],
        '\\parbox': ['[$p!]{$w!}{$1}', 'f0{1}', 'div class="Boxed" style="width: $w;"'],
        '\\raisebox': ['{$p!}{$1}', 'f0{$1.font}', 'span class="raisebox" style="vertical-align: $p;"'],
        '\\renewenvironment': ['{$1!}{$2!}{$3!}', ''],
        '\\rule': ['[$v!]{$w!}{$h!}', 'f0/', 'hr class="line" style="width: $w; height: $h;"'],
        '\\scriptscriptstyle': ['{$1}', 'f0{$1}', 'span class="scriptscriptstyle"'],
        '\\scriptstyle': ['{$1}', 'f0{$1}', 'span class="scriptstyle"'],
        # TODO: increase √-size with argument (\frac in display mode, ...)
        '\\sqrt': ['[$0]{$1}', 'f0{f1{$0}f2{√}f4{(}f3{$1}f4{)}}', 'span class="sqrt"', 'sup class="root"', 'span class="radical"', 'span class="root"', 'span class="ignored"'],
        '\\stackrel': ['{$1}{$2}', 'f0{f1{$1}f2{$2}}', 'span class="stackrel"', 'span class="upstackrel"', 'span class="downstackrel"'],
        '\\tbinom': ['{$1}{$2}', '(f0{f1{f2{$1}}f1{f2{ }}f1{f2{$2}}})', 'span class="binomial"', 'span class="binomrow"', 'span class="binomcell"'],
        '\\tfrac':  ['{$1}{$2}', 'f0{f3{(}f1{$1}f3{)/(}f2{$2}f3{)}}', 'span class="textfraction"', 'span class="numerator"', 'span class="denominator"', 'span class="ignored"'],
        '\\textcolor': ['{$p!}{$1}', 'f0{$1}', 'span style="color: $p;"'],
        '\\textstyle': ['{$1}', 'f0{$1}', 'span class="textstyle"'],
        '\\thispagestyle': ['{$p!}', 'f0{}', 'ignored'],
        '\\unit': ['[$0]{$1}', '$0f0{$1.font}', 'span class="unit"'],
        '\\unitfrac': ['[$0]{$1}{$2}', '$0f0{f1{$1.font}⁄f2{$2.font}}', 'span class="fraction"', 'sup class="unit"', 'sub class="unit"'],
        '\\uproot': ['{$p!}', 'f0{ }', 'span class="uproot" style="width: $p;px"'],
        '\\url': ['{$u!}', 'f0{$u}', 'a href="$u"'],
        '\\vspace': ['{$p!}', 'f0{ }', 'span class="vspace" style="height: $p;"'],
    }

    hybridsizes = {
        '\\binom': '$1+$2', '\\cfrac': '$1+$2', '\\dbinom': '$1+$2+1',
        '\\dfrac': '$1+$2', '\\frac': '$1+$2', '\\tbinom': '$1+$2+1',
    }

    labelfunctions = {
        '\\label': 'a name="#"',
    }

    limitcommands = {
        '\\biginterleave': '⫼',
        '\\inf': 'inf',
        '\\lim': 'lim',
        '\\max': 'max',
        '\\min': 'min',
        '\\sup': 'sup',
        '\\ointop':    '<span class="bigoperator integral">∮</span>',
        '\\bigcap':    '<span class="bigoperator">⋂</span>',
        '\\bigcup':    '<span class="bigoperator">⋃</span>',
        '\\bigodot':   '<span class="bigoperator">⨀</span>',
        '\\bigoplus':  '<span class="bigoperator">⨁</span>',
        '\\bigotimes': '<span class="bigoperator">⨂</span>',
        '\\bigsqcap':  '<span class="bigoperator">⨅</span>',
        '\\bigsqcup':  '<span class="bigoperator">⨆</span>',
        '\\biguplus':  '<span class="bigoperator">⨄</span>',
        '\\bigvee':    '<span class="bigoperator">⋁</span>',
        '\\bigwedge':  '<span class="bigoperator">⋀</span>',
        '\\coprod':    '<span class="bigoperator">∐</span>',
        '\\intop':     '<span class="bigoperator integral">∫</span>',
        '\\prod':      '<span class="bigoperator">∏</span>',
        '\\sum':       '<span class="bigoperator">∑</span>',
        '\\varprod':   '<span class="bigoperator">⨉</span>',
        '\\zcmp': '⨟', '\\zhide': '⧹', '\\zpipe': '⨠', '\\zproject': '⨡',
        # integrals have limits in index position with LaTeX default settings
        # TODO: move to commands?
        '\\int': '<span class="bigoperator integral">∫</span>',
        '\\iint': '<span class="bigoperator integral">∬</span>',
        '\\iiint': '<span class="bigoperator integral">∭</span>',
        '\\iiiint': '<span class="bigoperator integral">⨌</span>',
        '\\fint': '<span class="bigoperator integral">⨏</span>',
        '\\idotsint': '<span class="bigoperator integral">∫⋯∫</span>',
        '\\oint': '<span class="bigoperator integral">∮</span>',
        '\\oiint': '<span class="bigoperator integral">∯</span>',
        '\\oiiint': '<span class="bigoperator integral">∰</span>',
        '\\ointclockwise': '<span class="bigoperator integral">∲</span>',
        '\\ointctrclockwise': '<span class="bigoperator integral">∳</span>',
        '\\smallint': '<span class="smallsymbol integral">∫</span>',
        '\\sqint': '<span class="bigoperator integral">⨖</span>',
        '\\varointclockwise': '<span class="bigoperator integral">∲</span>',
    }

    modified = {
        '\n': '', ' ': '', '$': '', '&': '	', '\'': '’', '+': '\u2009+\u2009',
        ',': ',\u2009', '-': '\u2009−\u2009', '/': '\u2009⁄\u2009', ':': ' : ', '<': '\u2009&lt;\u2009',
        '=': '\u2009=\u2009', '>': '\u2009&gt;\u2009', '@': '', '~': '\u00a0',
    }

    onefunctions = {
        '\\big': 'span class="bigdelimiter size1"',
        '\\bigl': 'span class="bigdelimiter size1"',
        '\\bigr': 'span class="bigdelimiter size1"',
        '\\Big': 'span class="bigdelimiter size2"',
        '\\Bigl': 'span class="bigdelimiter size2"',
        '\\Bigr': 'span class="bigdelimiter size2"',
        '\\bigg': 'span class="bigdelimiter size3"',
        '\\biggl': 'span class="bigdelimiter size3"',
        '\\biggr': 'span class="bigdelimiter size3"',
        '\\Bigg': 'span class="bigdelimiter size4"',
        '\\Biggl': 'span class="bigdelimiter size4"',
        '\\Biggr': 'span class="bigdelimiter size4"',
        # '\\bar': 'span class="bar"',
        '\\begin{array}': 'span class="arraydef"',
        '\\centering': 'span class="align-center"',
        '\\ensuremath': 'span class="ensuremath"',
        '\\hphantom': 'span class="phantom"',
        '\\noindent': 'span class="noindent"',
        '\\overbrace': 'span class="overbrace"',
        '\\overline': 'span class="overline"',
        '\\phantom': 'span class="phantom"',
        '\\underbrace': 'span class="underbrace"',
        '\\underline': '',
        '\\vphantom': 'span class="phantom"',
    }

    # relations (put additional space before and after the symbol)
    spacedcommands = {
        # negated symbols without pre-composed Unicode character
        '\\nleqq':      '\u2266\u0338',  # ≦̸
        '\\ngeqq':      '\u2267\u0338',  # ≧̸
        '\\nleqslant':  '\u2a7d\u0338',  # ⩽̸
        '\\ngeqslant':  '\u2a7e\u0338',  # ⩾̸
        '\\nsubseteqq': '\u2AC5\u0338',  # ⫅̸
        '\\nsupseteqq': '\u2AC6\u0338',  # ⫆̸
        '\\nsqsubset':  '\u2276\u228F',  # ⊏̸
        # modified glyphs
        '\\shortmid': '<span class="smallsymbol">∣</span>',
        '\\shortparallel': '<span class="smallsymbol">∥</span>',
        '\\nshortmid': '<span class="smallsymbol">∤</span>',
        '\\nshortparallel': '<span class="smallsymbol">∦</span>',
        '\\smallfrown': '<span class="smallsymbol">⌢</span>',
        '\\smallsmile': '<span class="smallsymbol">⌣</span>',
        '\\thickapprox': '<span class="boldsymbol">≈</span>',
        '\\thicksim': '<span class="boldsymbol">∼</span>',
        '\\varpropto': '<span class="mathsf">\u221d</span>',  # ∝ PROPORTIONAL TO
    }
    for key, value in tex2unichar.mathrel.items():
        spacedcommands['\\'+key] = value
    starts = {
        'beginafter': '}', 'beginbefore': '\\begin{', 'bracket': '{',
        'command': '\\', 'comment': '%', 'complex': '\\[', 'simple': '$',
        'squarebracket': '[', 'unnumbered': '*',
    }

    symbolfunctions = {
        '^': 'sup', '_': 'sub',
    }

    textfunctions = {
        '\\mbox': 'span class="mbox"',
        '\\text': 'span class="text"',
        '\\textbf': 'span class="textbf"',
        '\\textit': 'span class="textit"',
        '\\textnormal': 'span class="textnormal"',
        '\\textrm': 'span class="textrm"',
        '\\textsc': 'span class="textsc"',
        '\\textsf': 'span class="textsf"',
        '\\textsl': 'span class="textsl"',
        '\\texttt': 'span class="texttt"',
        '\\textup': 'span class="normal"',
    }

    unmodified = {
        'characters': ['.', '*', '€', '(', ')', '[', ']',
                       '·', '!', ';', '|', '§', '"', '?'],
        }


class CommandLineParser:
    "A parser for runtime options"

    def __init__(self, options):
        self.options = options

    def parseoptions(self, args):
        "Parse command line options"
        if len(args) == 0:
            return None
        while len(args) > 0 and args[0].startswith('--'):
            key, value = self.readoption(args)
            if not key:
                return 'Option ' + value + ' not recognized'
            if not value:
                return 'Option ' + key + ' needs a value'
            setattr(self.options, key, value)
        return None

    def readoption(self, args):
        "Read the key and value for an option"
        arg = args[0][2:]
        del args[0]
        if '=' in arg:
            key = self.readequalskey(arg, args)
        else:
            key = arg.replace('-', '')
        if not hasattr(self.options, key):
            return None, key
        current = getattr(self.options, key)
        if isinstance(current, bool):
            return key, True
        # read value
        if len(args) == 0:
            return key, None
        if args[0].startswith('"'):
            initial = args[0]
            del args[0]
            return key, self.readquoted(args, initial)
        value = args[0].decode('utf-8')
        del args[0]
        if isinstance(current, list):
            current.append(value)
            return key, current
        return key, value

    def readquoted(self, args, initial):
        "Read a value between quotes"
        Trace.error('Oops')
        value = initial[1:]
        while len(args) > 0 and not args[0].endswith('"') and not args[0].startswith('--'):
            Trace.error('Appending ' + args[0])
            value += ' ' + args[0]
            del args[0]
        if len(args) == 0 or args[0].startswith('--'):
            return None
        value += ' ' + args[0:-1]
        return value

    def readequalskey(self, arg, args):
        "Read a key using equals"
        split = arg.split('=', 1)
        key = split[0]
        value = split[1]
        args.insert(0, value)
        return key


class Options:
    "A set of runtime options"

    location = None

    debug = False
    quiet = False
    version = False
    help = False
    simplemath = False
    showlines = True

    branches = {}

    def parseoptions(self, args):
        "Parse command line options"
        Options.location = args[0]
        del args[0]
        parser = CommandLineParser(Options)
        result = parser.parseoptions(args)
        if result:
            Trace.error(result)
            self.usage()
        self.processoptions()

    def processoptions(self):
        "Process all options parsed."
        if Options.help:
            self.usage()
        if Options.version:
            self.showversion()
        # set in Trace if necessary
        for param in dir(Trace):
            if param.endswith('mode'):
                setattr(Trace, param, getattr(self, param[:-4]))

    def usage(self):
        "Show correct usage"
        Trace.error(f'Usage: {pathlib.Path(Options.location).parent}'
                    ' [options] "input string"')
        Trace.error('Convert input string with LaTeX math to MathML')
        self.showoptions()

    def showoptions(self):
        "Show all possible options"
        Trace.error('    --help:                 show this online help')
        Trace.error('    --quiet:                disables all runtime messages')
        Trace.error('    --debug:                enable debugging messages (for developers)')
        Trace.error('    --version:              show version number and release date')
        Trace.error('    --simplemath:           do not generate fancy math constructions')
        sys.exit()

    def showversion(self):
        "Return the current eLyXer version string"
        Trace.error('math2html '+__version__)
        sys.exit()


class Cloner:
    "An object used to clone other objects."

    def clone(cls, original):
        "Return an exact copy of an object."
        "The original object must have an empty constructor."
        return cls.create(original.__class__)

    def create(cls, type):
        "Create an object of a given class."
        clone = type.__new__(type)
        clone.__init__()
        return clone

    clone = classmethod(clone)
    create = classmethod(create)


class ContainerExtractor:
    """A class to extract certain containers.

    The config parameter is a map containing three lists:
    allowed, copied and extracted.
    Each of the three is a list of class names for containers.
    Allowed containers are included as is into the result.
    Cloned containers are cloned and placed into the result.
    Extracted containers are looked into.
    All other containers are silently ignored.
    """

    def __init__(self, config):
        self.allowed = config['allowed']
        self.extracted = config['extracted']

    def extract(self, container):
        "Extract a group of selected containers from a container."
        list = []
        locate = lambda c: c.__class__.__name__ in self.allowed
        recursive = lambda c: c.__class__.__name__ in self.extracted
        process = lambda c: self.process(c, list)
        container.recursivesearch(locate, recursive, process)
        return list

    def process(self, container, list):
        "Add allowed containers."
        name = container.__class__.__name__
        if name in self.allowed:
            list.append(container)
        else:
            Trace.error('Unknown container class ' + name)

    def safeclone(self, container):
        "Return a new container with contents only in a safe list, recursively."
        clone = Cloner.clone(container)
        clone.output = container.output
        clone.contents = self.extract(container)
        return clone


class Parser:
    "A generic parser"

    def __init__(self):
        self.begin = 0
        self.parameters = {}

    def parseheader(self, reader):
        "Parse the header"
        header = reader.currentline().split()
        reader.nextline()
        self.begin = reader.linenumber
        return header

    def parseparameter(self, reader):
        "Parse a parameter"
        split = reader.currentline().strip().split(' ', 1)
        reader.nextline()
        if len(split) == 0:
            return
        key = split[0]
        if len(split) == 1:
            self.parameters[key] = True
            return
        if '"' not in split[1]:
            self.parameters[key] = split[1].strip()
            return
        doublesplit = split[1].split('"')
        self.parameters[key] = doublesplit[1]

    def parseending(self, reader, process):
        "Parse until the current ending is found"
        if not self.ending:
            Trace.error('No ending for ' + str(self))
            return
        while not reader.currentline().startswith(self.ending):
            process()

    def parsecontainer(self, reader, contents):
        container = self.factory.createcontainer(reader)
        if container:
            container.parent = self.parent
            contents.append(container)

    def __str__(self):
        "Return a description"
        return self.__class__.__name__ + ' (' + str(self.begin) + ')'


class LoneCommand(Parser):
    "A parser for just one command line"

    def parse(self, reader):
        "Read nothing"
        return []


class TextParser(Parser):
    "A parser for a command and a bit of text"

    stack = []

    def __init__(self, container):
        Parser.__init__(self)
        self.ending = None
        if container.__class__.__name__ in ContainerConfig.endings:
            self.ending = ContainerConfig.endings[container.__class__.__name__]
        self.endings = []

    def parse(self, reader):
        "Parse lines as long as they are text"
        TextParser.stack.append(self.ending)
        self.endings = TextParser.stack + [ContainerConfig.endings['Layout'],
                                           ContainerConfig.endings['Inset'],
                                           self.ending]
        contents = []
        while not self.isending(reader):
            self.parsecontainer(reader, contents)
        return contents

    def isending(self, reader):
        "Check if text is ending"
        current = reader.currentline().split()
        if len(current) == 0:
            return False
        if current[0] in self.endings:
            if current[0] in TextParser.stack:
                TextParser.stack.remove(current[0])
            else:
                TextParser.stack = []
            return True
        return False


class ExcludingParser(Parser):
    "A parser that excludes the final line"

    def parse(self, reader):
        "Parse everything up to (and excluding) the final line"
        contents = []
        self.parseending(reader, lambda: self.parsecontainer(reader, contents))
        return contents


class BoundedParser(ExcludingParser):
    "A parser bound by a final line"

    def parse(self, reader):
        "Parse everything, including the final line"
        contents = ExcludingParser.parse(self, reader)
        # skip last line
        reader.nextline()
        return contents


class BoundedDummy(Parser):
    "A bound parser that ignores everything"

    def parse(self, reader):
        "Parse the contents of the container"
        self.parseending(reader, lambda: reader.nextline())
        # skip last line
        reader.nextline()
        return []


class StringParser(Parser):
    "Parses just a string"

    def parseheader(self, reader):
        "Do nothing, just take note"
        self.begin = reader.linenumber + 1
        return []

    def parse(self, reader):
        "Parse a single line"
        contents = reader.currentline()
        reader.nextline()
        return contents


class ContainerOutput:
    "The generic HTML output for a container."

    def gethtml(self, container):
        "Show an error."
        Trace.error('gethtml() not implemented for ' + str(self))

    def isempty(self):
        "Decide if the output is empty: by default, not empty."
        return False


class EmptyOutput(ContainerOutput):

    def gethtml(self, container):
        "Return empty HTML code."
        return []

    def isempty(self):
        "This output is particularly empty."
        return True


class FixedOutput(ContainerOutput):
    "Fixed output"

    def gethtml(self, container):
        "Return constant HTML code"
        return container.html


class ContentsOutput(ContainerOutput):
    "Outputs the contents converted to HTML"

    def gethtml(self, container):
        "Return the HTML code"
        html = []
        if container.contents is None:
            return html
        for element in container.contents:
            if not hasattr(element, 'gethtml'):
                Trace.error('No html in ' + element.__class__.__name__ + ': ' + str(element))
                return html
            html += element.gethtml()
        return html


class TaggedOutput(ContentsOutput):
    "Outputs an HTML tag surrounding the contents."

    tag = None
    breaklines = False
    empty = False

    def settag(self, tag, breaklines=False, empty=False):
        "Set the value for the tag and other attributes."
        self.tag = tag
        if breaklines:
            self.breaklines = breaklines
        if empty:
            self.empty = empty
        return self

    def setbreaklines(self, breaklines):
        "Set the value for breaklines."
        self.breaklines = breaklines
        return self

    def gethtml(self, container):
        "Return the HTML code."
        if self.empty:
            return [self.selfclosing(container)]
        html = [self.open(container)]
        html += ContentsOutput.gethtml(self, container)
        html.append(self.close(container))
        return html

    def open(self, container):
        "Get opening line."
        if not self.checktag(container):
            return ''
        open = '<' + self.tag + '>'
        if self.breaklines:
            return open + '\n'
        return open

    def close(self, container):
        "Get closing line."
        if not self.checktag(container):
            return ''
        close = '</' + self.tag.split()[0] + '>'
        if self.breaklines:
            return '\n' + close + '\n'
        return close

    def selfclosing(self, container):
        "Get self-closing line."
        if not self.checktag(container):
            return ''
        selfclosing = '<' + self.tag + '/>'
        if self.breaklines:
            return selfclosing + '\n'
        return selfclosing

    def checktag(self, container):
        "Check that the tag is valid."
        if not self.tag:
            Trace.error('No tag in ' + str(container))
            return False
        if self.tag == '':
            return False
        return True


class FilteredOutput(ContentsOutput):
    "Returns the output in the contents, but filtered:"
    "some strings are replaced by others."

    def __init__(self):
        "Initialize the filters."
        self.filters = []

    def addfilter(self, original, replacement):
        "Add a new filter: replace the original by the replacement."
        self.filters.append((original, replacement))

    def gethtml(self, container):
        "Return the HTML code"
        result = []
        html = ContentsOutput.gethtml(self, container)
        for line in html:
            result.append(self.filter(line))
        return result

    def filter(self, line):
        "Filter a single line with all available filters."
        for original, replacement in self.filters:
            if original in line:
                line = line.replace(original, replacement)
        return line


class StringOutput(ContainerOutput):
    "Returns a bare string as output"

    def gethtml(self, container):
        "Return a bare string"
        return [container.string]


class Globable:
    """A bit of text which can be globbed (lumped together in bits).
    Methods current(), skipcurrent(), checkfor() and isout() have to be
    implemented by subclasses."""

    leavepending = False

    def __init__(self):
        self.endinglist = EndingList()

    def checkbytemark(self):
        "Check for a Unicode byte mark and skip it."
        if self.finished():
            return
        if ord(self.current()) == 0xfeff:
            self.skipcurrent()

    def isout(self):
        "Find out if we are out of the position yet."
        Trace.error('Unimplemented isout()')
        return True

    def current(self):
        "Return the current character."
        Trace.error('Unimplemented current()')
        return ''

    def checkfor(self, string):
        "Check for the given string in the current position."
        Trace.error('Unimplemented checkfor()')
        return False

    def finished(self):
        "Find out if the current text has finished."
        if self.isout():
            if not self.leavepending:
                self.endinglist.checkpending()
            return True
        return self.endinglist.checkin(self)

    def skipcurrent(self):
        "Return the current character and skip it."
        Trace.error('Unimplemented skipcurrent()')
        return ''

    def glob(self, currentcheck):
        "Glob a bit of text that satisfies a check on the current char."
        glob = ''
        while not self.finished() and currentcheck():
            glob += self.skipcurrent()
        return glob

    def globalpha(self):
        "Glob a bit of alpha text"
        return self.glob(lambda: self.current().isalpha())

    def globnumber(self):
        "Glob a row of digits."
        return self.glob(lambda: self.current().isdigit())

    def isidentifier(self):
        "Return if the current character is alphanumeric or _."
        if self.current().isalnum() or self.current() == '_':
            return True
        return False

    def globidentifier(self):
        "Glob alphanumeric and _ symbols."
        return self.glob(self.isidentifier)

    def isvalue(self):
        "Return if the current character is a value character:"
        "not a bracket or a space."
        if self.current().isspace():
            return False
        if self.current() in '{}()':
            return False
        return True

    def globvalue(self):
        "Glob a value: any symbols but brackets."
        return self.glob(self.isvalue)

    def skipspace(self):
        "Skip all whitespace at current position."
        return self.glob(lambda: self.current().isspace())

    def globincluding(self, magicchar):
        "Glob a bit of text up to (including) the magic char."
        glob = self.glob(lambda: self.current() != magicchar) + magicchar
        self.skip(magicchar)
        return glob

    def globexcluding(self, excluded):
        "Glob a bit of text up until (excluding) any excluded character."
        return self.glob(lambda: self.current() not in excluded)

    def pushending(self, ending, optional=False):
        "Push a new ending to the bottom"
        self.endinglist.add(ending, optional)

    def popending(self, expected=None):
        "Pop the ending found at the current position"
        if self.isout() and self.leavepending:
            return expected
        ending = self.endinglist.pop(self)
        if expected and expected != ending:
            Trace.error('Expected ending ' + expected + ', got ' + ending)
        self.skip(ending)
        return ending

    def nextending(self):
        "Return the next ending in the queue."
        nextending = self.endinglist.findending(self)
        if not nextending:
            return None
        return nextending.ending


class EndingList:
    "A list of position endings"

    def __init__(self):
        self.endings = []

    def add(self, ending, optional=False):
        "Add a new ending to the list"
        self.endings.append(PositionEnding(ending, optional))

    def pickpending(self, pos):
        "Pick any pending endings from a parse position."
        self.endings += pos.endinglist.endings

    def checkin(self, pos):
        "Search for an ending"
        if self.findending(pos):
            return True
        return False

    def pop(self, pos):
        "Remove the ending at the current position"
        if pos.isout():
            Trace.error('No ending out of bounds')
            return ''
        ending = self.findending(pos)
        if not ending:
            Trace.error('No ending at ' + pos.current())
            return ''
        for each in reversed(self.endings):
            self.endings.remove(each)
            if each == ending:
                return each.ending
            elif not each.optional:
                Trace.error('Removed non-optional ending ' + each)
        Trace.error('No endings left')
        return ''

    def findending(self, pos):
        "Find the ending at the current position"
        if len(self.endings) == 0:
            return None
        for index, ending in enumerate(reversed(self.endings)):
            if ending.checkin(pos):
                return ending
            if not ending.optional:
                return None
        return None

    def checkpending(self):
        "Check if there are any pending endings"
        if len(self.endings) != 0:
            Trace.error('Pending ' + str(self) + ' left open')

    def __str__(self):
        "Printable representation"
        string = 'endings ['
        for ending in self.endings:
            string += str(ending) + ','
        if len(self.endings) > 0:
            string = string[:-1]
        return string + ']'


class PositionEnding:
    "An ending for a parsing position"

    def __init__(self, ending, optional):
        self.ending = ending
        self.optional = optional

    def checkin(self, pos):
        "Check for the ending"
        return pos.checkfor(self.ending)

    def __str__(self):
        "Printable representation"
        string = 'Ending ' + self.ending
        if self.optional:
            string += ' (optional)'
        return string


class Position(Globable):
    """A position in a text to parse.
    Including those in Globable, functions to implement by subclasses are:
    skip(), identifier(), extract(), isout() and current()."""

    def __init__(self):
        Globable.__init__(self)

    def skip(self, string):
        "Skip a string"
        Trace.error('Unimplemented skip()')

    def identifier(self):
        "Return an identifier for the current position."
        Trace.error('Unimplemented identifier()')
        return 'Error'

    def extract(self, length):
        "Extract the next string of the given length, or None if not enough text,"
        "without advancing the parse position."
        Trace.error('Unimplemented extract()')
        return None

    def checkfor(self, string):
        "Check for a string at the given position."
        return string == self.extract(len(string))

    def checkforlower(self, string):
        "Check for a string in lower case."
        extracted = self.extract(len(string))
        if not extracted:
            return False
        return string.lower() == self.extract(len(string)).lower()

    def skipcurrent(self):
        "Return the current character and skip it."
        current = self.current()
        self.skip(current)
        return current

    def __next__(self):
        "Advance the position and return the next character."
        self.skipcurrent()
        return self.current()

    def checkskip(self, string):
        "Check for a string at the given position; if there, skip it"
        if not self.checkfor(string):
            return False
        self.skip(string)
        return True

    def error(self, message):
        "Show an error message and the position identifier."
        Trace.error(message + ': ' + self.identifier())


class TextPosition(Position):
    "A parse position based on a raw text."

    def __init__(self, text):
        "Create the position from some text."
        Position.__init__(self)
        self.pos = 0
        self.text = text
        self.checkbytemark()

    def skip(self, string):
        "Skip a string of characters."
        self.pos += len(string)

    def identifier(self):
        "Return a sample of the remaining text."
        length = 30
        if self.pos + length > len(self.text):
            length = len(self.text) - self.pos
        return '*' + self.text[self.pos:self.pos + length] + '*'

    def isout(self):
        "Find out if we are out of the text yet."
        return self.pos >= len(self.text)

    def current(self):
        "Return the current character, assuming we are not out."
        return self.text[self.pos]

    def extract(self, length):
        "Extract the next string of the given length, or None if not enough text."
        if self.pos + length > len(self.text):
            return None
        return self.text[self.pos : self.pos + length]                 # noqa: E203


class Container:
    "A container for text and objects in a lyx file"

    partkey = None
    parent = None
    begin = None

    def __init__(self):
        self.contents = list()

    def process(self):
        "Process contents"
        pass

    def gethtml(self):
        "Get the resulting HTML"
        html = self.output.gethtml(self)
        if isinstance(html, str):
            Trace.error('Raw string ' + html)
            html = [html]
        return html

    def escape(self, line, replacements=EscapeConfig.entities):
        "Escape a line with replacements from a map"
        pieces = sorted(replacements.keys())
        # do them in order
        for piece in pieces:
            if piece in line:
                line = line.replace(piece, replacements[piece])
        return line

    def escapeentities(self, line):
        "Escape all Unicode characters to HTML entities."
        result = ''
        pos = TextPosition(line)
        while not pos.finished():
            if ord(pos.current()) > 128:
                codepoint = hex(ord(pos.current()))
                if codepoint == '0xd835':
                    codepoint = hex(ord(next(pos)) + 0xf800)
                result += '&#' + codepoint[1:] + ';'
            else:
                result += pos.current()
            pos.skipcurrent()
        return result

    def searchall(self, type):
        "Search for all embedded containers of a given type"
        list = []
        self.searchprocess(type, lambda container: list.append(container))
        return list

    def searchremove(self, type):
        "Search for all containers of a type and remove them"
        list = self.searchall(type)
        for container in list:
            container.parent.contents.remove(container)
        return list

    def searchprocess(self, type, process):
        "Search for elements of a given type and process them"
        self.locateprocess(lambda container: isinstance(container, type), process)

    def locateprocess(self, locate, process):
        "Search for all embedded containers and process them"
        for container in self.contents:
            container.locateprocess(locate, process)
            if locate(container):
                process(container)

    def recursivesearch(self, locate, recursive, process):
        "Perform a recursive search in the container."
        for container in self.contents:
            if recursive(container):
                container.recursivesearch(locate, recursive, process)
            if locate(container):
                process(container)

    def extracttext(self):
        "Extract all text from allowed containers."
        constants = ContainerExtractor(ContainerConfig.extracttext).extract(self)
        return ''.join(constant.string for constant in constants)

    def group(self, index, group, isingroup):
        "Group some adjoining elements into a group"
        if index >= len(self.contents):
            return
        if hasattr(self.contents[index], 'grouped'):
            return
        while index < len(self.contents) and isingroup(self.contents[index]):
            self.contents[index].grouped = True
            group.contents.append(self.contents[index])
            self.contents.pop(index)
        self.contents.insert(index, group)

    def remove(self, index):
        "Remove a container but leave its contents"
        container = self.contents[index]
        self.contents.pop(index)
        while len(container.contents) > 0:
            self.contents.insert(index, container.contents.pop())

    def tree(self, level=0):
        "Show in a tree"
        Trace.debug("  " * level + str(self))
        for container in self.contents:
            container.tree(level + 1)

    def getparameter(self, name):
        "Get the value of a parameter, if present."
        if name not in self.parameters:
            return None
        return self.parameters[name]

    def getparameterlist(self, name):
        "Get the value of a comma-separated parameter as a list."
        paramtext = self.getparameter(name)
        if not paramtext:
            return []
        return paramtext.split(',')

    def hasemptyoutput(self):
        "Check if the parent's output is empty."
        current = self.parent
        while current:
            if current.output.isempty():
                return True
            current = current.parent
        return False

    def __str__(self):
        "Get a description"
        if not self.begin:
            return self.__class__.__name__
        return self.__class__.__name__ + '@' + str(self.begin)


class BlackBox(Container):
    "A container that does not output anything"

    def __init__(self):
        self.parser = LoneCommand()
        self.output = EmptyOutput()
        self.contents = []


class StringContainer(Container):
    "A container for a single string"

    parsed = None

    def __init__(self):
        self.parser = StringParser()
        self.output = StringOutput()
        self.string = ''

    def process(self):
        "Replace special chars from the contents."
        if self.parsed:
            self.string = self.replacespecial(self.parsed)
            self.parsed = None

    def replacespecial(self, line):
        "Replace all special chars from a line"
        replaced = self.escape(line, EscapeConfig.entities)
        replaced = self.changeline(replaced)
        if ContainerConfig.string['startcommand'] in replaced and len(replaced) > 1:
            # unprocessed commands
            if self.begin:
                message = 'Unknown command at ' + str(self.begin) + ': '
            else:
                message = 'Unknown command: '
            Trace.error(message + replaced.strip())
        return replaced

    def changeline(self, line):
        return self.escape(line, EscapeConfig.chars)

    def extracttext(self):
        "Return all text."
        return self.string

    def __str__(self):
        "Return a printable representation."
        result = 'StringContainer'
        if self.begin:
            result += '@' + str(self.begin)
        ellipsis = '...'
        if len(self.string.strip()) <= 15:
            ellipsis = ''
        return result + ' (' + self.string.strip()[:15] + ellipsis + ')'


class Constant(StringContainer):
    "A constant string"

    def __init__(self, text):
        self.contents = []
        self.string = text
        self.output = StringOutput()

    def __str__(self):
        return 'Constant: ' + self.string


class DocumentParameters:
    "Global parameters for the document."

    displaymode = False


class FormulaParser(Parser):
    "Parses a formula"

    def parseheader(self, reader):
        "See if the formula is inlined"
        self.begin = reader.linenumber + 1
        type = self.parsetype(reader)
        if not type:
            reader.nextline()
            type = self.parsetype(reader)
            if not type:
                Trace.error('Unknown formula type in ' + reader.currentline().strip())
                return ['unknown']
        return [type]

    def parsetype(self, reader):
        "Get the formula type from the first line."
        if reader.currentline().find(FormulaConfig.starts['simple']) >= 0:
            return 'inline'
        if reader.currentline().find(FormulaConfig.starts['complex']) >= 0:
            return 'block'
        if reader.currentline().find(FormulaConfig.starts['unnumbered']) >= 0:
            return 'block'
        if reader.currentline().find(FormulaConfig.starts['beginbefore']) >= 0:
            return 'numbered'
        return None

    def parse(self, reader):
        "Parse the formula until the end"
        formula = self.parseformula(reader)
        while not reader.currentline().startswith(self.ending):
            stripped = reader.currentline().strip()
            if len(stripped) > 0:
                Trace.error('Unparsed formula line ' + stripped)
            reader.nextline()
        reader.nextline()
        return formula

    def parseformula(self, reader):
        "Parse the formula contents"
        simple = FormulaConfig.starts['simple']
        if simple in reader.currentline():
            rest = reader.currentline().split(simple, 1)[1]
            if simple in rest:
                # formula is $...$
                return self.parsesingleliner(reader, simple, simple)
            # formula is multiline $...$
            return self.parsemultiliner(reader, simple, simple)
        if FormulaConfig.starts['complex'] in reader.currentline():
            # formula of the form \[...\]
            return self.parsemultiliner(reader, FormulaConfig.starts['complex'],
                                        FormulaConfig.endings['complex'])
        beginbefore = FormulaConfig.starts['beginbefore']
        beginafter = FormulaConfig.starts['beginafter']
        if beginbefore in reader.currentline():
            if reader.currentline().strip().endswith(beginafter):
                current = reader.currentline().strip()
                endsplit = current.split(beginbefore)[1].split(beginafter)
                startpiece = beginbefore + endsplit[0] + beginafter
                endbefore = FormulaConfig.endings['endbefore']
                endafter = FormulaConfig.endings['endafter']
                endpiece = endbefore + endsplit[0] + endafter
                return startpiece + self.parsemultiliner(reader, startpiece, endpiece) + endpiece
            Trace.error('Missing ' + beginafter + ' in ' + reader.currentline())
            return ''
        begincommand = FormulaConfig.starts['command']
        beginbracket = FormulaConfig.starts['bracket']
        if begincommand in reader.currentline() and beginbracket in reader.currentline():
            endbracket = FormulaConfig.endings['bracket']
            return self.parsemultiliner(reader, beginbracket, endbracket)
        Trace.error('Formula beginning ' + reader.currentline() + ' is unknown')
        return ''

    def parsesingleliner(self, reader, start, ending):
        "Parse a formula in one line"
        line = reader.currentline().strip()
        if start not in line:
            Trace.error('Line ' + line + ' does not contain formula start ' + start)
            return ''
        if not line.endswith(ending):
            Trace.error('Formula ' + line + ' does not end with ' + ending)
            return ''
        index = line.index(start)
        rest = line[index + len(start):-len(ending)]
        reader.nextline()
        return rest

    def parsemultiliner(self, reader, start, ending):
        "Parse a formula in multiple lines"
        formula = ''
        line = reader.currentline()
        if start not in line:
            Trace.error('Line ' + line.strip() + ' does not contain formula start ' + start)
            return ''
        index = line.index(start)
        line = line[index + len(start):].strip()
        while not line.endswith(ending):
            formula += line + '\n'
            reader.nextline()
            line = reader.currentline()
        formula += line[:-len(ending)]
        reader.nextline()
        return formula


class FormulaBit(Container):
    "A bit of a formula"

    type = None
    size = 1
    original = ''

    def __init__(self):
        "The formula bit type can be 'alpha', 'number', 'font'."
        self.contents = []
        self.output = ContentsOutput()

    def setfactory(self, factory):
        "Set the internal formula factory."
        self.factory = factory
        return self

    def add(self, bit):
        "Add any kind of formula bit already processed"
        self.contents.append(bit)
        self.original += bit.original
        bit.parent = self

    def skiporiginal(self, string, pos):
        "Skip a string and add it to the original formula"
        self.original += string
        if not pos.checkskip(string):
            Trace.error('String ' + string + ' not at ' + pos.identifier())

    def computesize(self):
        "Compute the size of the bit as the max of the sizes of all contents."
        if len(self.contents) == 0:
            return 1
        self.size = max(element.size for element in self.contents)
        return self.size

    def clone(self):
        "Return a copy of itself."
        return self.factory.parseformula(self.original)

    def __str__(self):
        "Get a string representation"
        return self.__class__.__name__ + ' read in ' + self.original


class TaggedBit(FormulaBit):
    "A tagged string in a formula"

    def constant(self, constant, tag):
        "Set the constant and the tag"
        self.output = TaggedOutput().settag(tag)
        self.add(FormulaConstant(constant))
        return self

    def complete(self, contents, tag, breaklines=False):
        "Set the constant and the tag"
        self.contents = contents
        self.output = TaggedOutput().settag(tag, breaklines)
        return self

    def selfcomplete(self, tag):
        "Set the self-closing tag, no contents (as in <hr/>)."
        self.output = TaggedOutput().settag(tag, empty=True)
        return self


class FormulaConstant(Constant):
    "A constant string in a formula"

    def __init__(self, string):
        "Set the constant string"
        Constant.__init__(self, string)
        self.original = string
        self.size = 1
        self.type = None

    def computesize(self):
        "Compute the size of the constant: always 1."
        return self.size

    def clone(self):
        "Return a copy of itself."
        return FormulaConstant(self.original)

    def __str__(self):
        "Return a printable representation."
        return 'Formula constant: ' + self.string


class RawText(FormulaBit):
    "A bit of text inside a formula"

    def detect(self, pos):
        "Detect a bit of raw text"
        return pos.current().isalpha()

    def parsebit(self, pos):
        "Parse alphabetic text"
        alpha = pos.globalpha()
        self.add(FormulaConstant(alpha))
        self.type = 'alpha'


class FormulaSymbol(FormulaBit):
    "A symbol inside a formula"

    modified = FormulaConfig.modified
    unmodified = FormulaConfig.unmodified['characters']

    def detect(self, pos):
        "Detect a symbol"
        if pos.current() in FormulaSymbol.unmodified:
            return True
        if pos.current() in FormulaSymbol.modified:
            return True
        return False

    def parsebit(self, pos):
        "Parse the symbol"
        if pos.current() in FormulaSymbol.unmodified:
            self.addsymbol(pos.current(), pos)
            return
        if pos.current() in FormulaSymbol.modified:
            self.addsymbol(FormulaSymbol.modified[pos.current()], pos)
            return
        Trace.error('Symbol ' + pos.current() + ' not found')

    def addsymbol(self, symbol, pos):
        "Add a symbol"
        self.skiporiginal(pos.current(), pos)
        self.contents.append(FormulaConstant(symbol))


class FormulaNumber(FormulaBit):
    "A string of digits in a formula"

    def detect(self, pos):
        "Detect a digit"
        return pos.current().isdigit()

    def parsebit(self, pos):
        "Parse a bunch of digits"
        digits = pos.glob(lambda: pos.current().isdigit())
        self.add(FormulaConstant(digits))
        self.type = 'number'


class Comment(FormulaBit):
    "A LaTeX comment: % to the end of the line."

    start = FormulaConfig.starts['comment']

    def detect(self, pos):
        "Detect the %."
        return pos.current() == self.start

    def parsebit(self, pos):
        "Parse to the end of the line."
        self.original += pos.globincluding('\n')


class WhiteSpace(FormulaBit):
    "Some white space inside a formula."

    def detect(self, pos):
        "Detect the white space."
        return pos.current().isspace()

    def parsebit(self, pos):
        "Parse all whitespace."
        self.original += pos.skipspace()

    def __str__(self):
        "Return a printable representation."
        return 'Whitespace: *' + self.original + '*'


class Bracket(FormulaBit):
    "A {} bracket inside a formula"

    start = FormulaConfig.starts['bracket']
    ending = FormulaConfig.endings['bracket']

    def __init__(self):
        "Create a (possibly literal) new bracket"
        FormulaBit.__init__(self)
        self.inner = None

    def detect(self, pos):
        "Detect the start of a bracket"
        return pos.checkfor(self.start)

    def parsebit(self, pos):
        "Parse the bracket"
        self.parsecomplete(pos, self.innerformula)
        return self

    def parsetext(self, pos):
        "Parse a text bracket"
        self.parsecomplete(pos, self.innertext)
        return self

    def parseliteral(self, pos):
        "Parse a literal bracket"
        self.parsecomplete(pos, self.innerliteral)
        return self

    def parsecomplete(self, pos, innerparser):
        "Parse the start and end marks"
        if not pos.checkfor(self.start):
            Trace.error('Bracket should start with ' + self.start + ' at ' + pos.identifier())
            return None
        self.skiporiginal(self.start, pos)
        pos.pushending(self.ending)
        innerparser(pos)
        self.original += pos.popending(self.ending)
        self.computesize()

    def innerformula(self, pos):
        "Parse a whole formula inside the bracket"
        while not pos.finished():
            self.add(self.factory.parseany(pos))

    def innertext(self, pos):
        "Parse some text inside the bracket, following textual rules."
        specialchars = list(FormulaConfig.symbolfunctions.keys())
        specialchars.append(FormulaConfig.starts['command'])
        specialchars.append(FormulaConfig.starts['bracket'])
        specialchars.append(Comment.start)
        while not pos.finished():
            if pos.current() in specialchars:
                self.add(self.factory.parseany(pos))
                if pos.checkskip(' '):
                    self.original += ' '
            else:
                self.add(FormulaConstant(pos.skipcurrent()))

    def innerliteral(self, pos):
        "Parse a literal inside the bracket, which does not generate HTML."
        self.literal = ''
        while not pos.finished() and not pos.current() == self.ending:
            if pos.current() == self.start:
                self.parseliteral(pos)
            else:
                self.literal += pos.skipcurrent()
        self.original += self.literal


class SquareBracket(Bracket):
    "A [] bracket inside a formula"

    start = FormulaConfig.starts['squarebracket']
    ending = FormulaConfig.endings['squarebracket']

    def clone(self):
        "Return a new square bracket with the same contents."
        bracket = SquareBracket()
        bracket.contents = self.contents
        return bracket


class MathsProcessor:
    "A processor for a maths construction inside the FormulaProcessor."

    def process(self, contents, index):
        "Process an element inside a formula."
        Trace.error('Unimplemented process() in ' + str(self))

    def __str__(self):
        "Return a printable description."
        return 'Maths processor ' + self.__class__.__name__


class FormulaProcessor:
    "A processor specifically for formulas."

    processors = []

    def process(self, bit):
        "Process the contents of every formula bit, recursively."
        self.processcontents(bit)
        self.processinsides(bit)
        self.traversewhole(bit)

    def processcontents(self, bit):
        "Process the contents of a formula bit."
        if not isinstance(bit, FormulaBit):
            return
        bit.process()
        for element in bit.contents:
            self.processcontents(element)

    def processinsides(self, bit):
        "Process the insides (limits, brackets) in a formula bit."
        if not isinstance(bit, FormulaBit):
            return
        for index, element in enumerate(bit.contents):
            for processor in self.processors:
                processor.process(bit.contents, index)
            # continue with recursive processing
            self.processinsides(element)

    def traversewhole(self, formula):
        "Traverse over the contents to alter variables and space units."
        last = None
        for bit, contents in self.traverse(formula):
            if bit.type == 'alpha':
                self.italicize(bit, contents)
            elif bit.type == 'font' and last and last.type == 'number':
                bit.contents.insert(0, FormulaConstant('\u2009'))
            last = bit

    def traverse(self, bit):
        "Traverse a formula and yield a flattened structure of (bit, list) pairs."
        for element in bit.contents:
            if hasattr(element, 'type') and element.type:
                yield element, bit.contents
            elif isinstance(element, FormulaBit):
                yield from self.traverse(element)

    def italicize(self, bit, contents):
        "Italicize the given bit of text."
        index = contents.index(bit)
        contents[index] = TaggedBit().complete([bit], 'i')


class Formula(Container):
    "A LaTeX formula"

    def __init__(self):
        self.parser = FormulaParser()
        self.output = TaggedOutput().settag('span class="formula"')

    def process(self):
        "Convert the formula to tags"
        if self.header[0] == 'inline':
            DocumentParameters.displaymode = False
        else:
            DocumentParameters.displaymode = True
            self.output.settag('div class="formula"', True)
        self.classic()

    def classic(self):
        "Make the contents using classic output generation with XHTML and CSS."
        whole = FormulaFactory().parseformula(self.parsed)
        FormulaProcessor().process(whole)
        whole.parent = self
        self.contents = [whole]

    def parse(self, pos):
        "Parse using a parse position instead of self.parser."
        if pos.checkskip('$$'):
            self.parsedollarblock(pos)
        elif pos.checkskip('$'):
            self.parsedollarinline(pos)
        elif pos.checkskip('\\('):
            self.parseinlineto(pos, '\\)')
        elif pos.checkskip('\\['):
            self.parseblockto(pos, '\\]')
        else:
            pos.error('Unparseable formula')
        self.process()
        return self

    def parsedollarinline(self, pos):
        "Parse a $...$ formula."
        self.header = ['inline']
        self.parsedollar(pos)

    def parsedollarblock(self, pos):
        "Parse a $$...$$ formula."
        self.header = ['block']
        self.parsedollar(pos)
        if not pos.checkskip('$'):
            pos.error('Formula should be $$...$$, but last $ is missing.')

    def parsedollar(self, pos):
        "Parse to the next $."
        pos.pushending('$')
        self.parsed = pos.globexcluding('$')
        pos.popending('$')

    def parseinlineto(self, pos, limit):
        "Parse a \\(...\\) formula."
        self.header = ['inline']
        self.parseupto(pos, limit)

    def parseblockto(self, pos, limit):
        "Parse a \\[...\\] formula."
        self.header = ['block']
        self.parseupto(pos, limit)

    def parseupto(self, pos, limit):
        "Parse a formula that ends with the given command."
        pos.pushending(limit)
        self.parsed = pos.glob(lambda: True)
        pos.popending(limit)

    def __str__(self):
        "Return a printable representation."
        if self.partkey and self.partkey.number:
            return 'Formula (' + self.partkey.number + ')'
        return 'Unnumbered formula'


class WholeFormula(FormulaBit):
    "Parse a whole formula"

    def detect(self, pos):
        "Not outside the formula is enough."
        return not pos.finished()

    def parsebit(self, pos):
        "Parse with any formula bit"
        while not pos.finished():
            self.add(self.factory.parseany(pos))


class FormulaFactory:
    "Construct bits of formula"

    # bit types will be appended later
    types = [FormulaSymbol, RawText, FormulaNumber, Bracket, Comment, WhiteSpace]
    skippedtypes = [Comment, WhiteSpace]
    defining = False

    def __init__(self):
        "Initialize the map of instances."
        self.instances = {}

    def detecttype(self, type, pos):
        "Detect a bit of a given type."
        if pos.finished():
            return False
        return self.instance(type).detect(pos)

    def instance(self, type):
        "Get an instance of the given type."
        if type not in self.instances or not self.instances[type]:
            self.instances[type] = self.create(type)
        return self.instances[type]

    def create(self, type):
        "Create a new formula bit of the given type."
        return Cloner.create(type).setfactory(self)

    def clearskipped(self, pos):
        "Clear any skipped types."
        while not pos.finished():
            if not self.skipany(pos):
                return
        return

    def skipany(self, pos):
        "Skip any skipped types."
        for type in self.skippedtypes:
            if self.instance(type).detect(pos):
                return self.parsetype(type, pos)
        return None

    def parseany(self, pos):
        "Parse any formula bit at the current location."
        for type in self.types + self.skippedtypes:
            if self.detecttype(type, pos):
                return self.parsetype(type, pos)
        Trace.error('Unrecognized formula at ' + pos.identifier())
        return FormulaConstant(pos.skipcurrent())

    def parsetype(self, type, pos):
        "Parse the given type and return it."
        bit = self.instance(type)
        self.instances[type] = None
        returnedbit = bit.parsebit(pos)
        if returnedbit:
            return returnedbit.setfactory(self)
        return bit

    def parseformula(self, formula):
        "Parse a string of text that contains a whole formula."
        pos = TextPosition(formula)
        whole = self.create(WholeFormula)
        if whole.detect(pos):
            whole.parsebit(pos)
            return whole
        # no formula found
        if not pos.finished():
            Trace.error('Unknown formula at: ' + pos.identifier())
            whole.add(TaggedBit().constant(formula, 'span class="unknown"'))
        return whole


class FormulaCommand(FormulaBit):
    "A LaTeX command inside a formula"

    types = []
    start = FormulaConfig.starts['command']
    commandmap = None

    def detect(self, pos):
        "Find the current command."
        return pos.checkfor(FormulaCommand.start)

    def parsebit(self, pos):
        "Parse the command."
        command = self.extractcommand(pos)
        bit = self.parsewithcommand(command, pos)
        if bit:
            return bit
        if command.startswith('\\up') or command.startswith('\\Up'):
            upgreek = self.parseupgreek(command, pos)
            if upgreek:
                return upgreek
        if not self.factory.defining:
            Trace.error('Unknown command ' + command)
        self.output = TaggedOutput().settag('span class="unknown"')
        self.add(FormulaConstant(command))
        return None

    def parsewithcommand(self, command, pos):
        "Parse the command type once we have the command."
        for type in FormulaCommand.types:
            if command in type.commandmap:
                return self.parsecommandtype(command, type, pos)
        return None

    def parsecommandtype(self, command, type, pos):
        "Parse a given command type."
        bit = self.factory.create(type)
        bit.setcommand(command)
        returned = bit.parsebit(pos)
        if returned:
            return returned
        return bit

    def extractcommand(self, pos):
        "Extract the command from the current position."
        if not pos.checkskip(FormulaCommand.start):
            pos.error('Missing command start ' + FormulaCommand.start)
            return
        if pos.finished():
            return self.emptycommand(pos)
        if pos.current().isalpha():
            # alpha command
            command = FormulaCommand.start + pos.globalpha()
            # skip mark of short command
            pos.checkskip('*')
            return command
        # symbol command
        return FormulaCommand.start + pos.skipcurrent()

    def emptycommand(self, pos):
        """Check for an empty command: look for command disguised as ending.
        Special case against '{ \\{ \\} }' situation."""
        command = ''
        if not pos.isout():
            ending = pos.nextending()
            if ending and pos.checkskip(ending):
                command = ending
        return FormulaCommand.start + command

    def parseupgreek(self, command, pos):
        "Parse the Greek \\up command.."
        if len(command) < 4:
            return None
        if command.startswith('\\up'):
            upcommand = '\\' + command[3:]
        elif pos.checkskip('\\Up'):
            upcommand = '\\' + command[3:4].upper() + command[4:]
        else:
            Trace.error('Impossible upgreek command: ' + command)
            return
        upgreek = self.parsewithcommand(upcommand, pos)
        if upgreek:
            upgreek.type = 'font'
        return upgreek


class CommandBit(FormulaCommand):
    "A formula bit that includes a command"

    def setcommand(self, command):
        "Set the command in the bit"
        self.command = command
        if self.commandmap:
            self.original += command
            self.translated = self.commandmap[self.command]

    def parseparameter(self, pos):
        "Parse a parameter at the current position"
        self.factory.clearskipped(pos)
        if pos.finished():
            return None
        parameter = self.factory.parseany(pos)
        self.add(parameter)
        return parameter

    def parsesquare(self, pos):
        "Parse a square bracket"
        self.factory.clearskipped(pos)
        if not self.factory.detecttype(SquareBracket, pos):
            return None
        bracket = self.factory.parsetype(SquareBracket, pos)
        self.add(bracket)
        return bracket

    def parseliteral(self, pos):
        "Parse a literal bracket."
        self.factory.clearskipped(pos)
        if not self.factory.detecttype(Bracket, pos):
            if not pos.isvalue():
                Trace.error('No literal parameter found at: ' + pos.identifier())
                return None
            return pos.globvalue()
        bracket = Bracket().setfactory(self.factory)
        self.add(bracket.parseliteral(pos))
        return bracket.literal

    def parsesquareliteral(self, pos):
        "Parse a square bracket literally."
        self.factory.clearskipped(pos)
        if not self.factory.detecttype(SquareBracket, pos):
            return None
        bracket = SquareBracket().setfactory(self.factory)
        self.add(bracket.parseliteral(pos))
        return bracket.literal

    def parsetext(self, pos):
        "Parse a text parameter."
        self.factory.clearskipped(pos)
        if not self.factory.detecttype(Bracket, pos):
            Trace.error('No text parameter for ' + self.command)
            return None
        bracket = Bracket().setfactory(self.factory).parsetext(pos)
        self.add(bracket)
        return bracket


class EmptyCommand(CommandBit):
    "An empty command (without parameters)"

    commandmap = FormulaConfig.commands

    def parsebit(self, pos):
        "Parse a command without parameters"
        self.contents = [FormulaConstant(self.translated)]


class SpacedCommand(CommandBit):
    """An empty command which should have math spacing in formulas."""

    commandmap = FormulaConfig.spacedcommands

    def parsebit(self, pos):
        "Place as contents the command translated and spaced."
        # pad with MEDIUM MATHEMATICAL SPACE (4/18 em): too wide in STIX fonts :(
        # self.contents = [FormulaConstant('\u205f' + self.translated + '\u205f')]
        # pad with THIN SPACE (1/5 em)
        self.contents = [FormulaConstant('\u2009' + self.translated + '\u2009')]


class AlphaCommand(EmptyCommand):
    """A command without parameters whose result is alphabetical."""

    commandmap = FormulaConfig.alphacommands
    greek_capitals = ('\\Xi', '\\Theta', '\\Pi', '\\Sigma', '\\Gamma',
                      '\\Lambda', '\\Phi', '\\Psi', '\\Delta',
                      '\\Upsilon', '\\Omega')

    def parsebit(self, pos):
        "Parse the command and set type to alpha"
        EmptyCommand.parsebit(self, pos)
        if self.command not in self.greek_capitals:
            # Greek Capital letters are upright in LaTeX default math-style.
            # TODO: use italic, like in MathML and "iso" math-style?
            self.type = 'alpha'


class OneParamFunction(CommandBit):
    "A function of one parameter"

    commandmap = FormulaConfig.onefunctions
    simplified = False

    def parsebit(self, pos):
        "Parse a function with one parameter"
        self.output = TaggedOutput().settag(self.translated)
        self.parseparameter(pos)
        self.simplifyifpossible()

    def simplifyifpossible(self):
        "Try to simplify to a single character."
        if self.original in self.commandmap:
            self.output = FixedOutput()
            self.html = [self.commandmap[self.original]]
            self.simplified = True


class SymbolFunction(CommandBit):
    "Find a function which is represented by a symbol (like _ or ^)"

    commandmap = FormulaConfig.symbolfunctions

    def detect(self, pos):
        "Find the symbol"
        return pos.current() in SymbolFunction.commandmap

    def parsebit(self, pos):
        "Parse the symbol"
        self.setcommand(pos.current())
        pos.skip(self.command)
        self.output = TaggedOutput().settag(self.translated)
        self.parseparameter(pos)


class TextFunction(CommandBit):
    "A function where parameters are read as text."

    commandmap = FormulaConfig.textfunctions

    def parsebit(self, pos):
        "Parse a text parameter"
        self.output = TaggedOutput().settag(self.translated)
        self.parsetext(pos)

    def process(self):
        "Set the type to font"
        self.type = 'font'


class FontFunction(OneParamFunction):
    """A function of one parameter that changes the font."""
    # TODO: keep letters italic with \boldsymbol.

    commandmap = FormulaConfig.fontfunctions

    def process(self):
        "Simplify if possible using a single character."
        self.type = 'font'
        self.simplifyifpossible()


FormulaFactory.types += [FormulaCommand, SymbolFunction]
FormulaCommand.types = [
    AlphaCommand, EmptyCommand, OneParamFunction, FontFunction,
    TextFunction, SpacedCommand]


class BigBracket:
    "A big bracket generator."

    def __init__(self, size, bracket, alignment='l'):
        "Set the size and symbol for the bracket."
        self.size = size
        self.original = bracket
        self.alignment = alignment
        self.pieces = None
        if bracket in FormulaConfig.bigbrackets:
            self.pieces = FormulaConfig.bigbrackets[bracket]

    def getpiece(self, index):
        "Return the nth piece for the bracket."
        function = getattr(self, 'getpiece' + str(len(self.pieces)))
        return function(index)

    def getpiece1(self, index):
        "Return the only piece for a single-piece bracket."
        return self.pieces[0]

    def getpiece3(self, index):
        "Get the nth piece for a 3-piece bracket: parenthesis or square bracket."
        if index == 0:
            return self.pieces[0]
        if index == self.size - 1:
            return self.pieces[-1]
        return self.pieces[1]

    def getpiece4(self, index):
        "Get the nth piece for a 4-piece bracket: curly bracket."
        if index == 0:
            return self.pieces[0]
        if index == self.size - 1:
            return self.pieces[3]
        if index == (self.size - 1)/2:
            return self.pieces[2]
        return self.pieces[1]

    def getcell(self, index):
        "Get the bracket piece as an array cell."
        piece = self.getpiece(index)
        span = 'span class="bracket align-' + self.alignment + '"'
        return TaggedBit().constant(piece, span)

    def getcontents(self):
        "Get the bracket as an array or as a single bracket."
        if self.size == 1 or not self.pieces:
            return self.getsinglebracket()
        rows = []
        for index in range(self.size):
            cell = self.getcell(index)
            rows.append(TaggedBit().complete([cell], 'span class="arrayrow"'))
        return [TaggedBit().complete(rows, 'span class="array"')]

    def getsinglebracket(self):
        "Return the bracket as a single sign."
        if self.original == '.':
            return [TaggedBit().constant('', 'span class="emptydot"')]
        return [TaggedBit().constant(self.original, 'span class="stretchy"')]


class FormulaEquation(CommandBit):
    "A simple numbered equation."

    piece = 'equation'

    def parsebit(self, pos):
        "Parse the array"
        self.output = ContentsOutput()
        self.add(self.factory.parsetype(WholeFormula, pos))


class FormulaCell(FormulaCommand):
    "An array cell inside a row"

    def setalignment(self, alignment):
        self.alignment = alignment
        self.output = TaggedOutput().settag('span class="arraycell align-'
                                            + alignment + '"', True)
        return self

    def parsebit(self, pos):
        self.factory.clearskipped(pos)
        if pos.finished():
            return
        self.add(self.factory.parsetype(WholeFormula, pos))


class FormulaRow(FormulaCommand):
    "An array row inside an array"

    cellseparator = FormulaConfig.array['cellseparator']

    def setalignments(self, alignments):
        self.alignments = alignments
        self.output = TaggedOutput().settag('span class="arrayrow"', True)
        return self

    def parsebit(self, pos):
        "Parse a whole row"
        index = 0
        pos.pushending(self.cellseparator, optional=True)
        while not pos.finished():
            cell = self.createcell(index)
            cell.parsebit(pos)
            self.add(cell)
            index += 1
            pos.checkskip(self.cellseparator)
        if len(self.contents) == 0:
            self.output = EmptyOutput()

    def createcell(self, index):
        "Create the cell that corresponds to the given index."
        alignment = self.alignments[index % len(self.alignments)]
        return self.factory.create(FormulaCell).setalignment(alignment)


class MultiRowFormula(CommandBit):
    "A formula with multiple rows."

    def parserows(self, pos):
        "Parse all rows, finish when no more row ends"
        self.rows = []
        first = True
        for row in self.iteraterows(pos):
            if first:
                first = False
            else:
                # intersparse empty rows
                self.addempty()
            row.parsebit(pos)
            self.addrow(row)
        self.size = len(self.rows)

    def iteraterows(self, pos):
        "Iterate over all rows, end when no more row ends"
        rowseparator = FormulaConfig.array['rowseparator']
        while True:
            pos.pushending(rowseparator, True)
            row = self.factory.create(FormulaRow)
            yield row.setalignments(self.alignments)
            if pos.checkfor(rowseparator):
                self.original += pos.popending(rowseparator)
            else:
                return

    def addempty(self):
        "Add an empty row."
        row = self.factory.create(FormulaRow).setalignments(self.alignments)
        for index, originalcell in enumerate(self.rows[-1].contents):
            cell = row.createcell(index)
            cell.add(FormulaConstant(' '))
            row.add(cell)
        self.addrow(row)

    def addrow(self, row):
        "Add a row to the contents and to the list of rows."
        self.rows.append(row)
        self.add(row)


class FormulaArray(MultiRowFormula):
    "An array within a formula"

    piece = 'array'

    def parsebit(self, pos):
        "Parse the array"
        self.output = TaggedOutput().settag('span class="array"', False)
        self.parsealignments(pos)
        self.parserows(pos)

    def parsealignments(self, pos):
        "Parse the different alignments"
        # vertical
        self.valign = 'c'
        literal = self.parsesquareliteral(pos)
        if literal:
            self.valign = literal
        # horizontal
        literal = self.parseliteral(pos)
        self.alignments = []
        for s in literal:
            self.alignments.append(s)


class FormulaMatrix(MultiRowFormula):
    "A matrix (array with center alignment)."

    piece = 'matrix'

    def parsebit(self, pos):
        "Parse the matrix, set alignments to 'c'."
        self.output = TaggedOutput().settag('span class="array"', False)
        self.valign = 'c'
        self.alignments = ['c']
        self.parserows(pos)


class FormulaCases(MultiRowFormula):
    "A cases statement"

    piece = 'cases'

    def parsebit(self, pos):
        "Parse the cases"
        self.output = ContentsOutput()
        self.alignments = ['l', 'l']
        self.parserows(pos)
        for row in self.contents:
            for cell in row.contents:
                cell.output.settag('span class="case align-l"', True)
                cell.contents.append(FormulaConstant(' '))
        array = TaggedBit().complete(self.contents, 'span class="bracketcases"', True)
        brace = BigBracket(len(self.contents), '{', 'l')
        self.contents = brace.getcontents() + [array]


class EquationEnvironment(MultiRowFormula):
    "A \\begin{}...\\end equation environment with rows and cells."

    def parsebit(self, pos):
        "Parse the whole environment."
        environment = self.piece.replace('*', '')
        self.output = TaggedOutput().settag(
                        'span class="environment %s"'%environment, False)
        if environment in FormulaConfig.environments:
            self.alignments = FormulaConfig.environments[environment]
        else:
            Trace.error('Unknown equation environment ' + self.piece)
            # print in red
            self.output = TaggedOutput().settag('span class="unknown"')
            self.add(FormulaConstant('\\begin{%s} '%environment))

            self.alignments = ['l']
        self.parserows(pos)


class BeginCommand(CommandBit):
    "A \\begin{}...\\end command and what it entails (array, cases, aligned)"

    commandmap = {FormulaConfig.array['begin']: ''}

    types = [FormulaEquation, FormulaArray, FormulaCases, FormulaMatrix]

    def parsebit(self, pos):
        "Parse the begin command"
        command = self.parseliteral(pos)
        bit = self.findbit(command)
        ending = FormulaConfig.array['end'] + '{' + command + '}'
        pos.pushending(ending)
        bit.parsebit(pos)
        self.add(bit)
        self.original += pos.popending(ending)
        self.size = bit.size

    def findbit(self, piece):
        "Find the command bit corresponding to the \\begin{piece}"
        for type in BeginCommand.types:
            if piece.replace('*', '') == type.piece:
                return self.factory.create(type)
        bit = self.factory.create(EquationEnvironment)
        bit.piece = piece
        return bit


FormulaCommand.types += [BeginCommand]


class CombiningFunction(OneParamFunction):

    commandmap = FormulaConfig.combiningfunctions

    def parsebit(self, pos):
        "Parse a combining function."
        combining = self.translated
        parameter = self.parsesingleparameter(pos)
        if not parameter:
            Trace.error('Missing parameter for combining function ' + self.command)
            return
        # Trace.message('apply %s to %r'%(self.command, parameter.extracttext()))
        # parameter.tree()
        if not isinstance(parameter, FormulaConstant):
            try:
                extractor = ContainerExtractor(ContainerConfig.extracttext)
                parameter = extractor.extract(parameter)[0]
            except IndexError:
                Trace.error('No base character found for "%s".' % self.command)
                return
        # Trace.message('  basechar: %r' % parameter.string)
        # Insert combining character after the first character:
        if parameter.string.startswith('\u2009'):
            i = 2  # skip padding by SpacedCommand and FormulaConfig.modified
        else:
            i = 1
        parameter.string = parameter.string[:i] + combining + parameter.string[i:]
        # Use pre-composed characters if possible: \not{=} -> ≠, say.
        parameter.string = unicodedata.normalize('NFC', parameter.string)

    def parsesingleparameter(self, pos):
        "Parse a parameter, or a single letter."
        self.factory.clearskipped(pos)
        if pos.finished():
            return None
        return self.parseparameter(pos)


class OversetFunction(OneParamFunction):
    "A function that decorates some bit of text with an overset."

    commandmap = FormulaConfig.oversetfunctions

    def parsebit(self, pos):
        "Parse an overset-function"
        symbol = self.translated
        self.symbol = TaggedBit().constant(symbol, 'sup')
        self.parameter = self.parseparameter(pos)
        self.output = TaggedOutput().settag('span class="embellished"')
        self.contents.insert(0, self.symbol)
        self.parameter.output = TaggedOutput().settag('span class="base"')
        self.simplifyifpossible()


class UndersetFunction(OneParamFunction):
    "A function that decorates some bit of text with an underset."

    commandmap = FormulaConfig.undersetfunctions

    def parsebit(self, pos):
        "Parse an underset-function"
        symbol = self.translated
        self.symbol = TaggedBit().constant(symbol, 'sub')
        self.parameter = self.parseparameter(pos)
        self.output = TaggedOutput().settag('span class="embellished"')
        self.contents.insert(0, self.symbol)
        self.parameter.output = TaggedOutput().settag('span class="base"')
        self.simplifyifpossible()


class LimitCommand(EmptyCommand):
    "A command which accepts limits above and below, in display mode."

    commandmap = FormulaConfig.limitcommands

    def parsebit(self, pos):
        "Parse a limit command."
        self.output = TaggedOutput().settag('span class="limits"')
        symbol = self.translated
        self.contents.append(TaggedBit().constant(symbol, 'span class="limit"'))


class LimitPreviousCommand(LimitCommand):
    "A command to limit the previous command."

    commandmap = None

    def parsebit(self, pos):
        "Do nothing."
        self.output = TaggedOutput().settag('span class="limits"')
        self.factory.clearskipped(pos)

    def __str__(self):
        "Return a printable representation."
        return 'Limit previous command'


class LimitsProcessor(MathsProcessor):
    "A processor for limits inside an element."

    def process(self, contents, index):
        "Process the limits for an element."
        if Options.simplemath:
            return
        if self.checklimits(contents, index):
            self.modifylimits(contents, index)
        if self.checkscript(contents, index) and self.checkscript(contents, index + 1):
            self.modifyscripts(contents, index)

    def checklimits(self, contents, index):
        "Check if the current position has a limits command."
        # TODO: check for \limits macro
        if not DocumentParameters.displaymode:
            return False
        if self.checkcommand(contents, index + 1, LimitPreviousCommand):
            self.limitsahead(contents, index)
            return False
        if not isinstance(contents[index], LimitCommand):
            return False
        return self.checkscript(contents, index + 1)

    def limitsahead(self, contents, index):
        "Limit the current element based on the next."
        contents[index + 1].add(contents[index].clone())
        contents[index].output = EmptyOutput()

    def modifylimits(self, contents, index):
        "Modify a limits commands so that the limits appear above and below."
        limited = contents[index]
        subscript = self.getlimit(contents, index + 1)
        if self.checkscript(contents, index + 1):
            superscript = self.getlimit(contents, index + 1)
        else:
            superscript = TaggedBit().constant('\u2009', 'sup class="limit"')
        # fix order if source is x^i
        if subscript.command == '^':
            superscript, subscript = subscript, superscript
        limited.contents.append(subscript)
        limited.contents.insert(0, superscript)

    def getlimit(self, contents, index):
        "Get the limit for a limits command."
        limit = self.getscript(contents, index)
        limit.output.tag = limit.output.tag.replace('script', 'limit')
        return limit

    def modifyscripts(self, contents, index):
        "Modify the super- and subscript to appear vertically aligned."
        subscript = self.getscript(contents, index)
        # subscript removed so instead of index + 1 we get index again
        superscript = self.getscript(contents, index)
        # super-/subscript are reversed if source is x^i_j
        if subscript.command == '^':
            superscript, subscript = subscript, superscript
        scripts = TaggedBit().complete([superscript, subscript], 'span class="scripts"')
        contents.insert(index, scripts)

    def checkscript(self, contents, index):
        "Check if the current element is a sub- or superscript."
        return self.checkcommand(contents, index, SymbolFunction)

    def checkcommand(self, contents, index, type):
        "Check for the given type as the current element."
        if len(contents) <= index:
            return False
        return isinstance(contents[index], type)

    def getscript(self, contents, index):
        "Get the sub- or superscript."
        bit = contents[index]
        bit.output.tag += ' class="script"'
        del contents[index]
        return bit


class BracketCommand(OneParamFunction):
    "A command which defines a bracket."

    commandmap = FormulaConfig.bracketcommands

    def parsebit(self, pos):
        "Parse the bracket."
        OneParamFunction.parsebit(self, pos)

    def create(self, direction, character):
        "Create the bracket for the given character."
        self.original = character
        self.command = '\\' + direction
        self.contents = [FormulaConstant(character)]
        return self


class BracketProcessor(MathsProcessor):
    "A processor for bracket commands."

    def process(self, contents, index):
        "Convert the bracket using Unicode pieces, if possible."
        if Options.simplemath:
            return
        if self.checkleft(contents, index):
            return self.processleft(contents, index)

    def processleft(self, contents, index):
        "Process a left bracket."
        rightindex = self.findright(contents, index + 1)
        if not rightindex:
            return
        size = self.findmax(contents, index, rightindex)
        self.resize(contents[index], size)
        self.resize(contents[rightindex], size)

    def checkleft(self, contents, index):
        "Check if the command at the given index is left."
        return self.checkdirection(contents[index], '\\left')

    def checkright(self, contents, index):
        "Check if the command at the given index is right."
        return self.checkdirection(contents[index], '\\right')

    def checkdirection(self, bit, command):
        "Check if the given bit is the desired bracket command."
        if not isinstance(bit, BracketCommand):
            return False
        return bit.command == command

    def findright(self, contents, index):
        "Find the right bracket starting at the given index, or 0."
        depth = 1
        while index < len(contents):
            if self.checkleft(contents, index):
                depth += 1
            if self.checkright(contents, index):
                depth -= 1
            if depth == 0:
                return index
            index += 1
        return None

    def findmax(self, contents, leftindex, rightindex):
        "Find the max size of the contents between the two given indices."
        sliced = contents[leftindex:rightindex]
        return max(element.size for element in sliced)

    def resize(self, command, size):
        "Resize a bracket command to the given size."
        character = command.extracttext()
        alignment = command.command.replace('\\', '')
        bracket = BigBracket(size, character, alignment)
        command.output = ContentsOutput()
        command.contents = bracket.getcontents()


FormulaCommand.types += [OversetFunction, UndersetFunction,
                         CombiningFunction, LimitCommand, BracketCommand]

FormulaProcessor.processors += [
    LimitsProcessor(), BracketProcessor(),
]


class ParameterDefinition:
    "The definition of a parameter in a hybrid function."
    "[] parameters are optional, {} parameters are mandatory."
    "Each parameter has a one-character name, like {$1} or {$p}."
    "A parameter that ends in ! like {$p!} is a literal."
    "Example: [$1]{$p!} reads an optional parameter $1 and a literal mandatory parameter p."

    parambrackets = [('[', ']'), ('{', '}')]

    def __init__(self):
        self.name = None
        self.literal = False
        self.optional = False
        self.value = None
        self.literalvalue = None

    def parse(self, pos):
        "Parse a parameter definition: [$0], {$x}, {$1!}..."
        for (opening, closing) in ParameterDefinition.parambrackets:
            if pos.checkskip(opening):
                if opening == '[':
                    self.optional = True
                if not pos.checkskip('$'):
                    Trace.error('Wrong parameter name, did you mean $' + pos.current() + '?')
                    return None
                self.name = pos.skipcurrent()
                if pos.checkskip('!'):
                    self.literal = True
                if not pos.checkskip(closing):
                    Trace.error('Wrong parameter closing ' + pos.skipcurrent())
                    return None
                return self
        Trace.error('Wrong character in parameter template: ' + pos.skipcurrent())
        return None

    def read(self, pos, function):
        "Read the parameter itself using the definition."
        if self.literal:
            if self.optional:
                self.literalvalue = function.parsesquareliteral(pos)
            else:
                self.literalvalue = function.parseliteral(pos)
            if self.literalvalue:
                self.value = FormulaConstant(self.literalvalue)
        elif self.optional:
            self.value = function.parsesquare(pos)
        else:
            self.value = function.parseparameter(pos)

    def __str__(self):
        "Return a printable representation."
        result = 'param ' + self.name
        if self.value:
            result += ': ' + str(self.value)
        else:
            result += ' (empty)'
        return result


class ParameterFunction(CommandBit):
    "A function with a variable number of parameters defined in a template."
    "The parameters are defined as a parameter definition."

    def readparams(self, readtemplate, pos):
        "Read the params according to the template."
        self.params = {}
        for paramdef in self.paramdefs(readtemplate):
            paramdef.read(pos, self)
            self.params['$' + paramdef.name] = paramdef

    def paramdefs(self, readtemplate):
        "Read each param definition in the template"
        pos = TextPosition(readtemplate)
        while not pos.finished():
            paramdef = ParameterDefinition().parse(pos)
            if paramdef:
                yield paramdef

    def getparam(self, name):
        "Get a parameter as parsed."
        if name not in self.params:
            return None
        return self.params[name]

    def getvalue(self, name):
        "Get the value of a parameter."
        return self.getparam(name).value

    def getliteralvalue(self, name):
        "Get the literal value of a parameter."
        param = self.getparam(name)
        if not param or not param.literalvalue:
            return None
        return param.literalvalue


class HybridFunction(ParameterFunction):
    """
    A parameter function where the output is also defined using a template.
    The template can use a number of functions; each function has an associated
    tag.
    Example: [f0{$1},span class="fbox"] defines a function f0 which corresponds
    to a span of class fbox, yielding <span class="fbox">$1</span>.
    Literal parameters can be used in tags definitions:
      [f0{$1},span style="color: $p;"]
    yields <span style="color: $p;">$1</span>, where $p is a literal parameter.
    Sizes can be specified in hybridsizes, e.g. adding parameter sizes. By
    default the resulting size is the max of all arguments. Sizes are used
    to generate the right parameters.
    A function followed by a single / is output as a self-closing XHTML tag:
      [f0/,hr]
    will generate <hr/>.
    """

    commandmap = FormulaConfig.hybridfunctions

    def parsebit(self, pos):
        "Parse a function with [] and {} parameters"
        readtemplate = self.translated[0]
        writetemplate = self.translated[1]
        self.readparams(readtemplate, pos)
        self.contents = self.writeparams(writetemplate)
        self.computehybridsize()

    def writeparams(self, writetemplate):
        "Write all params according to the template"
        return self.writepos(TextPosition(writetemplate))

    def writepos(self, pos):
        "Write all params as read in the parse position."
        result = []
        while not pos.finished():
            if pos.checkskip('$'):
                param = self.writeparam(pos)
                if param:
                    result.append(param)
            elif pos.checkskip('f'):
                function = self.writefunction(pos)
                if function:
                    function.type = None
                    result.append(function)
            elif pos.checkskip('('):
                result.append(self.writebracket('left', '('))
            elif pos.checkskip(')'):
                result.append(self.writebracket('right', ')'))
            else:
                result.append(FormulaConstant(pos.skipcurrent()))
        return result

    def writeparam(self, pos):
        "Write a single param of the form $0, $x..."
        name = '$' + pos.skipcurrent()
        if name not in self.params:
            Trace.error('Unknown parameter ' + name)
            return None
        if not self.params[name]:
            return None
        if pos.checkskip('.'):
            self.params[name].value.type = pos.globalpha()
        return self.params[name].value

    def writefunction(self, pos):
        "Write a single function f0,...,fn."
        tag = self.readtag(pos)
        if not tag:
            return None
        if pos.checkskip('/'):
            # self-closing XHTML tag, such as <hr/>
            return TaggedBit().selfcomplete(tag)
        if not pos.checkskip('{'):
            Trace.error('Function should be defined in {}')
            return None
        pos.pushending('}')
        contents = self.writepos(pos)
        pos.popending()
        if len(contents) == 0:
            return None
        return TaggedBit().complete(contents, tag)

    def readtag(self, pos):
        "Get the tag corresponding to the given index. Does parameter substitution."
        if not pos.current().isdigit():
            Trace.error('Function should be f0,...,f9: f' + pos.current())
            return None
        index = int(pos.skipcurrent())
        if 2 + index > len(self.translated):
            Trace.error('Function f' + str(index) + ' is not defined')
            return None
        tag = self.translated[2 + index]
        if '$' not in tag:
            return tag
        for variable in self.params:
            if variable in tag:
                param = self.params[variable]
                if not param.literal:
                    Trace.error('Parameters in tag ' + tag + ' should be literal: {' + variable + '!}')
                    continue
                if param.literalvalue:
                    value = param.literalvalue
                else:
                    value = ''
                tag = tag.replace(variable, value)
        return tag

    def writebracket(self, direction, character):
        "Return a new bracket looking at the given direction."
        return self.factory.create(BracketCommand).create(direction, character)

    def computehybridsize(self):
        "Compute the size of the hybrid function."
        if self.command not in HybridSize.configsizes:
            self.computesize()
            return
        self.size = HybridSize().getsize(self)
        # set the size in all elements at first level
        for element in self.contents:
            element.size = self.size


class HybridSize:
    "The size associated with a hybrid function."

    configsizes = FormulaConfig.hybridsizes

    def getsize(self, function):
        "Read the size for a function and parse it."
        sizestring = self.configsizes[function.command]
        for name in function.params:
            if name in sizestring:
                size = function.params[name].value.computesize()
                sizestring = sizestring.replace(name, str(size))
        if '$' in sizestring:
            Trace.error('Unconverted variable in hybrid size: ' + sizestring)
            return 1
        return eval(sizestring)


FormulaCommand.types += [HybridFunction]


def math2html(formula):
    "Convert some TeX math to HTML."
    factory = FormulaFactory()
    whole = factory.parseformula(formula)
    FormulaProcessor().process(whole)
    whole.process()
    return ''.join(whole.gethtml())


def main():
    "Main function, called if invoked from the command line"
    args = sys.argv
    Options().parseoptions(args)
    if len(args) != 1:
        Trace.error('Usage: math2html.py escaped_string')
        exit()
    result = math2html(args[0])
    Trace.message(result)


if __name__ == '__main__':
    main()

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/math/math2html.py
NikolaBlagojevic/pyrecodes
# :Id: $Id: tex2mathml_extern.py 9250 2022-11-16 14:01:31Z milde $
# :Copyright: © 2015 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause

"""Wrappers for TeX->MathML conversion by external tools

This module is provisional:
the API is not settled and may change with any minor Docutils version.
"""

import subprocess

document_template = r"""\documentclass{article}
\usepackage{amsmath}
\begin{document}
%s
\end{document}
"""


def latexml(math_code, reporter=None):
    """Convert LaTeX math code to MathML with LaTeXML_

    .. _LaTeXML: http://dlmf.nist.gov/LaTeXML/
    """
    p = subprocess.Popen(['latexml',
                          '-',  # read from stdin
                          '--preload=amsfonts',
                          '--preload=amsmath',
                          '--inputencoding=utf8',
                          ],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         close_fds=True)
    p.stdin.write((document_template % math_code).encode('utf-8'))
    p.stdin.close()
    latexml_code = p.stdout.read()
    latexml_err = p.stderr.read().decode('utf-8')
    if reporter and (latexml_err.find('Error') >= 0 or not latexml_code):
        reporter.error(latexml_err)

    post_p = subprocess.Popen(['latexmlpost',
                               '-',
                               '--nonumbersections',
                               '--format=xhtml',
                               # '--linelength=78',  # experimental
                               '--'
                               ],
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              close_fds=True)
    post_p.stdin.write(latexml_code)
    post_p.stdin.close()
    result = post_p.stdout.read().decode('utf-8')
    post_p_err = post_p.stderr.read().decode('utf-8')
    if reporter and (post_p_err.find('Error') >= 0 or not result):
        reporter.error(post_p_err)

    # extract MathML code:
    start, end = result.find('<math'), result.find('</math>')+7
    result = result[start:end]
    if 'class="ltx_ERROR' in result:
        raise SyntaxError(result)
    return result


def ttm(math_code, reporter=None):
    """Convert LaTeX math code to MathML with TtM_

    .. _TtM: http://hutchinson.belmont.ma.us/tth/mml/
    """
    p = subprocess.Popen(['ttm',
                          # '-i',  # italic font for equations. Default roman.
                          '-u',    # unicode encoding. (Default iso-8859-1).
                          '-r',    # output raw MathML (no wrapper)
                          ],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         close_fds=True)
    p.stdin.write((document_template % math_code).encode('utf-8'))
    p.stdin.close()
    result = p.stdout.read()
    err = p.stderr.read().decode('utf-8')
    if err.find('**** Unknown') >= 0:
        msg = '\n'.join(line for line in err.splitlines()
                        if line.startswith('****'))
        raise SyntaxError('\nMessage from external converter TtM:\n'+msg)
    if reporter and err.find('**** Error') >= 0 or not result:
        reporter.error(err)
    start, end = result.find('<math'), result.find('</math>')+7
    return result[start:end]


def blahtexml(math_code, inline=True, reporter=None):
    """Convert LaTeX math code to MathML with blahtexml_

    .. _blahtexml: http://gva.noekeon.org/blahtexml/
    """
    options = ['--mathml',
               '--indented',
               '--spacing', 'moderate',
               '--mathml-encoding', 'raw',
               '--other-encoding', 'raw',
               '--doctype-xhtml+mathml',
               '--annotate-TeX',
               ]
    if inline:
        mathmode_arg = ''
    else:
        mathmode_arg = ' display="block"'
        options.append('--displaymath')

    p = subprocess.Popen(['blahtexml']+options,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         close_fds=True)
    p.stdin.write(math_code.encode('utf-8'))
    p.stdin.close()
    result = p.stdout.read().decode('utf-8')
    err = p.stderr.read().decode('utf-8')

    if result.find('<error>') >= 0:
        msg = result[result.find('<message>')+9:result.find('</message>')]
        raise SyntaxError('\nMessage from external converter blahtexml:\n%s'
                          % msg)
    if reporter and (err.find('**** Error') >= 0 or not result):
        reporter.error(err)
    start, end = result.find('<markup>')+9, result.find('</markup>')
    result = ('<math xmlns="http://www.w3.org/1998/Math/MathML"%s>\n'
              '%s</math>\n') % (mathmode_arg, result[start:end])
    return result


def pandoc(math_code, reporter=None):
    """Convert LaTeX math code to MathML with pandoc_

    .. _pandoc: https://pandoc.org/
    """
    p = subprocess.Popen(['pandoc',
                          '--mathml',
                          '--from=latex',
                          ],
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         close_fds=True)
    p.stdin.write(math_code.encode('utf-8'))
    p.stdin.close()
    result = p.stdout.read().decode('utf-8')
    err = p.stderr.read().decode('utf-8').strip()
    x = p.wait()

    if err:
        if reporter:
            reporter.error(err)
        raise SyntaxError('\nError message from external converter pandoc:\n%s'
                          % err)
    if x != 0:
        raise SyntaxError('\nError code from external converter pandoc:\n%s'
                          % x)

    start, end = result.find('<math'), result.find('</math>')+7
    return result[start:end]


# self-test

if __name__ == "__main__":
    example = ('\\frac{\\partial \\sin^2(\\alpha)}{\\partial \\vec r}'
               '\\varpi \\mathbb{R} \\, \\text{Grüße}')
    # print(latexml('$'+example+'$'))
    # print(ttm('$'+example.replace('\\mathbb{R}', '')+'$'))
    print(blahtexml(example))
    # print(pandoc('$'+example+'$'))

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/math/tex2mathml_extern.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3

# LaTeX math to Unicode symbols translation dictionaries.
# Generated with ``write_tex2unichar.py`` from the data in
# http://milde.users.sourceforge.net/LUCR/Math/

# Includes commands from:
#   standard LaTeX
#   amssymb
#   amsmath
#   amsxtra
#   bbold
#   esint
#   mathabx
#   mathdots
#   txfonts
#   stmaryrd
#   wasysym

mathaccent = {
    'acute': '\u0301',  #  ́ COMBINING ACUTE ACCENT
    'bar': '\u0304',  #  ̄ COMBINING MACRON
    'breve': '\u0306',  #  ̆ COMBINING BREVE
    'check': '\u030c',  #  ̌ COMBINING CARON
    'ddddot': '\u20dc',  #  ⃜ COMBINING FOUR DOTS ABOVE
    'dddot': '\u20db',  #  ⃛ COMBINING THREE DOTS ABOVE
    'ddot': '\u0308',  #  ̈ COMBINING DIAERESIS
    'dot': '\u0307',  #  ̇ COMBINING DOT ABOVE
    'grave': '\u0300',  #  ̀ COMBINING GRAVE ACCENT
    'hat': '\u0302',  #  ̂ COMBINING CIRCUMFLEX ACCENT
    'mathring': '\u030a',  #  ̊ COMBINING RING ABOVE
    'not': '\u0338',  #  ̸ COMBINING LONG SOLIDUS OVERLAY
    'overleftrightarrow': '\u20e1',  #  ⃡ COMBINING LEFT RIGHT ARROW ABOVE
    'overline': '\u0305',  #  ̅ COMBINING OVERLINE
    'tilde': '\u0303',  #  ̃ COMBINING TILDE
    'underbar': '\u0331',  #  ̱ COMBINING MACRON BELOW
    'underleftarrow': '\u20ee',  #  ⃮ COMBINING LEFT ARROW BELOW
    'underline': '\u0332',  #  ̲ COMBINING LOW LINE
    'underrightarrow': '\u20ef',  #  ⃯ COMBINING RIGHT ARROW BELOW
    'vec': '\u20d7',  #  ⃗ COMBINING RIGHT ARROW ABOVE
    }

mathalpha = {
    'Bbbk': '\U0001d55c',  # 𝕜 MATHEMATICAL DOUBLE-STRUCK SMALL K
    'Delta': '\u0394',  # Δ GREEK CAPITAL LETTER DELTA
    'Gamma': '\u0393',  # Γ GREEK CAPITAL LETTER GAMMA
    'Im': '\u2111',  # ℑ BLACK-LETTER CAPITAL I
    'Lambda': '\u039b',  # Λ GREEK CAPITAL LETTER LAMDA
    'Omega': '\u03a9',  # Ω GREEK CAPITAL LETTER OMEGA
    'Phi': '\u03a6',  # Φ GREEK CAPITAL LETTER PHI
    'Pi': '\u03a0',  # Π GREEK CAPITAL LETTER PI
    'Psi': '\u03a8',  # Ψ GREEK CAPITAL LETTER PSI
    'Re': '\u211c',  # ℜ BLACK-LETTER CAPITAL R
    'Sigma': '\u03a3',  # Σ GREEK CAPITAL LETTER SIGMA
    'Theta': '\u0398',  # Θ GREEK CAPITAL LETTER THETA
    'Upsilon': '\u03a5',  # Υ GREEK CAPITAL LETTER UPSILON
    'Xi': '\u039e',  # Ξ GREEK CAPITAL LETTER XI
    'aleph': '\u2135',  # ℵ ALEF SYMBOL
    'alpha': '\u03b1',  # α GREEK SMALL LETTER ALPHA
    'beta': '\u03b2',  # β GREEK SMALL LETTER BETA
    'beth': '\u2136',  # ℶ BET SYMBOL
    'chi': '\u03c7',  # χ GREEK SMALL LETTER CHI
    'daleth': '\u2138',  # ℸ DALET SYMBOL
    'delta': '\u03b4',  # δ GREEK SMALL LETTER DELTA
    'digamma': '\u03dd',  # ϝ GREEK SMALL LETTER DIGAMMA
    'ell': '\u2113',  # ℓ SCRIPT SMALL L
    'epsilon': '\u03f5',  # ϵ GREEK LUNATE EPSILON SYMBOL
    'eta': '\u03b7',  # η GREEK SMALL LETTER ETA
    'eth': '\xf0',  # ð LATIN SMALL LETTER ETH
    'gamma': '\u03b3',  # γ GREEK SMALL LETTER GAMMA
    'gimel': '\u2137',  # ℷ GIMEL SYMBOL
    'hslash': '\u210f',  # ℏ PLANCK CONSTANT OVER TWO PI
    'imath': '\u0131',  # ı LATIN SMALL LETTER DOTLESS I
    'iota': '\u03b9',  # ι GREEK SMALL LETTER IOTA
    'jmath': '\u0237',  # ȷ LATIN SMALL LETTER DOTLESS J
    'kappa': '\u03ba',  # κ GREEK SMALL LETTER KAPPA
    'lambda': '\u03bb',  # λ GREEK SMALL LETTER LAMDA
    'mu': '\u03bc',  # μ GREEK SMALL LETTER MU
    'nu': '\u03bd',  # ν GREEK SMALL LETTER NU
    'omega': '\u03c9',  # ω GREEK SMALL LETTER OMEGA
    'phi': '\u03d5',  # ϕ GREEK PHI SYMBOL
    'pi': '\u03c0',  # π GREEK SMALL LETTER PI
    'psi': '\u03c8',  # ψ GREEK SMALL LETTER PSI
    'rho': '\u03c1',  # ρ GREEK SMALL LETTER RHO
    'sigma': '\u03c3',  # σ GREEK SMALL LETTER SIGMA
    'tau': '\u03c4',  # τ GREEK SMALL LETTER TAU
    'theta': '\u03b8',  # θ GREEK SMALL LETTER THETA
    'upsilon': '\u03c5',  # υ GREEK SMALL LETTER UPSILON
    'varDelta': '\U0001d6e5',  # 𝛥 MATHEMATICAL ITALIC CAPITAL DELTA
    'varGamma': '\U0001d6e4',  # 𝛤 MATHEMATICAL ITALIC CAPITAL GAMMA
    'varLambda': '\U0001d6ec',  # 𝛬 MATHEMATICAL ITALIC CAPITAL LAMDA
    'varOmega': '\U0001d6fa',  # 𝛺 MATHEMATICAL ITALIC CAPITAL OMEGA
    'varPhi': '\U0001d6f7',  # 𝛷 MATHEMATICAL ITALIC CAPITAL PHI
    'varPi': '\U0001d6f1',  # 𝛱 MATHEMATICAL ITALIC CAPITAL PI
    'varPsi': '\U0001d6f9',  # 𝛹 MATHEMATICAL ITALIC CAPITAL PSI
    'varSigma': '\U0001d6f4',  # 𝛴 MATHEMATICAL ITALIC CAPITAL SIGMA
    'varTheta': '\U0001d6e9',  # 𝛩 MATHEMATICAL ITALIC CAPITAL THETA
    'varUpsilon': '\U0001d6f6',  # 𝛶 MATHEMATICAL ITALIC CAPITAL UPSILON
    'varXi': '\U0001d6ef',  # 𝛯 MATHEMATICAL ITALIC CAPITAL XI
    'varepsilon': '\u03b5',  # ε GREEK SMALL LETTER EPSILON
    'varkappa': '\u03f0',  # ϰ GREEK KAPPA SYMBOL
    'varphi': '\u03c6',  # φ GREEK SMALL LETTER PHI
    'varpi': '\u03d6',  # ϖ GREEK PI SYMBOL
    'varrho': '\u03f1',  # ϱ GREEK RHO SYMBOL
    'varsigma': '\u03c2',  # ς GREEK SMALL LETTER FINAL SIGMA
    'vartheta': '\u03d1',  # ϑ GREEK THETA SYMBOL
    'wp': '\u2118',  # ℘ SCRIPT CAPITAL P
    'xi': '\u03be',  # ξ GREEK SMALL LETTER XI
    'zeta': '\u03b6',  # ζ GREEK SMALL LETTER ZETA
    }

mathbin = {
    'Cap': '\u22d2',  # ⋒ DOUBLE INTERSECTION
    'Circle': '\u25cb',  # ○ WHITE CIRCLE
    'Cup': '\u22d3',  # ⋓ DOUBLE UNION
    'LHD': '\u25c0',  # ◀ BLACK LEFT-POINTING TRIANGLE
    'RHD': '\u25b6',  # ▶ BLACK RIGHT-POINTING TRIANGLE
    'amalg': '\u2a3f',  # ⨿ AMALGAMATION OR COPRODUCT
    'ast': '\u2217',  # ∗ ASTERISK OPERATOR
    'barwedge': '\u22bc',  # ⊼ NAND
    'bigcirc': '\u25ef',  # ◯ LARGE CIRCLE
    'bigtriangledown': '\u25bd',  # ▽ WHITE DOWN-POINTING TRIANGLE
    'bigtriangleup': '\u25b3',  # △ WHITE UP-POINTING TRIANGLE
    'bindnasrepma': '\u214b',  # ⅋ TURNED AMPERSAND
    'blacklozenge': '\u29eb',  # ⧫ BLACK LOZENGE
    'boxast': '\u29c6',  # ⧆ SQUARED ASTERISK
    'boxbar': '\u25eb',  # ◫ WHITE SQUARE WITH VERTICAL BISECTING LINE
    'boxbox': '\u29c8',  # ⧈ SQUARED SQUARE
    'boxbslash': '\u29c5',  # ⧅ SQUARED FALLING DIAGONAL SLASH
    'boxcircle': '\u29c7',  # ⧇ SQUARED SMALL CIRCLE
    'boxdot': '\u22a1',  # ⊡ SQUARED DOT OPERATOR
    'boxminus': '\u229f',  # ⊟ SQUARED MINUS
    'boxplus': '\u229e',  # ⊞ SQUARED PLUS
    'boxslash': '\u29c4',  # ⧄ SQUARED RISING DIAGONAL SLASH
    'boxtimes': '\u22a0',  # ⊠ SQUARED TIMES
    'bullet': '\u2022',  # • BULLET
    'cap': '\u2229',  # ∩ INTERSECTION
    'cdot': '\u22c5',  # ⋅ DOT OPERATOR
    'circ': '\u2218',  # ∘ RING OPERATOR
    'circledast': '\u229b',  # ⊛ CIRCLED ASTERISK OPERATOR
    'circledbslash': '\u29b8',  # ⦸ CIRCLED REVERSE SOLIDUS
    'circledcirc': '\u229a',  # ⊚ CIRCLED RING OPERATOR
    'circleddash': '\u229d',  # ⊝ CIRCLED DASH
    'circledgtr': '\u29c1',  # ⧁ CIRCLED GREATER-THAN
    'circledless': '\u29c0',  # ⧀ CIRCLED LESS-THAN
    'cup': '\u222a',  # ∪ UNION
    'curlyvee': '\u22ce',  # ⋎ CURLY LOGICAL OR
    'curlywedge': '\u22cf',  # ⋏ CURLY LOGICAL AND
    'dagger': '\u2020',  # † DAGGER
    'ddagger': '\u2021',  # ‡ DOUBLE DAGGER
    'diamond': '\u22c4',  # ⋄ DIAMOND OPERATOR
    'div': '\xf7',  # ÷ DIVISION SIGN
    'divideontimes': '\u22c7',  # ⋇ DIVISION TIMES
    'dotplus': '\u2214',  # ∔ DOT PLUS
    'doublebarwedge': '\u2a5e',  # ⩞ LOGICAL AND WITH DOUBLE OVERBAR
    'gtrdot': '\u22d7',  # ⋗ GREATER-THAN WITH DOT
    'intercal': '\u22ba',  # ⊺ INTERCALATE
    'interleave': '\u2af4',  # ⫴ TRIPLE VERTICAL BAR BINARY RELATION
    'invamp': '\u214b',  # ⅋ TURNED AMPERSAND
    'land': '\u2227',  # ∧ LOGICAL AND
    'leftthreetimes': '\u22cb',  # ⋋ LEFT SEMIDIRECT PRODUCT
    'lessdot': '\u22d6',  # ⋖ LESS-THAN WITH DOT
    'lor': '\u2228',  # ∨ LOGICAL OR
    'ltimes': '\u22c9',  # ⋉ LEFT NORMAL FACTOR SEMIDIRECT PRODUCT
    'mp': '\u2213',  # ∓ MINUS-OR-PLUS SIGN
    'odot': '\u2299',  # ⊙ CIRCLED DOT OPERATOR
    'ominus': '\u2296',  # ⊖ CIRCLED MINUS
    'oplus': '\u2295',  # ⊕ CIRCLED PLUS
    'oslash': '\u2298',  # ⊘ CIRCLED DIVISION SLASH
    'otimes': '\u2297',  # ⊗ CIRCLED TIMES
    'pm': '\xb1',  # ± PLUS-MINUS SIGN
    'rightthreetimes': '\u22cc',  # ⋌ RIGHT SEMIDIRECT PRODUCT
    'rtimes': '\u22ca',  # ⋊ RIGHT NORMAL FACTOR SEMIDIRECT PRODUCT
    'setminus': '\u29f5',  # ⧵ REVERSE SOLIDUS OPERATOR
    'slash': '\u2215',  # ∕ DIVISION SLASH
    'smallsetminus': '\u2216',  # ∖ SET MINUS
    'smalltriangledown': '\u25bf',  # ▿ WHITE DOWN-POINTING SMALL TRIANGLE
    'smalltriangleleft': '\u25c3',  # ◃ WHITE LEFT-POINTING SMALL TRIANGLE
    'smalltriangleright': '\u25b9',  # ▹ WHITE RIGHT-POINTING SMALL TRIANGLE
    'sqcap': '\u2293',  # ⊓ SQUARE CAP
    'sqcup': '\u2294',  # ⊔ SQUARE CUP
    'sslash': '\u2afd',  # ⫽ DOUBLE SOLIDUS OPERATOR
    'star': '\u22c6',  # ⋆ STAR OPERATOR
    'talloblong': '\u2afe',  # ⫾ WHITE VERTICAL BAR
    'times': '\xd7',  # × MULTIPLICATION SIGN
    'triangleleft': '\u25c3',  # ◃ WHITE LEFT-POINTING SMALL TRIANGLE
    'triangleright': '\u25b9',  # ▹ WHITE RIGHT-POINTING SMALL TRIANGLE
    'uplus': '\u228e',  # ⊎ MULTISET UNION
    'vee': '\u2228',  # ∨ LOGICAL OR
    'veebar': '\u22bb',  # ⊻ XOR
    'wedge': '\u2227',  # ∧ LOGICAL AND
    'wr': '\u2240',  # ≀ WREATH PRODUCT
    }

mathclose = {
    'Rbag': '\u27c6',  # ⟆ RIGHT S-SHAPED BAG DELIMITER
    'lrcorner': '\u231f',  # ⌟ BOTTOM RIGHT CORNER
    'rangle': '\u27e9',  # ⟩ MATHEMATICAL RIGHT ANGLE BRACKET
    'rbag': '\u27c6',  # ⟆ RIGHT S-SHAPED BAG DELIMITER
    'rbrace': '}',  # } RIGHT CURLY BRACKET
    'rbrack': ']',  # ] RIGHT SQUARE BRACKET
    'rceil': '\u2309',  # ⌉ RIGHT CEILING
    'rfloor': '\u230b',  # ⌋ RIGHT FLOOR
    'rgroup': '\u27ef',  # ⟯ MATHEMATICAL RIGHT FLATTENED PARENTHESIS
    'rrbracket': '\u27e7',  # ⟧ MATHEMATICAL RIGHT WHITE SQUARE BRACKET
    'rrparenthesis': '\u2988',  # ⦈ Z NOTATION RIGHT IMAGE BRACKET
    'urcorner': '\u231d',  # ⌝ TOP RIGHT CORNER
    '}': '}',  # } RIGHT CURLY BRACKET
    }

mathfence = {
    'Vert': '\u2016',  # ‖ DOUBLE VERTICAL LINE
    'vert': '|',  # | VERTICAL LINE
    '|': '\u2016',  # ‖ DOUBLE VERTICAL LINE
    }

mathop = {
    'bigcap': '\u22c2',  # ⋂ N-ARY INTERSECTION
    'bigcup': '\u22c3',  # ⋃ N-ARY UNION
    'biginterleave': '\u2afc',  # ⫼ LARGE TRIPLE VERTICAL BAR OPERATOR
    'bigodot': '\u2a00',  # ⨀ N-ARY CIRCLED DOT OPERATOR
    'bigoplus': '\u2a01',  # ⨁ N-ARY CIRCLED PLUS OPERATOR
    'bigotimes': '\u2a02',  # ⨂ N-ARY CIRCLED TIMES OPERATOR
    'bigsqcap': '\u2a05',  # ⨅ N-ARY SQUARE INTERSECTION OPERATOR
    'bigsqcup': '\u2a06',  # ⨆ N-ARY SQUARE UNION OPERATOR
    'biguplus': '\u2a04',  # ⨄ N-ARY UNION OPERATOR WITH PLUS
    'bigvee': '\u22c1',  # ⋁ N-ARY LOGICAL OR
    'bigwedge': '\u22c0',  # ⋀ N-ARY LOGICAL AND
    'coprod': '\u2210',  # ∐ N-ARY COPRODUCT
    'fatsemi': '\u2a1f',  # ⨟ Z NOTATION SCHEMA COMPOSITION
    'fint': '\u2a0f',  # ⨏ INTEGRAL AVERAGE WITH SLASH
    'iiiint': '\u2a0c',  # ⨌ QUADRUPLE INTEGRAL OPERATOR
    'iiint': '\u222d',  # ∭ TRIPLE INTEGRAL
    'iint': '\u222c',  # ∬ DOUBLE INTEGRAL
    'int': '\u222b',  # ∫ INTEGRAL
    'intop': '\u222b',  # ∫ INTEGRAL
    'oiiint': '\u2230',  # ∰ VOLUME INTEGRAL
    'oiint': '\u222f',  # ∯ SURFACE INTEGRAL
    'oint': '\u222e',  # ∮ CONTOUR INTEGRAL
    'ointctrclockwise': '\u2233',  # ∳ ANTICLOCKWISE CONTOUR INTEGRAL
    'ointop': '\u222e',  # ∮ CONTOUR INTEGRAL
    'prod': '\u220f',  # ∏ N-ARY PRODUCT
    'sqint': '\u2a16',  # ⨖ QUATERNION INTEGRAL OPERATOR
    'sum': '\u2211',  # ∑ N-ARY SUMMATION
    'varointclockwise': '\u2232',  # ∲ CLOCKWISE CONTOUR INTEGRAL
    'varprod': '\u2a09',  # ⨉ N-ARY TIMES OPERATOR
    }

mathopen = {
    'Lbag': '\u27c5',  # ⟅ LEFT S-SHAPED BAG DELIMITER
    'langle': '\u27e8',  # ⟨ MATHEMATICAL LEFT ANGLE BRACKET
    'lbag': '\u27c5',  # ⟅ LEFT S-SHAPED BAG DELIMITER
    'lbrace': '{',  # { LEFT CURLY BRACKET
    'lbrack': '[',  # [ LEFT SQUARE BRACKET
    'lceil': '\u2308',  # ⌈ LEFT CEILING
    'lfloor': '\u230a',  # ⌊ LEFT FLOOR
    'lgroup': '\u27ee',  # ⟮ MATHEMATICAL LEFT FLATTENED PARENTHESIS
    'llbracket': '\u27e6',  # ⟦ MATHEMATICAL LEFT WHITE SQUARE BRACKET
    'llcorner': '\u231e',  # ⌞ BOTTOM LEFT CORNER
    'llparenthesis': '\u2987',  # ⦇ Z NOTATION LEFT IMAGE BRACKET
    'ulcorner': '\u231c',  # ⌜ TOP LEFT CORNER
    '{': '{',  # { LEFT CURLY BRACKET
    }

mathord = {
    '#': '#',  # # NUMBER SIGN
    '$': '$',  # $ DOLLAR SIGN
    '%': '%',  # % PERCENT SIGN
    '&': '&',  # & AMPERSAND
    'AC': '\u223f',  # ∿ SINE WAVE
    'APLcomment': '\u235d',  # ⍝ APL FUNCTIONAL SYMBOL UP SHOE JOT
    'APLdownarrowbox': '\u2357',  # ⍗ APL FUNCTIONAL SYMBOL QUAD DOWNWARDS ARROW
    'APLinput': '\u235e',  # ⍞ APL FUNCTIONAL SYMBOL QUOTE QUAD
    'APLinv': '\u2339',  # ⌹ APL FUNCTIONAL SYMBOL QUAD DIVIDE
    'APLleftarrowbox': '\u2347',  # ⍇ APL FUNCTIONAL SYMBOL QUAD LEFTWARDS ARROW
    'APLlog': '\u235f',  # ⍟ APL FUNCTIONAL SYMBOL CIRCLE STAR
    'APLrightarrowbox': '\u2348',  # ⍈ APL FUNCTIONAL SYMBOL QUAD RIGHTWARDS ARROW
    'APLuparrowbox': '\u2350',  # ⍐ APL FUNCTIONAL SYMBOL QUAD UPWARDS ARROW
    'Aries': '\u2648',  # ♈ ARIES
    'Box': '\u2b1c',  # ⬜ WHITE LARGE SQUARE
    'CIRCLE': '\u25cf',  # ● BLACK CIRCLE
    'CheckedBox': '\u2611',  # ☑ BALLOT BOX WITH CHECK
    'Diamond': '\u25c7',  # ◇ WHITE DIAMOND
    'Diamondblack': '\u25c6',  # ◆ BLACK DIAMOND
    'Diamonddot': '\u27d0',  # ⟐ WHITE DIAMOND WITH CENTRED DOT
    'Finv': '\u2132',  # Ⅎ TURNED CAPITAL F
    'Game': '\u2141',  # ⅁ TURNED SANS-SERIF CAPITAL G
    'Gemini': '\u264a',  # ♊ GEMINI
    'Jupiter': '\u2643',  # ♃ JUPITER
    'LEFTCIRCLE': '\u25d6',  # ◖ LEFT HALF BLACK CIRCLE
    'LEFTcircle': '\u25d0',  # ◐ CIRCLE WITH LEFT HALF BLACK
    'Leo': '\u264c',  # ♌ LEO
    'Libra': '\u264e',  # ♎ LIBRA
    'Mars': '\u2642',  # ♂ MALE SIGN
    'Mercury': '\u263f',  # ☿ MERCURY
    'Neptune': '\u2646',  # ♆ NEPTUNE
    'P': '\xb6',  # ¶ PILCROW SIGN
    'Pluto': '\u2647',  # ♇ PLUTO
    'RIGHTCIRCLE': '\u25d7',  # ◗ RIGHT HALF BLACK CIRCLE
    'RIGHTcircle': '\u25d1',  # ◑ CIRCLE WITH RIGHT HALF BLACK
    'S': '\xa7',  # § SECTION SIGN
    'Saturn': '\u2644',  # ♄ SATURN
    'Scorpio': '\u264f',  # ♏ SCORPIUS
    'Square': '\u2610',  # ☐ BALLOT BOX
    'Sun': '\u2609',  # ☉ SUN
    'Taurus': '\u2649',  # ♉ TAURUS
    'Uranus': '\u2645',  # ♅ URANUS
    'Venus': '\u2640',  # ♀ FEMALE SIGN
    'XBox': '\u2612',  # ☒ BALLOT BOX WITH X
    'Yup': '\u2144',  # ⅄ TURNED SANS-SERIF CAPITAL Y
    '_': '_',  # _ LOW LINE
    'angle': '\u2220',  # ∠ ANGLE
    'aquarius': '\u2652',  # ♒ AQUARIUS
    'aries': '\u2648',  # ♈ ARIES
    'arrowvert': '\u23d0',  # ⏐ VERTICAL LINE EXTENSION
    'backprime': '\u2035',  # ‵ REVERSED PRIME
    'backslash': '\\',  # \ REVERSE SOLIDUS
    'bigstar': '\u2605',  # ★ BLACK STAR
    'blacksmiley': '\u263b',  # ☻ BLACK SMILING FACE
    'blacksquare': '\u25fc',  # ◼ BLACK MEDIUM SQUARE
    'blacktriangle': '\u25b4',  # ▴ BLACK UP-POINTING SMALL TRIANGLE
    'blacktriangledown': '\u25be',  # ▾ BLACK DOWN-POINTING SMALL TRIANGLE
    'blacktriangleup': '\u25b4',  # ▴ BLACK UP-POINTING SMALL TRIANGLE
    'bot': '\u22a5',  # ⊥ UP TACK
    'boy': '\u2642',  # ♂ MALE SIGN
    'bracevert': '\u23aa',  # ⎪ CURLY BRACKET EXTENSION
    'cancer': '\u264b',  # ♋ CANCER
    'capricornus': '\u2651',  # ♑ CAPRICORN
    'cdots': '\u22ef',  # ⋯ MIDLINE HORIZONTAL ELLIPSIS
    'cent': '\xa2',  # ¢ CENT SIGN
    'checkmark': '\u2713',  # ✓ CHECK MARK
    'circledR': '\u24c7',  # Ⓡ CIRCLED LATIN CAPITAL LETTER R
    'circledS': '\u24c8',  # Ⓢ CIRCLED LATIN CAPITAL LETTER S
    'clubsuit': '\u2663',  # ♣ BLACK CLUB SUIT
    'complement': '\u2201',  # ∁ COMPLEMENT
    'diagdown': '\u27cd',  # ⟍ MATHEMATICAL FALLING DIAGONAL
    'diagup': '\u27cb',  # ⟋ MATHEMATICAL RISING DIAGONAL
    'diameter': '\u2300',  # ⌀ DIAMETER SIGN
    'diamondsuit': '\u2662',  # ♢ WHITE DIAMOND SUIT
    'earth': '\u2641',  # ♁ EARTH
    'emptyset': '\u2205',  # ∅ EMPTY SET
    'exists': '\u2203',  # ∃ THERE EXISTS
    'female': '\u2640',  # ♀ FEMALE SIGN
    'flat': '\u266d',  # ♭ MUSIC FLAT SIGN
    'forall': '\u2200',  # ∀ FOR ALL
    'fourth': '\u2057',  # ⁗ QUADRUPLE PRIME
    'frownie': '\u2639',  # ☹ WHITE FROWNING FACE
    'gemini': '\u264a',  # ♊ GEMINI
    'girl': '\u2640',  # ♀ FEMALE SIGN
    'heartsuit': '\u2661',  # ♡ WHITE HEART SUIT
    'infty': '\u221e',  # ∞ INFINITY
    'invdiameter': '\u2349',  # ⍉ APL FUNCTIONAL SYMBOL CIRCLE BACKSLASH
    'invneg': '\u2310',  # ⌐ REVERSED NOT SIGN
    'jupiter': '\u2643',  # ♃ JUPITER
    'ldots': '\u2026',  # … HORIZONTAL ELLIPSIS
    'leftmoon': '\u263e',  # ☾ LAST QUARTER MOON
    'leo': '\u264c',  # ♌ LEO
    'libra': '\u264e',  # ♎ LIBRA
    'lmoustache': '\u23b0',  # ⎰ UPPER LEFT OR LOWER RIGHT CURLY BRACKET SECTION
    'lnot': '\xac',  # ¬ NOT SIGN
    'lozenge': '\u25ca',  # ◊ LOZENGE
    'male': '\u2642',  # ♂ MALE SIGN
    'maltese': '\u2720',  # ✠ MALTESE CROSS
    'mathcent': '\xa2',  # ¢ CENT SIGN
    'mathdollar': '$',  # $ DOLLAR SIGN
    'mathsterling': '\xa3',  # £ POUND SIGN
    'measuredangle': '\u2221',  # ∡ MEASURED ANGLE
    'medbullet': '\u26ab',  # ⚫ MEDIUM BLACK CIRCLE
    'medcirc': '\u26aa',  # ⚪ MEDIUM WHITE CIRCLE
    'mercury': '\u263f',  # ☿ MERCURY
    'mho': '\u2127',  # ℧ INVERTED OHM SIGN
    'nabla': '\u2207',  # ∇ NABLA
    'natural': '\u266e',  # ♮ MUSIC NATURAL SIGN
    'neg': '\xac',  # ¬ NOT SIGN
    'neptune': '\u2646',  # ♆ NEPTUNE
    'nexists': '\u2204',  # ∄ THERE DOES NOT EXIST
    'notbackslash': '\u2340',  # ⍀ APL FUNCTIONAL SYMBOL BACKSLASH BAR
    'partial': '\u2202',  # ∂ PARTIAL DIFFERENTIAL
    'pisces': '\u2653',  # ♓ PISCES
    'pluto': '\u2647',  # ♇ PLUTO
    'pounds': '\xa3',  # £ POUND SIGN
    'prime': '\u2032',  # ′ PRIME
    'quarternote': '\u2669',  # ♩ QUARTER NOTE
    'rightmoon': '\u263d',  # ☽ FIRST QUARTER MOON
    'rmoustache': '\u23b1',  # ⎱ UPPER RIGHT OR LOWER LEFT CURLY BRACKET SECTION
    'sagittarius': '\u2650',  # ♐ SAGITTARIUS
    'saturn': '\u2644',  # ♄ SATURN
    'scorpio': '\u264f',  # ♏ SCORPIUS
    'second': '\u2033',  # ″ DOUBLE PRIME
    'sharp': '\u266f',  # ♯ MUSIC SHARP SIGN
    'smiley': '\u263a',  # ☺ WHITE SMILING FACE
    'spadesuit': '\u2660',  # ♠ BLACK SPADE SUIT
    'spddot': '\xa8',  # ¨ DIAERESIS
    'sphat': '^',  # ^ CIRCUMFLEX ACCENT
    'sphericalangle': '\u2222',  # ∢ SPHERICAL ANGLE
    'sptilde': '~',  # ~ TILDE
    'square': '\u25fb',  # ◻ WHITE MEDIUM SQUARE
    'sun': '\u263c',  # ☼ WHITE SUN WITH RAYS
    'surd': '\u221a',  # √ SQUARE ROOT
    'taurus': '\u2649',  # ♉ TAURUS
    'third': '\u2034',  # ‴ TRIPLE PRIME
    'top': '\u22a4',  # ⊤ DOWN TACK
    'twonotes': '\u266b',  # ♫ BEAMED EIGHTH NOTES
    'uranus': '\u2645',  # ♅ URANUS
    'varEarth': '\u2641',  # ♁ EARTH
    'varclubsuit': '\u2667',  # ♧ WHITE CLUB SUIT
    'vardiamondsuit': '\u2666',  # ♦ BLACK DIAMOND SUIT
    'varheartsuit': '\u2665',  # ♥ BLACK HEART SUIT
    'varspadesuit': '\u2664',  # ♤ WHITE SPADE SUIT
    'virgo': '\u264d',  # ♍ VIRGO
    'wasylozenge': '\u2311',  # ⌑ SQUARE LOZENGE
    'yen': '\xa5',  # ¥ YEN SIGN
    }

mathover = {
    'overbrace': '\u23de',  # ⏞ TOP CURLY BRACKET
    'wideparen': '\u23dc',  # ⏜ TOP PARENTHESIS
    }

mathpunct = {
    'ddots': '\u22f1',  # ⋱ DOWN RIGHT DIAGONAL ELLIPSIS
    'vdots': '\u22ee',  # ⋮ VERTICAL ELLIPSIS
    }

mathradical = {
    'sqrt[3]': '\u221b',  # ∛ CUBE ROOT
    'sqrt[4]': '\u221c',  # ∜ FOURTH ROOT
    }

mathrel = {
    'Bot': '\u2aeb',  # ⫫ DOUBLE UP TACK
    'Bumpeq': '\u224e',  # ≎ GEOMETRICALLY EQUIVALENT TO
    'Coloneqq': '\u2a74',  # ⩴ DOUBLE COLON EQUAL
    'Doteq': '\u2251',  # ≑ GEOMETRICALLY EQUAL TO
    'Downarrow': '\u21d3',  # ⇓ DOWNWARDS DOUBLE ARROW
    'Leftarrow': '\u21d0',  # ⇐ LEFTWARDS DOUBLE ARROW
    'Leftrightarrow': '\u21d4',  # ⇔ LEFT RIGHT DOUBLE ARROW
    'Lleftarrow': '\u21da',  # ⇚ LEFTWARDS TRIPLE ARROW
    'Longleftarrow': '\u27f8',  # ⟸ LONG LEFTWARDS DOUBLE ARROW
    'Longleftrightarrow': '\u27fa',  # ⟺ LONG LEFT RIGHT DOUBLE ARROW
    'Longmapsfrom': '\u27fd',  # ⟽ LONG LEFTWARDS DOUBLE ARROW FROM BAR
    'Longmapsto': '\u27fe',  # ⟾ LONG RIGHTWARDS DOUBLE ARROW FROM BAR
    'Longrightarrow': '\u27f9',  # ⟹ LONG RIGHTWARDS DOUBLE ARROW
    'Lsh': '\u21b0',  # ↰ UPWARDS ARROW WITH TIP LEFTWARDS
    'Mapsfrom': '\u2906',  # ⤆ LEFTWARDS DOUBLE ARROW FROM BAR
    'Mapsto': '\u2907',  # ⤇ RIGHTWARDS DOUBLE ARROW FROM BAR
    'Nearrow': '\u21d7',  # ⇗ NORTH EAST DOUBLE ARROW
    'Nwarrow': '\u21d6',  # ⇖ NORTH WEST DOUBLE ARROW
    'Perp': '\u2aeb',  # ⫫ DOUBLE UP TACK
    'Rightarrow': '\u21d2',  # ⇒ RIGHTWARDS DOUBLE ARROW
    'Rrightarrow': '\u21db',  # ⇛ RIGHTWARDS TRIPLE ARROW
    'Rsh': '\u21b1',  # ↱ UPWARDS ARROW WITH TIP RIGHTWARDS
    'Searrow': '\u21d8',  # ⇘ SOUTH EAST DOUBLE ARROW
    'Subset': '\u22d0',  # ⋐ DOUBLE SUBSET
    'Supset': '\u22d1',  # ⋑ DOUBLE SUPERSET
    'Swarrow': '\u21d9',  # ⇙ SOUTH WEST DOUBLE ARROW
    'Top': '\u2aea',  # ⫪ DOUBLE DOWN TACK
    'Uparrow': '\u21d1',  # ⇑ UPWARDS DOUBLE ARROW
    'Updownarrow': '\u21d5',  # ⇕ UP DOWN DOUBLE ARROW
    'VDash': '\u22ab',  # ⊫ DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE
    'Vdash': '\u22a9',  # ⊩ FORCES
    'Vvdash': '\u22aa',  # ⊪ TRIPLE VERTICAL BAR RIGHT TURNSTILE
    'apprge': '\u2273',  # ≳ GREATER-THAN OR EQUIVALENT TO
    'apprle': '\u2272',  # ≲ LESS-THAN OR EQUIVALENT TO
    'approx': '\u2248',  # ≈ ALMOST EQUAL TO
    'approxeq': '\u224a',  # ≊ ALMOST EQUAL OR EQUAL TO
    'asymp': '\u224d',  # ≍ EQUIVALENT TO
    'backepsilon': '\u220d',  # ∍ SMALL CONTAINS AS MEMBER
    'backsim': '\u223d',  # ∽ REVERSED TILDE
    'backsimeq': '\u22cd',  # ⋍ REVERSED TILDE EQUALS
    'barin': '\u22f6',  # ⋶ ELEMENT OF WITH OVERBAR
    'barleftharpoon': '\u296b',  # ⥫ LEFTWARDS HARPOON WITH BARB DOWN BELOW LONG DASH
    'barrightharpoon': '\u296d',  # ⥭ RIGHTWARDS HARPOON WITH BARB DOWN BELOW LONG DASH
    'because': '\u2235',  # ∵ BECAUSE
    'between': '\u226c',  # ≬ BETWEEN
    'blacktriangleleft': '\u25c2',  # ◂ BLACK LEFT-POINTING SMALL TRIANGLE
    'blacktriangleright': '\u25b8',  # ▸ BLACK RIGHT-POINTING SMALL TRIANGLE
    'bowtie': '\u22c8',  # ⋈ BOWTIE
    'bumpeq': '\u224f',  # ≏ DIFFERENCE BETWEEN
    'circeq': '\u2257',  # ≗ RING EQUAL TO
    'circlearrowleft': '\u21ba',  # ↺ ANTICLOCKWISE OPEN CIRCLE ARROW
    'circlearrowright': '\u21bb',  # ↻ CLOCKWISE OPEN CIRCLE ARROW
    'coloneq': '\u2254',  # ≔ COLON EQUALS
    'coloneqq': '\u2254',  # ≔ COLON EQUALS
    'cong': '\u2245',  # ≅ APPROXIMATELY EQUAL TO
    'corresponds': '\u2259',  # ≙ ESTIMATES
    'curlyeqprec': '\u22de',  # ⋞ EQUAL TO OR PRECEDES
    'curlyeqsucc': '\u22df',  # ⋟ EQUAL TO OR SUCCEEDS
    'curvearrowleft': '\u21b6',  # ↶ ANTICLOCKWISE TOP SEMICIRCLE ARROW
    'curvearrowright': '\u21b7',  # ↷ CLOCKWISE TOP SEMICIRCLE ARROW
    'dasharrow': '\u21e2',  # ⇢ RIGHTWARDS DASHED ARROW
    'dashleftarrow': '\u21e0',  # ⇠ LEFTWARDS DASHED ARROW
    'dashrightarrow': '\u21e2',  # ⇢ RIGHTWARDS DASHED ARROW
    'dashv': '\u22a3',  # ⊣ LEFT TACK
    'dlsh': '\u21b2',  # ↲ DOWNWARDS ARROW WITH TIP LEFTWARDS
    'doteq': '\u2250',  # ≐ APPROACHES THE LIMIT
    'doteqdot': '\u2251',  # ≑ GEOMETRICALLY EQUAL TO
    'downarrow': '\u2193',  # ↓ DOWNWARDS ARROW
    'downdownarrows': '\u21ca',  # ⇊ DOWNWARDS PAIRED ARROWS
    'downdownharpoons': '\u2965',  # ⥥ DOWNWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT
    'downharpoonleft': '\u21c3',  # ⇃ DOWNWARDS HARPOON WITH BARB LEFTWARDS
    'downharpoonright': '\u21c2',  # ⇂ DOWNWARDS HARPOON WITH BARB RIGHTWARDS
    'downuparrows': '\u21f5',  # ⇵ DOWNWARDS ARROW LEFTWARDS OF UPWARDS ARROW
    'downupharpoons': '\u296f',  # ⥯ DOWNWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT
    'drsh': '\u21b3',  # ↳ DOWNWARDS ARROW WITH TIP RIGHTWARDS
    'eqcirc': '\u2256',  # ≖ RING IN EQUAL TO
    'eqcolon': '\u2255',  # ≕ EQUALS COLON
    'eqqcolon': '\u2255',  # ≕ EQUALS COLON
    'eqsim': '\u2242',  # ≂ MINUS TILDE
    'eqslantgtr': '\u2a96',  # ⪖ SLANTED EQUAL TO OR GREATER-THAN
    'eqslantless': '\u2a95',  # ⪕ SLANTED EQUAL TO OR LESS-THAN
    'equiv': '\u2261',  # ≡ IDENTICAL TO
    'fallingdotseq': '\u2252',  # ≒ APPROXIMATELY EQUAL TO OR THE IMAGE OF
    'frown': '\u2322',  # ⌢ FROWN
    'ge': '\u2265',  # ≥ GREATER-THAN OR EQUAL TO
    'geq': '\u2265',  # ≥ GREATER-THAN OR EQUAL TO
    'geqq': '\u2267',  # ≧ GREATER-THAN OVER EQUAL TO
    'geqslant': '\u2a7e',  # ⩾ GREATER-THAN OR SLANTED EQUAL TO
    'gets': '\u2190',  # ← LEFTWARDS ARROW
    'gg': '\u226b',  # ≫ MUCH GREATER-THAN
    'ggcurly': '\u2abc',  # ⪼ DOUBLE SUCCEEDS
    'ggg': '\u22d9',  # ⋙ VERY MUCH GREATER-THAN
    'gggtr': '\u22d9',  # ⋙ VERY MUCH GREATER-THAN
    'gnapprox': '\u2a8a',  # ⪊ GREATER-THAN AND NOT APPROXIMATE
    'gneq': '\u2a88',  # ⪈ GREATER-THAN AND SINGLE-LINE NOT EQUAL TO
    'gneqq': '\u2269',  # ≩ GREATER-THAN BUT NOT EQUAL TO
    'gnsim': '\u22e7',  # ⋧ GREATER-THAN BUT NOT EQUIVALENT TO
    'gtrapprox': '\u2a86',  # ⪆ GREATER-THAN OR APPROXIMATE
    'gtreqless': '\u22db',  # ⋛ GREATER-THAN EQUAL TO OR LESS-THAN
    'gtreqqless': '\u2a8c',  # ⪌ GREATER-THAN ABOVE DOUBLE-LINE EQUAL ABOVE LESS-THAN
    'gtrless': '\u2277',  # ≷ GREATER-THAN OR LESS-THAN
    'gtrsim': '\u2273',  # ≳ GREATER-THAN OR EQUIVALENT TO
    'hash': '\u22d5',  # ⋕ EQUAL AND PARALLEL TO
    'hookleftarrow': '\u21a9',  # ↩ LEFTWARDS ARROW WITH HOOK
    'hookrightarrow': '\u21aa',  # ↪ RIGHTWARDS ARROW WITH HOOK
    'iddots': '\u22f0',  # ⋰ UP RIGHT DIAGONAL ELLIPSIS
    'impliedby': '\u27f8',  # ⟸ LONG LEFTWARDS DOUBLE ARROW
    'implies': '\u27f9',  # ⟹ LONG RIGHTWARDS DOUBLE ARROW
    'in': '\u2208',  # ∈ ELEMENT OF
    'le': '\u2264',  # ≤ LESS-THAN OR EQUAL TO
    'leadsto': '\u2933',  # ⤳ WAVE ARROW POINTING DIRECTLY RIGHT
    'leftarrow': '\u2190',  # ← LEFTWARDS ARROW
    'leftarrowtail': '\u21a2',  # ↢ LEFTWARDS ARROW WITH TAIL
    'leftarrowtriangle': '\u21fd',  # ⇽ LEFTWARDS OPEN-HEADED ARROW
    'leftbarharpoon': '\u296a',  # ⥪ LEFTWARDS HARPOON WITH BARB UP ABOVE LONG DASH
    'leftharpoondown': '\u21bd',  # ↽ LEFTWARDS HARPOON WITH BARB DOWNWARDS
    'leftharpoonup': '\u21bc',  # ↼ LEFTWARDS HARPOON WITH BARB UPWARDS
    'leftleftarrows': '\u21c7',  # ⇇ LEFTWARDS PAIRED ARROWS
    'leftleftharpoons': '\u2962',  # ⥢ LEFTWARDS HARPOON WITH BARB UP ABOVE LEFTWARDS HARPOON WITH BARB DOWN
    'leftrightarrow': '\u2194',  # ↔ LEFT RIGHT ARROW
    'leftrightarrows': '\u21c6',  # ⇆ LEFTWARDS ARROW OVER RIGHTWARDS ARROW
    'leftrightarrowtriangle': '\u21ff',  # ⇿ LEFT RIGHT OPEN-HEADED ARROW
    'leftrightharpoon': '\u294a',  # ⥊ LEFT BARB UP RIGHT BARB DOWN HARPOON
    'leftrightharpoons': '\u21cb',  # ⇋ LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON
    'leftrightsquigarrow': '\u21ad',  # ↭ LEFT RIGHT WAVE ARROW
    'leftslice': '\u2aa6',  # ⪦ LESS-THAN CLOSED BY CURVE
    'leftsquigarrow': '\u21dc',  # ⇜ LEFTWARDS SQUIGGLE ARROW
    'leftturn': '\u21ba',  # ↺ ANTICLOCKWISE OPEN CIRCLE ARROW
    'leq': '\u2264',  # ≤ LESS-THAN OR EQUAL TO
    'leqq': '\u2266',  # ≦ LESS-THAN OVER EQUAL TO
    'leqslant': '\u2a7d',  # ⩽ LESS-THAN OR SLANTED EQUAL TO
    'lessapprox': '\u2a85',  # ⪅ LESS-THAN OR APPROXIMATE
    'lesseqgtr': '\u22da',  # ⋚ LESS-THAN EQUAL TO OR GREATER-THAN
    'lesseqqgtr': '\u2a8b',  # ⪋ LESS-THAN ABOVE DOUBLE-LINE EQUAL ABOVE GREATER-THAN
    'lessgtr': '\u2276',  # ≶ LESS-THAN OR GREATER-THAN
    'lesssim': '\u2272',  # ≲ LESS-THAN OR EQUIVALENT TO
    'lhd': '\u22b2',  # ⊲ NORMAL SUBGROUP OF
    'lightning': '\u21af',  # ↯ DOWNWARDS ZIGZAG ARROW
    'll': '\u226a',  # ≪ MUCH LESS-THAN
    'llcurly': '\u2abb',  # ⪻ DOUBLE PRECEDES
    'lll': '\u22d8',  # ⋘ VERY MUCH LESS-THAN
    'llless': '\u22d8',  # ⋘ VERY MUCH LESS-THAN
    'lnapprox': '\u2a89',  # ⪉ LESS-THAN AND NOT APPROXIMATE
    'lneq': '\u2a87',  # ⪇ LESS-THAN AND SINGLE-LINE NOT EQUAL TO
    'lneqq': '\u2268',  # ≨ LESS-THAN BUT NOT EQUAL TO
    'lnsim': '\u22e6',  # ⋦ LESS-THAN BUT NOT EQUIVALENT TO
    'longleftarrow': '\u27f5',  # ⟵ LONG LEFTWARDS ARROW
    'longleftrightarrow': '\u27f7',  # ⟷ LONG LEFT RIGHT ARROW
    'longmapsfrom': '\u27fb',  # ⟻ LONG LEFTWARDS ARROW FROM BAR
    'longmapsto': '\u27fc',  # ⟼ LONG RIGHTWARDS ARROW FROM BAR
    'longrightarrow': '\u27f6',  # ⟶ LONG RIGHTWARDS ARROW
    'looparrowleft': '\u21ab',  # ↫ LEFTWARDS ARROW WITH LOOP
    'looparrowright': '\u21ac',  # ↬ RIGHTWARDS ARROW WITH LOOP
    'lrtimes': '\u22c8',  # ⋈ BOWTIE
    'mapsfrom': '\u21a4',  # ↤ LEFTWARDS ARROW FROM BAR
    'mapsto': '\u21a6',  # ↦ RIGHTWARDS ARROW FROM BAR
    'mid': '\u2223',  # ∣ DIVIDES
    'models': '\u22a7',  # ⊧ MODELS
    'multimap': '\u22b8',  # ⊸ MULTIMAP
    'multimapboth': '\u29df',  # ⧟ DOUBLE-ENDED MULTIMAP
    'multimapdotbothA': '\u22b6',  # ⊶ ORIGINAL OF
    'multimapdotbothB': '\u22b7',  # ⊷ IMAGE OF
    'multimapinv': '\u27dc',  # ⟜ LEFT MULTIMAP
    'nLeftarrow': '\u21cd',  # ⇍ LEFTWARDS DOUBLE ARROW WITH STROKE
    'nLeftrightarrow': '\u21ce',  # ⇎ LEFT RIGHT DOUBLE ARROW WITH STROKE
    'nRightarrow': '\u21cf',  # ⇏ RIGHTWARDS DOUBLE ARROW WITH STROKE
    'nVDash': '\u22af',  # ⊯ NEGATED DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE
    'nVdash': '\u22ae',  # ⊮ DOES NOT FORCE
    'ncong': '\u2247',  # ≇ NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO
    'ne': '\u2260',  # ≠ NOT EQUAL TO
    'nearrow': '\u2197',  # ↗ NORTH EAST ARROW
    'neq': '\u2260',  # ≠ NOT EQUAL TO
    'ngeq': '\u2271',  # ≱ NEITHER GREATER-THAN NOR EQUAL TO
    'ngtr': '\u226f',  # ≯ NOT GREATER-THAN
    'ngtrless': '\u2279',  # ≹ NEITHER GREATER-THAN NOR LESS-THAN
    'ni': '\u220b',  # ∋ CONTAINS AS MEMBER
    'nleftarrow': '\u219a',  # ↚ LEFTWARDS ARROW WITH STROKE
    'nleftrightarrow': '\u21ae',  # ↮ LEFT RIGHT ARROW WITH STROKE
    'nleq': '\u2270',  # ≰ NEITHER LESS-THAN NOR EQUAL TO
    'nless': '\u226e',  # ≮ NOT LESS-THAN
    'nlessgtr': '\u2278',  # ≸ NEITHER LESS-THAN NOR GREATER-THAN
    'nmid': '\u2224',  # ∤ DOES NOT DIVIDE
    'notasymp': '\u226d',  # ≭ NOT EQUIVALENT TO
    'notin': '\u2209',  # ∉ NOT AN ELEMENT OF
    'notni': '\u220c',  # ∌ DOES NOT CONTAIN AS MEMBER
    'notowner': '\u220c',  # ∌ DOES NOT CONTAIN AS MEMBER
    'notslash': '\u233f',  # ⌿ APL FUNCTIONAL SYMBOL SLASH BAR
    'nparallel': '\u2226',  # ∦ NOT PARALLEL TO
    'nprec': '\u2280',  # ⊀ DOES NOT PRECEDE
    'npreceq': '\u22e0',  # ⋠ DOES NOT PRECEDE OR EQUAL
    'nrightarrow': '\u219b',  # ↛ RIGHTWARDS ARROW WITH STROKE
    'nsim': '\u2241',  # ≁ NOT TILDE
    'nsimeq': '\u2244',  # ≄ NOT ASYMPTOTICALLY EQUAL TO
    'nsubseteq': '\u2288',  # ⊈ NEITHER A SUBSET OF NOR EQUAL TO
    'nsucc': '\u2281',  # ⊁ DOES NOT SUCCEED
    'nsucceq': '\u22e1',  # ⋡ DOES NOT SUCCEED OR EQUAL
    'nsupseteq': '\u2289',  # ⊉ NEITHER A SUPERSET OF NOR EQUAL TO
    'ntriangleleft': '\u22ea',  # ⋪ NOT NORMAL SUBGROUP OF
    'ntrianglelefteq': '\u22ec',  # ⋬ NOT NORMAL SUBGROUP OF OR EQUAL TO
    'ntriangleright': '\u22eb',  # ⋫ DOES NOT CONTAIN AS NORMAL SUBGROUP
    'ntrianglerighteq': '\u22ed',  # ⋭ DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL
    'nvDash': '\u22ad',  # ⊭ NOT TRUE
    'nvdash': '\u22ac',  # ⊬ DOES NOT PROVE
    'nwarrow': '\u2196',  # ↖ NORTH WEST ARROW
    'owns': '\u220b',  # ∋ CONTAINS AS MEMBER
    'parallel': '\u2225',  # ∥ PARALLEL TO
    'perp': '\u27c2',  # ⟂ PERPENDICULAR
    'pitchfork': '\u22d4',  # ⋔ PITCHFORK
    'prec': '\u227a',  # ≺ PRECEDES
    'precapprox': '\u2ab7',  # ⪷ PRECEDES ABOVE ALMOST EQUAL TO
    'preccurlyeq': '\u227c',  # ≼ PRECEDES OR EQUAL TO
    'preceq': '\u2aaf',  # ⪯ PRECEDES ABOVE SINGLE-LINE EQUALS SIGN
    'preceqq': '\u2ab3',  # ⪳ PRECEDES ABOVE EQUALS SIGN
    'precnapprox': '\u2ab9',  # ⪹ PRECEDES ABOVE NOT ALMOST EQUAL TO
    'precneqq': '\u2ab5',  # ⪵ PRECEDES ABOVE NOT EQUAL TO
    'precnsim': '\u22e8',  # ⋨ PRECEDES BUT NOT EQUIVALENT TO
    'precsim': '\u227e',  # ≾ PRECEDES OR EQUIVALENT TO
    'propto': '\u221d',  # ∝ PROPORTIONAL TO
    'restriction': '\u21be',  # ↾ UPWARDS HARPOON WITH BARB RIGHTWARDS
    'rhd': '\u22b3',  # ⊳ CONTAINS AS NORMAL SUBGROUP
    'rightarrow': '\u2192',  # → RIGHTWARDS ARROW
    'rightarrowtail': '\u21a3',  # ↣ RIGHTWARDS ARROW WITH TAIL
    'rightarrowtriangle': '\u21fe',  # ⇾ RIGHTWARDS OPEN-HEADED ARROW
    'rightbarharpoon': '\u296c',  # ⥬ RIGHTWARDS HARPOON WITH BARB UP ABOVE LONG DASH
    'rightharpoondown': '\u21c1',  # ⇁ RIGHTWARDS HARPOON WITH BARB DOWNWARDS
    'rightharpoonup': '\u21c0',  # ⇀ RIGHTWARDS HARPOON WITH BARB UPWARDS
    'rightleftarrows': '\u21c4',  # ⇄ RIGHTWARDS ARROW OVER LEFTWARDS ARROW
    'rightleftharpoon': '\u294b',  # ⥋ LEFT BARB DOWN RIGHT BARB UP HARPOON
    'rightleftharpoons': '\u21cc',  # ⇌ RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON
    'rightrightarrows': '\u21c9',  # ⇉ RIGHTWARDS PAIRED ARROWS
    'rightrightharpoons': '\u2964',  # ⥤ RIGHTWARDS HARPOON WITH BARB UP ABOVE RIGHTWARDS HARPOON WITH BARB DOWN
    'rightslice': '\u2aa7',  # ⪧ GREATER-THAN CLOSED BY CURVE
    'rightsquigarrow': '\u21dd',  # ⇝ RIGHTWARDS SQUIGGLE ARROW
    'rightturn': '\u21bb',  # ↻ CLOCKWISE OPEN CIRCLE ARROW
    'risingdotseq': '\u2253',  # ≓ IMAGE OF OR APPROXIMATELY EQUAL TO
    'searrow': '\u2198',  # ↘ SOUTH EAST ARROW
    'sim': '\u223c',  # ∼ TILDE OPERATOR
    'simeq': '\u2243',  # ≃ ASYMPTOTICALLY EQUAL TO
    'smile': '\u2323',  # ⌣ SMILE
    'sqsubset': '\u228f',  # ⊏ SQUARE IMAGE OF
    'sqsubseteq': '\u2291',  # ⊑ SQUARE IMAGE OF OR EQUAL TO
    'sqsupset': '\u2290',  # ⊐ SQUARE ORIGINAL OF
    'sqsupseteq': '\u2292',  # ⊒ SQUARE ORIGINAL OF OR EQUAL TO
    'strictfi': '\u297c',  # ⥼ LEFT FISH TAIL
    'strictif': '\u297d',  # ⥽ RIGHT FISH TAIL
    'subset': '\u2282',  # ⊂ SUBSET OF
    'subseteq': '\u2286',  # ⊆ SUBSET OF OR EQUAL TO
    'subseteqq': '\u2ac5',  # ⫅ SUBSET OF ABOVE EQUALS SIGN
    'subsetneq': '\u228a',  # ⊊ SUBSET OF WITH NOT EQUAL TO
    'subsetneqq': '\u2acb',  # ⫋ SUBSET OF ABOVE NOT EQUAL TO
    'succ': '\u227b',  # ≻ SUCCEEDS
    'succapprox': '\u2ab8',  # ⪸ SUCCEEDS ABOVE ALMOST EQUAL TO
    'succcurlyeq': '\u227d',  # ≽ SUCCEEDS OR EQUAL TO
    'succeq': '\u2ab0',  # ⪰ SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN
    'succeqq': '\u2ab4',  # ⪴ SUCCEEDS ABOVE EQUALS SIGN
    'succnapprox': '\u2aba',  # ⪺ SUCCEEDS ABOVE NOT ALMOST EQUAL TO
    'succneqq': '\u2ab6',  # ⪶ SUCCEEDS ABOVE NOT EQUAL TO
    'succnsim': '\u22e9',  # ⋩ SUCCEEDS BUT NOT EQUIVALENT TO
    'succsim': '\u227f',  # ≿ SUCCEEDS OR EQUIVALENT TO
    'supset': '\u2283',  # ⊃ SUPERSET OF
    'supseteq': '\u2287',  # ⊇ SUPERSET OF OR EQUAL TO
    'supseteqq': '\u2ac6',  # ⫆ SUPERSET OF ABOVE EQUALS SIGN
    'supsetneq': '\u228b',  # ⊋ SUPERSET OF WITH NOT EQUAL TO
    'supsetneqq': '\u2acc',  # ⫌ SUPERSET OF ABOVE NOT EQUAL TO
    'swarrow': '\u2199',  # ↙ SOUTH WEST ARROW
    'therefore': '\u2234',  # ∴ THEREFORE
    'to': '\u2192',  # → RIGHTWARDS ARROW
    'trianglelefteq': '\u22b4',  # ⊴ NORMAL SUBGROUP OF OR EQUAL TO
    'triangleq': '\u225c',  # ≜ DELTA EQUAL TO
    'trianglerighteq': '\u22b5',  # ⊵ CONTAINS AS NORMAL SUBGROUP OR EQUAL TO
    'twoheadleftarrow': '\u219e',  # ↞ LEFTWARDS TWO HEADED ARROW
    'twoheadrightarrow': '\u21a0',  # ↠ RIGHTWARDS TWO HEADED ARROW
    'uparrow': '\u2191',  # ↑ UPWARDS ARROW
    'updownarrow': '\u2195',  # ↕ UP DOWN ARROW
    'updownarrows': '\u21c5',  # ⇅ UPWARDS ARROW LEFTWARDS OF DOWNWARDS ARROW
    'updownharpoons': '\u296e',  # ⥮ UPWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT
    'upharpoonleft': '\u21bf',  # ↿ UPWARDS HARPOON WITH BARB LEFTWARDS
    'upharpoonright': '\u21be',  # ↾ UPWARDS HARPOON WITH BARB RIGHTWARDS
    'upuparrows': '\u21c8',  # ⇈ UPWARDS PAIRED ARROWS
    'upupharpoons': '\u2963',  # ⥣ UPWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT
    'vDash': '\u22a8',  # ⊨ TRUE
    'vartriangle': '\u25b5',  # ▵ WHITE UP-POINTING SMALL TRIANGLE
    'vartriangleleft': '\u22b2',  # ⊲ NORMAL SUBGROUP OF
    'vartriangleright': '\u22b3',  # ⊳ CONTAINS AS NORMAL SUBGROUP
    'vdash': '\u22a2',  # ⊢ RIGHT TACK
    'wasytherefore': '\u2234',  # ∴ THEREFORE
    }

mathunder = {
    'underbrace': '\u23df',  # ⏟ BOTTOM CURLY BRACKET
    }

space = {
    ' ': ' ',  #   SPACE
    ',': '\u2006',  #   SIX-PER-EM SPACE
    ':': '\u205f',  #   MEDIUM MATHEMATICAL SPACE
    'medspace': '\u205f',  #   MEDIUM MATHEMATICAL SPACE
    'quad': '\u2001',  #   EM QUAD
    'thinspace': '\u2006',  #   SIX-PER-EM SPACE
    }

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/math/tex2unichar.py
NikolaBlagojevic/pyrecodes
# :Id: $Id: punctuation_chars.py 9270 2022-11-24 20:28:03Z milde $
# :Copyright: © 2011, 2017 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause
#
# This file is generated by
# ``docutils/tools/dev/generate_punctuation_chars.py``.
# ::

"""Docutils character category patterns.

   Patterns for the implementation of the `inline markup recognition rules`_
   in the reStructuredText parser `docutils.parsers.rst.states.py` based
   on Unicode character categories.
   The patterns are used inside ``[ ]`` in regular expressions.

   Rule (5) requires determination of matching open/close pairs. However, the
   pairing of open/close quotes is ambiguous due to  different typographic
   conventions in different languages. The ``quote_pairs`` function tests
   whether two characters form an open/close pair.

   The patterns are generated by
   ``docutils/tools/dev/generate_punctuation_chars.py`` to  prevent dependence
   on the Python version and avoid the time-consuming generation with every
   Docutils run. See there for motives and implementation details.

   The category of some characters changed with the development of the
   Unicode standard. The current lists are generated with the help of the
   "unicodedata" module of Python 2.7.13 (based on Unicode version 5.2.0).

   .. _inline markup recognition rules:
      https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html
      #inline-markup-recognition-rules
"""

openers = (
    '"\'(<\\[{\u0f3a\u0f3c\u169b\u2045\u207d\u208d\u2329\u2768'
    '\u276a\u276c\u276e\u2770\u2772\u2774\u27c5\u27e6\u27e8\u27ea'
    '\u27ec\u27ee\u2983\u2985\u2987\u2989\u298b\u298d\u298f\u2991'
    '\u2993\u2995\u2997\u29d8\u29da\u29fc\u2e22\u2e24\u2e26\u2e28'
    '\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d'
    '\u301d\ufd3e\ufe17\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41'
    '\ufe43\ufe47\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff5f\uff62'
    '\xab\u2018\u201c\u2039\u2e02\u2e04\u2e09\u2e0c\u2e1c\u2e20'
    '\u201a\u201e\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d'
    '\u2e1d\u2e21\u201b\u201f'
    )
closers = (
    '"\')>\\]}\u0f3b\u0f3d\u169c\u2046\u207e\u208e\u232a\u2769'
    '\u276b\u276d\u276f\u2771\u2773\u2775\u27c6\u27e7\u27e9\u27eb'
    '\u27ed\u27ef\u2984\u2986\u2988\u298a\u298c\u298e\u2990\u2992'
    '\u2994\u2996\u2998\u29d9\u29db\u29fd\u2e23\u2e25\u2e27\u2e29'
    '\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e'
    '\u301f\ufd3f\ufe18\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42'
    '\ufe44\ufe48\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff60\uff63'
    '\xbb\u2019\u201d\u203a\u2e03\u2e05\u2e0a\u2e0d\u2e1d\u2e21'
    '\u201b\u201f\xab\u2018\u201c\u2039\u2e02\u2e04\u2e09\u2e0c'
    '\u2e1c\u2e20\u201a\u201e'
    )
delimiters = (
    '\\-/:\u058a\xa1\xb7\xbf\u037e\u0387\u055a-\u055f\u0589'
    '\u05be\u05c0\u05c3\u05c6\u05f3\u05f4\u0609\u060a\u060c'
    '\u060d\u061b\u061e\u061f\u066a-\u066d\u06d4\u0700-\u070d'
    '\u07f7-\u07f9\u0830-\u083e\u0964\u0965\u0970\u0df4\u0e4f'
    '\u0e5a\u0e5b\u0f04-\u0f12\u0f85\u0fd0-\u0fd4\u104a-\u104f'
    '\u10fb\u1361-\u1368\u1400\u166d\u166e\u16eb-\u16ed\u1735'
    '\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u180a\u1944\u1945'
    '\u19de\u19df\u1a1e\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-'
    '\u1b60\u1c3b-\u1c3f\u1c7e\u1c7f\u1cd3\u2010-\u2017\u2020-'
    '\u2027\u2030-\u2038\u203b-\u203e\u2041-\u2043\u2047-'
    '\u2051\u2053\u2055-\u205e\u2cf9-\u2cfc\u2cfe\u2cff\u2e00'
    '\u2e01\u2e06-\u2e08\u2e0b\u2e0e-\u2e1b\u2e1e\u2e1f\u2e2a-'
    '\u2e2e\u2e30\u2e31\u3001-\u3003\u301c\u3030\u303d\u30a0'
    '\u30fb\ua4fe\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7'
    '\ua874-\ua877\ua8ce\ua8cf\ua8f8-\ua8fa\ua92e\ua92f\ua95f'
    '\ua9c1-\ua9cd\ua9de\ua9df\uaa5c-\uaa5f\uaade\uaadf\uabeb'
    '\ufe10-\ufe16\ufe19\ufe30-\ufe32\ufe45\ufe46\ufe49-\ufe4c'
    '\ufe50-\ufe52\ufe54-\ufe58\ufe5f-\ufe61\ufe63\ufe68\ufe6a'
    '\ufe6b\uff01-\uff03\uff05-\uff07\uff0a\uff0c-\uff0f\uff1a'
    '\uff1b\uff1f\uff20\uff3c\uff61\uff64\uff65'
    '\U00010100\U00010101\U0001039f\U000103d0\U00010857'
    '\U0001091f\U0001093f\U00010a50-\U00010a58\U00010a7f'
    '\U00010b39-\U00010b3f\U000110bb\U000110bc\U000110be-'
    '\U000110c1\U00012470-\U00012473'
    )
closing_delimiters = r'\\.,;!?'


# Matching open/close quotes
# --------------------------

# Matching open/close pairs are at the same position in
# `punctuation_chars.openers` and `punctuation_chars.closers`.
# Additional matches (due to different typographic conventions
# in different languages) are stored in `quote_pairs`.

quote_pairs = {
    # open char: matching closing characters # use case
    '\xbb': '\xbb',            # » » Swedish
    '\u2018': '\u201a',        # ‘ ‚ Albanian/Greek/Turkish
    '\u2019': '\u2019',        # ’ ’ Swedish
    '\u201a': '\u2018\u2019',  # ‚ ‘ German, ‚ ’ Polish
    '\u201c': '\u201e',        # “ „ Albanian/Greek/Turkish
    '\u201e': '\u201c\u201d',  # „ “ German, „ ” Polish
    '\u201d': '\u201d',        # ” ” Swedish
    '\u203a': '\u203a',        # › › Swedish
    }
"""Additional open/close quote pairs."""


def match_chars(c1, c2):
    """Test whether `c1` and `c2` are a matching open/close character pair."""
    try:
        i = openers.index(c1)
    except ValueError:  # c1 not in openers
        return False
    return c2 == closers[i] or c2 in quote_pairs.get(c1, '')

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/punctuation_chars.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/python3
# :Id: $Id: smartquotes.py 9335 2023-04-06 18:40:57Z milde $
# :Copyright: © 2010 Günter Milde,
#             original `SmartyPants`_: © 2003 John Gruber
#             smartypants.py:          © 2004, 2007 Chad Miller
# :Maintainer: docutils-develop@lists.sourceforge.net
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notices and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause


r"""
=========================
Smart Quotes for Docutils
=========================

Synopsis
========

"SmartyPants" is a free web publishing plug-in for Movable Type, Blosxom, and
BBEdit that easily translates plain ASCII punctuation characters into "smart"
typographic punctuation characters.

``smartquotes.py`` is an adaption of "SmartyPants" to Docutils_.

* Using Unicode instead of HTML entities for typographic punctuation
  characters, it works for any output format that supports Unicode.
* Supports `language specific quote characters`__.

__ https://en.wikipedia.org/wiki/Non-English_usage_of_quotation_marks


Authors
=======

`John Gruber`_ did all of the hard work of writing this software in Perl for
`Movable Type`_ and almost all of this useful documentation.  `Chad Miller`_
ported it to Python to use with Pyblosxom_.
Adapted to Docutils_ by Günter Milde.

Additional Credits
==================

Portions of the SmartyPants original work are based on Brad Choate's nifty
MTRegex plug-in.  `Brad Choate`_ also contributed a few bits of source code to
this plug-in.  Brad Choate is a fine hacker indeed.

`Jeremy Hedley`_ and `Charles Wiltgen`_ deserve mention for exemplary beta
testing of the original SmartyPants.

`Rael Dornfest`_ ported SmartyPants to Blosxom.

.. _Brad Choate: http://bradchoate.com/
.. _Jeremy Hedley: http://antipixel.com/
.. _Charles Wiltgen: http://playbacktime.com/
.. _Rael Dornfest: http://raelity.org/


Copyright and License
=====================

SmartyPants_ license (3-Clause BSD license):

  Copyright (c) 2003 John Gruber (http://daringfireball.net/)
  All rights reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are
  met:

  * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.

  * Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in
    the documentation and/or other materials provided with the
    distribution.

  * Neither the name "SmartyPants" nor the names of its contributors
    may be used to endorse or promote products derived from this
    software without specific prior written permission.

  This software is provided by the copyright holders and contributors
  "as is" and any express or implied warranties, including, but not
  limited to, the implied warranties of merchantability and fitness for
  a particular purpose are disclaimed. In no event shall the copyright
  owner or contributors be liable for any direct, indirect, incidental,
  special, exemplary, or consequential damages (including, but not
  limited to, procurement of substitute goods or services; loss of use,
  data, or profits; or business interruption) however caused and on any
  theory of liability, whether in contract, strict liability, or tort
  (including negligence or otherwise) arising in any way out of the use
  of this software, even if advised of the possibility of such damage.

smartypants.py license (2-Clause BSD license):

  smartypants.py is a derivative work of SmartyPants.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are
  met:

  * Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.

  * Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in
    the documentation and/or other materials provided with the
    distribution.

  This software is provided by the copyright holders and contributors
  "as is" and any express or implied warranties, including, but not
  limited to, the implied warranties of merchantability and fitness for
  a particular purpose are disclaimed. In no event shall the copyright
  owner or contributors be liable for any direct, indirect, incidental,
  special, exemplary, or consequential damages (including, but not
  limited to, procurement of substitute goods or services; loss of use,
  data, or profits; or business interruption) however caused and on any
  theory of liability, whether in contract, strict liability, or tort
  (including negligence or otherwise) arising in any way out of the use
  of this software, even if advised of the possibility of such damage.

.. _John Gruber: http://daringfireball.net/
.. _Chad Miller: http://web.chad.org/

.. _Pyblosxom: http://pyblosxom.bluesock.org/
.. _SmartyPants: http://daringfireball.net/projects/smartypants/
.. _Movable Type: http://www.movabletype.org/
.. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause
.. _Docutils: https://docutils.sourceforge.io/

Description
===========

SmartyPants can perform the following transformations:

- Straight quotes ( " and ' ) into "curly" quote characters
- Backticks-style quotes (\`\`like this'') into "curly" quote characters
- Dashes (``--`` and ``---``) into en- and em-dash entities
- Three consecutive dots (``...`` or ``. . .``) into an ellipsis entity

This means you can write, edit, and save your posts using plain old
ASCII straight quotes, plain dashes, and plain dots, but your published
posts (and final HTML output) will appear with smart quotes, em-dashes,
and proper ellipses.

SmartyPants does not modify characters within ``<pre>``, ``<code>``, ``<kbd>``,
``<math>`` or ``<script>`` tag blocks. Typically, these tags are used to
display text where smart quotes and other "smart punctuation" would not be
appropriate, such as source code or example markup.


Backslash Escapes
=================

If you need to use literal straight quotes (or plain hyphens and periods),
`smartquotes` accepts the following backslash escape sequences to force
ASCII-punctuation. Mind, that you need two backslashes as Docutils expands it,
too.

========  =========
Escape    Character
========  =========
``\\``    \\
``\\"``   \\"
``\\'``   \\'
``\\.``   \\.
``\\-``   \\-
``\\```   \\`
========  =========

This is useful, for example, when you want to use straight quotes as
foot and inch marks: 6\\'2\\" tall; a 17\\" iMac.


Caveats
=======

Why You Might Not Want to Use Smart Quotes in Your Weblog
---------------------------------------------------------

For one thing, you might not care.

Most normal, mentally stable individuals do not take notice of proper
typographic punctuation. Many design and typography nerds, however, break
out in a nasty rash when they encounter, say, a restaurant sign that uses
a straight apostrophe to spell "Joe's".

If you're the sort of person who just doesn't care, you might well want to
continue not caring. Using straight quotes -- and sticking to the 7-bit
ASCII character set in general -- is certainly a simpler way to live.

Even if you *do* care about accurate typography, you still might want to
think twice before educating the quote characters in your weblog. One side
effect of publishing curly quote characters is that it makes your
weblog a bit harder for others to quote from using copy-and-paste. What
happens is that when someone copies text from your blog, the copied text
contains the 8-bit curly quote characters (as well as the 8-bit characters
for em-dashes and ellipses, if you use these options). These characters
are not standard across different text encoding methods, which is why they
need to be encoded as characters.

People copying text from your weblog, however, may not notice that you're
using curly quotes, and they'll go ahead and paste the unencoded 8-bit
characters copied from their browser into an email message or their own
weblog. When pasted as raw "smart quotes", these characters are likely to
get mangled beyond recognition.

That said, my own opinion is that any decent text editor or email client
makes it easy to stupefy smart quote characters into their 7-bit
equivalents, and I don't consider it my problem if you're using an
indecent text editor or email client.


Algorithmic Shortcomings
------------------------

One situation in which quotes will get curled the wrong way is when
apostrophes are used at the start of leading contractions. For example::

  'Twas the night before Christmas.

In the case above, SmartyPants will turn the apostrophe into an opening
secondary quote, when in fact it should be the `RIGHT SINGLE QUOTATION MARK`
character which is also "the preferred character to use for apostrophe"
(Unicode). I don't think this problem can be solved in the general case --
every word processor I've tried gets this wrong as well. In such cases, it's
best to inset the `RIGHT SINGLE QUOTATION MARK` (’) by hand.

In English, the same character is used for apostrophe and  closing secondary
quote (both plain and "smart" ones). For other locales (French, Italean,
Swiss, ...) "smart" secondary closing quotes differ from the curly apostrophe.

   .. class:: language-fr

   Il dit : "C'est 'super' !"

If the apostrophe is used at the end of a word, it cannot be distinguished
from a secondary quote by the algorithm. Therefore, a text like::

   .. class:: language-de-CH

   "Er sagt: 'Ich fass' es nicht.'"

will get a single closing guillemet instead of an apostrophe.

This can be prevented by use use of the `RIGHT SINGLE QUOTATION MARK` in
the source::

   -  "Er sagt: 'Ich fass' es nicht.'"
   +  "Er sagt: 'Ich fass’ es nicht.'"


Version History
===============

1.9     2022-03-04
        - Code cleanup. Require Python 3.

1.8.1   2017-10-25
        - Use open quote after Unicode whitespace, ZWSP, and ZWNJ.
        - Code cleanup.

1.8:    2017-04-24
        - Command line front-end.

1.7.1:  2017-03-19
        - Update and extend language-dependent quotes.
        - Differentiate apostrophe from single quote.

1.7:    2012-11-19
        - Internationalization: language-dependent quotes.

1.6.1:  2012-11-06
        - Refactor code, code cleanup,
        - `educate_tokens()` generator as interface for Docutils.

1.6:    2010-08-26
        - Adaption to Docutils:
          - Use Unicode instead of HTML entities,
          - Remove code special to pyblosxom.

1.5_1.6: Fri, 27 Jul 2007 07:06:40 -0400
        - Fixed bug where blocks of precious unalterable text was instead
          interpreted.  Thanks to Le Roux and Dirk van Oosterbosch.

1.5_1.5: Sat, 13 Aug 2005 15:50:24 -0400
        - Fix bogus magical quotation when there is no hint that the
          user wants it, e.g., in "21st century".  Thanks to Nathan Hamblen.
        - Be smarter about quotes before terminating numbers in an en-dash'ed
          range.

1.5_1.4: Thu, 10 Feb 2005 20:24:36 -0500
        - Fix a date-processing bug, as reported by jacob childress.
        - Begin a test-suite for ensuring correct output.
        - Removed import of "string", since I didn't really need it.
          (This was my first every Python program.  Sue me!)

1.5_1.3: Wed, 15 Sep 2004 18:25:58 -0400
        - Abort processing if the flavour is in forbidden-list.  Default of
          [ "rss" ]   (Idea of Wolfgang SCHNERRING.)
        - Remove stray virgules from en-dashes.  Patch by Wolfgang SCHNERRING.

1.5_1.2: Mon, 24 May 2004 08:14:54 -0400
        - Some single quotes weren't replaced properly.  Diff-tesuji played
          by Benjamin GEIGER.

1.5_1.1: Sun, 14 Mar 2004 14:38:28 -0500
        - Support upcoming pyblosxom 0.9 plugin verification feature.

1.5_1.0: Tue, 09 Mar 2004 08:08:35 -0500
        - Initial release
"""

import re
import sys


options = r"""
Options
=======

Numeric values are the easiest way to configure SmartyPants' behavior:

:0:     Suppress all transformations. (Do nothing.)

:1:     Performs default SmartyPants transformations: quotes (including
        \`\`backticks'' -style), em-dashes, and ellipses. "``--``" (dash dash)
        is used to signify an em-dash; there is no support for en-dashes

:2:     Same as smarty_pants="1", except that it uses the old-school typewriter
        shorthand for dashes:  "``--``" (dash dash) for en-dashes, "``---``"
        (dash dash dash)
        for em-dashes.

:3:     Same as smarty_pants="2", but inverts the shorthand for dashes:
        "``--``" (dash dash) for em-dashes, and "``---``" (dash dash dash) for
        en-dashes.

:-1:    Stupefy mode. Reverses the SmartyPants transformation process, turning
        the characters produced by SmartyPants into their ASCII equivalents.
        E.g. the LEFT DOUBLE QUOTATION MARK (“) is turned into a simple
        double-quote (\"), "—" is turned into two dashes, etc.


The following single-character attribute values can be combined to toggle
individual transformations from within the smarty_pants attribute. For
example, ``"1"`` is equivalent to ``"qBde"``.

:q:     Educates normal quote characters: (") and (').

:b:     Educates \`\`backticks'' -style double quotes.

:B:     Educates \`\`backticks'' -style double quotes and \`single' quotes.

:d:     Educates em-dashes.

:D:     Educates em-dashes and en-dashes, using old-school typewriter
        shorthand: (dash dash) for en-dashes, (dash dash dash) for em-dashes.

:i:     Educates em-dashes and en-dashes, using inverted old-school typewriter
        shorthand: (dash dash) for em-dashes, (dash dash dash) for en-dashes.

:e:     Educates ellipses.

:w:     Translates any instance of ``&quot;`` into a normal double-quote
        character. This should be of no interest to most people, but
        of particular interest to anyone who writes their posts using
        Dreamweaver, as Dreamweaver inexplicably uses this entity to represent
        a literal double-quote character. SmartyPants only educates normal
        quotes, not entities (because ordinarily, entities are used for
        the explicit purpose of representing the specific character they
        represent). The "w" option must be used in conjunction with one (or
        both) of the other quote options ("q" or "b"). Thus, if you wish to
        apply all SmartyPants transformations (quotes, en- and em-dashes, and
        ellipses) and also translate ``&quot;`` entities into regular quotes
        so SmartyPants can educate them, you should pass the following to the
        smarty_pants attribute:
"""


class smartchars:
    """Smart quotes and dashes"""

    endash = '–'      # "&#8211;" EN DASH
    emdash = '—'      # "&#8212;" EM DASH
    ellipsis = '…'    # "&#8230;" HORIZONTAL ELLIPSIS
    apostrophe = '’'  # "&#8217;" RIGHT SINGLE QUOTATION MARK

    # quote characters (language-specific, set in __init__())
    # https://en.wikipedia.org/wiki/Non-English_usage_of_quotation_marks
    # https://de.wikipedia.org/wiki/Anf%C3%BChrungszeichen#Andere_Sprachen
    # https://fr.wikipedia.org/wiki/Guillemet
    # https://typographisme.net/post/Les-espaces-typographiques-et-le-web
    # https://www.btb.termiumplus.gc.ca/tpv2guides/guides/redac/index-fra.html
    # https://en.wikipedia.org/wiki/Hebrew_punctuation#Quotation_marks
    # [7] https://www.tustep.uni-tuebingen.de/bi/bi00/bi001t1-anfuehrung.pdf
    # [8] https://www.korrekturavdelingen.no/anforselstegn.htm
    # [9] Typografisk håndbok. Oslo: Spartacus. 2000. s. 67. ISBN 8243001530.
    # [10] https://www.typografi.org/sitat/sitatart.html
    # [11] https://mk.wikipedia.org/wiki/Правопис_и_правоговор_на_македонскиот_јазик  # noqa:E501
    # [12] https://hrvatska-tipografija.com/polunavodnici/
    # [13] https://pl.wikipedia.org/wiki/Cudzys%C5%82%C3%B3w
    #
    # See also configuration option "smartquote-locales".
    quotes = {
        'af':           '“”‘’',
        'af-x-altquot': '„”‚’',
        'bg':           '„“‚‘',  # https://bg.wikipedia.org/wiki/Кавички
        'ca':           '«»“”',
        'ca-x-altquot': '“”‘’',
        'cs':           '„“‚‘',
        'cs-x-altquot': '»«›‹',
        'da':           '»«›‹',
        'da-x-altquot': '„“‚‘',
        # 'da-x-altquot2': '””’’',
        'de':           '„“‚‘',
        'de-x-altquot': '»«›‹',
        'de-ch':        '«»‹›',
        'el':           '«»“”',  # '«»‟”' https://hal.science/hal-02101618
        'en':           '“”‘’',
        'en-uk-x-altquot': '‘’“”',  # Attention: " → ‘ and ' → “ !
        'eo':           '“”‘’',
        'es':           '«»“”',
        'es-x-altquot': '“”‘’',
        'et':           '„“‚‘',  # no secondary quote listed in
        'et-x-altquot': '«»‹›',  # the sources above (wikipedia.org)
        'eu':           '«»‹›',
        'fi':           '””’’',
        'fi-x-altquot': '»»››',
        'fr':           ('« ', ' »', '“', '”'),  # full no-break space
        'fr-x-altquot': ('« ', ' »', '“', '”'),  # narrow no-break space
        'fr-ch':        '«»‹›',                  # https://typoguide.ch/
        'fr-ch-x-altquot': ('« ',  ' »', '‹ ', ' ›'),  # narrow no-break space  # noqa:E501
        'gl':           '«»“”',
        'he':           '”“»«',  # Hebrew is RTL, test position:
        'he-x-altquot': '„”‚’',  # low quotation marks are opening.
        # 'he-x-altquot': '“„‘‚',  # RTL: low quotation marks opening
        'hr':           '„”‘’',  # Croatian [12]
        'hr-x-altquot': '»«›‹',
        'hsb':          '„“‚‘',
        'hsb-x-altquot': '»«›‹',
        'hu':           '„”«»',
        'is':           '„“‚‘',
        'it':           '«»“”',
        'it-ch':        '«»‹›',
        'it-x-altquot': '“”‘’',
        # 'it-x-altquot2': '“„‘‚',  # [7] in headlines
        'ja':           '「」『』',
        'ko':           '“”‘’',
        'lt':           '„“‚‘',
        'lv':           '„“‚‘',
        'mk':           '„“‚‘',  # Macedonian [11]
        'nl':           '“”‘’',
        'nl-x-altquot': '„”‚’',
        # 'nl-x-altquot2': '””’’',
        'nb':           '«»’’',     # Norsk bokmål (canonical form 'no')
        'nn':           '«»’’',     # Nynorsk [10]
        'nn-x-altquot': '«»‘’',     # [8], [10]
        # 'nn-x-altquot2': '«»«»',  # [9], [10]
        # 'nn-x-altquot3': '„“‚‘',  # [10]
        'no':           '«»’’',     # Norsk bokmål [10]
        'no-x-altquot': '«»‘’',     # [8], [10]
        # 'no-x-altquot2': '«»«»',  # [9], [10
        # 'no-x-altquot3': '„“‚‘',  # [10]
        'pl':           '„”«»',
        'pl-x-altquot': '«»‚’',
        # 'pl-x-altquot2': '„”‚’',  # [13]
        'pt':           '«»“”',
        'pt-br':        '“”‘’',
        'ro':           '„”«»',
        'ru':           '«»„“',
        'sh':           '„”‚’',  # Serbo-Croatian
        'sh-x-altquot': '»«›‹',
        'sk':           '„“‚‘',  # Slovak
        'sk-x-altquot': '»«›‹',
        'sl':           '„“‚‘',  # Slovenian
        'sl-x-altquot': '»«›‹',
        'sq':           '«»‹›',  # Albanian
        'sq-x-altquot': '“„‘‚',
        'sr':           '„”’’',
        'sr-x-altquot': '»«›‹',
        'sv':           '””’’',
        'sv-x-altquot': '»»››',
        'tr':           '“”‘’',
        'tr-x-altquot': '«»‹›',
        # 'tr-x-altquot2': '“„‘‚',  # [7] antiquated?
        'uk':           '«»„“',
        'uk-x-altquot': '„“‚‘',
        'zh-cn':        '“”‘’',
        'zh-tw':        '「」『』',
        }

    def __init__(self, language='en'):
        self.language = language
        try:
            (self.opquote, self.cpquote,
             self.osquote, self.csquote) = self.quotes[language.lower()]
        except KeyError:
            self.opquote, self.cpquote, self.osquote, self.csquote = '""\'\''


default_smartypants_attr = '1'


def smartyPants(text, attr=default_smartypants_attr, language='en'):
    """Main function for "traditional" use."""

    return "".join(t for t in educate_tokens(tokenize(text), attr, language))


def educate_tokens(text_tokens, attr=default_smartypants_attr, language='en'):
    """Return iterator that "educates" the items of `text_tokens`."""
    # Parse attributes:
    # 0 : do nothing
    # 1 : set all
    # 2 : set all, using old school en- and em- dash shortcuts
    # 3 : set all, using inverted old school en and em- dash shortcuts
    #
    # q : quotes
    # b : backtick quotes (``double'' only)
    # B : backtick quotes (``double'' and `single')
    # d : dashes
    # D : old school dashes
    # i : inverted old school dashes
    # e : ellipses
    # w : convert &quot; entities to " for Dreamweaver users

    convert_quot = False  # translate &quot; entities into normal quotes?
    do_dashes = False
    do_backticks = False
    do_quotes = False
    do_ellipses = False
    do_stupefy = False

    # if attr == "0":  # pass tokens unchanged (see below).
    if attr == '1':  # Do everything, turn all options on.
        do_quotes = True
        do_backticks = True
        do_dashes = 1
        do_ellipses = True
    elif attr == '2':
        # Do everything, turn all options on, use old school dash shorthand.
        do_quotes = True
        do_backticks = True
        do_dashes = 2
        do_ellipses = True
    elif attr == '3':
        # Do everything, use inverted old school dash shorthand.
        do_quotes = True
        do_backticks = True
        do_dashes = 3
        do_ellipses = True
    elif attr == '-1':  # Special "stupefy" mode.
        do_stupefy = True
    else:
        if 'q' in attr: do_quotes = True     # noqa: E701
        if 'b' in attr: do_backticks = True  # noqa: E701
        if 'B' in attr: do_backticks = 2     # noqa: E701
        if 'd' in attr: do_dashes = 1        # noqa: E701
        if 'D' in attr: do_dashes = 2        # noqa: E701
        if 'i' in attr: do_dashes = 3        # noqa: E701
        if 'e' in attr: do_ellipses = True   # noqa: E701
        if 'w' in attr: convert_quot = True  # noqa: E701

    prev_token_last_char = ' '
    # Last character of the previous text token. Used as
    # context to curl leading quote characters correctly.

    for (ttype, text) in text_tokens:

        # skip HTML and/or XML tags as well as empty text tokens
        # without updating the last character
        if ttype == 'tag' or not text:
            yield text
            continue

        # skip literal text (math, literal, raw, ...)
        if ttype == 'literal':
            prev_token_last_char = text[-1:]
            yield text
            continue

        last_char = text[-1:]  # Remember last char before processing.

        text = processEscapes(text)

        if convert_quot:
            text = text.replace('&quot;', '"')

        if do_dashes == 1:
            text = educateDashes(text)
        elif do_dashes == 2:
            text = educateDashesOldSchool(text)
        elif do_dashes == 3:
            text = educateDashesOldSchoolInverted(text)

        if do_ellipses:
            text = educateEllipses(text)

        # Note: backticks need to be processed before quotes.
        if do_backticks:
            text = educateBackticks(text, language)

        if do_backticks == 2:
            text = educateSingleBackticks(text, language)

        if do_quotes:
            # Replace plain quotes in context to prevent conversion to
            # 2-character sequence in French.
            context = prev_token_last_char.replace('"', ';').replace("'", ';')
            text = educateQuotes(context+text, language)[1:]

        if do_stupefy:
            text = stupefyEntities(text, language)

        # Remember last char as context for the next token
        prev_token_last_char = last_char

        text = processEscapes(text, restore=True)

        yield text


def educateQuotes(text, language='en'):
    """
    Parameter:  - text string (unicode or bytes).
                - language (`BCP 47` language tag.)
    Returns:    The `text`, with "educated" curly quote characters.

    Example input:  "Isn't this fun?"
    Example output: “Isn’t this fun?“;
    """

    smart = smartchars(language)
    ch_classes = {'open': '[([{]',    # opening braces
                  'close': r'[^\s]',  # everything except whitespace
                  'punct': r"""[-!" #\$\%'()*+,.\/:;<=>?\@\[\\\]\^_`{|}~]""",
                  'dash': '[-–—]'     # hyphen and em/en dashes
                          r'|&[mn]dash;|&\#8211;|&\#8212;|&\#x201[34];',
                  'sep': '[\\s\u200B\u200C]|&nbsp;',  # Whitespace, ZWSP, ZWNJ
                  }

    # Special case if the very first character is a quote
    # followed by punctuation at a non-word-break. Use closing quotes.
    # TODO: example (when does this match?)
    text = re.sub(r"^'(?=%s\\B)" % ch_classes['punct'], smart.csquote, text)
    text = re.sub(r'^"(?=%s\\B)' % ch_classes['punct'], smart.cpquote, text)

    # Special case for adjacent quotes
    # like "'Quoted' words in a larger quote."
    text = re.sub('"\'(?=\\w)', smart.opquote+smart.osquote, text)
    text = re.sub('\'"(?=\\w)', smart.osquote+smart.opquote, text)

    # Special case: "opening character" followed by quote,
    # optional punctuation and space like "[", '(', or '-'.
    text = re.sub(r"(%(open)s|%(dash)s)'(?=%(punct)s? )" % ch_classes,
                  r'\1%s'%smart.csquote, text)
    text = re.sub(r'(%(open)s|%(dash)s)"(?=%(punct)s? )' % ch_classes,
                  r'\1%s'%smart.cpquote, text)

    # Special case for decade abbreviations (the '80s):
    if language.startswith('en'):  # TODO similar cases in other languages?
        text = re.sub(r"'(?=\d{2}s)", smart.apostrophe, text)

    # Get most opening secondary quotes:
    opening_secondary_quotes_regex = re.compile("""
                    (# ?<=  # look behind fails: requires fixed-width pattern
                      %(sep)s     |  # a whitespace char, or
                      %(open)s    |  # opening brace, or
                      %(dash)s       # em/en-dash
                    )
                    '                  # the quote
                    (?=\\w|%(punct)s)  # word character or punctuation
                    """ % ch_classes, re.VERBOSE)

    text = opening_secondary_quotes_regex.sub(r'\1'+smart.osquote, text)

    # In many locales, secondary closing quotes are different from apostrophe:
    if smart.csquote != smart.apostrophe:
        apostrophe_regex = re.compile(r"(?<=(\w|\d))'(?=\w)")
        text = apostrophe_regex.sub(smart.apostrophe, text)
    # TODO: keep track of quoting level to recognize apostrophe in, e.g.,
    # "Ich fass' es nicht."

    closing_secondary_quotes_regex = re.compile(r"(?<!\s)'")
    text = closing_secondary_quotes_regex.sub(smart.csquote, text)

    # Any remaining secondary quotes should be opening ones:
    text = text.replace(r"'", smart.osquote)

    # Get most opening primary quotes:
    opening_primary_quotes_regex = re.compile("""
                    (
                      %(sep)s     |  # a whitespace char, or
                      %(open)s    |  # zero width separating char, or
                      %(dash)s       # em/en-dash
                    )
                    "                 # the quote, followed by
                    (?=\\w|%(punct)s) # a word character or punctuation
                    """ % ch_classes, re.VERBOSE)

    text = opening_primary_quotes_regex.sub(r'\1'+smart.opquote, text)

    # primary closing quotes:
    closing_primary_quotes_regex = re.compile(r"""
                    (
                    (?<!\s)" | # no whitespace before
                    "(?=\s)    # whitespace behind
                    )
                    """, re.VERBOSE)
    text = closing_primary_quotes_regex.sub(smart.cpquote, text)

    # Any remaining quotes should be opening ones.
    text = text.replace(r'"', smart.opquote)

    return text


def educateBackticks(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with ``backticks'' -style double quotes
                translated into HTML curly quote entities.
    Example input:  ``Isn't this fun?''
    Example output: “Isn't this fun?“;
    """
    smart = smartchars(language)

    text = text.replace(r'``', smart.opquote)
    text = text.replace(r"''", smart.cpquote)
    return text


def educateSingleBackticks(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with `backticks' -style single quotes
                translated into HTML curly quote entities.

    Example input:  `Isn't this fun?'
    Example output: ‘Isn’t this fun?’
    """
    smart = smartchars(language)

    text = text.replace(r'`', smart.osquote)
    text = text.replace(r"'", smart.csquote)
    return text


def educateDashes(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an em-dash character.
    """

    text = text.replace(r'---', smartchars.endash)  # en (yes, backwards)
    text = text.replace(r'--', smartchars.emdash)   # em (yes, backwards)
    return text


def educateDashesOldSchool(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an en-dash character, and each "---" translated to
                an em-dash character.
    """

    text = text.replace(r'---', smartchars.emdash)
    text = text.replace(r'--', smartchars.endash)
    return text


def educateDashesOldSchoolInverted(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "--" translated to
                an em-dash character, and each "---" translated to
                an en-dash character. Two reasons why: First, unlike the
                en- and em-dash syntax supported by
                EducateDashesOldSchool(), it's compatible with existing
                entries written before SmartyPants 1.1, back when "--" was
                only used for em-dashes.  Second, em-dashes are more
                common than en-dashes, and so it sort of makes sense that
                the shortcut should be shorter to type. (Thanks to Aaron
                Swartz for the idea.)
    """
    text = text.replace(r'---', smartchars.endash)    # em
    text = text.replace(r'--', smartchars.emdash)    # en
    return text


def educateEllipses(text):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each instance of "..." translated to
                an ellipsis character.

    Example input:  Huh...?
    Example output: Huh&#8230;?
    """

    text = text.replace(r'...', smartchars.ellipsis)
    text = text.replace(r'. . .', smartchars.ellipsis)
    return text


def stupefyEntities(text, language='en'):
    """
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with each SmartyPants character translated to
                its ASCII counterpart.

    Example input:  “Hello — world.”
    Example output: "Hello -- world."
    """
    smart = smartchars(language)

    text = text.replace(smart.endash, "-")
    text = text.replace(smart.emdash, "--")
    text = text.replace(smart.osquote, "'")  # open secondary quote
    text = text.replace(smart.csquote, "'")  # close secondary quote
    text = text.replace(smart.opquote, '"')  # open primary quote
    text = text.replace(smart.cpquote, '"')  # close primary quote
    text = text.replace(smart.ellipsis, '...')

    return text


def processEscapes(text, restore=False):
    r"""
    Parameter:  String (unicode or bytes).
    Returns:    The `text`, with after processing the following backslash
                escape sequences. This is useful if you want to force a "dumb"
                quote or other character to appear.

                Escape  Value
                ------  -----
                \\      &#92;
                \"      &#34;
                \'      &#39;
                \.      &#46;
                \-      &#45;
                \`      &#96;
    """
    replacements = ((r'\\', r'&#92;'),
                    (r'\"', r'&#34;'),
                    (r"\'", r'&#39;'),
                    (r'\.', r'&#46;'),
                    (r'\-', r'&#45;'),
                    (r'\`', r'&#96;'))
    if restore:
        for (ch, rep) in replacements:
            text = text.replace(rep, ch[1])
    else:
        for (ch, rep) in replacements:
            text = text.replace(ch, rep)

    return text


def tokenize(text):
    """
    Parameter:  String containing HTML markup.
    Returns:    An iterator that yields the tokens comprising the input
                string. Each token is either a tag (possibly with nested,
                tags contained therein, such as <a href="<MTFoo>">, or a
                run of text between tags. Each yielded element is a
                two-element tuple; the first is either 'tag' or 'text';
                the second is the actual value.

    Based on the _tokenize() subroutine from Brad Choate's MTRegex plugin.
    """
    tag_soup = re.compile(r'([^<]*)(<[^>]*>)')
    token_match = tag_soup.search(text)
    previous_end = 0

    while token_match is not None:
        if token_match.group(1):
            yield 'text', token_match.group(1)
        yield 'tag', token_match.group(2)
        previous_end = token_match.end()
        token_match = tag_soup.search(text, token_match.end())

    if previous_end < len(text):
        yield 'text', text[previous_end:]


if __name__ == "__main__":

    import itertools
    import locale
    try:
        locale.setlocale(locale.LC_ALL, '')  # set to user defaults
        defaultlanguage = locale.getlocale()[0]
    except:  # noqa  catchall
        defaultlanguage = 'en'

    # Normalize and drop unsupported subtags:
    defaultlanguage = defaultlanguage.lower().replace('-', '_')
    # split (except singletons, which mark the following tag as non-standard):
    defaultlanguage = re.sub(r'_([a-zA-Z0-9])_', r'_\1-', defaultlanguage)
    _subtags = [subtag for subtag in defaultlanguage.split('_')]
    _basetag = _subtags.pop(0)
    # find all combinations of subtags
    for n in range(len(_subtags), 0, -1):
        for tags in itertools.combinations(_subtags, n):
            _tag = '-'.join((_basetag, *tags))
            if _tag in smartchars.quotes:
                defaultlanguage = _tag
                break
        else:
            if _basetag in smartchars.quotes:
                defaultlanguage = _basetag
            else:
                defaultlanguage = 'en'

    import argparse
    parser = argparse.ArgumentParser(
                description='Filter <input> making ASCII punctuation "smart".')
    # TODO: require input arg or other means to print USAGE instead of waiting.
    # parser.add_argument("input", help="Input stream, use '-' for stdin.")
    parser.add_argument("-a", "--action", default="1",
                        help="what to do with the input (see --actionhelp)")
    parser.add_argument("-e", "--encoding", default="utf-8",
                        help="text encoding")
    parser.add_argument("-l", "--language", default=defaultlanguage,
                        help="text language (BCP47 tag), "
                             f"Default: {defaultlanguage}")
    parser.add_argument("-q", "--alternative-quotes", action="store_true",
                        help="use alternative quote style")
    parser.add_argument("--doc", action="store_true",
                        help="print documentation")
    parser.add_argument("--actionhelp", action="store_true",
                        help="list available actions")
    parser.add_argument("--stylehelp", action="store_true",
                        help="list available quote styles")
    parser.add_argument("--test", action="store_true",
                        help="perform short self-test")
    args = parser.parse_args()

    if args.doc:
        print(__doc__)
    elif args.actionhelp:
        print(options)
    elif args.stylehelp:
        print()
        print("Available styles (primary open/close, secondary open/close)")
        print("language tag   quotes")
        print("============   ======")
        for key in sorted(smartchars.quotes.keys()):
            print("%-14s %s" % (key, smartchars.quotes[key]))
    elif args.test:
        # Unit test output goes to stderr.
        import unittest

        class TestSmartypantsAllAttributes(unittest.TestCase):
            # the default attribute is "1", which means "all".
            def test_dates(self):
                self.assertEqual(smartyPants("1440-80's"), "1440-80’s")
                self.assertEqual(smartyPants("1440-'80s"), "1440-’80s")
                self.assertEqual(smartyPants("1440---'80s"), "1440–’80s")
                self.assertEqual(smartyPants("1960's"), "1960’s")
                self.assertEqual(smartyPants("one two '60s"), "one two ’60s")
                self.assertEqual(smartyPants("'60s"), "’60s")

            def test_educated_quotes(self):
                self.assertEqual(smartyPants('"Isn\'t this fun?"'),
                                 '“Isn’t this fun?”')

            def test_html_tags(self):
                text = '<a src="foo">more</a>'
                self.assertEqual(smartyPants(text), text)

        suite = unittest.TestLoader().loadTestsFromTestCase(
                                            TestSmartypantsAllAttributes)
        unittest.TextTestRunner().run(suite)

    else:
        if args.alternative_quotes:
            if '-x-altquot' in args.language:
                args.language = args.language.replace('-x-altquot', '')
            else:
                args.language += '-x-altquot'
        text = sys.stdin.read()
        print(smartyPants(text, attr=args.action, language=args.language))

env_pyrecodes/lib/python3.9/site-packages/docutils/utils/smartquotes.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9368 2023-04-28 21:26:36Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
This package contains Docutils Writer modules.
"""

__docformat__ = 'reStructuredText'

from importlib import import_module

import docutils
from docutils import languages, Component
from docutils.transforms import universal


class Writer(Component):

    """
    Abstract base class for docutils Writers.

    Each writer module or package must export a subclass also called 'Writer'.
    Each writer must support all standard node types listed in
    `docutils.nodes.node_class_names`.

    The `write()` method is the main entry point.
    """

    component_type = 'writer'
    config_section = 'writers'

    def get_transforms(self):
        return super().get_transforms() + [universal.Messages,
                                           universal.FilterMessages,
                                           universal.StripClassesAndElements]

    document = None
    """The document to write (Docutils doctree); set by `write()`."""

    output = None
    """Final translated form of `document`

    (`str` for text, `bytes` for binary formats); set by `translate()`.
    """

    language = None
    """Language module for the document; set by `write()`."""

    destination = None
    """`docutils.io` Output object; where to write the document.

    Set by `write()`.
    """

    def __init__(self):

        self.parts = {}
        """Mapping of document part names to fragments of `self.output`.

        See `Writer.assemble_parts()` below and
        <https://docutils.sourceforge.io/docs/api/publisher.html>.
        """

    def write(self, document, destination):
        """
        Process a document into its final form.

        Translate `document` (a Docutils document tree) into the Writer's
        native format, and write it out to its `destination` (a
        `docutils.io.Output` subclass object).

        Normally not overridden or extended in subclasses.
        """
        self.document = document
        self.language = languages.get_language(
            document.settings.language_code,
            document.reporter)
        self.destination = destination
        self.translate()
        return self.destination.write(self.output)

    def translate(self):
        """
        Do final translation of `self.document` into `self.output`.  Called
        from `write`.  Override in subclasses.

        Usually done with a `docutils.nodes.NodeVisitor` subclass, in
        combination with a call to `docutils.nodes.Node.walk()` or
        `docutils.nodes.Node.walkabout()`.  The ``NodeVisitor`` subclass must
        support all standard elements (listed in
        `docutils.nodes.node_class_names`) and possibly non-standard elements
        used by the current Reader as well.
        """
        raise NotImplementedError('subclass must override this method')

    def assemble_parts(self):
        """Assemble the `self.parts` dictionary.  Extend in subclasses.

        See <https://docutils.sourceforge.io/docs/api/publisher.html>.
        """
        self.parts['whole'] = self.output
        self.parts['encoding'] = self.document.settings.output_encoding
        self.parts['errors'] = (
            self.document.settings.output_encoding_error_handler)
        self.parts['version'] = docutils.__version__


class UnfilteredWriter(Writer):

    """
    A writer that passes the document tree on unchanged (e.g. a
    serializer.)

    Documents written by UnfilteredWriters are typically reused at a
    later date using a subclass of `readers.ReReader`.
    """

    def get_transforms(self):
        # Do not add any transforms.  When the document is reused
        # later, the then-used writer will add the appropriate
        # transforms.
        return Component.get_transforms(self)


_writer_aliases = {
      'html': 'html4css1',  # may change to html5 some day
      'html4': 'html4css1',
      'xhtml10': 'html4css1',
      'html5': 'html5_polyglot',
      'xhtml': 'html5_polyglot',
      's5': 's5_html',
      'latex': 'latex2e',
      'xelatex': 'xetex',
      'luatex': 'xetex',
      'lualatex': 'xetex',
      'odf': 'odf_odt',
      'odt': 'odf_odt',
      'ooffice': 'odf_odt',
      'openoffice': 'odf_odt',
      'libreoffice': 'odf_odt',
      'pprint': 'pseudoxml',
      'pformat': 'pseudoxml',
      'pdf': 'rlpdf',
      'xml': 'docutils_xml'}


def get_writer_class(writer_name):
    """Return the Writer class from the `writer_name` module."""
    name = writer_name.lower()
    name = _writer_aliases.get(name, name)
    try:
        module = import_module('docutils.writers.'+name)
    except ImportError:
        try:
            module = import_module(name)
        except ImportError as err:
            raise ImportError(f'Writer "{writer_name}" not found. {err}')
    return module.Writer

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
# :Author: David Goodger, Günter Milde
#          Based on the html4css1 writer by David Goodger.
# :Maintainer: docutils-develop@lists.sourceforge.net
# :Revision: $Revision: 9292 $
# :Date: $Date: 2005-06-28$
# :Copyright: © 2016 David Goodger, Günter Milde
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause

"""common definitions for Docutils HTML writers"""

import base64
import mimetypes
import os
import os.path
import re
from urllib.request import unquote as unquote_url
from urllib.request import url2pathname  # unquote and use local path sep
import warnings

import docutils
from docutils import frontend, languages, nodes, utils, writers
from docutils.parsers.rst.directives import length_or_percentage_or_unitless
from docutils.parsers.rst.directives.images import PIL
from docutils.transforms import writer_aux
from docutils.utils.math import (unichar2tex, pick_math_environment,
                                 math2html, latex2mathml, tex2mathml_extern)


class Writer(writers.Writer):

    supported = ('html', 'xhtml')  # update in subclass
    """Formats this writer supports."""

    settings_spec = (
        'HTML Writer Options',
        None,
        (('Specify the template file (UTF-8 encoded). '
          '(default: writer dependent)',
          ['--template'],
          {'metavar': '<file>'}),
         ('Comma separated list of stylesheet URLs. '
          'Overrides previous --stylesheet and --stylesheet-path settings.',
          ['--stylesheet'],
          {'metavar': '<URL[,URL,...]>', 'overrides': 'stylesheet_path',
           'validator': frontend.validate_comma_separated_list}),
         ('Comma separated list of stylesheet paths. '
          'Relative paths are expanded if a matching file is found in '
          'the --stylesheet-dirs. With --link-stylesheet, '
          'the path is rewritten relative to the output HTML file. '
          '(default: writer dependent)',
          ['--stylesheet-path'],
          {'metavar': '<file[,file,...]>', 'overrides': 'stylesheet',
           'validator': frontend.validate_comma_separated_list}),
         ('Comma-separated list of directories where stylesheets are found. '
          'Used by --stylesheet-path when expanding relative path arguments. '
          '(default: writer dependent)',
          ['--stylesheet-dirs'],
          {'metavar': '<dir[,dir,...]>',
           'validator': frontend.validate_comma_separated_list}),
         ('Embed the stylesheet(s) in the output HTML file.  The stylesheet '
          'files must be accessible during processing. (default)',
          ['--embed-stylesheet'],
          {'default': 1, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Link to the stylesheet(s) in the output HTML file. ',
          ['--link-stylesheet'],
          {'dest': 'embed_stylesheet', 'action': 'store_false'}),
         ('Specify the initial header level. '
          'Does not affect document title & subtitle (see --no-doc-title).'
          '(default: writer dependent).',
          ['--initial-header-level'],
          {'choices': '1 2 3 4 5 6'.split(), 'default': '2',
           'metavar': '<level>'}),
         ('Format for footnote references: one of "superscript" or '
          '"brackets". (default: "brackets")',
          ['--footnote-references'],
          {'choices': ['superscript', 'brackets'], 'default': 'brackets',
           'metavar': '<format>',
           'overrides': 'trim_footnote_reference_space'}),
         ('Format for block quote attributions: '
          'one of "dash" (em-dash prefix), "parentheses"/"parens", or "none". '
          '(default: "dash")',
          ['--attribution'],
          {'choices': ['dash', 'parentheses', 'parens', 'none'],
           'default': 'dash', 'metavar': '<format>'}),
         ('Remove extra vertical whitespace between items of "simple" bullet '
          'lists and enumerated lists. (default)',
          ['--compact-lists'],
          {'default': True, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Disable compact simple bullet and enumerated lists.',
          ['--no-compact-lists'],
          {'dest': 'compact_lists', 'action': 'store_false'}),
         ('Remove extra vertical whitespace between items of simple field '
          'lists. (default)',
          ['--compact-field-lists'],
          {'default': True, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Disable compact simple field lists.',
          ['--no-compact-field-lists'],
          {'dest': 'compact_field_lists', 'action': 'store_false'}),
         ('Added to standard table classes. '
          'Defined styles: borderless, booktabs, '
          'align-left, align-center, align-right, '
          'colwidths-auto, colwidths-grid.',
          ['--table-style'],
          {'default': ''}),
         ('Math output format (one of "MathML", "HTML", "MathJax", '
          'or "LaTeX") and option(s). '
          '(default: "HTML math.css")',
          ['--math-output'],
          {'default': 'HTML math.css'}),
         ('Prepend an XML declaration. ',
          ['--xml-declaration'],
          {'default': False, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Omit the XML declaration.',
          ['--no-xml-declaration'],
          {'dest': 'xml_declaration', 'action': 'store_false'}),
         ('Obfuscate email addresses to confuse harvesters while still '
          'keeping email links usable with standards-compliant browsers.',
          ['--cloak-email-addresses'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         )
        )

    settings_defaults = {'output_encoding_error_handler': 'xmlcharrefreplace'}

    config_section = 'html base writer'  # overwrite in subclass
    config_section_dependencies = ('writers', 'html writers')

    visitor_attributes = (
        'head_prefix', 'head', 'stylesheet', 'body_prefix',
        'body_pre_docinfo', 'docinfo', 'body', 'body_suffix',
        'title', 'subtitle', 'header', 'footer', 'meta', 'fragment',
        'html_prolog', 'html_head', 'html_title', 'html_subtitle',
        'html_body')

    def get_transforms(self):
        return super().get_transforms() + [writer_aux.Admonitions]

    def translate(self):
        self.visitor = visitor = self.translator_class(self.document)
        self.document.walkabout(visitor)
        for attr in self.visitor_attributes:
            setattr(self, attr, getattr(visitor, attr))
        self.output = self.apply_template()

    def apply_template(self):
        with open(self.document.settings.template, encoding='utf-8') as fp:
            template = fp.read()
        subs = self.interpolation_dict()
        return template % subs

    def interpolation_dict(self):
        subs = {}
        settings = self.document.settings
        for attr in self.visitor_attributes:
            subs[attr] = ''.join(getattr(self, attr)).rstrip('\n')
        subs['encoding'] = settings.output_encoding
        subs['version'] = docutils.__version__
        return subs

    def assemble_parts(self):
        writers.Writer.assemble_parts(self)
        for part in self.visitor_attributes:
            self.parts[part] = ''.join(getattr(self, part))


class HTMLTranslator(nodes.NodeVisitor):

    """
    Generic Docutils to HTML translator.

    See the `html4css1` and `html5_polyglot` writers for full featured
    HTML writers.

    .. IMPORTANT::
      The `visit_*` and `depart_*` methods use a
      heterogeneous stack, `self.context`.
      When subclassing, make sure to be consistent in its use!

      Examples for robust coding:

      a) Override both `visit_*` and `depart_*` methods, don't call the
         parent functions.

      b) Extend both and unconditionally call the parent functions::

           def visit_example(self, node):
               if foo:
                   self.body.append('<div class="foo">')
               html4css1.HTMLTranslator.visit_example(self, node)

           def depart_example(self, node):
               html4css1.HTMLTranslator.depart_example(self, node)
               if foo:
                   self.body.append('</div>')

      c) Extend both, calling the parent functions under the same
         conditions::

           def visit_example(self, node):
               if foo:
                   self.body.append('<div class="foo">\n')
               else: # call the parent method
                   _html_base.HTMLTranslator.visit_example(self, node)

           def depart_example(self, node):
               if foo:
                   self.body.append('</div>\n')
               else: # call the parent method
                   _html_base.HTMLTranslator.depart_example(self, node)

      d) Extend one method (call the parent), but don't otherwise use the
         `self.context` stack::

           def depart_example(self, node):
               _html_base.HTMLTranslator.depart_example(self, node)
               if foo:
                   # implementation-specific code
                   # that does not use `self.context`
                   self.body.append('</div>\n')

      This way, changes in stack use will not bite you.
    """

    doctype = '<!DOCTYPE html>\n'
    doctype_mathml = doctype

    head_prefix_template = ('<html xmlns="http://www.w3.org/1999/xhtml"'
                            ' xml:lang="%(lang)s" lang="%(lang)s">\n<head>\n')
    content_type = '<meta charset="%s" />\n'
    generator = (
        f'<meta name="generator" content="Docutils {docutils.__version__}: '
        'https://docutils.sourceforge.io/" />\n')
    # `starttag()` arguments for the main document (HTML5 uses <main>)
    documenttag_args = {'tagname': 'div', 'CLASS': 'document'}

    # Template for the MathJax script in the header:
    mathjax_script = '<script type="text/javascript" src="%s"></script>\n'

    mathjax_url = 'file:/usr/share/javascript/mathjax/MathJax.js'
    """
    URL of the MathJax javascript library.

    The MathJax library ought to be installed on the same
    server as the rest of the deployed site files and specified
    in the `math-output` setting appended to "mathjax".
    See `Docutils Configuration`__.

    __ https://docutils.sourceforge.io/docs/user/config.html#math-output

    The fallback tries a local MathJax installation at
    ``/usr/share/javascript/mathjax/MathJax.js``.
    """

    stylesheet_link = '<link rel="stylesheet" href="%s" type="text/css" />\n'
    embedded_stylesheet = '<style type="text/css">\n\n%s\n</style>\n'
    words_and_spaces = re.compile(r'[^ \n]+| +|\n')
    # wrap point inside word:
    in_word_wrap_point = re.compile(r'.+\W\W.+|[-?].+')
    lang_attribute = 'lang'  # name changes to 'xml:lang' in XHTML 1.1

    special_characters = {ord('&'): '&amp;',
                          ord('<'): '&lt;',
                          ord('"'): '&quot;',
                          ord('>'): '&gt;',
                          ord('@'): '&#64;',  # may thwart address harvesters
                          }
    """Character references for characters with a special meaning in HTML."""

    def __init__(self, document):
        nodes.NodeVisitor.__init__(self, document)
        # process settings
        self.settings = settings = document.settings
        self.language = languages.get_language(
                            settings.language_code, document.reporter)
        self.initial_header_level = int(settings.initial_header_level)
        # image_loading (only defined for HTML5 writer)
        self.image_loading = getattr(settings, 'image_loading', None)
        # legacy setting embed_images:
        if getattr(settings, 'embed_images', None) is True:
            warnings.warn('The configuration setting "embed_images" '
                          'will be removed in Docutils 2.0. '
                          'Use "image_loading: embed".',
                          FutureWarning, stacklevel=8)
            if self.image_loading is None:
                self.image_loading = 'embed'
        if getattr(settings, 'embed_images', None) is False:
            warnings.warn('The configuration setting "embed_images" '
                          'will be removed in Docutils 2.0. '
                          'Use "image_loading: link".',
                          FutureWarning, stacklevel=8)
        if self.image_loading is None:
            self.image_loading = 'link'  # default
        self.math_output = settings.math_output.split()
        self.math_output_options = self.math_output[1:]
        self.math_output = self.math_output[0].lower()

        # set up "parts" (cf. docs/api/publisher.html#publish-parts-details)
        #
        self.body = []  # equivalent to `fragment`, ≠ `html_body`
        self.body_prefix = ['</head>\n<body>\n']  # + optional header
        self.body_pre_docinfo = []  # document heading (title and subtitle)
        self.body_suffix = ['</body>\n</html>\n']  # + optional footer
        self.docinfo = []
        self.footer = []
        self.fragment = []  # main content of the document ("naked" body)
        self.head = []
        self.head_prefix = []  # everything up to and including <head>
        self.header = []
        self.html_body = []
        self.html_head = [self.content_type]  # charset not interpolated
        self.html_prolog = []
        self.html_subtitle = []
        self.html_title = []
        self.meta = [self.generator]
        self.stylesheet = [self.stylesheet_call(path)
                           for path in utils.get_stylesheet_list(settings)]
        self.title = []
        self.subtitle = []
        if settings.xml_declaration:
            self.head_prefix.append(
                utils.xml_declaration(settings.output_encoding))
            self.html_prolog.append(
                utils.xml_declaration('%s'))  # encoding not interpolated
        if (settings.output_encoding
            and settings.output_encoding.lower() != 'unicode'):
            self.meta.insert(0, self.content_type % settings.output_encoding)

        # bookkeeping attributes; reflect state of translator
        #
        self.context = []
        """Heterogeneous stack.

        Used by visit_* and depart_* functions in conjunction with the tree
        traversal. Make sure that the pops correspond to the pushes.
        """
        self.section_level = 0
        self.colspecs = []
        self.compact_p = True
        self.compact_simple = False
        self.compact_field_list = False
        self.in_docinfo = False
        self.in_sidebar = False
        self.in_document_title = 0  # len(self.body) or 0
        self.in_mailto = False
        self.author_in_authors = False  # for html4css1
        self.math_header = []

    def astext(self):
        return ''.join(self.head_prefix + self.head
                       + self.stylesheet + self.body_prefix
                       + self.body_pre_docinfo + self.docinfo
                       + self.body + self.body_suffix)

    def encode(self, text):
        """Encode special characters in `text` & return."""
        # Use only named entities known in both XML and HTML
        # other characters are automatically encoded "by number" if required.
        # @@@ A codec to do these and all other HTML entities would be nice.
        text = str(text)
        return text.translate(self.special_characters)

    def cloak_mailto(self, uri):
        """Try to hide a mailto: URL from harvesters."""
        # Encode "@" using a URL octet reference (see RFC 1738).
        # Further cloaking with HTML entities will be done in the
        # `attval` function.
        return uri.replace('@', '%40')

    def cloak_email(self, addr):
        """Try to hide the link text of a email link from harversters."""
        # Surround at-signs and periods with <span> tags.  ("@" has
        # already been encoded to "&#64;" by the `encode` method.)
        addr = addr.replace('&#64;', '<span>&#64;</span>')
        return addr.replace('.', '<span>&#46;</span>')

    def attval(self, text,
               whitespace=re.compile('[\n\r\t\v\f]')):
        """Cleanse, HTML encode, and return attribute value text."""
        encoded = self.encode(whitespace.sub(' ', text))
        if self.in_mailto and self.settings.cloak_email_addresses:
            # Cloak at-signs ("%40") and periods with HTML entities.
            encoded = encoded.replace('%40', '&#37;&#52;&#48;')
            encoded = encoded.replace('.', '&#46;')
        return encoded

    def stylesheet_call(self, path, adjust_path=None):
        """Return code to reference or embed stylesheet file `path`"""
        if adjust_path is None:
            adjust_path = bool(self.settings.stylesheet_path)
        if self.settings.embed_stylesheet:
            try:
                with open(path, encoding='utf-8') as f:
                    content = f.read()
            except OSError as err:
                msg = f'Cannot embed stylesheet: {err}'
                self.document.reporter.error(msg)
                return '<--- %s --->\n' % msg
            else:
                self.settings.record_dependencies.add(path)
            return self.embedded_stylesheet % content
        # else link to style file:
        if adjust_path:
            # rewrite path relative to output (cf. config.html#stylesheet-path)
            path = utils.relative_path(self.settings._destination, path)
        return self.stylesheet_link % self.encode(path)

    def starttag(self, node, tagname, suffix='\n', empty=False, **attributes):
        """
        Construct and return a start tag given a node (id & class attributes
        are extracted), tag name, and optional attributes.
        """
        tagname = tagname.lower()
        prefix = []
        atts = {}
        for (name, value) in attributes.items():
            atts[name.lower()] = value
        classes = atts.pop('classes', [])
        languages = []
        # unify class arguments and move language specification
        for cls in node.get('classes', []) + atts.pop('class', '').split():
            if cls.startswith('language-'):
                languages.append(cls[9:])
            elif cls.strip() and cls not in classes:
                classes.append(cls)
        if languages:
            # attribute name is 'lang' in XHTML 1.0 but 'xml:lang' in 1.1
            atts[self.lang_attribute] = languages[0]
        # filter classes that are processed by the writer:
        internal = ('colwidths-auto', 'colwidths-given', 'colwidths-grid')
        if isinstance(node, nodes.table):
            classes = [cls for cls in classes if cls not in internal]
        if classes:
            atts['class'] = ' '.join(classes)
        assert 'id' not in atts
        ids = node.get('ids', [])
        ids.extend(atts.pop('ids', []))
        if ids:
            atts['id'] = ids[0]
            for id in ids[1:]:
                # Add empty "span" elements for additional IDs.  Note
                # that we cannot use empty "a" elements because there
                # may be targets inside of references, but nested "a"
                # elements aren't allowed in XHTML (even if they do
                # not all have a "href" attribute).
                if empty or isinstance(node, (nodes.Sequential,
                                              nodes.docinfo,
                                              nodes.table)):
                    # Insert target right in front of element.
                    prefix.append('<span id="%s"></span>' % id)
                else:
                    # Non-empty tag.  Place the auxiliary <span> tag
                    # *inside* the element, as the first child.
                    suffix += '<span id="%s"></span>' % id
        attlist = sorted(atts.items())
        parts = [tagname]
        for name, value in attlist:
            # value=None was used for boolean attributes without
            # value, but this isn't supported by XHTML.
            assert value is not None
            if isinstance(value, list):
                values = [str(v) for v in value]
                parts.append('%s="%s"' % (name.lower(),
                                          self.attval(' '.join(values))))
            else:
                parts.append('%s="%s"' % (name.lower(),
                                          self.attval(str(value))))
        if empty:
            infix = ' /'
        else:
            infix = ''
        return ''.join(prefix) + '<%s%s>' % (' '.join(parts), infix) + suffix

    def emptytag(self, node, tagname, suffix='\n', **attributes):
        """Construct and return an XML-compatible empty tag."""
        return self.starttag(node, tagname, suffix, empty=True, **attributes)

    def set_class_on_child(self, node, class_, index=0):
        """
        Set class `class_` on the visible child no. index of `node`.
        Do nothing if node has fewer children than `index`.
        """
        children = [n for n in node if not isinstance(n, nodes.Invisible)]
        try:
            child = children[index]
        except IndexError:
            return
        child['classes'].append(class_)

    def visit_Text(self, node):
        text = node.astext()
        encoded = self.encode(text)
        if self.in_mailto and self.settings.cloak_email_addresses:
            encoded = self.cloak_email(encoded)
        self.body.append(encoded)

    def depart_Text(self, node):
        pass

    def visit_abbreviation(self, node):
        # @@@ implementation incomplete ("title" attribute)
        self.body.append(self.starttag(node, 'abbr', ''))

    def depart_abbreviation(self, node):
        self.body.append('</abbr>')

    def visit_acronym(self, node):
        # @@@ implementation incomplete ("title" attribute)
        self.body.append(self.starttag(node, 'acronym', ''))

    def depart_acronym(self, node):
        self.body.append('</acronym>')

    def visit_address(self, node):
        self.visit_docinfo_item(node, 'address', meta=False)
        self.body.append(self.starttag(node, 'pre',
                                       suffix='', CLASS='address'))

    def depart_address(self, node):
        self.body.append('\n</pre>\n')
        self.depart_docinfo_item()

    def visit_admonition(self, node):
        self.body.append(self.starttag(node, 'aside', classes=['admonition']))

    def depart_admonition(self, node=None):
        self.body.append('</aside>\n')

    attribution_formats = {'dash': ('\u2014', ''),
                           'parentheses': ('(', ')'),
                           'parens': ('(', ')'),
                           'none': ('', '')}

    def visit_attribution(self, node):
        prefix, suffix = self.attribution_formats[self.settings.attribution]
        self.context.append(suffix)
        self.body.append(
            self.starttag(node, 'p', prefix, CLASS='attribution'))

    def depart_attribution(self, node):
        self.body.append(self.context.pop() + '</p>\n')

    def visit_author(self, node):
        if not isinstance(node.parent, nodes.authors):
            self.visit_docinfo_item(node, 'author')
        self.body.append('<p>')

    def depart_author(self, node):
        self.body.append('</p>')
        if isinstance(node.parent, nodes.authors):
            self.body.append('\n')
        else:
            self.depart_docinfo_item()

    def visit_authors(self, node):
        self.visit_docinfo_item(node, 'authors')

    def depart_authors(self, node):
        self.depart_docinfo_item()

    def visit_block_quote(self, node):
        self.body.append(self.starttag(node, 'blockquote'))

    def depart_block_quote(self, node):
        self.body.append('</blockquote>\n')

    def check_simple_list(self, node):
        """Check for a simple list that can be rendered compactly."""
        visitor = SimpleListChecker(self.document)
        try:
            node.walk(visitor)
        except nodes.NodeFound:
            return False
        else:
            return True

    # Compact lists
    # ------------
    # Include definition lists and field lists (in addition to ordered
    # and unordered lists) in the test if a list is "simple"  (cf. the
    # html4css1.HTMLTranslator docstring and the SimpleListChecker class at
    # the end of this file).

    def is_compactable(self, node):
        # explicit class arguments have precedence
        if 'compact' in node['classes']:
            return True
        if 'open' in node['classes']:
            return False
        # check config setting:
        if (isinstance(node, (nodes.field_list, nodes.definition_list))
            and not self.settings.compact_field_lists):
            return False
        if (isinstance(node, (nodes.enumerated_list, nodes.bullet_list))
            and not self.settings.compact_lists):
            return False
        # Table of Contents:
        if 'contents' in node.parent['classes']:
            return True
        # check the list items:
        return self.check_simple_list(node)

    def visit_bullet_list(self, node):
        atts = {}
        old_compact_simple = self.compact_simple
        self.context.append((self.compact_simple, self.compact_p))
        self.compact_p = None
        self.compact_simple = self.is_compactable(node)
        if self.compact_simple and not old_compact_simple:
            atts['class'] = 'simple'
        self.body.append(self.starttag(node, 'ul', **atts))

    def depart_bullet_list(self, node):
        self.compact_simple, self.compact_p = self.context.pop()
        self.body.append('</ul>\n')

    def visit_caption(self, node):
        self.body.append(self.starttag(node, 'p', '', CLASS='caption'))

    def depart_caption(self, node):
        self.body.append('</p>\n')

    # Use semantic tag and DPub role (HTML4 uses a table)
    def visit_citation(self, node):
        # role 'doc-bibloentry' requires wrapping in an element with
        # role 'list' and an element with role 'doc-bibliography'
        # https://www.w3.org/TR/dpub-aria-1.0/#doc-biblioentry)
        if not isinstance(node.previous_sibling(), type(node)):
            self.body.append('<div role="list" class="citation-list">\n')
        self.body.append(self.starttag(node, 'div', classes=[node.tagname],
                                       role="doc-biblioentry"))

    def depart_citation(self, node):
        self.body.append('</div>\n')
        if not isinstance(node.next_node(descend=False, siblings=True),
                          type(node)):
            self.body.append('</div>\n')

    # Use DPub role (overwritten in HTML4)
    def visit_citation_reference(self, node):
        href = '#'
        if 'refid' in node:
            href += node['refid']
        elif 'refname' in node:
            href += self.document.nameids[node['refname']]
        # else: # TODO system message (or already in the transform)?
        # 'Citation reference missing.'
        self.body.append(self.starttag(node, 'a', suffix='[', href=href,
                                       classes=['citation-reference'],
                                       role='doc-biblioref'))

    def depart_citation_reference(self, node):
        self.body.append(']</a>')

    # classifier
    # ----------
    # don't insert classifier-delimiter here (done by CSS)

    def visit_classifier(self, node):
        self.body.append(self.starttag(node, 'span', '', CLASS='classifier'))

    def depart_classifier(self, node):
        self.body.append('</span>')

    def visit_colspec(self, node):
        self.colspecs.append(node)
        # "stubs" list is an attribute of the tgroup element:
        node.parent.stubs.append(node.attributes.get('stub'))

    def depart_colspec(self, node):
        # write out <colgroup> when all colspecs are processed
        if isinstance(node.next_node(descend=False, siblings=True),
                      nodes.colspec):
            return
        if 'colwidths-auto' in node.parent.parent['classes'] or (
            'colwidths-grid' not in self.settings.table_style
            and 'colwidths-given' not in node.parent.parent['classes']):
            return
        self.body.append(self.starttag(node, 'colgroup'))
        total_width = sum(node['colwidth'] for node in self.colspecs)
        for node in self.colspecs:
            colwidth = node['colwidth'] / total_width
            self.body.append(self.emptytag(node, 'col',
                                           style=f'width: {colwidth:.1%}'))
        self.body.append('</colgroup>\n')

    def visit_comment(self, node,
                      sub=re.compile('-(?=-)').sub):
        """Escape double-dashes in comment text."""
        self.body.append('<!-- %s -->\n' % sub('- ', node.astext()))
        # Content already processed:
        raise nodes.SkipNode

    def visit_compound(self, node):
        self.body.append(self.starttag(node, 'div', CLASS='compound'))

    def depart_compound(self, node):
        self.body.append('</div>\n')

    def visit_container(self, node):
        self.body.append(self.starttag(node, 'div',
                                       CLASS='docutils container'))

    def depart_container(self, node):
        self.body.append('</div>\n')

    def visit_contact(self, node):
        self.visit_docinfo_item(node, 'contact', meta=False)

    def depart_contact(self, node):
        self.depart_docinfo_item()

    def visit_copyright(self, node):
        self.visit_docinfo_item(node, 'copyright')

    def depart_copyright(self, node):
        self.depart_docinfo_item()

    def visit_date(self, node):
        self.visit_docinfo_item(node, 'date')

    def depart_date(self, node):
        self.depart_docinfo_item()

    def visit_decoration(self, node):
        pass

    def depart_decoration(self, node):
        pass

    def visit_definition(self, node):
        if 'details' in node.parent.parent['classes']:
            self.body.append('</summary>\n')
        else:
            self.body.append('</dt>\n')
            self.body.append(self.starttag(node, 'dd', ''))

    def depart_definition(self, node):
        if 'details' not in node.parent.parent['classes']:
            self.body.append('</dd>\n')

    def visit_definition_list(self, node):
        if 'details' in node['classes']:
            self.body.append(self.starttag(node, 'div'))
        else:
            classes = ['simple'] if self.is_compactable(node) else []
            self.body.append(self.starttag(node, 'dl', classes=classes))

    def depart_definition_list(self, node):
        if 'details' in node['classes']:
            self.body.append('</div>\n')
        else:
            self.body.append('</dl>\n')

    # Use a "details" disclosure element if parent has "class" arg "details".
    def visit_definition_list_item(self, node):
        if 'details' in node.parent['classes']:
            atts = {}
            if "open" in node.parent['classes']:
                atts['open'] = 'open'
            self.body.append(self.starttag(node, 'details', **atts))

    def depart_definition_list_item(self, node):
        if 'details' in node.parent['classes']:
            self.body.append('</details>\n')

    def visit_description(self, node):
        self.body.append(self.starttag(node, 'dd', ''))

    def depart_description(self, node):
        self.body.append('</dd>\n')

    def visit_docinfo(self, node):
        self.context.append(len(self.body))
        classes = ['docinfo']
        if self.is_compactable(node):
            classes.append('simple')
        self.body.append(self.starttag(node, 'dl', classes=classes))

    def depart_docinfo(self, node):
        self.body.append('</dl>\n')
        start = self.context.pop()
        self.docinfo = self.body[start:]
        self.body = []

    def visit_docinfo_item(self, node, name, meta=True):
        if meta:
            self.meta.append(f'<meta name="{name}" '
                             f'content="{self.attval(node.astext())}" />\n')
        self.body.append(f'<dt class="{name}">{self.language.labels[name]}'
                         '<span class="colon">:</span></dt>\n')
        self.body.append(self.starttag(node, 'dd', '', CLASS=name))

    def depart_docinfo_item(self):
        self.body.append('</dd>\n')

    def visit_doctest_block(self, node):
        self.body.append(self.starttag(node, 'pre', suffix='',
                                       classes=['code', 'python', 'doctest']))

    def depart_doctest_block(self, node):
        self.body.append('\n</pre>\n')

    def visit_document(self, node):
        title = (node.get('title') or os.path.basename(node['source'])
                 or 'untitled Docutils document')
        self.head.append(f'<title>{self.encode(title)}</title>\n')

    def depart_document(self, node):
        self.head_prefix.extend([self.doctype,
                                 self.head_prefix_template %
                                 {'lang': self.settings.language_code}])
        self.html_prolog.append(self.doctype)
        self.head = self.meta[:] + self.head
        if 'name="dcterms.' in ''.join(self.meta):
            self.head.append('<link rel="schema.dcterms"'
                             ' href="http://purl.org/dc/terms/"/>')
        if self.math_header:
            if self.math_output == 'mathjax':
                self.head.extend(self.math_header)
            else:
                self.stylesheet.extend(self.math_header)
        # skip content-type meta tag with interpolated charset value:
        self.html_head.extend(self.head[1:])
        self.body_prefix.append(self.starttag(node, **self.documenttag_args))
        self.body_suffix.insert(0, f'</{self.documenttag_args["tagname"]}>\n')
        self.fragment.extend(self.body)  # self.fragment is the "naked" body
        self.html_body.extend(self.body_prefix[1:] + self.body_pre_docinfo
                              + self.docinfo + self.body
                              + self.body_suffix[:-1])
        assert not self.context, f'len(context) = {len(self.context)}'

    def visit_emphasis(self, node):
        self.body.append(self.starttag(node, 'em', ''))

    def depart_emphasis(self, node):
        self.body.append('</em>')

    def visit_entry(self, node):
        atts = {'classes': []}
        if isinstance(node.parent.parent, nodes.thead):
            atts['classes'].append('head')
        if node.parent.parent.parent.stubs[node.parent.column]:
            # "stubs" list is an attribute of the tgroup element
            atts['classes'].append('stub')
        if atts['classes']:
            tagname = 'th'
        else:
            tagname = 'td'
        node.parent.column += 1
        if 'morerows' in node:
            atts['rowspan'] = node['morerows'] + 1
        if 'morecols' in node:
            atts['colspan'] = node['morecols'] + 1
            node.parent.column += node['morecols']
        self.body.append(self.starttag(node, tagname, '', **atts))
        self.context.append('</%s>\n' % tagname.lower())

    def depart_entry(self, node):
        self.body.append(self.context.pop())

    def visit_enumerated_list(self, node):
        atts = {'classes': []}
        if 'start' in node:
            atts['start'] = node['start']
        if 'enumtype' in node:
            atts['classes'].append(node['enumtype'])
        if self.is_compactable(node):
            atts['classes'].append('simple')
        self.body.append(self.starttag(node, 'ol', **atts))

    def depart_enumerated_list(self, node):
        self.body.append('</ol>\n')

    def visit_field_list(self, node):
        atts = {}
        classes = node.setdefault('classes', [])
        for i, cls in enumerate(classes):
            if cls.startswith('field-indent-'):
                try:
                    indent_length = length_or_percentage_or_unitless(
                                                        cls[13:], 'px')
                except ValueError:
                    break
                atts['style'] = '--field-indent: %s;' % indent_length
                classes.pop(i)
                break
        classes.append('field-list')
        if self.is_compactable(node):
            classes.append('simple')
        self.body.append(self.starttag(node, 'dl', **atts))

    def depart_field_list(self, node):
        self.body.append('</dl>\n')

    def visit_field(self, node):
        pass

    def depart_field(self, node):
        pass

    # as field is ignored, pass class arguments to field-name and field-body:
    def visit_field_name(self, node):
        self.body.append(self.starttag(node, 'dt', '',
                                       classes=node.parent['classes']))

    def depart_field_name(self, node):
        self.body.append('<span class="colon">:</span></dt>\n')

    def visit_field_body(self, node):
        self.body.append(self.starttag(node, 'dd', '',
                                       classes=node.parent['classes']))
        # prevent misalignment of following content if the field is empty:
        if not node.children:
            self.body.append('<p></p>')

    def depart_field_body(self, node):
        self.body.append('</dd>\n')

    def visit_figure(self, node):
        atts = {'class': 'figure'}
        if node.get('width'):
            atts['style'] = 'width: %s' % node['width']
        if node.get('align'):
            atts['class'] += " align-" + node['align']
        self.body.append(self.starttag(node, 'div', **atts))

    def depart_figure(self, node):
        self.body.append('</div>\n')

    def visit_footer(self, node):
        self.context.append(len(self.body))

    def depart_footer(self, node):
        start = self.context.pop()
        footer = [self.starttag(node, 'div', CLASS='footer'),
                  '<hr class="footer" />\n']
        footer.extend(self.body[start:])
        footer.append('\n</div>\n')
        self.footer.extend(footer)
        self.body_suffix[:0] = footer
        del self.body[start:]

    def visit_footnote(self, node):
        # No native HTML element: use <aside> with ARIA role
        # (html4css1 uses tables).
        # Wrap groups of footnotes for easier styling.
        label_style = self.settings.footnote_references  # brackets/superscript
        if not isinstance(node.previous_sibling(), type(node)):
            self.body.append(f'<aside class="footnote-list {label_style}">\n')
        self.body.append(self.starttag(node, 'aside',
                                       classes=[node.tagname, label_style],
                                       role="doc-footnote"))

    def depart_footnote(self, node):
        self.body.append('</aside>\n')
        if not isinstance(node.next_node(descend=False, siblings=True),
                          type(node)):
            self.body.append('</aside>\n')

    def visit_footnote_reference(self, node):
        href = '#' + node['refid']
        classes = ['footnote-reference', self.settings.footnote_references]
        self.body.append(self.starttag(node, 'a', suffix='', classes=classes,
                                       role='doc-noteref', href=href))
        self.body.append('<span class="fn-bracket">[</span>')

    def depart_footnote_reference(self, node):
        self.body.append('<span class="fn-bracket">]</span>')
        self.body.append('</a>')

    # Docutils-generated text: put section numbers in a span for CSS styling:
    def visit_generated(self, node):
        if 'sectnum' in node['classes']:
            # get section number (strip trailing no-break-spaces)
            sectnum = node.astext().rstrip(' ')
            self.body.append('<span class="sectnum">%s </span>'
                             % self.encode(sectnum))
            # Content already processed:
            raise nodes.SkipNode

    def depart_generated(self, node):
        pass

    def visit_header(self, node):
        self.context.append(len(self.body))

    def depart_header(self, node):
        start = self.context.pop()
        header = [self.starttag(node, 'div', CLASS='header')]
        header.extend(self.body[start:])
        header.append('\n<hr class="header"/>\n</div>\n')
        self.body_prefix.extend(header)
        self.header.extend(header)
        del self.body[start:]

    def visit_image(self, node):
        atts = {}
        uri = node['uri']
        mimetype = mimetypes.guess_type(uri)[0]
        # image size
        if 'width' in node:
            atts['width'] = node['width']
        if 'height' in node:
            atts['height'] = node['height']
        if 'scale' in node:
            if (PIL and ('width' not in node or 'height' not in node)
                and self.settings.file_insertion_enabled):
                imagepath = url2pathname(uri)
                try:
                    with PIL.Image.open(imagepath) as img:
                        imgsize = img.size
                except (OSError, UnicodeEncodeError):
                    pass  # TODO: warn?
                else:
                    self.settings.record_dependencies.add(
                        imagepath.replace('\\', '/'))
                    if 'width' not in atts:
                        atts['width'] = '%dpx' % imgsize[0]
                    if 'height' not in atts:
                        atts['height'] = '%dpx' % imgsize[1]
                    del img
            for att_name in 'width', 'height':
                if att_name in atts:
                    match = re.match(r'([0-9.]+)(\S*)$', atts[att_name])
                    assert match
                    atts[att_name] = '%s%s' % (
                        float(match.group(1)) * (float(node['scale']) / 100),
                        match.group(2))
        style = []
        for att_name in 'width', 'height':
            if att_name in atts:
                if re.match(r'^[0-9.]+$', atts[att_name]):
                    # Interpret unitless values as pixels.
                    atts[att_name] += 'px'
                style.append('%s: %s;' % (att_name, atts[att_name]))
                del atts[att_name]
        if style:
            atts['style'] = ' '.join(style)
        if (isinstance(node.parent, nodes.TextElement)
            or (isinstance(node.parent, nodes.reference)
                and not isinstance(node.parent.parent, nodes.TextElement))):
            # Inline context or surrounded by <a>...</a>.
            suffix = ''
        else:
            suffix = '\n'
        if 'align' in node:
            atts['class'] = 'align-%s' % node['align']
        # Embed image file (embedded SVG or data URI):
        if self.image_loading == 'embed':
            try:
                with open(url2pathname(uri), 'rb') as imagefile:
                    imagedata = imagefile.read()
            except OSError as err:
                self.document.reporter.error('Cannot embed image %r: %s'
                                             % (uri, err.strerror))
            else:
                self.settings.record_dependencies.add(unquote_url(uri))
                # TODO: insert SVG as-is?
                # if mimetype == 'image/svg+xml':
                # read/parse, apply arguments,
                # insert as <svg ....> ... </svg> # (about 1/3 less data)
                data64 = base64.b64encode(imagedata).decode()
                uri = 'data:%s;base64,%s' % (mimetype, data64)
        elif self.image_loading == 'lazy':
            atts['loading'] = 'lazy'
        if mimetype == 'application/x-shockwave-flash':
            atts['type'] = mimetype
            # do NOT use an empty tag: incorrect rendering in browsers
            tag = (self.starttag(node, 'object', '', data=uri, **atts)
                   + node.get('alt', uri) + '</object>' + suffix)
        else:
            atts['alt'] = node.get('alt', node['uri'])
            tag = self.emptytag(node, 'img', suffix, src=uri, **atts)
        self.body.append(tag)

    def depart_image(self, node):
        pass

    def visit_inline(self, node):
        self.body.append(self.starttag(node, 'span', ''))

    def depart_inline(self, node):
        self.body.append('</span>')

    # footnote and citation labels:
    def visit_label(self, node):
        self.body.append('<span class="label">')
        self.body.append('<span class="fn-bracket">[</span>')
        # footnote/citation backrefs:
        if self.settings.footnote_backlinks:
            backrefs = node.parent.get('backrefs', [])
            if len(backrefs) == 1:
                self.body.append('<a role="doc-backlink"'
                                 ' href="#%s">' % backrefs[0])

    def depart_label(self, node):
        backrefs = []
        if self.settings.footnote_backlinks:
            backrefs = node.parent.get('backrefs', backrefs)
        if len(backrefs) == 1:
            self.body.append('</a>')
        self.body.append('<span class="fn-bracket">]</span></span>\n')
        if len(backrefs) > 1:
            backlinks = ['<a role="doc-backlink" href="#%s">%s</a>' % (ref, i)
                         for (i, ref) in enumerate(backrefs, 1)]
            self.body.append('<span class="backrefs">(%s)</span>\n'
                             % ','.join(backlinks))

    def visit_legend(self, node):
        self.body.append(self.starttag(node, 'div', CLASS='legend'))

    def depart_legend(self, node):
        self.body.append('</div>\n')

    def visit_line(self, node):
        self.body.append(self.starttag(node, 'div', suffix='', CLASS='line'))
        if not len(node):
            self.body.append('<br />')

    def depart_line(self, node):
        self.body.append('</div>\n')

    def visit_line_block(self, node):
        self.body.append(self.starttag(node, 'div', CLASS='line-block'))

    def depart_line_block(self, node):
        self.body.append('</div>\n')

    def visit_list_item(self, node):
        self.body.append(self.starttag(node, 'li', ''))

    def depart_list_item(self, node):
        self.body.append('</li>\n')

    # inline literal
    def visit_literal(self, node):
        # special case: "code" role
        classes = node['classes']
        if 'code' in classes:
            # filter 'code' from class arguments
            classes.pop(classes.index('code'))
            self.body.append(self.starttag(node, 'code', ''))
            return
        self.body.append(
            self.starttag(node, 'span', '', CLASS='docutils literal'))
        text = node.astext()
        if not isinstance(node.parent, nodes.literal_block):
            text = text.replace('\n', ' ')
        # Protect text like ``--an-option`` and the regular expression
        # ``[+]?(\d+(\.\d*)?|\.\d+)`` from bad line wrapping
        for token in self.words_and_spaces.findall(text):
            if token.strip() and self.in_word_wrap_point.search(token):
                self.body.append('<span class="pre">%s</span>'
                                 % self.encode(token))
            else:
                self.body.append(self.encode(token))
        self.body.append('</span>')
        raise nodes.SkipNode  # content already processed

    def depart_literal(self, node):
        # skipped unless literal element is from "code" role:
        self.body.append('</code>')

    def visit_literal_block(self, node):
        self.body.append(self.starttag(node, 'pre', '', CLASS='literal-block'))
        if 'code' in node['classes']:
            self.body.append('<code>')

    def depart_literal_block(self, node):
        if 'code' in node['classes']:
            self.body.append('</code>')
        self.body.append('</pre>\n')

    # Mathematics:
    # As there is no native HTML math support, we provide alternatives
    # for the math-output: LaTeX and MathJax simply wrap the content,
    # HTML and MathML also convert the math_code.
    # HTML container
    math_tags = {
                 # math_output: (block, inline, class-arguments)
                 'html': ('div', 'span', 'formula'),
                 'latex': ('pre', 'tt', 'math'),
                 'mathml': ('div', '', ''),
                 'mathjax': ('div', 'span', 'math'),
                }

    def visit_math(self, node, math_env=''):
        # If the method is called from visit_math_block(), math_env != ''.

        if self.math_output not in self.math_tags:
            self.document.reporter.error(
                f'math-output format "{self.math_output}" not supported '
                'falling back to "latex"')
            self.math_output = 'latex'
        tag = self.math_tags[self.math_output][math_env == '']
        clsarg = self.math_tags[self.math_output][2]
        # LaTeX container
        wrappers = {
                    # math_mode: (inline, block)
                    'html': ('$%s$', '\\begin{%s}\n%s\n\\end{%s}'),
                    'latex': (None, None),
                    'mathml': ('$%s$', '\\begin{%s}\n%s\n\\end{%s}'),
                    'mathjax': (r'\(%s\)', '\\begin{%s}\n%s\n\\end{%s}'),
                   }
        wrapper = wrappers[self.math_output][math_env != '']
        if (self.math_output == 'mathml'
            and (not self.math_output_options
                 or self.math_output_options[0] == 'blahtexml')):
            wrapper = None
        # get and wrap content
        math_code = node.astext().translate(unichar2tex.uni2tex_table)
        if wrapper:
            try:  # wrapper with three "%s"
                math_code = wrapper % (math_env, math_code, math_env)
            except TypeError:  # wrapper with one "%s"
                math_code = wrapper % math_code
        # settings and conversion
        if self.math_output in ('latex', 'mathjax'):
            math_code = self.encode(math_code)
        if self.math_output == 'mathjax' and not self.math_header:
            try:
                self.mathjax_url = self.math_output_options[0]
            except IndexError:
                self.document.reporter.warning('No MathJax URL specified, '
                                               'using local fallback '
                                               '(see config.html)')
            # append configuration, if not already present in the URL:
            # input LaTeX with AMS, output common HTML
            if '?' not in self.mathjax_url:
                self.mathjax_url += '?config=TeX-AMS_CHTML'
            self.math_header = [self.mathjax_script % self.mathjax_url]
        elif self.math_output == 'html':
            if self.math_output_options and not self.math_header:
                self.math_header = [self.stylesheet_call(
                    utils.find_file_in_dirs(s, self.settings.stylesheet_dirs),
                    adjust_path=True)
                    for s in self.math_output_options[0].split(',')]
            # TODO: fix display mode in matrices and fractions
            math2html.DocumentParameters.displaymode = (math_env != '')
            math_code = math2html.math2html(math_code)
        elif self.math_output == 'mathml':
            if 'XHTML 1' in self.doctype:
                self.doctype = self.doctype_mathml
                self.content_type = self.content_type_mathml
            converter = ' '.join(self.math_output_options).lower()
            try:
                if converter == 'latexml':
                    math_code = tex2mathml_extern.latexml(
                                    math_code, self.document.reporter)
                elif converter == 'ttm':
                    math_code = tex2mathml_extern.ttm(
                                    math_code, self.document.reporter)
                elif converter == 'blahtexml':
                    math_code = tex2mathml_extern.blahtexml(
                                    math_code,
                                    inline=(not math_env),
                                    reporter=self.document.reporter)
                elif converter == 'pandoc':
                    math_code = tex2mathml_extern.pandoc(
                                    math_code,
                                    reporter=self.document.reporter)
                elif not converter:
                    math_code = latex2mathml.tex2mathml(
                                    math_code, inline=(not math_env))
                else:
                    self.document.reporter.error('option "%s" not supported '
                                                 'with math-output "MathML"')
            except OSError:
                raise OSError('is "latexmlmath" in your PATH?')
            except SyntaxError as err:
                err_node = self.document.reporter.error(err, base_node=node)
                self.visit_system_message(err_node)
                self.body.append(self.starttag(node, 'p'))
                self.body.append(','.join(err.args))
                self.body.append('</p>\n')
                self.body.append(self.starttag(node, 'pre',
                                               CLASS='literal-block'))
                self.body.append(self.encode(math_code))
                self.body.append('\n</pre>\n')
                self.depart_system_message(err_node)
                raise nodes.SkipNode
        # append to document body
        if tag:
            self.body.append(self.starttag(node, tag,
                                           suffix='\n'*bool(math_env),
                                           CLASS=clsarg))
        self.body.append(math_code)
        if math_env:  # block mode (equation, display)
            self.body.append('\n')
        if tag:
            self.body.append('</%s>' % tag)
        if math_env:
            self.body.append('\n')
        # Content already processed:
        raise nodes.SkipNode

    def depart_math(self, node):
        pass  # never reached

    def visit_math_block(self, node):
        math_env = pick_math_environment(node.astext())
        self.visit_math(node, math_env=math_env)

    def depart_math_block(self, node):
        pass  # never reached

    # Meta tags: 'lang' attribute replaced by 'xml:lang' in XHTML 1.1
    # HTML5/polyglot recommends using both
    def visit_meta(self, node):
        self.meta.append(self.emptytag(node, 'meta',
                                       **node.non_default_attributes()))

    def depart_meta(self, node):
        pass

    def visit_option(self, node):
        self.body.append(self.starttag(node, 'span', '', CLASS='option'))

    def depart_option(self, node):
        self.body.append('</span>')
        if isinstance(node.next_node(descend=False, siblings=True),
                      nodes.option):
            self.body.append(', ')

    def visit_option_argument(self, node):
        self.body.append(node.get('delimiter', ' '))
        self.body.append(self.starttag(node, 'var', ''))

    def depart_option_argument(self, node):
        self.body.append('</var>')

    def visit_option_group(self, node):
        self.body.append(self.starttag(node, 'dt', ''))
        self.body.append('<kbd>')

    def depart_option_group(self, node):
        self.body.append('</kbd></dt>\n')

    def visit_option_list(self, node):
        self.body.append(
            self.starttag(node, 'dl', CLASS='option-list'))

    def depart_option_list(self, node):
        self.body.append('</dl>\n')

    def visit_option_list_item(self, node):
        pass

    def depart_option_list_item(self, node):
        pass

    def visit_option_string(self, node):
        pass

    def depart_option_string(self, node):
        pass

    def visit_organization(self, node):
        self.visit_docinfo_item(node, 'organization')

    def depart_organization(self, node):
        self.depart_docinfo_item()

    # Do not omit <p> tags
    # --------------------
    #
    # The HTML4CSS1 writer does this to "produce
    # visually compact lists (less vertical whitespace)". This writer
    # relies on CSS rules for visual compactness.
    #
    # * In XHTML 1.1, e.g., a <blockquote> element may not contain
    #   character data, so you cannot drop the <p> tags.
    # * Keeping simple paragraphs in the field_body enables a CSS
    #   rule to start the field-body on a new line if the label is too long
    # * it makes the code simpler.
    #
    # TODO: omit paragraph tags in simple table cells?

    def visit_paragraph(self, node):
        self.body.append(self.starttag(node, 'p', ''))

    def depart_paragraph(self, node):
        self.body.append('</p>')
        if not (isinstance(node.parent, (nodes.list_item, nodes.entry))
                and (len(node.parent) == 1)):
            self.body.append('\n')

    def visit_problematic(self, node):
        if node.hasattr('refid'):
            self.body.append('<a href="#%s">' % node['refid'])
            self.context.append('</a>')
        else:
            self.context.append('')
        self.body.append(self.starttag(node, 'span', '', CLASS='problematic'))

    def depart_problematic(self, node):
        self.body.append('</span>')
        self.body.append(self.context.pop())

    def visit_raw(self, node):
        if 'html' in node.get('format', '').split():
            if isinstance(node.parent, nodes.TextElement):
                tagname = 'span'
            else:
                tagname = 'div'
            if node['classes']:
                self.body.append(self.starttag(node, tagname, suffix=''))
            self.body.append(node.astext())
            if node['classes']:
                self.body.append('</%s>' % tagname)
        # Keep non-HTML raw text out of output:
        raise nodes.SkipNode

    def visit_reference(self, node):
        atts = {'class': 'reference'}
        if 'refuri' in node:
            atts['href'] = node['refuri']
            if (self.settings.cloak_email_addresses
                and atts['href'].startswith('mailto:')):
                atts['href'] = self.cloak_mailto(atts['href'])
                self.in_mailto = True
            atts['class'] += ' external'
        else:
            assert 'refid' in node, \
                   'References must have "refuri" or "refid" attribute.'
            atts['href'] = '#' + node['refid']
            atts['class'] += ' internal'
        if len(node) == 1 and isinstance(node[0], nodes.image):
            atts['class'] += ' image-reference'
        if not isinstance(node.parent, nodes.TextElement):
            assert len(node) == 1 and isinstance(node[0], nodes.image)
            atts['class'] += ' image-reference'
        self.body.append(self.starttag(node, 'a', '', **atts))

    def depart_reference(self, node):
        self.body.append('</a>')
        if not isinstance(node.parent, nodes.TextElement):
            self.body.append('\n')
        self.in_mailto = False

    def visit_revision(self, node):
        self.visit_docinfo_item(node, 'revision', meta=False)

    def depart_revision(self, node):
        self.depart_docinfo_item()

    def visit_row(self, node):
        self.body.append(self.starttag(node, 'tr', ''))
        node.column = 0

    def depart_row(self, node):
        self.body.append('</tr>\n')

    def visit_rubric(self, node):
        self.body.append(self.starttag(node, 'p', '', CLASS='rubric'))

    def depart_rubric(self, node):
        self.body.append('</p>\n')

    def visit_section(self, node):
        self.section_level += 1
        self.body.append(
            self.starttag(node, 'div', CLASS='section'))

    def depart_section(self, node):
        self.section_level -= 1
        self.body.append('</div>\n')

    # TODO: use the new HTML5 element <aside>
    def visit_sidebar(self, node):
        self.body.append(
            self.starttag(node, 'div', CLASS='sidebar'))
        self.in_sidebar = True

    def depart_sidebar(self, node):
        self.body.append('</div>\n')
        self.in_sidebar = False

    def visit_status(self, node):
        self.visit_docinfo_item(node, 'status', meta=False)

    def depart_status(self, node):
        self.depart_docinfo_item()

    def visit_strong(self, node):
        self.body.append(self.starttag(node, 'strong', ''))

    def depart_strong(self, node):
        self.body.append('</strong>')

    def visit_subscript(self, node):
        self.body.append(self.starttag(node, 'sub', ''))

    def depart_subscript(self, node):
        self.body.append('</sub>')

    def visit_substitution_definition(self, node):
        """Internal only."""
        raise nodes.SkipNode

    def visit_substitution_reference(self, node):
        self.unimplemented_visit(node)

    # h1–h6 elements must not be used to markup subheadings, subtitles,
    # alternative titles and taglines unless intended to be the heading for a
    # new section or subsection.
    # -- http://www.w3.org/TR/html51/sections.html#headings-and-sections
    def visit_subtitle(self, node):
        if isinstance(node.parent, nodes.sidebar):
            classes = ['sidebar-subtitle']
        elif isinstance(node.parent, nodes.document):
            classes = ['subtitle']
            self.in_document_title = len(self.body) + 1
        elif isinstance(node.parent, nodes.section):
            classes = ['section-subtitle']
        self.body.append(self.starttag(node, 'p', '', classes=classes))

    def depart_subtitle(self, node):
        self.body.append('</p>\n')
        if isinstance(node.parent, nodes.document):
            self.subtitle = self.body[self.in_document_title:-1]
            self.in_document_title = 0
            self.body_pre_docinfo.extend(self.body)
            self.html_subtitle.extend(self.body)
            del self.body[:]

    def visit_superscript(self, node):
        self.body.append(self.starttag(node, 'sup', ''))

    def depart_superscript(self, node):
        self.body.append('</sup>')

    def visit_system_message(self, node):
        self.body.append(self.starttag(node, 'aside', CLASS='system-message'))
        self.body.append('<p class="system-message-title">')
        backref_text = ''
        if len(node['backrefs']):
            backrefs = node['backrefs']
            if len(backrefs) == 1:
                backref_text = ('; <em><a href="#%s">backlink</a></em>'
                                % backrefs[0])
            else:
                i = 1
                backlinks = []
                for backref in backrefs:
                    backlinks.append('<a href="#%s">%s</a>' % (backref, i))
                    i += 1
                backref_text = ('; <em>backlinks: %s</em>'
                                % ', '.join(backlinks))
        if node.hasattr('line'):
            line = ', line %s' % node['line']
        else:
            line = ''
        self.body.append('System Message: %s/%s '
                         '(<span class="docutils literal">%s</span>%s)%s</p>\n'
                         % (node['type'], node['level'],
                            self.encode(node['source']), line, backref_text))

    def depart_system_message(self, node):
        self.body.append('</aside>\n')

    def visit_table(self, node):
        atts = {'classes': self.settings.table_style.replace(',', ' ').split()}
        if 'align' in node:
            atts['classes'].append('align-%s' % node['align'])
        if 'width' in node:
            atts['style'] = 'width: %s;' % node['width']
        tag = self.starttag(node, 'table', **atts)
        self.body.append(tag)

    def depart_table(self, node):
        self.body.append('</table>\n')

    def visit_target(self, node):
        if ('refuri' not in node
                and 'refid' not in node
                and 'refname' not in node):
            self.body.append(self.starttag(node, 'span', '', CLASS='target'))
            self.context.append('</span>')
        else:
            self.context.append('')

    def depart_target(self, node):
        self.body.append(self.context.pop())

    # no hard-coded vertical alignment in table body
    def visit_tbody(self, node):
        self.body.append(self.starttag(node, 'tbody'))

    def depart_tbody(self, node):
        self.body.append('</tbody>\n')

    def visit_term(self, node):
        if 'details' in node.parent.parent['classes']:
            self.body.append(self.starttag(node, 'summary', suffix=''))
        else:
            # The parent node (definition_list_item) is omitted in HTML.
            self.body.append(self.starttag(node, 'dt', suffix='',
                                           classes=node.parent['classes'],
                                           ids=node.parent['ids']))

    def depart_term(self, node):
        # Leave the end tag to `self.visit_definition()`,
        # in case there's a classifier.
        pass

    def visit_tgroup(self, node):
        self.colspecs = []
        node.stubs = []

    def depart_tgroup(self, node):
        pass

    def visit_thead(self, node):
        self.body.append(self.starttag(node, 'thead'))

    def depart_thead(self, node):
        self.body.append('</thead>\n')

    def section_title_tags(self, node):
        atts = {}
        h_level = self.section_level + self.initial_header_level - 1
        # Only 6 heading levels have dedicated HTML tags.
        tagname = 'h%i' % min(h_level, 6)
        if h_level > 6:
            atts['aria-level'] = h_level
        start_tag = self.starttag(node, tagname, '', **atts)
        if node.hasattr('refid'):
            atts = {}
            atts['class'] = 'toc-backref'
            atts['role'] = 'doc-backlink'  # HTML5 only
            atts['href'] = '#' + node['refid']
            start_tag += self.starttag(nodes.reference(), 'a', '', **atts)
            close_tag = '</a></%s>\n' % tagname
        else:
            close_tag = '</%s>\n' % tagname
        return start_tag, close_tag

    def visit_title(self, node):
        close_tag = '</p>\n'
        if isinstance(node.parent, nodes.topic):
            # TODO: use role="heading" or <h1>? (HTML5 only)
            self.body.append(
                self.starttag(node, 'p', '', CLASS='topic-title'))
        elif isinstance(node.parent, nodes.sidebar):
            # TODO: use role="heading" or <h1>? (HTML5 only)
            self.body.append(
                self.starttag(node, 'p', '', CLASS='sidebar-title'))
        elif isinstance(node.parent, nodes.Admonition):
            self.body.append(
                  self.starttag(node, 'p', '', CLASS='admonition-title'))
        elif isinstance(node.parent, nodes.table):
            self.body.append(
                  self.starttag(node, 'caption', ''))
            close_tag = '</caption>\n'
        elif isinstance(node.parent, nodes.document):
            self.body.append(self.starttag(node, 'h1', '', CLASS='title'))
            close_tag = '</h1>\n'
            self.in_document_title = len(self.body)
        else:
            assert isinstance(node.parent, nodes.section)
            # Get correct heading and evt. backlink tags
            start_tag, close_tag = self.section_title_tags(node)
            self.body.append(start_tag)
        self.context.append(close_tag)

    def depart_title(self, node):
        self.body.append(self.context.pop())
        if self.in_document_title:
            self.title = self.body[self.in_document_title:-1]
            self.in_document_title = 0
            self.body_pre_docinfo.extend(self.body)
            self.html_title.extend(self.body)
            del self.body[:]

    def visit_title_reference(self, node):
        self.body.append(self.starttag(node, 'cite', ''))

    def depart_title_reference(self, node):
        self.body.append('</cite>')

    def visit_topic(self, node):
        self.body.append(self.starttag(node, 'div', CLASS='topic'))

    def depart_topic(self, node):
        self.body.append('</div>\n')

    def visit_transition(self, node):
        self.body.append(self.emptytag(node, 'hr', CLASS='docutils'))

    def depart_transition(self, node):
        pass

    def visit_version(self, node):
        self.visit_docinfo_item(node, 'version', meta=False)

    def depart_version(self, node):
        self.depart_docinfo_item()

    def unimplemented_visit(self, node):
        raise NotImplementedError('visiting unimplemented node type: %s'
                                  % node.__class__.__name__)


class SimpleListChecker(nodes.GenericNodeVisitor):

    """
    Raise `nodes.NodeFound` if non-simple list item is encountered.

    Here "simple" means a list item containing nothing other than a single
    paragraph, a simple list, or a paragraph followed by a simple list.

    This version also checks for simple field lists and docinfo.
    """

    def default_visit(self, node):
        raise nodes.NodeFound

    def visit_list_item(self, node):
        children = [child for child in node.children
                    if not isinstance(child, nodes.Invisible)]
        if (children and isinstance(children[0], nodes.paragraph)
            and (isinstance(children[-1], nodes.bullet_list)
                 or isinstance(children[-1], nodes.enumerated_list)
                 or isinstance(children[-1], nodes.field_list))):
            children.pop()
        if len(children) <= 1:
            return
        else:
            raise nodes.NodeFound

    def pass_node(self, node):
        pass

    def ignore_node(self, node):
        # ignore nodes that are never complex (can contain only inline nodes)
        raise nodes.SkipNode

    # Paragraphs and text
    visit_Text = ignore_node
    visit_paragraph = ignore_node

    # Lists
    visit_bullet_list = pass_node
    visit_enumerated_list = pass_node
    visit_docinfo = pass_node

    # Docinfo nodes:
    visit_author = ignore_node
    visit_authors = visit_list_item
    visit_address = visit_list_item
    visit_contact = pass_node
    visit_copyright = ignore_node
    visit_date = ignore_node
    visit_organization = ignore_node
    visit_status = ignore_node
    visit_version = visit_list_item

    # Definition list:
    visit_definition_list = pass_node
    visit_definition_list_item = pass_node
    visit_term = ignore_node
    visit_classifier = pass_node
    visit_definition = visit_list_item

    # Field list:
    visit_field_list = pass_node
    visit_field = pass_node
    # the field body corresponds to a list item
    visit_field_body = visit_list_item
    visit_field_name = ignore_node

    # Invisible nodes should be ignored.
    visit_comment = ignore_node
    visit_substitution_definition = ignore_node
    visit_target = ignore_node
    visit_pending = ignore_node

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/_html_base.py
NikolaBlagojevic/pyrecodes
# $Id: docutils_xml.py 9239 2022-11-13 16:14:59Z milde $
# Author: David Goodger, Paul Tremblay, Guenter Milde
# Maintainer: docutils-develop@lists.sourceforge.net
# Copyright: This module has been placed in the public domain.

"""
Simple document tree Writer, writes Docutils XML according to
https://docutils.sourceforge.io/docs/ref/docutils.dtd.
"""

__docformat__ = 'reStructuredText'

from io import StringIO
import xml.sax.saxutils

import docutils
from docutils import frontend, nodes, writers, utils


class RawXmlError(docutils.ApplicationError):
    pass


class Writer(writers.Writer):

    supported = ('xml',)
    """Formats this writer supports."""

    settings_spec = (
        '"Docutils XML" Writer Options',
        None,
        (('Generate XML with newlines before and after tags.',
          ['--newlines'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Generate XML with indents and newlines.',
          ['--indents'],  # TODO use integer value for number of spaces?
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Omit the XML declaration.  Use with caution.',
          ['--no-xml-declaration'],
          {'dest': 'xml_declaration', 'default': 1, 'action': 'store_false',
           'validator': frontend.validate_boolean}),
         ('Omit the DOCTYPE declaration.',
          ['--no-doctype'],
          {'dest': 'doctype_declaration', 'default': 1,
           'action': 'store_false', 'validator': frontend.validate_boolean}),))

    settings_defaults = {'output_encoding_error_handler': 'xmlcharrefreplace'}

    config_section = 'docutils_xml writer'
    config_section_dependencies = ('writers',)

    output = None
    """Final translated form of `document`."""

    def __init__(self):
        writers.Writer.__init__(self)
        self.translator_class = XMLTranslator

    def translate(self):
        self.visitor = visitor = self.translator_class(self.document)
        self.document.walkabout(visitor)
        self.output = ''.join(visitor.output)


class XMLTranslator(nodes.GenericNodeVisitor):

    # TODO: add stylesheet options similar to HTML and LaTeX writers?
    # xml_stylesheet = '<?xml-stylesheet type="text/xsl" href="%s"?>\n'
    doctype = (
        '<!DOCTYPE document PUBLIC'
        ' "+//IDN docutils.sourceforge.net//DTD Docutils Generic//EN//XML"'
        ' "http://docutils.sourceforge.net/docs/ref/docutils.dtd">\n')
    generator = '<!-- Generated by Docutils %s -->\n'

    xmlparser = xml.sax.make_parser()
    """SAX parser instance to check/extract raw XML."""
    xmlparser.setFeature(
        "http://xml.org/sax/features/external-general-entities", True)

    def __init__(self, document):
        nodes.NodeVisitor.__init__(self, document)

        # Reporter
        self.warn = self.document.reporter.warning
        self.error = self.document.reporter.error

        # Settings
        self.settings = settings = document.settings
        self.indent = self.newline = ''
        if settings.newlines:
            self.newline = '\n'
        if settings.indents:
            self.newline = '\n'
            self.indent = '    '  # TODO make this configurable?
        self.level = 0       # indentation level
        self.in_simple = 0   # level of nesting inside mixed-content elements
        self.fixed_text = 0  # level of nesting inside FixedText elements

        # Output
        self.output = []
        if settings.xml_declaration:
            self.output.append(utils.xml_declaration(settings.output_encoding))
        if settings.doctype_declaration:
            self.output.append(self.doctype)
        self.output.append(self.generator % docutils.__version__)

        # initialize XML parser
        self.the_handle = TestXml()
        self.xmlparser.setContentHandler(self.the_handle)

    # generic visit and depart methods
    # --------------------------------

    simple_nodes = (nodes.TextElement,
                    nodes.image, nodes.colspec, nodes.transition)

    def default_visit(self, node):
        """Default node visit method."""
        if not self.in_simple:
            self.output.append(self.indent*self.level)
        self.output.append(node.starttag(xml.sax.saxutils.quoteattr))
        self.level += 1
        # @@ make nodes.literal an instance of FixedTextElement?
        if isinstance(node, (nodes.FixedTextElement, nodes.literal)):
            self.fixed_text += 1
        if isinstance(node, self.simple_nodes):
            self.in_simple += 1
        if not self.in_simple:
            self.output.append(self.newline)

    def default_departure(self, node):
        """Default node depart method."""
        self.level -= 1
        if not self.in_simple:
            self.output.append(self.indent*self.level)
        self.output.append(node.endtag())
        if isinstance(node, (nodes.FixedTextElement, nodes.literal)):
            self.fixed_text -= 1
        if isinstance(node, self.simple_nodes):
            self.in_simple -= 1
        if not self.in_simple:
            self.output.append(self.newline)

    # specific visit and depart methods
    # ---------------------------------

    def visit_Text(self, node):
        text = xml.sax.saxutils.escape(node.astext())
        # indent text if we are not in a FixedText element:
        if not self.fixed_text:
            text = text.replace('\n', '\n'+self.indent*self.level)
        self.output.append(text)

    def depart_Text(self, node):
        pass

    def visit_raw(self, node):
        if 'xml' not in node.get('format', '').split():
            # skip other raw content?
            # raise nodes.SkipNode
            self.default_visit(node)
            return
        # wrap in <raw> element
        self.default_visit(node)      # or not?
        xml_string = node.astext()
        self.output.append(xml_string)
        self.default_departure(node)  # or not?
        # Check validity of raw XML:
        try:
            self.xmlparser.parse(StringIO(xml_string))
        except xml.sax._exceptions.SAXParseException:
            col_num = self.the_handle.locator.getColumnNumber()
            line_num = self.the_handle.locator.getLineNumber()
            srcline = node.line
            if not isinstance(node.parent, nodes.TextElement):
                srcline += 2  # directive content start line
            msg = 'Invalid raw XML in column %d, line offset %d:\n%s' % (
                   col_num, line_num, node.astext())
            self.warn(msg, source=node.source, line=srcline+line_num-1)
        raise nodes.SkipNode  # content already processed


class TestXml(xml.sax.handler.ContentHandler):

    def setDocumentLocator(self, locator):
        self.locator = locator

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/docutils_xml.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9282 2022-11-28 23:55:46Z milde $
# Author: David Goodger
# Maintainer: docutils-develop@lists.sourceforge.net
# Copyright: This module has been placed in the public domain.

"""
Simple HyperText Markup Language document tree Writer.

The output conforms to the XHTML version 1.0 Transitional DTD
(*almost* strict).  The output contains a minimum of formatting
information.  The cascading style sheet "html4css1.css" is required
for proper viewing with a modern graphical browser.
"""

__docformat__ = 'reStructuredText'

import os.path
import re

from docutils import frontend, nodes, writers
from docutils.writers import _html_base
from docutils.writers._html_base import PIL, url2pathname


class Writer(writers._html_base.Writer):

    supported = ('html', 'html4', 'html4css1', 'xhtml', 'xhtml10')
    """Formats this writer supports."""

    default_stylesheets = ['html4css1.css']
    default_stylesheet_dirs = ['.',
                               os.path.abspath(os.path.dirname(__file__)),
                               os.path.abspath(os.path.join(
                                   os.path.dirname(os.path.dirname(__file__)),
                                   'html5_polyglot'))  # for math.css
                               ]
    default_template = os.path.join(
        os.path.dirname(os.path.abspath(__file__)), 'template.txt')

    # use a copy of the parent spec with some modifications
    settings_spec = frontend.filter_settings_spec(
        writers._html_base.Writer.settings_spec,
        template=(
            'Template file. (UTF-8 encoded, default: "%s")' % default_template,
            ['--template'],
            {'default': default_template, 'metavar': '<file>'}),
        stylesheet_path=(
            'Comma separated list of stylesheet paths. '
            'Relative paths are expanded if a matching file is found in '
            'the --stylesheet-dirs. With --link-stylesheet, '
            'the path is rewritten relative to the output HTML file. '
            '(default: "%s")' % ','.join(default_stylesheets),
            ['--stylesheet-path'],
            {'metavar': '<file[,file,...]>', 'overrides': 'stylesheet',
             'validator': frontend.validate_comma_separated_list,
             'default': default_stylesheets}),
        stylesheet_dirs=(
            'Comma-separated list of directories where stylesheets are found. '
            'Used by --stylesheet-path when expanding relative path '
            'arguments. (default: "%s")' % ','.join(default_stylesheet_dirs),
            ['--stylesheet-dirs'],
            {'metavar': '<dir[,dir,...]>',
             'validator': frontend.validate_comma_separated_list,
             'default': default_stylesheet_dirs}),
        initial_header_level=(
            'Specify the initial header level. Does not affect document '
            'title & subtitle (see --no-doc-title). (default: 1 for "<h1>")',
            ['--initial-header-level'],
            {'choices': '1 2 3 4 5 6'.split(), 'default': '1',
             'metavar': '<level>'}),
        xml_declaration=(
            'Prepend an XML declaration (default). ',
            ['--xml-declaration'],
            {'default': True, 'action': 'store_true',
             'validator': frontend.validate_boolean}),
        )
    settings_spec = settings_spec + (
        'HTML4 Writer Options',
        '',
        (('Specify the maximum width (in characters) for one-column field '
          'names.  Longer field names will span an entire row of the table '
          'used to render the field list.  Default is 14 characters.  '
          'Use 0 for "no limit".',
          ['--field-name-limit'],
          {'default': 14, 'metavar': '<level>',
           'validator': frontend.validate_nonnegative_int}),
         ('Specify the maximum width (in characters) for options in option '
          'lists.  Longer options will span an entire row of the table used '
          'to render the option list.  Default is 14 characters.  '
          'Use 0 for "no limit".',
          ['--option-limit'],
          {'default': 14, 'metavar': '<level>',
           'validator': frontend.validate_nonnegative_int}),
         )
        )

    config_section = 'html4css1 writer'

    def __init__(self):
        self.parts = {}
        self.translator_class = HTMLTranslator


class HTMLTranslator(writers._html_base.HTMLTranslator):
    """
    The html4css1 writer has been optimized to produce visually compact
    lists (less vertical whitespace).  HTML's mixed content models
    allow list items to contain "<li><p>body elements</p></li>" or
    "<li>just text</li>" or even "<li>text<p>and body
    elements</p>combined</li>", each with different effects.  It would
    be best to stick with strict body elements in list items, but they
    affect vertical spacing in older browsers (although they really
    shouldn't).
    The html5_polyglot writer solves this using CSS2.

    Here is an outline of the optimization:

    - Check for and omit <p> tags in "simple" lists: list items
      contain either a single paragraph, a nested simple list, or a
      paragraph followed by a nested simple list.  This means that
      this list can be compact:

          - Item 1.
          - Item 2.

      But this list cannot be compact:

          - Item 1.

            This second paragraph forces space between list items.

          - Item 2.

    - In non-list contexts, omit <p> tags on a paragraph if that
      paragraph is the only child of its parent (footnotes & citations
      are allowed a label first).

    - Regardless of the above, in definitions, table cells, field bodies,
      option descriptions, and list items, mark the first child with
      'class="first"' and the last child with 'class="last"'.  The stylesheet
      sets the margins (top & bottom respectively) to 0 for these elements.

    The ``no_compact_lists`` setting (``--no-compact-lists`` command-line
    option) disables list whitespace optimization.
    """

    # The following definitions are required for display in browsers limited
    # to CSS1 or backwards compatible behaviour of the writer:

    doctype = (
        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"'
        ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n')

    content_type = ('<meta http-equiv="Content-Type"'
                    ' content="text/html; charset=%s" />\n')
    content_type_mathml = ('<meta http-equiv="Content-Type"'
                           ' content="application/xhtml+xml; charset=%s" />\n')

    # encode also non-breaking space
    special_characters = _html_base.HTMLTranslator.special_characters.copy()
    special_characters[0xa0] = '&nbsp;'

    # use character reference for dash (not valid in HTML5)
    attribution_formats = {'dash': ('&mdash;', ''),
                           'parentheses': ('(', ')'),
                           'parens': ('(', ')'),
                           'none': ('', '')}

    # ersatz for first/last pseudo-classes missing in CSS1
    def set_first_last(self, node):
        self.set_class_on_child(node, 'first', 0)
        self.set_class_on_child(node, 'last', -1)

    # add newline after opening tag
    def visit_address(self, node):
        self.visit_docinfo_item(node, 'address', meta=False)
        self.body.append(self.starttag(node, 'pre', CLASS='address'))

    def depart_address(self, node):
        self.body.append('\n</pre>\n')
        self.depart_docinfo_item()

    # ersatz for first/last pseudo-classes
    def visit_admonition(self, node):
        node['classes'].insert(0, 'admonition')
        self.body.append(self.starttag(node, 'div'))
        self.set_first_last(node)

    def depart_admonition(self, node=None):
        self.body.append('</div>\n')

    # author, authors: use <br> instead of paragraphs
    def visit_author(self, node):
        if isinstance(node.parent, nodes.authors):
            if self.author_in_authors:
                self.body.append('\n<br />')
        else:
            self.visit_docinfo_item(node, 'author')

    def depart_author(self, node):
        if isinstance(node.parent, nodes.authors):
            self.author_in_authors = True
        else:
            self.depart_docinfo_item()

    def visit_authors(self, node):
        self.visit_docinfo_item(node, 'authors')
        self.author_in_authors = False  # initialize

    def depart_authors(self, node):
        self.depart_docinfo_item()

    # use "width" argument instead of "style: 'width'":
    def visit_colspec(self, node):
        self.colspecs.append(node)
        # "stubs" list is an attribute of the tgroup element:
        node.parent.stubs.append(node.attributes.get('stub'))

    def depart_colspec(self, node):
        # write out <colgroup> when all colspecs are processed
        if isinstance(node.next_node(descend=False, siblings=True),
                      nodes.colspec):
            return
        if ('colwidths-auto' in node.parent.parent['classes']
            or ('colwidths-auto' in self.settings.table_style
                and 'colwidths-given' not in node.parent.parent['classes'])):
            return
        total_width = sum(node['colwidth'] for node in self.colspecs)
        self.body.append(self.starttag(node, 'colgroup'))
        for node in self.colspecs:
            colwidth = int(node['colwidth'] * 100.0 / total_width + 0.5)
            self.body.append(self.emptytag(node, 'col',
                                           width='%i%%' % colwidth))
        self.body.append('</colgroup>\n')

    # Compact lists:
    # exclude definition lists and field lists (non-compact by default)

    def is_compactable(self, node):
        return ('compact' in node['classes']
                or (self.settings.compact_lists
                    and 'open' not in node['classes']
                    and (self.compact_simple
                         or 'contents' in node.parent['classes']
                         # TODO: self.in_contents
                         or self.check_simple_list(node))))

    # citations: Use table for bibliographic references.
    def visit_citation(self, node):
        self.body.append(self.starttag(node, 'table',
                                       CLASS='docutils citation',
                                       frame="void", rules="none"))
        self.body.append('<colgroup><col class="label" /><col /></colgroup>\n'
                         '<tbody valign="top">\n'
                         '<tr>')
        self.footnote_backrefs(node)

    def depart_citation(self, node):
        self.body.append('</td></tr>\n'
                         '</tbody>\n</table>\n')

    def visit_citation_reference(self, node):
        href = '#'
        if 'refid' in node:
            href += node['refid']
        elif 'refname' in node:
            href += self.document.nameids[node['refname']]
        self.body.append(self.starttag(node, 'a', suffix='[', href=href,
                                       classes=['citation-reference']))

    def depart_citation_reference(self, node):
        self.body.append(']</a>')

    # insert classifier-delimiter (not required with CSS2)
    def visit_classifier(self, node):
        self.body.append(' <span class="classifier-delimiter">:</span> ')
        self.body.append(self.starttag(node, 'span', '', CLASS='classifier'))

    def depart_classifier(self, node):
        self.body.append('</span>')

    # ersatz for first/last pseudo-classes
    def visit_compound(self, node):
        self.body.append(self.starttag(node, 'div', CLASS='compound'))
        if len(node) > 1:
            node[0]['classes'].append('compound-first')
            node[-1]['classes'].append('compound-last')
            for child in node[1:-1]:
                child['classes'].append('compound-middle')

    def depart_compound(self, node):
        self.body.append('</div>\n')

    # ersatz for first/last pseudo-classes, no special handling of "details"
    def visit_definition(self, node):
        self.body.append('</dt>\n')
        self.body.append(self.starttag(node, 'dd', ''))
        self.set_first_last(node)

    def depart_definition(self, node):
        self.body.append('</dd>\n')

    # don't add "simple" class value, no special handling of "details"
    def visit_definition_list(self, node):
        self.body.append(self.starttag(node, 'dl', CLASS='docutils'))

    def depart_definition_list(self, node):
        self.body.append('</dl>\n')

    # no special handling of "details"
    def visit_definition_list_item(self, node):
        pass

    def depart_definition_list_item(self, node):
        pass

    # use a table for description lists
    def visit_description(self, node):
        self.body.append(self.starttag(node, 'td', ''))
        self.set_first_last(node)

    def depart_description(self, node):
        self.body.append('</td>')

    # use table for docinfo
    def visit_docinfo(self, node):
        self.context.append(len(self.body))
        self.body.append(self.starttag(node, 'table',
                                       CLASS='docinfo',
                                       frame="void", rules="none"))
        self.body.append('<col class="docinfo-name" />\n'
                         '<col class="docinfo-content" />\n'
                         '<tbody valign="top">\n')
        self.in_docinfo = True

    def depart_docinfo(self, node):
        self.body.append('</tbody>\n</table>\n')
        self.in_docinfo = False
        start = self.context.pop()
        self.docinfo = self.body[start:]
        self.body = []

    def visit_docinfo_item(self, node, name, meta=True):
        if meta:
            meta_tag = '<meta name="%s" content="%s" />\n' \
                       % (name, self.attval(node.astext()))
            self.meta.append(meta_tag)
        self.body.append(self.starttag(node, 'tr', ''))
        self.body.append('<th class="docinfo-name">%s:</th>\n<td>'
                         % self.language.labels[name])
        if len(node):
            if isinstance(node[0], nodes.Element):
                node[0]['classes'].append('first')
            if isinstance(node[-1], nodes.Element):
                node[-1]['classes'].append('last')

    def depart_docinfo_item(self):
        self.body.append('</td></tr>\n')

    # add newline after opening tag
    def visit_doctest_block(self, node):
        self.body.append(self.starttag(node, 'pre', CLASS='doctest-block'))

    def depart_doctest_block(self, node):
        self.body.append('\n</pre>\n')

    # insert an NBSP into empty cells, ersatz for first/last
    def visit_entry(self, node):
        writers._html_base.HTMLTranslator.visit_entry(self, node)
        if len(node) == 0:              # empty cell
            self.body.append('&nbsp;')
        self.set_first_last(node)

    def depart_entry(self, node):
        self.body.append(self.context.pop())

    # ersatz for first/last pseudo-classes
    def visit_enumerated_list(self, node):
        """
        The 'start' attribute does not conform to HTML 4.01's strict.dtd, but
        cannot be emulated in CSS1 (HTML 5 reincludes it).
        """
        atts = {}
        if 'start' in node:
            atts['start'] = node['start']
        if 'enumtype' in node:
            atts['class'] = node['enumtype']
        # @@@ To do: prefix, suffix. How? Change prefix/suffix to a
        # single "format" attribute? Use CSS2?
        old_compact_simple = self.compact_simple
        self.context.append((self.compact_simple, self.compact_p))
        self.compact_p = None
        self.compact_simple = self.is_compactable(node)
        if self.compact_simple and not old_compact_simple:
            atts['class'] = (atts.get('class', '') + ' simple').strip()
        self.body.append(self.starttag(node, 'ol', **atts))

    def depart_enumerated_list(self, node):
        self.compact_simple, self.compact_p = self.context.pop()
        self.body.append('</ol>\n')

    # use table for field-list:
    def visit_field(self, node):
        self.body.append(self.starttag(node, 'tr', '', CLASS='field'))

    def depart_field(self, node):
        self.body.append('</tr>\n')

    def visit_field_body(self, node):
        self.body.append(self.starttag(node, 'td', '', CLASS='field-body'))
        self.set_class_on_child(node, 'first', 0)
        field = node.parent
        if (self.compact_field_list
            or isinstance(field.parent, nodes.docinfo)
            or field.parent.index(field) == len(field.parent) - 1):
            # If we are in a compact list, the docinfo, or if this is
            # the last field of the field list, do not add vertical
            # space after last element.
            self.set_class_on_child(node, 'last', -1)

    def depart_field_body(self, node):
        self.body.append('</td>\n')

    def visit_field_list(self, node):
        self.context.append((self.compact_field_list, self.compact_p))
        self.compact_p = None
        if 'compact' in node['classes']:
            self.compact_field_list = True
        elif (self.settings.compact_field_lists
              and 'open' not in node['classes']):
            self.compact_field_list = True
        if self.compact_field_list:
            for field in node:
                field_body = field[-1]
                assert isinstance(field_body, nodes.field_body)
                children = [n for n in field_body
                            if not isinstance(n, nodes.Invisible)]
                if not (len(children) == 0
                        or len(children) == 1
                        and isinstance(children[0],
                                       (nodes.paragraph, nodes.line_block))):
                    self.compact_field_list = False
                    break
        self.body.append(self.starttag(node, 'table', frame='void',
                                       rules='none',
                                       CLASS='docutils field-list'))
        self.body.append('<col class="field-name" />\n'
                         '<col class="field-body" />\n'
                         '<tbody valign="top">\n')

    def depart_field_list(self, node):
        self.body.append('</tbody>\n</table>\n')
        self.compact_field_list, self.compact_p = self.context.pop()

    def visit_field_name(self, node):
        atts = {}
        if self.in_docinfo:
            atts['class'] = 'docinfo-name'
        else:
            atts['class'] = 'field-name'
        if (self.settings.field_name_limit
            and len(node.astext()) > self.settings.field_name_limit):
            atts['colspan'] = 2
            self.context.append('</tr>\n'
                                + self.starttag(node.parent, 'tr', '',
                                                CLASS='field')
                                + '<td>&nbsp;</td>')
        else:
            self.context.append('')
        self.body.append(self.starttag(node, 'th', '', **atts))

    def depart_field_name(self, node):
        self.body.append(':</th>')
        self.body.append(self.context.pop())

    # use table for footnote text
    def visit_footnote(self, node):
        self.body.append(self.starttag(node, 'table',
                                       CLASS='docutils footnote',
                                       frame="void", rules="none"))
        self.body.append('<colgroup><col class="label" /><col /></colgroup>\n'
                         '<tbody valign="top">\n'
                         '<tr>')
        self.footnote_backrefs(node)

    def footnote_backrefs(self, node):
        backlinks = []
        backrefs = node['backrefs']
        if self.settings.footnote_backlinks and backrefs:
            if len(backrefs) == 1:
                self.context.append('')
                self.context.append('</a>')
                self.context.append('<a class="fn-backref" href="#%s">'
                                    % backrefs[0])
            else:
                for (i, backref) in enumerate(backrefs, 1):
                    backlinks.append('<a class="fn-backref" href="#%s">%s</a>'
                                     % (backref, i))
                self.context.append('<em>(%s)</em> ' % ', '.join(backlinks))
                self.context += ['', '']
        else:
            self.context.append('')
            self.context += ['', '']
        # If the node does not only consist of a label.
        if len(node) > 1:
            # If there are preceding backlinks, we do not set class
            # 'first', because we need to retain the top-margin.
            if not backlinks:
                node[1]['classes'].append('first')
            node[-1]['classes'].append('last')

    def depart_footnote(self, node):
        self.body.append('</td></tr>\n'
                         '</tbody>\n</table>\n')

    # insert markers in text (pseudo-classes are not supported in CSS1):
    def visit_footnote_reference(self, node):
        href = '#' + node['refid']
        format = self.settings.footnote_references
        if format == 'brackets':
            suffix = '['
            self.context.append(']')
        else:
            assert format == 'superscript'
            suffix = '<sup>'
            self.context.append('</sup>')
        self.body.append(self.starttag(node, 'a', suffix,
                                       CLASS='footnote-reference', href=href))

    def depart_footnote_reference(self, node):
        self.body.append(self.context.pop() + '</a>')

    # just pass on generated text
    def visit_generated(self, node):
        pass

    # Backwards-compatibility implementation:
    # * Do not use <video>,
    # * don't embed images,
    # * use <object> instead of <img> for SVG.
    #   (SVG not supported by IE up to version 8,
    #   html4css1 strives for IE6 compatibility.)
    object_image_types = {'.svg': 'image/svg+xml',
                          '.swf': 'application/x-shockwave-flash'}

    def visit_image(self, node):
        atts = {}
        uri = node['uri']
        ext = os.path.splitext(uri)[1].lower()
        if ext in self.object_image_types:
            atts['data'] = uri
            atts['type'] = self.object_image_types[ext]
        else:
            atts['src'] = uri
            atts['alt'] = node.get('alt', uri)
        # image size
        if 'width' in node:
            atts['width'] = node['width']
        if 'height' in node:
            atts['height'] = node['height']
        if 'scale' in node:
            if (PIL and ('width' not in node or 'height' not in node)
                and self.settings.file_insertion_enabled):
                imagepath = url2pathname(uri)
                try:
                    with PIL.Image.open(imagepath) as img:
                        img_size = img.size
                except (OSError, UnicodeEncodeError):
                    pass  # TODO: warn/info?
                else:
                    self.settings.record_dependencies.add(
                        imagepath.replace('\\', '/'))
                    if 'width' not in atts:
                        atts['width'] = '%dpx' % img_size[0]
                    if 'height' not in atts:
                        atts['height'] = '%dpx' % img_size[1]
            for att_name in 'width', 'height':
                if att_name in atts:
                    match = re.match(r'([0-9.]+)(\S*)$', atts[att_name])
                    assert match
                    atts[att_name] = '%s%s' % (
                        float(match.group(1)) * (float(node['scale']) / 100),
                        match.group(2))
        style = []
        for att_name in 'width', 'height':
            if att_name in atts:
                if re.match(r'^[0-9.]+$', atts[att_name]):
                    # Interpret unitless values as pixels.
                    atts[att_name] += 'px'
                style.append('%s: %s;' % (att_name, atts[att_name]))
                del atts[att_name]
        if style:
            atts['style'] = ' '.join(style)
        if (isinstance(node.parent, nodes.TextElement)
            or (isinstance(node.parent, nodes.reference)
                and not isinstance(node.parent.parent, nodes.TextElement))):
            # Inline context or surrounded by <a>...</a>.
            suffix = ''
        else:
            suffix = '\n'
        if 'align' in node:
            atts['class'] = 'align-%s' % node['align']
        if ext in self.object_image_types:
            # do NOT use an empty tag: incorrect rendering in browsers
            self.body.append(self.starttag(node, 'object', '', **atts)
                             + node.get('alt', uri) + '</object>' + suffix)
        else:
            self.body.append(self.emptytag(node, 'img', suffix, **atts))

    def depart_image(self, node):
        pass

    # use table for footnote text,
    # context added in footnote_backrefs.
    def visit_label(self, node):
        self.body.append(self.starttag(node, 'td', '%s[' % self.context.pop(),
                                       CLASS='label'))

    def depart_label(self, node):
        self.body.append(f']{self.context.pop()}</td><td>{self.context.pop()}')

    # ersatz for first/last pseudo-classes
    def visit_list_item(self, node):
        self.body.append(self.starttag(node, 'li', ''))
        if len(node):
            node[0]['classes'].append('first')

    def depart_list_item(self, node):
        self.body.append('</li>\n')

    # use <tt> (not supported by HTML5),
    # cater for limited styling options in CSS1 using hard-coded NBSPs
    def visit_literal(self, node):
        # special case: "code" role
        classes = node['classes']
        if 'code' in classes:
            # filter 'code' from class arguments
            node['classes'] = [cls for cls in classes if cls != 'code']
            self.body.append(self.starttag(node, 'code', ''))
            return
        self.body.append(
            self.starttag(node, 'tt', '', CLASS='docutils literal'))
        text = node.astext()
        for token in self.words_and_spaces.findall(text):
            if token.strip():
                # Protect text like "--an-option" and the regular expression
                # ``[+]?(\d+(\.\d*)?|\.\d+)`` from bad line wrapping
                if self.in_word_wrap_point.search(token):
                    self.body.append('<span class="pre">%s</span>'
                                     % self.encode(token))
                else:
                    self.body.append(self.encode(token))
            elif token in ('\n', ' '):
                # Allow breaks at whitespace:
                self.body.append(token)
            else:
                # Protect runs of multiple spaces; the last space can wrap:
                self.body.append('&nbsp;' * (len(token) - 1) + ' ')
        self.body.append('</tt>')
        # Content already processed:
        raise nodes.SkipNode

    def depart_literal(self, node):
        # skipped unless literal element is from "code" role:
        self.body.append('</code>')

    # add newline after wrapper tags, don't use <code> for code
    def visit_literal_block(self, node):
        self.body.append(self.starttag(node, 'pre', CLASS='literal-block'))

    def depart_literal_block(self, node):
        self.body.append('\n</pre>\n')

    # use table for option list
    def visit_option_group(self, node):
        atts = {}
        if (self.settings.option_limit
            and len(node.astext()) > self.settings.option_limit):
            atts['colspan'] = 2
            self.context.append('</tr>\n<tr><td>&nbsp;</td>')
        else:
            self.context.append('')
        self.body.append(
            self.starttag(node, 'td', CLASS='option-group', **atts))
        self.body.append('<kbd>')
        self.context.append(0)          # count number of options

    def depart_option_group(self, node):
        self.context.pop()
        self.body.append('</kbd></td>\n')
        self.body.append(self.context.pop())

    def visit_option_list(self, node):
        self.body.append(
              self.starttag(node, 'table', CLASS='docutils option-list',
                            frame="void", rules="none"))
        self.body.append('<col class="option" />\n'
                         '<col class="description" />\n'
                         '<tbody valign="top">\n')

    def depart_option_list(self, node):
        self.body.append('</tbody>\n</table>\n')

    def visit_option_list_item(self, node):
        self.body.append(self.starttag(node, 'tr', ''))

    def depart_option_list_item(self, node):
        self.body.append('</tr>\n')

    # Omit <p> tags to produce visually compact lists (less vertical
    # whitespace) as CSS styling requires CSS2.
    def should_be_compact_paragraph(self, node):
        """
        Determine if the <p> tags around paragraph ``node`` can be omitted.
        """
        if (isinstance(node.parent, nodes.document)
            or isinstance(node.parent, nodes.compound)):
            # Never compact paragraphs in document or compound.
            return False
        for key, value in node.attlist():
            if (node.is_not_default(key)
                and not (key == 'classes'
                         and value in ([], ['first'],
                                       ['last'], ['first', 'last']))):
                # Attribute which needs to survive.
                return False
        first = isinstance(node.parent[0], nodes.label)  # skip label
        for child in node.parent.children[first:]:
            # only first paragraph can be compact
            if isinstance(child, nodes.Invisible):
                continue
            if child is node:
                break
            return False
        parent_length = len([n for n in node.parent if not isinstance(
            n, (nodes.Invisible, nodes.label))])
        if (self.compact_simple
            or self.compact_field_list
            or self.compact_p and parent_length == 1):
            return True
        return False

    def visit_paragraph(self, node):
        if self.should_be_compact_paragraph(node):
            self.context.append('')
        else:
            self.body.append(self.starttag(node, 'p', ''))
            self.context.append('</p>\n')

    def depart_paragraph(self, node):
        self.body.append(self.context.pop())

    # ersatz for first/last pseudo-classes
    def visit_sidebar(self, node):
        self.body.append(
            self.starttag(node, 'div', CLASS='sidebar'))
        self.set_first_last(node)
        self.in_sidebar = True

    def depart_sidebar(self, node):
        self.body.append('</div>\n')
        self.in_sidebar = False

    # <sub> not allowed in <pre>
    def visit_subscript(self, node):
        if isinstance(node.parent, nodes.literal_block):
            self.body.append(self.starttag(node, 'span', '',
                                           CLASS='subscript'))
        else:
            self.body.append(self.starttag(node, 'sub', ''))

    def depart_subscript(self, node):
        if isinstance(node.parent, nodes.literal_block):
            self.body.append('</span>')
        else:
            self.body.append('</sub>')

    # Use <h*> for subtitles (deprecated in HTML 5)
    def visit_subtitle(self, node):
        if isinstance(node.parent, nodes.sidebar):
            self.body.append(self.starttag(node, 'p', '',
                                           CLASS='sidebar-subtitle'))
            self.context.append('</p>\n')
        elif isinstance(node.parent, nodes.document):
            self.body.append(self.starttag(node, 'h2', '', CLASS='subtitle'))
            self.context.append('</h2>\n')
            self.in_document_title = len(self.body)
        elif isinstance(node.parent, nodes.section):
            tag = 'h%s' % (self.section_level + self.initial_header_level - 1)
            self.body.append(
                self.starttag(node, tag, '', CLASS='section-subtitle')
                + self.starttag({}, 'span', '', CLASS='section-subtitle'))
            self.context.append('</span></%s>\n' % tag)

    def depart_subtitle(self, node):
        self.body.append(self.context.pop())
        if self.in_document_title:
            self.subtitle = self.body[self.in_document_title:-1]
            self.in_document_title = 0
            self.body_pre_docinfo.extend(self.body)
            self.html_subtitle.extend(self.body)
            del self.body[:]

    # <sup> not allowed in <pre> in HTML 4
    def visit_superscript(self, node):
        if isinstance(node.parent, nodes.literal_block):
            self.body.append(self.starttag(node, 'span', '',
                                           CLASS='superscript'))
        else:
            self.body.append(self.starttag(node, 'sup', ''))

    def depart_superscript(self, node):
        if isinstance(node.parent, nodes.literal_block):
            self.body.append('</span>')
        else:
            self.body.append('</sup>')

    # <tt> element deprecated in HTML 5
    def visit_system_message(self, node):
        self.body.append(self.starttag(node, 'div', CLASS='system-message'))
        self.body.append('<p class="system-message-title">')
        backref_text = ''
        if len(node['backrefs']):
            backrefs = node['backrefs']
            if len(backrefs) == 1:
                backref_text = ('; <em><a href="#%s">backlink</a></em>'
                                % backrefs[0])
            else:
                i = 1
                backlinks = []
                for backref in backrefs:
                    backlinks.append('<a href="#%s">%s</a>' % (backref, i))
                    i += 1
                backref_text = ('; <em>backlinks: %s</em>'
                                % ', '.join(backlinks))
        if node.hasattr('line'):
            line = ', line %s' % node['line']
        else:
            line = ''
        self.body.append('System Message: %s/%s '
                         '(<tt class="docutils">%s</tt>%s)%s</p>\n'
                         % (node['type'], node['level'],
                            self.encode(node['source']), line, backref_text))

    def depart_system_message(self, node):
        self.body.append('</div>\n')

    # "hard coded" border setting
    def visit_table(self, node):
        self.context.append(self.compact_p)
        self.compact_p = True
        atts = {'border': 1}
        classes = ['docutils', self.settings.table_style]
        if 'align' in node:
            classes.append('align-%s' % node['align'])
        if 'width' in node:
            atts['style'] = 'width: %s' % node['width']
        self.body.append(
            self.starttag(node, 'table', CLASS=' '.join(classes), **atts))

    def depart_table(self, node):
        self.compact_p = self.context.pop()
        self.body.append('</table>\n')

    # hard-coded vertical alignment
    def visit_tbody(self, node):
        self.body.append(self.starttag(node, 'tbody', valign='top'))

    def depart_tbody(self, node):
        self.body.append('</tbody>\n')

    # no special handling of "details" in definition list
    def visit_term(self, node):
        self.body.append(self.starttag(node, 'dt', '',
                                       classes=node.parent['classes'],
                                       ids=node.parent['ids']))

    def depart_term(self, node):
        pass

    # hard-coded vertical alignment
    def visit_thead(self, node):
        self.body.append(self.starttag(node, 'thead', valign='bottom'))

    def depart_thead(self, node):
        self.body.append('</thead>\n')

    # auxiliary method, called by visit_title()
    # "with-subtitle" class, no ARIA roles
    def section_title_tags(self, node):
        classes = []
        h_level = self.section_level + self.initial_header_level - 1
        if (len(node.parent) >= 2
            and isinstance(node.parent[1], nodes.subtitle)):
            classes.append('with-subtitle')
        if h_level > 6:
            classes.append('h%i' % h_level)
        tagname = 'h%i' % min(h_level, 6)
        start_tag = self.starttag(node, tagname, '', classes=classes)
        if node.hasattr('refid'):
            atts = {}
            atts['class'] = 'toc-backref'
            atts['href'] = '#' + node['refid']
            start_tag += self.starttag({}, 'a', '', **atts)
            close_tag = '</a></%s>\n' % tagname
        else:
            close_tag = '</%s>\n' % tagname
        return start_tag, close_tag


class SimpleListChecker(writers._html_base.SimpleListChecker):

    """
    Raise `nodes.NodeFound` if non-simple list item is encountered.

    Here "simple" means a list item containing nothing other than a single
    paragraph, a simple list, or a paragraph followed by a simple list.
    """

    def visit_list_item(self, node):
        children = []
        for child in node.children:
            if not isinstance(child, nodes.Invisible):
                children.append(child)
        if (children and isinstance(children[0], nodes.paragraph)
            and (isinstance(children[-1], nodes.bullet_list)
                 or isinstance(children[-1], nodes.enumerated_list))):
            children.pop()
        if len(children) <= 1:
            return
        else:
            raise nodes.NodeFound

    # def visit_bullet_list(self, node):
    #     pass

    # def visit_enumerated_list(self, node):
    #     pass

    def visit_paragraph(self, node):
        raise nodes.SkipNode

    def visit_definition_list(self, node):
        raise nodes.NodeFound

    def visit_docinfo(self, node):
        raise nodes.NodeFound

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/html4css1/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9272 2022-11-24 20:28:40Z milde $
# :Author: Günter Milde <milde@users.sf.net>
#          Based on the html4css1 writer by David Goodger.
# :Maintainer: docutils-develop@lists.sourceforge.net
# :Copyright: © 2005, 2009, 2015 Günter Milde,
#             portions from html4css1 © David Goodger.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause

# Use "best practice" as recommended by the W3C:
# http://www.w3.org/2009/cheatsheet/

"""
Plain HyperText Markup Language document tree Writer.

The output conforms to the `HTML 5` specification.

The cascading style sheet "minimal.css" is required for proper viewing,
the style sheet "plain.css" improves reading experience.
"""
__docformat__ = 'reStructuredText'

import mimetypes
from pathlib import Path

from docutils import frontend, nodes
from docutils.writers import _html_base


class Writer(_html_base.Writer):

    supported = ('html5', 'xhtml', 'html')
    """Formats this writer supports."""

    default_stylesheets = ['minimal.css', 'plain.css']
    default_stylesheet_dirs = ['.', str(Path(__file__).parent)]
    default_template = Path(__file__).parent / 'template.txt'

    # use a copy of the parent spec with some modifications
    settings_spec = frontend.filter_settings_spec(
        _html_base.Writer.settings_spec,
        template=(
            f'Template file. (UTF-8 encoded, default: "{default_template}")',
            ['--template'],
            {'default': default_template, 'metavar': '<file>'}),
        stylesheet_path=(
            'Comma separated list of stylesheet paths. '
            'Relative paths are expanded if a matching file is found in '
            'the --stylesheet-dirs. With --link-stylesheet, '
            'the path is rewritten relative to the output HTML file. '
            '(default: "%s")' % ','.join(default_stylesheets),
            ['--stylesheet-path'],
            {'metavar': '<file[,file,...]>', 'overrides': 'stylesheet',
             'validator': frontend.validate_comma_separated_list,
             'default': default_stylesheets}),
        stylesheet_dirs=(
            'Comma-separated list of directories where stylesheets are found. '
            'Used by --stylesheet-path when expanding relative path '
            'arguments. (default: "%s")' % ','.join(default_stylesheet_dirs),
            ['--stylesheet-dirs'],
            {'metavar': '<dir[,dir,...]>',
             'validator': frontend.validate_comma_separated_list,
             'default': default_stylesheet_dirs}),
        initial_header_level=(
            'Specify the initial header level. Does not affect document '
            'title & subtitle (see --no-doc-title). (default: 2 for "<h2>")',
            ['--initial-header-level'],
            {'choices': '1 2 3 4 5 6'.split(), 'default': '2',
             'metavar': '<level>'}),
        no_xml_declaration=(
            'Omit the XML declaration.',
            ['--no-xml-declaration'],
            {'dest': 'xml_declaration', 'action': 'store_false'}),
    )
    settings_spec = settings_spec + (
        'HTML5 Writer Options',
        '',
        ((frontend.SUPPRESS_HELP,  # Obsoleted by "--image-loading"
          ['--embed-images'],
          {'action': 'store_true',
           'validator': frontend.validate_boolean}),
         (frontend.SUPPRESS_HELP,  # Obsoleted by "--image-loading"
          ['--link-images'],
          {'dest': 'embed_images', 'action': 'store_false'}),
         ('Suggest at which point images should be loaded: '
          '"embed", "link" (default), or "lazy".',
          ['--image-loading'],
          {'choices': ('embed', 'link', 'lazy'),
           # 'default': 'link'  # default set in _html_base.py
           }),
         ('Append a self-link to section headings.',
          ['--section-self-link'],
          {'default': 0, 'action': 'store_true'}),
         ('Do not append a self-link to section headings. (default)',
          ['--no-section-self-link'],
          {'dest': 'section_self_link', 'action': 'store_false'}),
         )
        )

    config_section = 'html5 writer'

    def __init__(self):
        self.parts = {}
        self.translator_class = HTMLTranslator


class HTMLTranslator(_html_base.HTMLTranslator):
    """
    This writer generates `polyglot markup`: HTML5 that is also valid XML.

    Safe subclassing: when overriding, treat ``visit_*`` and ``depart_*``
    methods as a unit to prevent breaks due to internal changes. See the
    docstring of docutils.writers._html_base.HTMLTranslator for details
    and examples.
    """

    # self.starttag() arguments for the main document
    documenttag_args = {'tagname': 'main'}

    # add meta tag to fix rendering in mobile browsers
    def __init__(self, document):
        super().__init__(document)
        self.meta.append('<meta name="viewport" '
                         'content="width=device-width, initial-scale=1" />\n')

    # <acronym> tag obsolete in HTML5. Use the <abbr> tag instead.
    def visit_acronym(self, node):
        # @@@ implementation incomplete ("title" attribute)
        self.body.append(self.starttag(node, 'abbr', ''))

    def depart_acronym(self, node):
        self.body.append('</abbr>')

    # no standard meta tag name in HTML5, use separate "author" meta tags
    # https://www.w3.org/TR/html5/document-metadata.html#standard-metadata-names
    def visit_authors(self, node):
        self.visit_docinfo_item(node, 'authors', meta=False)
        for subnode in node:
            self.meta.append('<meta name="author" content='
                             f'"{self.attval(subnode.astext())}" />\n')

    def depart_authors(self, node):
        self.depart_docinfo_item()

    # use the <figcaption> semantic tag.
    def visit_caption(self, node):
        if isinstance(node.parent, nodes.figure):
            self.body.append('<figcaption>\n')
        self.body.append(self.starttag(node, 'p', ''))

    def depart_caption(self, node):
        self.body.append('</p>\n')
        # <figcaption> is closed in depart_figure(), as legend may follow.

    # use HTML block-level tags if matching class value found
    supported_block_tags = {'ins', 'del'}

    def visit_container(self, node):
        # If there is exactly one of the "supported block tags" in
        # the list of class values, use it as tag name:
        classes = node['classes']
        tags = [cls for cls in classes
                if cls in self.supported_block_tags]
        if len(tags) == 1:
            node.html5tagname = tags[0]
            classes.remove(tags[0])
        else:
            node.html5tagname = 'div'
        self.body.append(self.starttag(node, node.html5tagname,
                                       CLASS='docutils container'))

    def depart_container(self, node):
        self.body.append(f'</{node.html5tagname}>\n')

    # no standard meta tag name in HTML5, use dcterms.rights
    # see https://wiki.whatwg.org/wiki/MetaExtensions
    def visit_copyright(self, node):
        self.visit_docinfo_item(node, 'copyright', meta=False)
        self.meta.append('<meta name="dcterms.rights" '
                         f'content="{self.attval(node.astext())}" />\n')

    def depart_copyright(self, node):
        self.depart_docinfo_item()

    # no standard meta tag name in HTML5, use dcterms.date
    def visit_date(self, node):
        self.visit_docinfo_item(node, 'date', meta=False)
        self.meta.append('<meta name="dcterms.date" '
                         f'content="{self.attval(node.astext())}" />\n')

    def depart_date(self, node):
        self.depart_docinfo_item()

    # use new HTML5 <figure> and <figcaption> elements
    def visit_figure(self, node):
        atts = {}
        if node.get('width'):
            atts['style'] = f"width: {node['width']}"
        if node.get('align'):
            atts['class'] = f"align-{node['align']}"
        self.body.append(self.starttag(node, 'figure', **atts))

    def depart_figure(self, node):
        if len(node) > 1:
            self.body.append('</figcaption>\n')
        self.body.append('</figure>\n')

    # use HTML5 <footer> element
    def visit_footer(self, node):
        self.context.append(len(self.body))

    def depart_footer(self, node):
        start = self.context.pop()
        footer = [self.starttag(node, 'footer')]
        footer.extend(self.body[start:])
        footer.append('</footer>\n')
        self.footer.extend(footer)
        self.body_suffix[:0] = footer
        del self.body[start:]

    # use HTML5 <header> element
    def visit_header(self, node):
        self.context.append(len(self.body))

    def depart_header(self, node):
        start = self.context.pop()
        header = [self.starttag(node, 'header')]
        header.extend(self.body[start:])
        header.append('</header>\n')
        self.body_prefix.extend(header)
        self.header.extend(header)
        del self.body[start:]

    # MIME types supported by the HTML5 <video> element
    videotypes = ('video/mp4', 'video/webm', 'video/ogg')

    def visit_image(self, node):
        atts = {}
        uri = node['uri']
        mimetype = mimetypes.guess_type(uri)[0]
        if mimetype not in self.videotypes:
            return super().visit_image(node)
        # image size
        if 'width' in node:
            atts['width'] = node['width'].replace('px', '')
        if 'height' in node:
            atts['height'] = node['height'].replace('px', '')
        if 'align' in node:
            atts['class'] = f"align-{node['align']}"
        if 'controls' in node['classes']:
            atts['controls'] = 'controls'
            node['classes'].remove('controls')
        atts['title'] = node.get('alt', uri)
        if getattr(self.settings, 'image_loading', None) == 'lazy':
            atts['loading'] = 'lazy'
        # No newline in inline context or if surrounded by <a>...</a>.
        if (isinstance(node.parent, nodes.TextElement)
            or (isinstance(node.parent, nodes.reference)
                and not isinstance(node.parent.parent, nodes.TextElement))):
            suffix = ''
        else:
            suffix = '\n'
        fallback = node.get('alt', uri)
        self.body.append(
            self.starttag(node, "video", suffix, src=uri, **atts)
            + f'<a href="{uri}">{fallback}</a>{suffix}</video>{suffix}')

    def depart_image(self, node):
        pass

    # use HTML text-level tags if matching class value found
    supported_inline_tags = {'code', 'kbd', 'dfn', 'samp', 'var',
                             'bdi', 'del', 'ins', 'mark', 'small',
                             'b', 'i', 'q', 's', 'u'}

    # Use `supported_inline_tags` if found in class values
    def visit_inline(self, node):
        classes = node['classes']
        tags = [cls for cls in self.supported_inline_tags
                if cls in classes]
        if len(tags):
            node.html5tagname = tags[0]
            classes.remove(tags[0])
        elif (classes == ['ln']
              and isinstance(node.parent, nodes.literal_block)
              and 'code' in node.parent.get('classes')):
            if self.body[-1] == '<code>':
                del self.body[-1]
            else:
                self.body.append('</code>')
            node.html5tagname = 'small'
        else:
            node.html5tagname = 'span'
        self.body.append(self.starttag(node, node.html5tagname, ''))

    def depart_inline(self, node):
        self.body.append(f'</{node.html5tagname}>')
        if (node.html5tagname == 'small' and node.get('classes') == ['ln']
            and isinstance(node.parent, nodes.literal_block)):
            self.body.append(f'<code data-lineno="{node.astext()}">')
        del node.html5tagname

    # place inside HTML5 <figcaption> element (together with caption)
    def visit_legend(self, node):
        if not isinstance(node.parent[1], nodes.caption):
            self.body.append('<figcaption>\n')
        self.body.append(self.starttag(node, 'div', CLASS='legend'))

    def depart_legend(self, node):
        self.body.append('</div>\n')
        # <figcaption> closed in visit_figure()

    # use HTML text-level tags if matching class value found
    def visit_literal(self, node):
        classes = node['classes']
        tags = [cls for cls in self.supported_inline_tags
                if cls in classes]
        if len(tags):
            tagname = tags[0]
            classes.remove(tags[0])
        else:
            tagname = 'span'
        if tagname == 'code':
            self.body.append(self.starttag(node, 'code', ''))
            return
        self.body.append(
            self.starttag(node, tagname, '', CLASS='docutils literal'))
        text = node.astext()
        # remove hard line breaks (except if in a parsed-literal block)
        if not isinstance(node.parent, nodes.literal_block):
            text = text.replace('\n', ' ')
        # Protect text like ``--an-option`` and the regular expression
        # ``[+]?(\d+(\.\d*)?|\.\d+)`` from bad line wrapping
        for token in self.words_and_spaces.findall(text):
            if token.strip() and self.in_word_wrap_point.search(token):
                self.body.append(
                    f'<span class="pre">{self.encode(token)}</span>')
            else:
                self.body.append(self.encode(token))
        self.body.append(f'</{tagname}>')
        # Content already processed:
        raise nodes.SkipNode

    def depart_literal(self, node):
        # skipped unless literal element is from "code" role:
        self.body.append('</code>')

    # Meta tags: 'lang' attribute replaced by 'xml:lang' in XHTML 1.1
    # HTML5/polyglot recommends using both
    def visit_meta(self, node):
        if node.hasattr('lang'):
            node['xml:lang'] = node['lang']
        self.meta.append(self.emptytag(node, 'meta',
                                       **node.non_default_attributes()))

    def depart_meta(self, node):
        pass

    # no standard meta tag name in HTML5
    def visit_organization(self, node):
        self.visit_docinfo_item(node, 'organization', meta=False)

    def depart_organization(self, node):
        self.depart_docinfo_item()

    # use the new HTML5 element <section>
    def visit_section(self, node):
        self.section_level += 1
        self.body.append(
            self.starttag(node, 'section'))

    def depart_section(self, node):
        self.section_level -= 1
        self.body.append('</section>\n')

    # use the new HTML5 element <aside>
    def visit_sidebar(self, node):
        self.body.append(
            self.starttag(node, 'aside', CLASS='sidebar'))
        self.in_sidebar = True

    def depart_sidebar(self, node):
        self.body.append('</aside>\n')
        self.in_sidebar = False

    # Use new HTML5 element <aside> or <nav>
    # Add class value to <body>, if there is a ToC in the document
    # (see responsive.css how this is used for a navigation sidebar).
    def visit_topic(self, node):
        atts = {'classes': ['topic']}
        if 'contents' in node['classes']:
            node.html_tagname = 'nav'
            del atts['classes']
            if isinstance(node.parent, nodes.document):
                atts['role'] = 'doc-toc'
                self.body_prefix[0] = '</head>\n<body class="with-toc">\n'
        elif 'abstract' in node['classes']:
            node.html_tagname = 'div'
            atts['role'] = 'doc-abstract'
        elif 'dedication' in node['classes']:
            node.html_tagname = 'div'
            atts['role'] = 'doc-dedication'
        else:
            node.html_tagname = 'aside'
        self.body.append(self.starttag(node, node.html_tagname, **atts))

    def depart_topic(self, node):
        self.body.append(f'</{node.html_tagname}>\n')
        del node.html_tagname

    # append self-link
    def section_title_tags(self, node):
        start_tag, close_tag = super().section_title_tags(node)
        ids = node.parent['ids']
        if (ids and getattr(self.settings, 'section_self_link', None)
            and not isinstance(node.parent, nodes.document)):
            self_link = ('<a class="self-link" title="link to this section"'
                         f' href="#{ids[0]}"></a>')
            close_tag = close_tag.replace('</h', self_link + '</h')
        return start_tag, close_tag

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/html5_polyglot/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9331 2023-03-25 17:54:40Z aa-turner $
# Author: Engelbert Gruber, Günter Milde
# Maintainer: docutils-develop@lists.sourceforge.net
# Copyright: This module has been placed in the public domain.

"""LaTeX2e document tree Writer."""

__docformat__ = 'reStructuredText'

# code contributions from several people included, thanks to all.
# some named: David Abrahams, Julien Letessier, Lele Gaifax, and others.
#
# convention deactivate code by two # i.e. ##.

from pathlib import Path
import re
import string
from urllib.request import url2pathname
import warnings
try:
    import roman
except ImportError:
    import docutils.utils.roman as roman

from docutils import frontend, nodes, languages, writers, utils
from docutils.transforms import writer_aux
from docutils.utils.math import pick_math_environment, unichar2tex

LATEX_WRITER_DIR = Path(__file__).parent


class Writer(writers.Writer):

    supported = ('latex', 'latex2e')
    """Formats this writer supports."""

    default_template = 'default.tex'
    default_template_path = LATEX_WRITER_DIR
    default_preamble = ('% PDF Standard Fonts\n'
                        '\\usepackage{mathptmx} % Times\n'
                        '\\usepackage[scaled=.90]{helvet}\n'
                        '\\usepackage{courier}')
    table_style_values = [  # TODO: align-left, align-center, align-right, ??
                          'booktabs', 'borderless', 'colwidths-auto',
                          'nolines', 'standard']

    settings_spec = (
        'LaTeX-Specific Options',
        None,
        (('Specify LaTeX documentclass.  Default: "article".',
          ['--documentclass'],
          {'default': 'article', }),
         ('Specify document options.  Multiple options can be given, '
          'separated by commas.  Default: "a4paper".',
          ['--documentoptions'],
          {'default': 'a4paper', }),
         ('Format for footnote references: one of "superscript" or '
          '"brackets".  Default: "superscript".',
          ['--footnote-references'],
          {'choices': ['superscript', 'brackets'], 'default': 'superscript',
           'metavar': '<format>',
           'overrides': 'trim_footnote_reference_space'}),
         ('Use \\cite command for citations. (future default)',
          ['--use-latex-citations'],
          {'default': None, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Use figure floats for citations '
          '(might get mixed with real figures). (provisional default)',
          ['--figure-citations'],
          {'dest': 'use_latex_citations', 'action': 'store_false',
           'validator': frontend.validate_boolean}),
         ('Format for block quote attributions: one of "dash" (em-dash '
          'prefix), "parentheses"/"parens", or "none".  Default: "dash".',
          ['--attribution'],
          {'choices': ['dash', 'parentheses', 'parens', 'none'],
           'default': 'dash', 'metavar': '<format>'}),
         ('Specify LaTeX packages/stylesheets. '
          'A style is referenced with "\\usepackage" if extension is '
          '".sty" or omitted and with "\\input" else. '
          ' Overrides previous --stylesheet and --stylesheet-path settings.',
          ['--stylesheet'],
          {'default': '', 'metavar': '<file[,file,...]>',
           'overrides': 'stylesheet_path',
           'validator': frontend.validate_comma_separated_list}),
         ('Comma separated list of LaTeX packages/stylesheets. '
          'Relative paths are expanded if a matching file is found in '
          'the --stylesheet-dirs. With --link-stylesheet, '
          'the path is rewritten relative to the output *.tex file. ',
          ['--stylesheet-path'],
          {'metavar': '<file[,file,...]>', 'overrides': 'stylesheet',
           'validator': frontend.validate_comma_separated_list}),
         ('Link to the stylesheet(s) in the output file. (default)',
          ['--link-stylesheet'],
          {'dest': 'embed_stylesheet', 'action': 'store_false'}),
         ('Embed the stylesheet(s) in the output file. '
          'Stylesheets must be accessible during processing. ',
          ['--embed-stylesheet'],
          {'default': False, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Comma-separated list of directories where stylesheets are found. '
          'Used by --stylesheet-path when expanding relative path arguments. '
          'Default: ".".',
          ['--stylesheet-dirs'],
          {'metavar': '<dir[,dir,...]>',
           'validator': frontend.validate_comma_separated_list,
           'default': ['.']}),
         ('Customization by LaTeX code in the preamble. '
          'Default: select PDF standard fonts (Times, Helvetica, Courier).',
          ['--latex-preamble'],
          {'default': default_preamble}),
         ('Specify the template file. Default: "%s".' % default_template,
          ['--template'],
          {'default': default_template, 'metavar': '<file>'}),
         ('Table of contents by LaTeX. (default)',
          ['--use-latex-toc'],
          {'default': True, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Table of contents by Docutils (without page numbers).',
          ['--use-docutils-toc'],
          {'dest': 'use_latex_toc', 'action': 'store_false',
           'validator': frontend.validate_boolean}),
         ('Add parts on top of the section hierarchy.',
          ['--use-part-section'],
          {'default': False, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Attach author and date to the document info table. (default)',
          ['--use-docutils-docinfo'],
          {'dest': 'use_latex_docinfo', 'action': 'store_false',
           'validator': frontend.validate_boolean}),
         ('Attach author and date to the document title.',
          ['--use-latex-docinfo'],
          {'default': False, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ("Typeset abstract as topic. (default)",
          ['--topic-abstract'],
          {'dest': 'use_latex_abstract', 'action': 'store_false',
           'validator': frontend.validate_boolean}),
         ("Use LaTeX abstract environment for the document's abstract.",
          ['--use-latex-abstract'],
          {'default': False, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Color of any hyperlinks embedded in text. '
          'Default: "blue" (use "false" to disable).',
          ['--hyperlink-color'], {'default': 'blue'}),
         ('Additional options to the "hyperref" package.',
          ['--hyperref-options'], {'default': ''}),
         ('Enable compound enumerators for nested enumerated lists '
          '(e.g. "1.2.a.ii").',
          ['--compound-enumerators'],
          {'default': False, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Disable compound enumerators for nested enumerated lists. '
          '(default)',
          ['--no-compound-enumerators'],
          {'action': 'store_false', 'dest': 'compound_enumerators'}),
         ('Enable section ("." subsection ...) prefixes for compound '
          'enumerators.  This has no effect without --compound-enumerators.',
          ['--section-prefix-for-enumerators'],
          {'default': None, 'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Disable section prefixes for compound enumerators. (default)',
          ['--no-section-prefix-for-enumerators'],
          {'action': 'store_false', 'dest': 'section_prefix_for_enumerators'}),
         ('Set the separator between section number and enumerator '
          'for compound enumerated lists.  Default: "-".',
          ['--section-enumerator-separator'],
          {'default': '-', 'metavar': '<char>'}),
         ('When possible, use the specified environment for literal-blocks. '
          'Default: "" (fall back to "alltt").',
          ['--literal-block-env'],
          {'default': ''}),
         ('Deprecated alias for "--literal-block-env=verbatim".',
          ['--use-verbatim-when-possible'],
          {'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Table style. "standard" with horizontal and vertical lines, '
          '"booktabs" (LaTeX booktabs style) only horizontal lines '
          'above and below the table and below the header, or "borderless". '
          'Default: "standard"',
          ['--table-style'],
          {'default': ['standard'],
           'metavar': '<format>',
           'action': 'append',
           'validator': frontend.validate_comma_separated_list,
           'choices': table_style_values}),
         ('LaTeX graphicx package option. '
          'Possible values are "dvipdfmx", "dvips", "dvisvgm", '
          '"luatex", "pdftex", and "xetex".'
          'Default: "".',
          ['--graphicx-option'],
          {'default': ''}),
         ('LaTeX font encoding. '
          'Possible values are "", "T1" (default), "OT1", "LGR,T1" or '
          'any other combination of options to the `fontenc` package. ',
          ['--font-encoding'],
          {'default': 'T1'}),
         ('Per default the latex-writer puts the reference title into '
          'hyperreferences. Specify "ref*" or "pageref*" to get the section '
          'number or the page number.',
          ['--reference-label'],
          {'default': ''}),
         ('Specify style and database(s) for bibtex, for example '
          '"--use-bibtex=unsrt,mydb1,mydb2". Provisional!',
          ['--use-bibtex'],
          {'default': '',
           'metavar': '<style,bibfile[,bibfile,...]>',
           'validator': frontend.validate_comma_separated_list}),
         ('Use legacy functions with class value list for '
          '\\DUtitle and \\DUadmonition.',
          ['--legacy-class-functions'],
          {'default': False,
           'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Use \\DUrole and "DUclass" wrappers for class values. '
          'Place admonition content in an environment. (default)',
          ['--new-class-functions'],
          {'dest': 'legacy_class_functions',
           'action': 'store_false',
           'validator': frontend.validate_boolean}),
         ('Use legacy algorithm to determine table column widths. '
          '(provisional default)',
          ['--legacy-column-widths'],
          {'default': None,
           'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ('Use new algorithm to determine table column widths. '
          '(future default)',
          ['--new-column-widths'],
          {'dest': 'legacy_column_widths',
           'action': 'store_false',
           'validator': frontend.validate_boolean}),
         # TODO: implement "latex footnotes" alternative
         ('Footnotes with numbers/symbols by Docutils. (default) '
          '(The alternative, --latex-footnotes, is not implemented yet.)',
          ['--docutils-footnotes'],
          {'default': True,
           'action': 'store_true',
           'validator': frontend.validate_boolean}),
         ),
        )

    settings_defaults = {'sectnum_depth': 0}  # updated by SectNum transform
    config_section = 'latex2e writer'
    config_section_dependencies = ('writers', 'latex writers')

    head_parts = ('head_prefix', 'requirements', 'latex_preamble',
                  'stylesheet', 'fallbacks', 'pdfsetup', 'titledata')
    visitor_attributes = head_parts + ('title', 'subtitle',
                                       'body_pre_docinfo', 'docinfo',
                                       'dedication', 'abstract', 'body')

    output = None
    """Final translated form of `document`."""

    def __init__(self):
        writers.Writer.__init__(self)
        self.translator_class = LaTeXTranslator

    def get_transforms(self):
        # Override parent method to add latex-specific transforms
        return super().get_transforms() + [
                   # Convert specific admonitions to generic one
                   writer_aux.Admonitions,
                   # TODO: footnote collection transform
                   ]

    def translate(self):
        visitor = self.translator_class(self.document)
        self.document.walkabout(visitor)
        # copy parts
        for part in self.visitor_attributes:
            setattr(self, part, getattr(visitor, part))
        # get template string from file
        templatepath = Path(self.document.settings.template)
        if not templatepath.exists():
            templatepath = self.default_template_path / templatepath
        template = templatepath.read_text(encoding='utf-8')
        # fill template
        self.assemble_parts()  # create dictionary of parts
        self.output = string.Template(template).substitute(self.parts)

    def assemble_parts(self):
        """Assemble the `self.parts` dictionary of output fragments."""
        writers.Writer.assemble_parts(self)
        for part in self.visitor_attributes:
            lines = getattr(self, part)
            if part in self.head_parts:
                if lines:
                    lines.append('')  # to get a trailing newline
                self.parts[part] = '\n'.join(lines)
            else:
                # body contains inline elements, so join without newline
                self.parts[part] = ''.join(lines)


class Babel:
    """Language specifics for LaTeX."""

    # TeX (babel) language names:
    # ! not all of these are supported by Docutils!
    #
    # based on LyX' languages file with adaptions to `BCP 47`_
    # (https://www.rfc-editor.org/rfc/bcp/bcp47.txt) and
    # http://www.tug.org/TUGboat/Articles/tb29-3/tb93miklavec.pdf
    # * the key without subtags is the default
    # * case is ignored
    # cf. https://docutils.sourceforge.io/docs/howto/i18n.html
    #     https://www.w3.org/International/articles/language-tags/
    # and http://www.iana.org/assignments/language-subtag-registry
    language_codes = {
        # code          TeX/Babel-name    comment
        'af':           'afrikaans',
        'ar':           'arabic',
        # 'be':           'belarusian',
        'bg':           'bulgarian',
        'br':           'breton',
        'ca':           'catalan',
        # 'cop':          'coptic',
        'cs':           'czech',
        'cy':           'welsh',
        'da':           'danish',
        'de':           'ngerman',        # new spelling (de_1996)
        'de-1901':      'german',         # old spelling
        'de-AT':        'naustrian',
        'de-AT-1901':   'austrian',
        'dsb':          'lowersorbian',
        'el':           'greek',          # monotonic (el-monoton)
        'el-polyton':   'polutonikogreek',
        'en':           'english',        # TeX' default language
        'en-AU':        'australian',
        'en-CA':        'canadian',
        'en-GB':        'british',
        'en-NZ':        'newzealand',
        'en-US':        'american',
        'eo':           'esperanto',
        'es':           'spanish',
        'et':           'estonian',
        'eu':           'basque',
        # 'fa':           'farsi',
        'fi':           'finnish',
        'fr':           'french',
        'fr-CA':        'canadien',
        'ga':           'irish',          # Irish Gaelic
        # 'grc':                          # Ancient Greek
        'grc-ibycus':   'ibycus',         # Ibycus encoding
        'gl':           'galician',
        'he':           'hebrew',
        'hr':           'croatian',
        'hsb':          'uppersorbian',
        'hu':           'magyar',
        'ia':           'interlingua',
        'id':           'bahasai',        # Bahasa (Indonesian)
        'is':           'icelandic',
        'it':           'italian',
        'ja':           'japanese',
        'kk':           'kazakh',
        'la':           'latin',
        'lt':           'lithuanian',
        'lv':           'latvian',
        'mn':           'mongolian',      # Mongolian, Cyrillic (mn-cyrl)
        'ms':           'bahasam',        # Bahasa (Malay)
        'nb':           'norsk',          # Norwegian Bokmal
        'nl':           'dutch',
        'nn':           'nynorsk',        # Norwegian Nynorsk
        'no':           'norsk',          # Norwegian (Bokmal)
        'pl':           'polish',
        'pt':           'portuges',
        'pt-BR':        'brazil',
        'ro':           'romanian',
        'ru':           'russian',
        'se':           'samin',          # North Sami
        'sh-Cyrl':      'serbianc',       # Serbo-Croatian, Cyrillic
        'sh-Latn':      'serbian',        # Serbo-Croatian, Latin (cf. 'hr')
        'sk':           'slovak',
        'sl':           'slovene',
        'sq':           'albanian',
        'sr':           'serbianc',       # Serbian, Cyrillic (contributed)
        'sr-Latn':      'serbian',        # Serbian, Latin script
        'sv':           'swedish',
        # 'th':           'thai',
        'tr':           'turkish',
        'uk':           'ukrainian',
        'vi':           'vietnam',
        # zh-Latn:      Chinese Pinyin
        }
    # normalize (downcase) keys
    language_codes = {k.lower(): v for k, v in language_codes.items()}

    warn_msg = 'Language "%s" not supported by LaTeX (babel)'

    # "Active characters" are shortcuts that start a LaTeX macro and may need
    # escaping for literals use. Characters that prevent literal use (e.g.
    # starting accent macros like "a -> ä) will be deactivated if one of the
    # defining languages is used in the document.
    # Special cases:
    #  ~ (tilde) -- used in estonian, basque, galician, and old versions of
    #    spanish -- cannot be deactivated as it denotes a no-break space macro,
    #  " (straight quote) -- used in albanian, austrian, basque
    #    brazil, bulgarian, catalan, czech, danish, dutch, estonian,
    #    finnish, galician, german, icelandic, italian, latin, naustrian,
    #    ngerman, norsk, nynorsk, polish, portuges, russian, serbian, slovak,
    #    slovene, spanish, swedish, ukrainian, and uppersorbian --
    #    is escaped as ``\textquotedbl``.
    active_chars = {
                    # TeX/Babel-name:  active characters to deactivate
                    # 'breton':        ':;!?'  # ensure whitespace
                    # 'esperanto':     '^',
                    # 'estonian':      '~"`',
                    # 'french':        ':;!?'  # ensure whitespace
                    'galician':        '.<>',  # also '~"'
                    # 'magyar':        '`',  # for special hyphenation cases
                    'spanish':         '.<>',  # old versions also '~'
                    # 'turkish':       ':!='  # ensure whitespace
                   }

    def __init__(self, language_code, reporter=None):
        self.reporter = reporter
        self.language = self.language_name(language_code)
        self.otherlanguages = {}

    def __call__(self):
        """Return the babel call with correct options and settings"""
        languages = sorted(self.otherlanguages.keys())
        languages.append(self.language or 'english')
        self.setup = [r'\usepackage[%s]{babel}' % ','.join(languages)]
        # Deactivate "active characters"
        shorthands = []
        for c in ''.join(self.active_chars.get(lng, '') for lng in languages):
            if c not in shorthands:
                shorthands.append(c)
        if shorthands:
            self.setup.append(r'\AtBeginDocument{\shorthandoff{%s}}'
                              % ''.join(shorthands))
        # Including '~' in shorthandoff prevents its use as no-break space
        if 'galician' in languages:
            self.setup.append(r'\deactivatetilden % restore ~ in Galician')
        if 'estonian' in languages:
            self.setup.extend([r'\makeatletter',
                               r'  \addto\extrasestonian{\bbl@deactivate{~}}',
                               r'\makeatother'])
        if 'basque' in languages:
            self.setup.extend([r'\makeatletter',
                               r'  \addto\extrasbasque{\bbl@deactivate{~}}',
                               r'\makeatother'])
        if (languages[-1] == 'english'
            and 'french' in self.otherlanguages.keys()):
            self.setup += ['% Prevent side-effects if French hyphenation '
                           'patterns are not loaded:',
                           r'\frenchbsetup{StandardLayout}',
                           r'\AtBeginDocument{\selectlanguage{%s}'
                           r'\noextrasfrench}' % self.language]
        return '\n'.join(self.setup)

    def language_name(self, language_code):
        """Return TeX language name for `language_code`"""
        for tag in utils.normalize_language_tag(language_code):
            try:
                return self.language_codes[tag]
            except KeyError:
                pass
        if self.reporter is not None:
            self.reporter.warning(self.warn_msg % language_code)
        return ''

    def get_language(self):
        # Obsolete, kept for backwards compatibility with Sphinx
        return self.language


# Building blocks for the latex preamble
# --------------------------------------

class SortableDict(dict):
    """Dictionary with additional sorting methods

    Tip: use key starting with with '_' for sorting before small letters
         and with '~' for sorting after small letters.
    """
    def sortedkeys(self):
        """Return sorted list of keys"""
        return sorted(self.keys())

    def sortedvalues(self):
        """Return list of values sorted by keys"""
        return [self[key] for key in self.sortedkeys()]


# PreambleCmds
# `````````````
# A container for LaTeX code snippets that can be
# inserted into the preamble if required in the document.
#
# .. The package 'makecmds' would enable shorter definitions using the
#    \providelength and \provideenvironment commands.
#    However, it is pretty non-standard (texlive-latex-extra).

class PreambleCmds:
    """Building blocks for the latex preamble."""


# Requirements and Setup

PreambleCmds.color = r"""\usepackage{color}"""

PreambleCmds.float = r"""\usepackage{float} % extended float configuration
\floatplacement{figure}{H} % place figures here definitely"""

PreambleCmds.linking = r"""%% hyperlinks:
\ifthenelse{\isundefined{\hypersetup}}{
  \usepackage[%s]{hyperref}
  \usepackage{bookmark}
  \urlstyle{same} %% normal text font (alternatives: tt, rm, sf)
}{}"""

PreambleCmds.minitoc = r"""%% local table of contents
\usepackage{minitoc}"""

PreambleCmds.table = r"""\usepackage{longtable,ltcaption,array}
\setlength{\extrarowheight}{2pt}
\newlength{\DUtablewidth} % internal use in tables"""

PreambleCmds.table_columnwidth = (
    r'\newcommand{\DUcolumnwidth}[1]'
    r'{\dimexpr#1\DUtablewidth-2\tabcolsep\relax}')

PreambleCmds.textcomp = r"""\usepackage{textcomp} % text symbol macros"""
# TODO? Options [force,almostfull] prevent spurious error messages,
# see de.comp.text.tex/2005-12/msg01855

# backwards compatibility definitions

PreambleCmds.abstract_legacy = r"""
% abstract title
\providecommand*{\DUtitleabstract}[1]{\centerline{\textbf{#1}}}"""

# see https://sourceforge.net/p/docutils/bugs/339/
PreambleCmds.admonition_legacy = r"""
% admonition (specially marked topic)
\providecommand{\DUadmonition}[2][class-arg]{%
  % try \DUadmonition#1{#2}:
  \ifcsname DUadmonition#1\endcsname%
    \csname DUadmonition#1\endcsname{#2}%
  \else
    \begin{center}
      \fbox{\parbox{0.9\linewidth}{#2}}
    \end{center}
  \fi
}"""

PreambleCmds.error_legacy = r"""
% error admonition title
\providecommand*{\DUtitleerror}[1]{\DUtitle{\color{red}#1}}"""

PreambleCmds.title_legacy = r"""
% title for topics, admonitions, unsupported section levels, and sidebar
\providecommand*{\DUtitle}[2][class-arg]{%
  % call \DUtitle#1{#2} if it exists:
  \ifcsname DUtitle#1\endcsname%
    \csname DUtitle#1\endcsname{#2}%
  \else
    \smallskip\noindent\textbf{#2}\smallskip%
  \fi
}"""

PreambleCmds.toc_list = r"""
\providecommand*{\DUCLASScontents}{%
  \renewenvironment{itemize}%
    {\begin{list}{}{\setlength{\partopsep}{0pt}
                    \setlength{\parsep}{0pt}}
                   }%
    {\end{list}}%
}"""

PreambleCmds.ttem = r"""
% character width in monospaced font
\newlength{\ttemwidth}
\settowidth{\ttemwidth}{\ttfamily M}"""

## PreambleCmds.caption = r"""% configure caption layout
## \usepackage{caption}
## \captionsetup{singlelinecheck=false}% no exceptions for one-liners"""


# Definitions from docutils.sty::

def _read_block(fp):
    block = [next(fp)]  # first line (empty)
    for line in fp:
        if not line.strip():
            break
        block.append(line)
    return ''.join(block).rstrip()


with open(LATEX_WRITER_DIR/'docutils.sty', encoding='utf-8') as fp:
    for line in fp:
        line = line.strip('% \n')
        if not line.endswith('::'):
            continue
        block_name = line.rstrip(':')
        if not block_name:
            continue
        definitions = _read_block(fp)
        if block_name in ('color', 'float', 'table', 'textcomp'):
            definitions = definitions.strip()
        # print('Block: `%s`'% block_name)
        # print(definitions)
        setattr(PreambleCmds, block_name, definitions)


# LaTeX encoding maps
# -------------------
# ::

class CharMaps:
    """LaTeX representations for active and Unicode characters."""

    # characters that need escaping even in `alltt` environments:
    alltt = {
        ord('\\'): '\\textbackslash{}',
        ord('{'): '\\{',
        ord('}'): '\\}',
    }
    # characters that normally need escaping:
    special = {
        ord('#'): '\\#',
        ord('$'): '\\$',
        ord('%'): '\\%',
        ord('&'): '\\&',
        ord('~'): '\\textasciitilde{}',
        ord('_'): '\\_',
        ord('^'): '\\textasciicircum{}',
        # straight double quotes are 'active' in many languages
        ord('"'): '\\textquotedbl{}',
        # Square brackets are ordinary chars and cannot be escaped with '\',
        # so we put them in a group '{[}'. (Alternative: ensure that all
        # macros with optional arguments are terminated with {} and text
        # inside any optional argument is put in a group ``[{text}]``).
        # Commands with optional args inside an optional arg must be put in a
        # group, e.g. ``\item[{\hyperref[label]{text}}]``.
        ord('['): '{[}',
        ord(']'): '{]}',
        # the soft hyphen is unknown in 8-bit text
        # and not properly handled by XeTeX
        0x00AD: '\\-',  # SOFT HYPHEN
    }
    # Unicode chars that are not recognized by LaTeX's utf8 encoding
    unsupported_unicode = {
        # TODO: ensure white space also at the beginning of a line?
        # 0x00A0: '\\leavevmode\\nobreak\\vadjust{}~'
        0x2000: '\\enskip',                        # EN QUAD
        0x2001: '\\quad',                          # EM QUAD
        0x2002: '\\enskip',                        # EN SPACE
        0x2003: '\\quad',                          # EM SPACE
        0x2008: '\\,',                             # PUNCTUATION SPACE
        0x200b: '\\hspace{0pt}',                   # ZERO WIDTH SPACE
        0x202F: '\\,',                             # NARROW NO-BREAK SPACE
        # 0x02d8: '\\\u{ }',                       # BREVE
        0x2011: '\\hbox{-}',                       # NON-BREAKING HYPHEN
        0x212b: '\\AA',                            # ANGSTROM SIGN
        0x21d4: '\\ensuremath{\\Leftrightarrow}',  # LEFT RIGHT DOUBLE ARROW
        0x2260: '\\ensuremath{\\neq}',             # NOT EQUAL TO
        0x2261: '\\ensuremath{\\equiv}',           # IDENTICAL TO
        0x2264: '\\ensuremath{\\le}',              # LESS-THAN OR EQUAL TO
        0x2265: '\\ensuremath{\\ge}',              # GREATER-THAN OR EQUAL TO
        # Docutils footnote symbols:
        0x2660: '\\ensuremath{\\spadesuit}',
        0x2663: '\\ensuremath{\\clubsuit}',
        0xfb00: 'ff',                              # LATIN SMALL LIGATURE FF
        0xfb01: 'fi',                              # LATIN SMALL LIGATURE FI
        0xfb02: 'fl',                              # LATIN SMALL LIGATURE FL
        0xfb03: 'ffi',                             # LATIN SMALL LIGATURE FFI
        0xfb04: 'ffl',                             # LATIN SMALL LIGATURE FFL
    }
    # Unicode chars that are recognized by LaTeX's utf8 encoding
    utf8_supported_unicode = {
        0x00A0: '~',                   # NO-BREAK SPACE
        0x00AB: '\\guillemotleft{}',   # LEFT-POINTING DOUBLE ANGLE QUOTATION
        0x00bb: '\\guillemotright{}',  # RIGHT-POINTING DOUBLE ANGLE QUOTATION
        0x200C: '\\textcompwordmark{}',  # ZERO WIDTH NON-JOINER
        0x2013: '\\textendash{}',
        0x2014: '\\textemdash{}',
        0x2018: '\\textquoteleft{}',
        0x2019: '\\textquoteright{}',
        0x201A: '\\quotesinglbase{}',    # SINGLE LOW-9 QUOTATION MARK
        0x201C: '\\textquotedblleft{}',
        0x201D: '\\textquotedblright{}',
        0x201E: '\\quotedblbase{}',      # DOUBLE LOW-9 QUOTATION MARK
        0x2030: '\\textperthousand{}',   # PER MILLE SIGN
        0x2031: '\\textpertenthousand{}',  # PER TEN THOUSAND SIGN
        0x2039: '\\guilsinglleft{}',
        0x203A: '\\guilsinglright{}',
        0x2423: '\\textvisiblespace{}',  # OPEN BOX
        0x2020: '\\dag{}',
        0x2021: '\\ddag{}',
        0x2026: '\\dots{}',
        0x2122: '\\texttrademark{}',
    }
    # recognized with 'utf8', if textcomp is loaded
    textcomp = {
        # Latin-1 Supplement
        0x00a2: '\\textcent{}',              # ¢ CENT SIGN
        0x00a4: '\\textcurrency{}',          # ¤ CURRENCY SYMBOL
        0x00a5: '\\textyen{}',               # ¥ YEN SIGN
        0x00a6: '\\textbrokenbar{}',         # ¦ BROKEN BAR
        0x00a7: '\\textsection{}',           # § SECTION SIGN
        0x00a8: '\\textasciidieresis{}',     # ¨ DIAERESIS
        0x00a9: '\\textcopyright{}',         # © COPYRIGHT SIGN
        0x00aa: '\\textordfeminine{}',       # ª FEMININE ORDINAL INDICATOR
        0x00ac: '\\textlnot{}',              # ¬ NOT SIGN
        0x00ae: '\\textregistered{}',        # ® REGISTERED SIGN
        0x00af: '\\textasciimacron{}',       # ¯ MACRON
        0x00b0: '\\textdegree{}',            # ° DEGREE SIGN
        0x00b1: '\\textpm{}',                # ± PLUS-MINUS SIGN
        0x00b2: '\\texttwosuperior{}',       # ² SUPERSCRIPT TWO
        0x00b3: '\\textthreesuperior{}',     # ³ SUPERSCRIPT THREE
        0x00b4: '\\textasciiacute{}',        # ´ ACUTE ACCENT
        0x00b5: '\\textmu{}',                # µ MICRO SIGN
        0x00b6: '\\textparagraph{}',         # ¶ PILCROW SIGN # != \textpilcrow
        0x00b9: '\\textonesuperior{}',       # ¹ SUPERSCRIPT ONE
        0x00ba: '\\textordmasculine{}',      # º MASCULINE ORDINAL INDICATOR
        0x00bc: '\\textonequarter{}',        # 1/4 FRACTION
        0x00bd: '\\textonehalf{}',           # 1/2 FRACTION
        0x00be: '\\textthreequarters{}',     # 3/4 FRACTION
        0x00d7: '\\texttimes{}',             # × MULTIPLICATION SIGN
        0x00f7: '\\textdiv{}',               # ÷ DIVISION SIGN
        # others
        0x0192: '\\textflorin{}',            # LATIN SMALL LETTER F WITH HOOK
        0x02b9: '\\textasciiacute{}',        # MODIFIER LETTER PRIME
        0x02ba: '\\textacutedbl{}',          # MODIFIER LETTER DOUBLE PRIME
        0x2016: '\\textbardbl{}',            # DOUBLE VERTICAL LINE
        0x2022: '\\textbullet{}',            # BULLET
        0x2032: '\\textasciiacute{}',        # PRIME
        0x2033: '\\textacutedbl{}',          # DOUBLE PRIME
        0x2035: '\\textasciigrave{}',        # REVERSED PRIME
        0x2036: '\\textgravedbl{}',          # REVERSED DOUBLE PRIME
        0x203b: '\\textreferencemark{}',     # REFERENCE MARK
        0x203d: '\\textinterrobang{}',       # INTERROBANG
        0x2044: '\\textfractionsolidus{}',   # FRACTION SLASH
        0x2045: '\\textlquill{}',            # LEFT SQUARE BRACKET WITH QUILL
        0x2046: '\\textrquill{}',            # RIGHT SQUARE BRACKET WITH QUILL
        0x2052: '\\textdiscount{}',          # COMMERCIAL MINUS SIGN
        0x20a1: '\\textcolonmonetary{}',     # COLON SIGN
        0x20a3: '\\textfrenchfranc{}',       # FRENCH FRANC SIGN
        0x20a4: '\\textlira{}',              # LIRA SIGN
        0x20a6: '\\textnaira{}',             # NAIRA SIGN
        0x20a9: '\\textwon{}',               # WON SIGN
        0x20ab: '\\textdong{}',              # DONG SIGN
        0x20ac: '\\texteuro{}',              # EURO SIGN
        0x20b1: '\\textpeso{}',              # PESO SIGN
        0x20b2: '\\textguarani{}',           # GUARANI SIGN
        0x2103: '\\textcelsius{}',           # DEGREE CELSIUS
        0x2116: '\\textnumero{}',            # NUMERO SIGN
        0x2117: '\\textcircledP{}',          # SOUND RECORDING COPYRIGHT
        0x211e: '\\textrecipe{}',            # PRESCRIPTION TAKE
        0x2120: '\\textservicemark{}',       # SERVICE MARK
        0x2122: '\\texttrademark{}',         # TRADE MARK SIGN
        0x2126: '\\textohm{}',               # OHM SIGN
        0x2127: '\\textmho{}',               # INVERTED OHM SIGN
        0x212e: '\\textestimated{}',         # ESTIMATED SYMBOL
        0x2190: '\\textleftarrow{}',         # LEFTWARDS ARROW
        0x2191: '\\textuparrow{}',           # UPWARDS ARROW
        0x2192: '\\textrightarrow{}',        # RIGHTWARDS ARROW
        0x2193: '\\textdownarrow{}',         # DOWNWARDS ARROW
        0x2212: '\\textminus{}',             # MINUS SIGN
        0x2217: '\\textasteriskcentered{}',  # ASTERISK OPERATOR
        0x221a: '\\textsurd{}',              # SQUARE ROOT
        0x2422: '\\textblank{}',             # BLANK SYMBOL
        0x25e6: '\\textopenbullet{}',        # WHITE BULLET
        0x25ef: '\\textbigcircle{}',         # LARGE CIRCLE
        0x266a: '\\textmusicalnote{}',       # EIGHTH NOTE
        0x26ad: '\\textmarried{}',           # MARRIAGE SYMBOL
        0x26ae: '\\textdivorced{}',          # DIVORCE SYMBOL
        0x27e8: '\\textlangle{}',            # MATHEMATICAL LEFT ANGLE BRACKET
        0x27e9: '\\textrangle{}',            # MATHEMATICAL RIGHT ANGLE BRACKET
    }
    # Unicode chars that require a feature/package to render
    pifont = {
        0x2665: '\\ding{170}',               # black heartsuit
        0x2666: '\\ding{169}',               # black diamondsuit
        0x2713: '\\ding{51}',                # check mark
        0x2717: '\\ding{55}',                # check mark
    }
    # TODO: greek alphabet ... ?
    # see also LaTeX codec
    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/252124
    # and unimap.py from TeXML


class DocumentClass:
    """Details of a LaTeX document class."""

    def __init__(self, document_class, with_part=False):
        self.document_class = document_class
        self._with_part = with_part
        self.sections = ['section', 'subsection', 'subsubsection',
                         'paragraph', 'subparagraph']
        if self.document_class in ('book', 'memoir', 'report',
                                   'scrbook', 'scrreprt'):
            self.sections.insert(0, 'chapter')
        if self._with_part:
            self.sections.insert(0, 'part')

    def section(self, level):
        """Return the LaTeX section name for section `level`.

        The name depends on the specific document class.
        Level is 1,2,3..., as level 0 is the title.
        """
        if level <= len(self.sections):
            return self.sections[level-1]
        # unsupported levels
        return 'DUtitle'

    def latex_section_depth(self, depth):
        """
        Return LaTeX equivalent of Docutils section level `depth`.

        Given the value of the ``:depth:`` option of the "contents" or
        "sectnum" directive, return the corresponding value for the
        LaTeX ``tocdepth`` or ``secnumdepth`` counters.
        """
        depth = min(depth, len(self.sections))  # limit to supported levels
        if 'chapter' in self.sections:
            depth -= 1
        if self.sections[0] == 'part':
            depth -= 1
        return depth


class Table:
    """Manage a table while traversing.

    Table style might be

    :standard:   horizontal and vertical lines
    :booktabs:   only horizontal lines (requires "booktabs" LaTeX package)
    :borderless: no borders around table cells
    :nolines:    alias for borderless

    :colwidths-auto:  column widths determined by LaTeX
    """
    def __init__(self, translator, latex_type):
        self._translator = translator
        self._latex_type = latex_type
        self.legacy_column_widths = False

        self.close()
        self._colwidths = []
        self._rowspan = []
        self._in_thead = 0

    def open(self):
        self._open = True
        self._col_specs = []
        self.caption = []
        self._attrs = {}
        self._in_head = False  # maybe context with search

    def close(self):
        self._open = False
        self._col_specs = None
        self.caption = []
        self._attrs = {}
        self.stubs = []
        self.colwidths_auto = False

    def is_open(self):
        return self._open

    def set_table_style(self, node, settings):
        self.legacy_column_widths = settings.legacy_column_widths
        if 'align' in node:
            self.set('align', node['align'])
        # TODO: elif 'align' in classes/settings.table-style:
        #           self.set('align', ...)
        borders = [cls.replace('nolines', 'borderless')
                   for cls in (['standard']
                               + settings.table_style
                               + node['classes'])
                   if cls in ('standard', 'booktabs', 'borderless', 'nolines')]
        self.borders = borders[-1]
        self.colwidths_auto = (('colwidths-auto' in node['classes']
                                or 'colwidths-auto' in settings.table_style)
                               and 'colwidths-given' not in node['classes']
                               and 'width' not in node)

    def get_latex_type(self):
        if self._latex_type == 'longtable' and not self.caption:
            # do not advance the "table" counter (requires "ltcaption" package)
            return 'longtable*'
        return self._latex_type

    def set(self, attr, value):
        self._attrs[attr] = value

    def get(self, attr):
        if attr in self._attrs:
            return self._attrs[attr]
        return None

    def get_vertical_bar(self):
        if self.borders == 'standard':
            return '|'
        return ''

    # horizontal lines are drawn below a row,
    def get_opening(self, width=r'\linewidth'):
        align_map = {'left': '[l]',
                     'center': '[c]',
                     'right': '[r]',
                     None: ''}
        align = align_map.get(self.get('align'))
        latex_type = self.get_latex_type()
        if align and latex_type not in ("longtable", "longtable*"):
            opening = [r'\noindent\makebox[\linewidth]%s{%%' % (align,),
                       r'\begin{%s}' % (latex_type,)]
        else:
            opening = [r'\begin{%s}%s' % (latex_type, align)]
        if not self.colwidths_auto:
            if self.borders == 'standard' and not self.legacy_column_widths:
                opening.insert(-1, r'\setlength{\DUtablewidth}'
                               r'{\dimexpr%s-%i\arrayrulewidth\relax}%%'
                               % (width, len(self._col_specs)+1))
            else:
                opening.insert(-1, r'\setlength{\DUtablewidth}{%s}%%' % width)
        return '\n'.join(opening)

    def get_closing(self):
        closing = []
        if self.borders == 'booktabs':
            closing.append(r'\bottomrule')
        # elif self.borders == 'standard':
        #     closing.append(r'\hline')
        closing.append(r'\end{%s}' % self.get_latex_type())
        if (self.get('align')
            and self.get_latex_type() not in ("longtable", "longtable*")):
            closing.append('}')
        return '\n'.join(closing)

    def visit_colspec(self, node):
        self._col_specs.append(node)
        # "stubs" list is an attribute of the tgroup element:
        self.stubs.append(node.attributes.get('stub'))

    def get_colspecs(self, node):
        """Return column specification for longtable.
        """
        bar = self.get_vertical_bar()
        self._rowspan = [0] * len(self._col_specs)
        if self.colwidths_auto:
            self._colwidths = []
            latex_colspecs = ['l'] * len(self._col_specs)
        elif self.legacy_column_widths:
            # use old algorithm for backwards compatibility
            width = 80  # assumed standard line length
            factor = 0.93  # do not make it full linewidth
            # first see if we get too wide.
            total_width = sum(node['colwidth']+1 for node in self._col_specs)
            if total_width > width:
                factor *= width / total_width
            self._colwidths = [(factor * (node['colwidth']+1)/width)
                               + 0.005 for node in self._col_specs]
            latex_colspecs = ['p{%.3f\\DUtablewidth}' % colwidth
                              for colwidth in self._colwidths]
        else:
            # No of characters corresponding to table width = 100%
            #   Characters/line with LaTeX article, A4, Times, default margins
            #   depends on character:  M: 40, A: 50, x: 70, i: 120.
            norm_length = 40
            # Allowance to prevent unpadded columns like
            #   === ==
            #   ABC DE
            #   === ==
            # getting too narrow:
            if 'colwidths-given' not in node.parent.parent['classes']:
                allowance = 1
            else:
                allowance = 0  # "widths" option specified, use exact ratio
            self._colwidths = [(node['colwidth']+allowance)/norm_length
                               for node in self._col_specs]
            total_width = sum(self._colwidths)
            # Limit to 100%, force 100% if table width is specified:
            if total_width > 1 or 'width' in node.parent.parent.attributes:
                self._colwidths = [colwidth/total_width
                                   for colwidth in self._colwidths]
            latex_colspecs = ['p{\\DUcolumnwidth{%.3f}}' % colwidth
                              for colwidth in self._colwidths]
        return bar + bar.join(latex_colspecs) + bar

    def get_column_width(self):
        """Return columnwidth for current cell (not multicell)."""
        try:
            if self.legacy_column_widths:
                return '%.2f\\DUtablewidth'%self._colwidths[self._cell_in_row]
            return '\\DUcolumnwidth{%.2f}'%self._colwidths[self._cell_in_row]
        except IndexError:
            return '*'

    def get_multicolumn_width(self, start, len_):
        """Return sum of columnwidths for multicell."""
        try:
            multicol_width = sum(self._colwidths[start + co]
                                 for co in range(len_))
            if self.legacy_column_widths:
                return 'p{%.2f\\DUtablewidth}' % multicol_width
            return 'p{\\DUcolumnwidth{%.3f}}' % multicol_width
        except IndexError:
            return 'l'

    def get_caption(self):
        if not self.caption:
            return ''
        caption = ''.join(self.caption)
        if 1 == self._translator.thead_depth():
            return r'\caption{%s}\\' '\n' % caption
        return r'\caption[]{%s (... continued)}\\' '\n' % caption

    def need_recurse(self):
        if self._latex_type == 'longtable':
            return 1 == self._translator.thead_depth()
        return 0

    def visit_thead(self):
        self._in_thead += 1
        if self.borders == 'standard':
            return ['\\hline\n']
        elif self.borders == 'booktabs':
            return ['\\toprule\n']
        return []

    def depart_thead(self):
        a = []
        ## if self.borders == 'standard':
        ##     a.append('\\hline\n')
        if self.borders == 'booktabs':
            a.append('\\midrule\n')
        if self._latex_type == 'longtable':
            if 1 == self._translator.thead_depth():
                a.append('\\endfirsthead\n')
            else:
                n_c = len(self._col_specs)
                a.append('\\endhead\n')
                # footer on all but last page (if it fits):
                twidth = sum(node['colwidth']+2 for node in self._col_specs)
                if twidth > 30 or (twidth > 12 and not self.colwidths_auto):
                    a.append(r'\multicolumn{%d}{%s}'
                             % (n_c, self.get_multicolumn_width(0, n_c))
                             + r'{\raggedleft\ldots continued on next page}\\'
                             + '\n')
                a.append('\\endfoot\n\\endlastfoot\n')
            # for longtable one could add firsthead, foot and lastfoot
        self._in_thead -= 1
        return a

    def visit_row(self):
        self._cell_in_row = 0

    def depart_row(self):
        res = [' \\\\\n']
        self._cell_in_row = None  # remove cell counter
        for i in range(len(self._rowspan)):
            if self._rowspan[i] > 0:
                self._rowspan[i] -= 1

        if self.borders == 'standard':
            rowspans = [i+1 for i in range(len(self._rowspan))
                        if self._rowspan[i] <= 0]
            if len(rowspans) == len(self._rowspan):
                res.append('\\hline\n')
            else:
                cline = ''
                rowspans.reverse()
                # TODO merge clines
                while True:
                    try:
                        c_start = rowspans.pop()
                    except IndexError:
                        break
                    cline += '\\cline{%d-%d}\n' % (c_start, c_start)
                res.append(cline)
        return res

    def set_rowspan(self, cell, value):
        try:
            self._rowspan[cell] = value
        except IndexError:
            pass

    def get_rowspan(self, cell):
        try:
            return self._rowspan[cell]
        except IndexError:
            return 0

    def get_entry_number(self):
        return self._cell_in_row

    def visit_entry(self):
        self._cell_in_row += 1

    def is_stub_column(self):
        if len(self.stubs) >= self._cell_in_row:
            return self.stubs[self._cell_in_row]
        return False


class LaTeXTranslator(nodes.NodeVisitor):
    """
    Generate code for 8-bit LaTeX from a Docutils document tree.

    See the docstring of docutils.writers._html_base.HTMLTranslator for
    notes on and examples of safe subclassing.
    """

    # When options are given to the documentclass, latex will pass them
    # to other packages, as done with babel.
    # Dummy settings might be taken from document settings

    # Generate code for typesetting with 8-bit latex/pdflatex vs.
    # xelatex/lualatex engine. Overwritten by the XeTeX writer
    is_xetex = False

    # Config setting defaults
    # -----------------------

    # TODO: use mixins for different implementations.
    # list environment for docinfo. else tabularx
    ## use_optionlist_for_docinfo = False # TODO: NOT YET IN USE

    # Use compound enumerations (1.A.1.)
    compound_enumerators = False

    # If using compound enumerations, include section information.
    section_prefix_for_enumerators = False

    # This is the character that separates the section ("." subsection ...)
    # prefix from the regular list enumerator.
    section_enumerator_separator = '-'

    # Auxiliary variables
    # -------------------

    has_latex_toc = False  # is there a toc in the doc? (needed by minitoc)
    section_level = 0

    # Flags to encode():
    # inside citation reference labels underscores dont need to be escaped
    inside_citation_reference_label = False
    verbatim = False                    # do not encode
    insert_non_breaking_blanks = False  # replace blanks by "~"
    insert_newline = False              # add latex newline commands
    literal = False                     # literal text (block or inline)
    alltt = False                       # inside `alltt` environment

    def __init__(self, document, babel_class=Babel):
        super().__init__(document)
        # Reporter
        # ~~~~~~~~
        self.warn = self.document.reporter.warning
        self.error = self.document.reporter.error

        # Settings
        # ~~~~~~~~
        self.settings = settings = document.settings
        # warn of deprecated settings and changing defaults:
        if settings.use_latex_citations is None and not settings.use_bibtex:
            settings.use_latex_citations = False
            warnings.warn('The default for the setting "use_latex_citations" '
                          'will change to "True" in Docutils 1.0.',
                          FutureWarning, stacklevel=7)
        if settings.legacy_column_widths is None:
            settings.legacy_column_widths = True
            warnings.warn('The default for the setting "legacy_column_widths" '
                          'will change to "False" in Docutils 1.0.)',
                          FutureWarning, stacklevel=7)
        if settings.use_verbatim_when_possible is not None:
            warnings.warn(
                'The configuration setting "use_verbatim_when_possible" '
                'will be removed in Docutils 2.0. '
                'Use "literal_block_env: verbatim".',
                FutureWarning, stacklevel=7)

        self.latex_encoding = self.to_latex_encoding(settings.output_encoding)
        self.use_latex_toc = settings.use_latex_toc
        self.use_latex_docinfo = settings.use_latex_docinfo
        self.use_latex_citations = settings.use_latex_citations
        self._reference_label = settings.reference_label
        self.hyperlink_color = settings.hyperlink_color
        self.compound_enumerators = settings.compound_enumerators
        self.font_encoding = getattr(settings, 'font_encoding', '')
        self.section_prefix_for_enumerators = (
            settings.section_prefix_for_enumerators)
        self.section_enumerator_separator = (
            settings.section_enumerator_separator.replace('_', r'\_'))
        # literal blocks:
        self.literal_block_env = ''
        self.literal_block_options = ''
        if settings.literal_block_env:
            (none,
             self.literal_block_env,
             self.literal_block_options,
             none) = re.split(r'(\w+)(.*)', settings.literal_block_env)
        elif settings.use_verbatim_when_possible:
            self.literal_block_env = 'verbatim'

        if settings.use_bibtex:
            self.use_latex_citations = True
        self.bibtex = settings.use_bibtex
        # language module for Docutils-generated text
        # (labels, bibliographic_fields, and author_separators)
        self.language_module = languages.get_language(settings.language_code,
                                                      document.reporter)
        self.babel = babel_class(settings.language_code, document.reporter)
        self.author_separator = self.language_module.author_separators[0]
        d_options = [settings.documentoptions]
        if self.babel.language not in ('english', ''):
            d_options.append(self.babel.language)
        self.documentoptions = ','.join(filter(None, d_options))
        self.d_class = DocumentClass(settings.documentclass,
                                     settings.use_part_section)
        # graphic package options:
        if settings.graphicx_option == '':
            self.graphicx_package = r'\usepackage{graphicx}'
        else:
            self.graphicx_package = (r'\usepackage[%s]{graphicx}' %
                                     settings.graphicx_option)
        # footnotes: TODO: implement LaTeX footnotes
        self.docutils_footnotes = settings.docutils_footnotes

        # Output collection stacks
        # ~~~~~~~~~~~~~~~~~~~~~~~~

        # Document parts
        self.head_prefix = [r'\documentclass[%s]{%s}' %
                            (self.documentoptions,
                             settings.documentclass)]
        self.requirements = SortableDict()  # made a list in depart_document()
        self.requirements['__static'] = r'\usepackage{ifthen}'
        self.latex_preamble = [settings.latex_preamble]
        self.fallbacks = SortableDict()  # made a list in depart_document()
        self.pdfsetup = []  # PDF properties (hyperref package)
        self.title = []
        self.subtitle = []
        self.titledata = []  # \title, \author, \date
        ## self.body_prefix = ['\\begin{document}\n']
        self.body_pre_docinfo = []  # \maketitle
        self.docinfo = []
        self.dedication = []
        self.abstract = []
        self.body = []
        ## self.body_suffix = ['\\end{document}\n']

        self.context = []
        """Heterogeneous stack.

        Used by visit_* and depart_* functions in conjunction with the tree
        traversal. Make sure that the pops correspond to the pushes."""

        # Title metadata:
        self.title_labels = []
        self.subtitle_labels = []
        # (if use_latex_docinfo: collects lists of
        # author/organization/contact/address lines)
        self.author_stack = []
        self.date = []

        # PDF properties: pdftitle, pdfauthor
        self.pdfauthor = []
        self.pdfinfo = []
        if settings.language_code != 'en':
            self.pdfinfo.append('  pdflang={%s},'%settings.language_code)

        # Stack of section counters so that we don't have to use_latex_toc.
        # This will grow and shrink as processing occurs.
        # Initialized for potential first-level sections.
        self._section_number = [0]

        # The current stack of enumerations so that we can expand
        # them into a compound enumeration.
        self._enumeration_counters = []
        # The maximum number of enumeration counters we've used.
        # If we go beyond this number, we need to create a new
        # counter; otherwise, just reuse an old one.
        self._max_enumeration_counters = 0

        self._bibitems = []

        # object for a table while processing.
        self.table_stack = []
        self.active_table = Table(self, 'longtable')

        # Where to collect the output of visitor methods (default: body)
        self.out = self.body
        self.out_stack = []  # stack of output collectors

        # Process settings
        # ~~~~~~~~~~~~~~~~
        # Encodings:
        # Docutils' output-encoding => TeX input encoding
        if self.latex_encoding not in ('ascii', 'unicode', 'utf8'):
            self.requirements['_inputenc'] = (r'\usepackage[%s]{inputenc}'
                                              % self.latex_encoding)
        # TeX font encoding
        if not self.is_xetex:
            if self.font_encoding:
                self.requirements['_fontenc'] = (r'\usepackage[%s]{fontenc}' %
                                                 self.font_encoding)
            # ensure \textquotedbl is defined:
            for enc in self.font_encoding.split(','):
                enc = enc.strip()
                if enc == 'OT1':
                    self.requirements['_textquotedblOT1'] = (
                        r'\DeclareTextSymbol{\textquotedbl}{OT1}{`\"}')
                elif enc not in ('T1', 'T2A', 'T2B', 'T2C', 'T4', 'T5'):
                    self.requirements['_textquotedbl'] = (
                        r'\DeclareTextSymbolDefault{\textquotedbl}{T1}')
        # page layout with typearea (if there are relevant document options)
        if (settings.documentclass.find('scr') == -1
            and (self.documentoptions.find('DIV') != -1
                 or self.documentoptions.find('BCOR') != -1)):
            self.requirements['typearea'] = r'\usepackage{typearea}'

        # Stylesheets
        # (the name `self.stylesheet` is singular because only one
        # stylesheet was supported before Docutils 0.6).
        stylesheet_list = utils.get_stylesheet_list(settings)
        self.fallback_stylesheet = 'docutils' in stylesheet_list
        if self.fallback_stylesheet:
            stylesheet_list.remove('docutils')
            if settings.legacy_class_functions:
                # docutils.sty is incompatible with legacy functions
                self.fallback_stylesheet = False
            else:
                # require a minimal version:
                self.fallbacks['docutils.sty'] = (
                    r'\usepackage{docutils}[2020/08/28]')

        self.stylesheet = [self.stylesheet_call(path)
                           for path in stylesheet_list]

        # PDF setup
        if self.hyperlink_color in ('0', 'false', 'False', ''):
            self.hyperref_options = ''
        else:
            self.hyperref_options = ('colorlinks=true,'
                                     f'linkcolor={self.hyperlink_color},'
                                     f'urlcolor={self.hyperlink_color}')
        if settings.hyperref_options:
            self.hyperref_options += ',' + settings.hyperref_options

        # LaTeX Toc
        # include all supported sections in toc and PDF bookmarks
        # (or use documentclass-default (as currently))?

        # Section numbering
        if settings.sectnum_xform:  # section numbering by Docutils
            PreambleCmds.secnumdepth = r'\setcounter{secnumdepth}{0}'
        else:  # section numbering by LaTeX:
            secnumdepth = settings.sectnum_depth
            # Possible values of settings.sectnum_depth:
            # None  "sectnum" directive without depth arg -> LaTeX default
            #  0    no "sectnum" directive -> no section numbers
            # >0    value of "depth" argument -> translate to LaTeX levels:
            #       -1  part    (0 with "article" document class)
            #        0  chapter (missing in "article" document class)
            #        1  section
            #        2  subsection
            #        3  subsubsection
            #        4  paragraph
            #        5  subparagraph
            if secnumdepth is not None:
                PreambleCmds.secnumdepth = (
                    r'\setcounter{secnumdepth}{%d}'
                    % self.d_class.latex_section_depth(secnumdepth))
            # start with specified number:
            if (hasattr(settings, 'sectnum_start')
                and settings.sectnum_start != 1):
                self.requirements['sectnum_start'] = (
                    r'\setcounter{%s}{%d}' % (self.d_class.sections[0],
                                              settings.sectnum_start-1))
            # TODO: currently ignored (configure in a stylesheet):
            ## settings.sectnum_prefix
            ## settings.sectnum_suffix

    # Auxiliary Methods
    # -----------------

    def stylesheet_call(self, path):
        """Return code to reference or embed stylesheet file `path`"""
        path = Path(path)
        # is it a package (no extension or *.sty) or "normal" tex code:
        is_package = path.suffix in ('.sty', '')
        # Embed content of style file:
        if self.settings.embed_stylesheet:
            if is_package:
                path = path.with_suffix('.sty')  # ensure extension
            try:
                content = path.read_text(encoding='utf-8')
            except OSError as err:
                msg = f'Cannot embed stylesheet:\n {err}'.replace('\\\\', '/')
                self.document.reporter.error(msg)
                return '% ' + msg.replace('\n', '\n% ')
            else:
                self.settings.record_dependencies.add(path.as_posix())
            if is_package:
                # allow '@' in macro names:
                content = (f'\\makeatletter\n{content}\n\\makeatother')
            return (f'% embedded stylesheet: {path.as_posix()}\n'
                    f'{content}')
        # Link to style file:
        if is_package:
            path = path.parent / path.stem  # drop extension
            cmd = r'\usepackage{%s}'
        else:
            cmd = r'\input{%s}'
        if self.settings.stylesheet_path:
            # adapt path relative to output (cf. config.html#stylesheet-path)
            return cmd % utils.relative_path(self.settings._destination, path)
        return cmd % path.as_posix()

    def to_latex_encoding(self, docutils_encoding):
        """Translate docutils encoding name into LaTeX's.

        Default method is remove "-" and "_" chars from docutils_encoding.
        """
        tr = {'iso-8859-1': 'latin1',     # west european
              'iso-8859-2': 'latin2',     # east european
              'iso-8859-3': 'latin3',     # esperanto, maltese
              'iso-8859-4': 'latin4',     # north european
              'iso-8859-5': 'iso88595',   # cyrillic (ISO)
              'iso-8859-9': 'latin5',     # turkish
              'iso-8859-15': 'latin9',    # latin9, update to latin1.
              'mac_cyrillic': 'maccyr',   # cyrillic (on Mac)
              'windows-1251': 'cp1251',   # cyrillic (on Windows)
              'koi8-r': 'koi8-r',         # cyrillic (Russian)
              'koi8-u': 'koi8-u',         # cyrillic (Ukrainian)
              'windows-1250': 'cp1250',   #
              'windows-1252': 'cp1252',   #
              'us-ascii': 'ascii',        # ASCII (US)
              # unmatched encodings
              # '': 'applemac',
              # '': 'ansinew',  # windows 3.1 ansi
              # '': 'ascii',    # ASCII encoding for the range 32--127.
              # '': 'cp437',    # dos latin us
              # '': 'cp850',    # dos latin 1
              # '': 'cp852',    # dos latin 2
              # '': 'decmulti',
              # '': 'latin10',
              # 'iso-8859-6': ''   # arabic
              # 'iso-8859-7': ''   # greek
              # 'iso-8859-8': ''   # hebrew
              # 'iso-8859-10': ''  # latin6, more complete iso-8859-4
              }
        encoding = docutils_encoding.lower()  # normalize case
        encoding = encoding.split(':')[0]     # strip the error handler
        if encoding in tr:
            return tr[encoding]
        # drop HYPHEN or LOW LINE from "latin_1", "utf-8" and similar
        return encoding.replace('_', '').replace('-', '')

    def language_label(self, docutil_label):
        return self.language_module.labels[docutil_label]

    def encode(self, text):
        """Return text with 'problematic' characters escaped.

        * Escape the special printing characters ``# $ % & ~ _ ^ \\ { }``,
          square brackets ``[ ]``, double quotes and (in OT1) ``< | >``.
        * Translate non-supported Unicode characters.
        * Separate ``-`` (and more in literal text) to prevent input ligatures.
        """
        if self.verbatim:
            return text
        # Set up the translation table:
        table = CharMaps.alltt.copy()
        if not self.alltt:
            table.update(CharMaps.special)
        # keep the underscore in citation references
        if self.inside_citation_reference_label and not self.alltt:
            del table[ord('_')]
        # Workarounds for OT1 font-encoding
        if self.font_encoding in ['OT1', ''] and not self.is_xetex:
            # * out-of-order characters in cmtt
            if self.literal:
                # replace underscore by underlined blank,
                # because this has correct width.
                table[ord('_')] = '\\underline{~}'
                # the backslash doesn't work, so we use a mirrored slash.
                # \reflectbox is provided by graphicx:
                self.requirements['graphicx'] = self.graphicx_package
                table[ord('\\')] = '\\reflectbox{/}'
            # * ``< | >`` come out as different chars (except for cmtt):
            else:
                table[ord('|')] = '\\textbar{}'
                table[ord('<')] = '\\textless{}'
                table[ord('>')] = '\\textgreater{}'
        if self.insert_non_breaking_blanks:
            table[ord(' ')] = '~'
            # tab chars may occur in included files (literal or code)
            # quick-and-dirty replacement with spaces
            # (for better results use `--literal-block-env=lstlisting`)
            table[ord('\t')] = '~' * self.settings.tab_width
        # Unicode replacements for 8-bit tex engines (not required with XeTeX)
        if not self.is_xetex:
            if not self.latex_encoding.startswith('utf8'):
                table.update(CharMaps.unsupported_unicode)
                table.update(CharMaps.utf8_supported_unicode)
                table.update(CharMaps.textcomp)
            table.update(CharMaps.pifont)
            # Characters that require a feature/package to render
            for ch in text:
                cp = ord(ch)
                if cp in CharMaps.textcomp and not self.fallback_stylesheet:
                    self.requirements['textcomp'] = PreambleCmds.textcomp
                elif cp in CharMaps.pifont:
                    self.requirements['pifont'] = '\\usepackage{pifont}'
                # preamble-definitions for unsupported Unicode characters
                elif (self.latex_encoding == 'utf8'
                      and cp in CharMaps.unsupported_unicode):
                    self.requirements['_inputenc'+str(cp)] = (
                        '\\DeclareUnicodeCharacter{%04X}{%s}'
                        % (cp, CharMaps.unsupported_unicode[cp]))
        text = text.translate(table)

        # Break up input ligatures e.g. '--' to '-{}-'.
        if not self.is_xetex:  # Not required with xetex/luatex
            separate_chars = '-'
            # In monospace-font, we also separate ',,', '``' and "''" and some
            # other characters which can't occur in non-literal text.
            if self.literal:
                separate_chars += ',`\'"<>'
            for char in separate_chars * 2:
                # Do it twice ("* 2") because otherwise we would replace
                # '---' by '-{}--'.
                text = text.replace(char + char, char + '{}' + char)

        # Literal line breaks (in address or literal blocks):
        if self.insert_newline:
            lines = text.split('\n')
            # Add a protected space to blank lines (except the last)
            # to avoid ``! LaTeX Error: There's no line here to end.``
            for i, line in enumerate(lines[:-1]):
                if not line.lstrip():
                    lines[i] += '~'
            text = (r'\\' + '\n').join(lines)
        if self.literal and not self.insert_non_breaking_blanks:
            # preserve runs of spaces but allow wrapping
            text = text.replace('  ', ' ~')
        return text

    def attval(self, text,
               whitespace=re.compile('[\n\r\t\v\f]')):
        """Cleanse, encode, and return attribute value text."""
        return self.encode(whitespace.sub(' ', text))

    # TODO: is this used anywhere? -> update (use template) or delete
    ## def astext(self):
    ##     """Assemble document parts and return as string."""
    ##     head = '\n'.join(self.head_prefix + self.stylesheet + self.head)
    ##     body = ''.join(self.body_prefix  + self.body + self.body_suffix)
    ##     return head + '\n' + body

    def is_inline(self, node):
        """Check whether a node represents an inline or block-level element"""
        return isinstance(node.parent, nodes.TextElement)

    def append_hypertargets(self, node):
        """Append hypertargets for all ids of `node`"""
        # hypertarget places the anchor at the target's baseline,
        # so we raise it explicitly
        self.out.append('%\n'.join('\\raisebox{1em}{\\hypertarget{%s}{}}' %
                                   id for id in node['ids']))

    def ids_to_labels(self, node, set_anchor=True, protect=False,
                      newline=False):
        """Return list of label definitions for all ids of `node`

        If `set_anchor` is True, an anchor is set with \\phantomsection.
        If `protect` is True, the \\label cmd is made robust.
        If `newline` is True, a newline is added if there are labels.
        """
        prefix = '\\protect' if protect else ''
        labels = [prefix + '\\label{%s}' % id for id in node['ids']]
        if set_anchor and labels:
            labels.insert(0, '\\phantomsection')
        if newline and labels:
            labels.append('\n')
        return labels

    def set_align_from_classes(self, node):
        """Convert ``align-*`` class arguments into alignment args."""
        # separate:
        align = [cls for cls in node['classes'] if cls.startswith('align-')]
        if align:
            node['align'] = align[-1].replace('align-', '')
            node['classes'] = [cls for cls in node['classes']
                               if not cls.startswith('align-')]

    def insert_align_declaration(self, node, default=None):
        align = node.get('align', default)
        if align == 'left':
            self.out.append('\\raggedright\n')
        elif align == 'center':
            self.out.append('\\centering\n')
        elif align == 'right':
            self.out.append('\\raggedleft\n')

    def duclass_open(self, node):
        """Open a group and insert declarations for class values."""
        if not isinstance(node.parent, nodes.compound):
            self.out.append('\n')
        for cls in node['classes']:
            if cls.startswith('language-'):
                language = self.babel.language_name(cls[9:])
                if language:
                    self.babel.otherlanguages[language] = True
                    self.out.append('\\begin{selectlanguage}{%s}\n' % language)
            elif (isinstance(node, nodes.table)
                  and cls in Writer.table_style_values + ['colwidths-given']):
                pass
            else:
                if not self.fallback_stylesheet:
                    self.fallbacks['DUclass'] = PreambleCmds.duclass
                self.out.append('\\begin{DUclass}{%s}\n' % cls)

    def duclass_close(self, node):
        """Close a group of class declarations."""
        for cls in reversed(node['classes']):
            if cls.startswith('language-'):
                language = self.babel.language_name(cls[9:])
                if language:
                    self.out.append('\\end{selectlanguage}\n')
            elif (isinstance(node, nodes.table)
                  and cls in Writer.table_style_values + ['colwidths-given']):
                pass
            else:
                if not self.fallback_stylesheet:
                    self.fallbacks['DUclass'] = PreambleCmds.duclass
                self.out.append('\\end{DUclass}\n')

    def push_output_collector(self, new_out):
        self.out_stack.append(self.out)
        self.out = new_out

    def pop_output_collector(self):
        self.out = self.out_stack.pop()

    def term_postfix(self, node):
        """
        Return LaTeX code required between term or field name and content.

        In a LaTeX "description" environment (used for definition
        lists and non-docinfo field lists), a ``\\leavevmode``
        between an item's label and content ensures the correct
        placement of certain block constructs.
        """
        for child in node:
            if not isinstance(child, (nodes.Invisible, nodes.footnote,
                                      nodes.citation)):
                break
        else:
            return ''
        if isinstance(child, (nodes.container, nodes.compound)):
            return self.term_postfix(child)
        if isinstance(child, nodes.image):
            return '\\leavevmode\n'  # Images get an additional newline.
        if not isinstance(child, (nodes.paragraph, nodes.math_block)):
            return '\\leavevmode'
        return ''

    # Visitor methods
    # ---------------

    def visit_Text(self, node):
        self.out.append(self.encode(node.astext()))

    def depart_Text(self, node):
        pass

    def visit_abbreviation(self, node):
        node['classes'].insert(0, 'abbreviation')
        self.visit_inline(node)

    def depart_abbreviation(self, node):
        self.depart_inline(node)

    def visit_acronym(self, node):
        node['classes'].insert(0, 'acronym')
        self.visit_inline(node)

    def depart_acronym(self, node):
        self.depart_inline(node)

    def visit_address(self, node):
        self.visit_docinfo_item(node, 'address')

    def depart_address(self, node):
        self.depart_docinfo_item(node)

    def visit_admonition(self, node):
        # strip the generic 'admonition' from the list of classes
        node['classes'] = [cls for cls in node['classes']
                           if cls != 'admonition']
        if self.settings.legacy_class_functions:
            self.fallbacks['admonition'] = PreambleCmds.admonition_legacy
            if 'error' in node['classes']:
                self.fallbacks['error'] = PreambleCmds.error_legacy
            self.out.append('\n\\DUadmonition[%s]{'%','.join(node['classes']))
            return
        if not self.fallback_stylesheet:
            self.fallbacks['admonition'] = PreambleCmds.admonition
        if 'error' in node['classes'] and not self.fallback_stylesheet:
            self.fallbacks['error'] = PreambleCmds.error
        self.duclass_open(node)
        self.out.append('\\begin{DUadmonition}')

    def depart_admonition(self, node):
        if self.settings.legacy_class_functions:
            self.out.append('}\n')
            return
        self.out.append('\\end{DUadmonition}\n')
        self.duclass_close(node)

    def visit_author(self, node):
        self.pdfauthor.append(self.attval(node.astext()))
        self.visit_docinfo_item(node, 'author')

    def depart_author(self, node):
        self.depart_docinfo_item(node)

    def visit_authors(self, node):
        # not used: visit_author is called anyway for each author.
        pass

    def depart_authors(self, node):
        pass

    def visit_block_quote(self, node):
        self.duclass_open(node)
        self.out.append('\\begin{quote}')

    def depart_block_quote(self, node):
        self.out.append('\\end{quote}\n')
        self.duclass_close(node)

    def visit_bullet_list(self, node):
        self.duclass_open(node)
        self.out.append('\\begin{itemize}')

    def depart_bullet_list(self, node):
        self.out.append('\\end{itemize}\n')
        self.duclass_close(node)

    def visit_superscript(self, node):
        self.out.append(r'\textsuperscript{')
        self.visit_inline(node)

    def depart_superscript(self, node):
        self.depart_inline(node)
        self.out.append('}')

    def visit_subscript(self, node):
        self.out.append(r'\textsubscript{')
        self.visit_inline(node)

    def depart_subscript(self, node):
        self.depart_inline(node)
        self.out.append('}')

    def visit_caption(self, node):
        self.out.append('\n\\caption{')

    def depart_caption(self, node):
        self.out.append('}\n')

    def visit_title_reference(self, node):
        if not self.fallback_stylesheet:
            self.fallbacks['titlereference'] = PreambleCmds.titlereference
        self.out.append(r'\DUroletitlereference{')
        self.visit_inline(node)

    def depart_title_reference(self, node):
        self.depart_inline(node)
        self.out.append('}')

    def visit_citation(self, node):
        if self.use_latex_citations:
            self.push_output_collector([])
        else:
            # self.requirements['~fnt_floats'] = PreambleCmds.footnote_floats
            self.out.append(r'\begin{figure}[b]')
            self.append_hypertargets(node)

    def depart_citation(self, node):
        if self.use_latex_citations:
            # TODO: normalize label
            label = self.out[0]
            text = ''.join(self.out[1:])
            self._bibitems.append([label, text])
            self.pop_output_collector()
        else:
            self.out.append('\\end{figure}\n')

    def visit_citation_reference(self, node):
        if self.bibtex:
            self._bibitems.append([node.astext()])
        if self.use_latex_citations:
            if not self.inside_citation_reference_label:
                self.out.append(r'\cite{')
                self.inside_citation_reference_label = 1
            else:
                assert self.out[-1] in (' ', '\n'),\
                        'unexpected non-whitespace while in reference label'
                del self.out[-1]
        else:
            href = ''
            if 'refid' in node:
                href = node['refid']
            elif 'refname' in node:
                href = self.document.nameids[node['refname']]
            self.out.append('\\hyperlink{%s}{[' % href)

    def depart_citation_reference(self, node):
        # TODO: normalize labels
        if self.use_latex_citations:
            followup_citation = False
            # check for a following citation separated by a space or newline
            sibling = node.next_node(descend=False, siblings=True)
            if (isinstance(sibling, nodes.Text)
                and sibling.astext() in (' ', '\n')):
                sibling2 = sibling.next_node(descend=False, siblings=True)
                if isinstance(sibling2, nodes.citation_reference):
                    followup_citation = True
            if followup_citation:
                self.out.append(',')
            else:
                self.out.append('}')
                self.inside_citation_reference_label = False
        else:
            self.out.append(']}')

    def visit_classifier(self, node):
        self.out.append('(\\textbf{')

    def depart_classifier(self, node):
        self.out.append('})')

    def visit_colspec(self, node):
        self.active_table.visit_colspec(node)

    def depart_colspec(self, node):
        pass

    def visit_comment(self, node):
        if not isinstance(node.parent, nodes.compound):
            self.out.append('\n')
        # Precede every line with a comment sign, wrap in newlines
        self.out.append('%% %s\n' % node.astext().replace('\n', '\n% '))
        raise nodes.SkipNode

    def depart_comment(self, node):
        pass

    def visit_compound(self, node):
        if isinstance(node.parent, nodes.compound):
            self.out.append('\n')
        node['classes'].insert(0, 'compound')
        self.duclass_open(node)

    def depart_compound(self, node):
        self.duclass_close(node)

    def visit_contact(self, node):
        self.visit_docinfo_item(node, 'contact')

    def depart_contact(self, node):
        self.depart_docinfo_item(node)

    def visit_container(self, node):
        self.duclass_open(node)

    def depart_container(self, node):
        self.duclass_close(node)

    def visit_copyright(self, node):
        self.visit_docinfo_item(node, 'copyright')

    def depart_copyright(self, node):
        self.depart_docinfo_item(node)

    def visit_date(self, node):
        self.visit_docinfo_item(node, 'date')

    def depart_date(self, node):
        self.depart_docinfo_item(node)

    def visit_decoration(self, node):
        # header and footer
        pass

    def depart_decoration(self, node):
        pass

    def visit_definition(self, node):
        pass

    def depart_definition(self, node):
        self.out.append('\n')                # TODO: just pass?

    def visit_definition_list(self, node):
        self.duclass_open(node)
        self.out.append('\\begin{description}\n')

    def depart_definition_list(self, node):
        self.out.append('\\end{description}\n')
        self.duclass_close(node)

    def visit_definition_list_item(self, node):
        pass

    def depart_definition_list_item(self, node):
        pass

    def visit_description(self, node):
        self.out.append(' ')

    def depart_description(self, node):
        pass

    def visit_docinfo(self, node):
        self.push_output_collector(self.docinfo)

    def depart_docinfo(self, node):
        self.pop_output_collector()
        # Some itmes (e.g. author) end up at other places
        if self.docinfo:
            # tabularx: automatic width of columns, no page breaks allowed.
            self.requirements['tabularx'] = r'\usepackage{tabularx}'
            if not self.fallback_stylesheet:
                self.fallbacks['_providelength'] = PreambleCmds.providelength
                self.fallbacks['docinfo'] = PreambleCmds.docinfo
            #
            self.docinfo.insert(0, '\n% Docinfo\n'
                                '\\begin{center}\n'
                                '\\begin{tabularx}{\\DUdocinfowidth}{lX}\n')
            self.docinfo.append('\\end{tabularx}\n'
                                '\\end{center}\n')

    def visit_docinfo_item(self, node, name):
        if self.use_latex_docinfo:
            if name in ('author', 'organization', 'contact', 'address'):
                # We attach these to the last author.  If any of them precedes
                # the first author, put them in a separate "author" group
                # (in lack of better semantics).
                if name == 'author' or not self.author_stack:
                    self.author_stack.append([])
                if name == 'address':   # newlines are meaningful
                    self.insert_newline = True
                    text = self.encode(node.astext())
                    self.insert_newline = False
                else:
                    text = self.attval(node.astext())
                self.author_stack[-1].append(text)
                raise nodes.SkipNode
            elif name == 'date':
                self.date.append(self.attval(node.astext()))
                raise nodes.SkipNode
        self.out.append('\\textbf{%s}: &\n\t' % self.language_label(name))
        if name == 'address':
            self.insert_newline = True
            self.out.append('{\\raggedright\n')
            self.context.append(' } \\\\\n')
        else:
            self.context.append(' \\\\\n')

    def depart_docinfo_item(self, node):
        self.out.append(self.context.pop())
        # for address we did set insert_newline
        self.insert_newline = False

    def visit_doctest_block(self, node):
        self.visit_literal_block(node)

    def depart_doctest_block(self, node):
        self.depart_literal_block(node)

    def visit_document(self, node):
        # titled document?
        if (self.use_latex_docinfo or len(node)
            and isinstance(node[0], nodes.title)):
            protect = (self.settings.documentclass == 'memoir')
            self.title_labels += self.ids_to_labels(node, set_anchor=False,
                                                    protect=protect)

    def depart_document(self, node):
        # Complete "parts" with information gained from walkabout
        # * language setup
        if (self.babel.otherlanguages
            or self.babel.language not in ('', 'english')):
            self.requirements['babel'] = self.babel()
        # * conditional requirements (before style sheet)
        self.requirements = self.requirements.sortedvalues()
        # * coditional fallback definitions (after style sheet)
        self.fallbacks = self.fallbacks.sortedvalues()
        # * PDF properties
        self.pdfsetup.append(PreambleCmds.linking % self.hyperref_options)
        if self.pdfauthor:
            authors = self.author_separator.join(self.pdfauthor)
            self.pdfinfo.append('  pdfauthor={%s}' % authors)
        if self.pdfinfo:
            self.pdfsetup += [r'\hypersetup{'] + self.pdfinfo + ['}']
        # * title (including author(s) and date if using "latex_docinfo")
        if self.title or (self.use_latex_docinfo
                          and (self.author_stack or self.date)):
            self.make_title()  # see below
        # * bibliography
        if self._bibitems:
            self.append_bibliogaphy()  # see below
        # * make sure to generate a toc file if needed for local contents:
        if 'minitoc' in self.requirements and not self.has_latex_toc:
            self.out.append('\n\\faketableofcontents % for local ToCs\n')

    def make_title(self):
        # Auxiliary function called by `self.depart_document()`.
        #
        # Append ``\title``, ``\author``, and ``\date`` to "titledata".
        # (We need all three, even if empty, to prevent errors
        # and/or automatic display of the current date by \maketitle.)
        # Append ``\maketitle`` to "body_pre_docinfo" parts.
        #
        # \title
        title_arg = [''.join(self.title)]  # ensure len == 1
        if self.title:
            title_arg += self.title_labels
        if self.subtitle:
            title_arg += [r'\\',
                          r'\DUdocumentsubtitle{%s}' % ''.join(self.subtitle),
                          ] + self.subtitle_labels
        self.titledata.append(r'\title{%s}' % '%\n  '.join(title_arg))
        # \author
        author_arg = ['\\\\\n'.join(author_entry)
                      for author_entry in self.author_stack]
        self.titledata.append(r'\author{%s}' %
                              ' \\and\n'.join(author_arg))
        # \date
        self.titledata.append(r'\date{%s}' % ', '.join(self.date))
        # \maketitle
        # Must be in the document body. We add it to `body_pre_docinfo`
        # to allow templates to put `titledata` into the document preamble.
        self.body_pre_docinfo.append('\\maketitle\n')

    def append_bibliogaphy(self):
        # Add bibliography at end of document.
        # TODO insertion point should be configurable.
        # Auxiliary function called by `depart_document`.
        if self.bibtex:
            self.out.append('\n\\bibliographystyle{%s}\n' % self.bibtex[0])
            self.out.append('\\bibliography{%s}\n' % ','.join(self.bibtex[1:]))
        elif self.use_latex_citations:
            # TODO: insert citations at point of definition.
            widest_label = ''
            for bibitem in self._bibitems:
                if len(widest_label) < len(bibitem[0]):
                    widest_label = bibitem[0]
            self.out.append('\n\\begin{thebibliography}{%s}\n' %
                            widest_label)
            for bibitem in self._bibitems:
                # cite_key: underscores must not be escaped
                cite_key = bibitem[0].replace(r'\_', '_')
                self.out.append('\\bibitem[%s]{%s}{%s}\n' %
                                (bibitem[0], cite_key, bibitem[1]))
            self.out.append('\\end{thebibliography}\n')

    def visit_emphasis(self, node):
        self.out.append('\\emph{')
        self.visit_inline(node)

    def depart_emphasis(self, node):
        self.depart_inline(node)
        self.out.append('}')

    # Append column delimiters and advance column counter,
    # if the current cell is a multi-row continuation."""
    def insert_additional_table_colum_delimiters(self):
        while self.active_table.get_rowspan(
                                self.active_table.get_entry_number()):
            self.out.append(' & ')
            self.active_table.visit_entry()  # increment cell count

    def visit_entry(self, node):
        # cell separation
        if self.active_table.get_entry_number() == 0:
            self.insert_additional_table_colum_delimiters()
        else:
            self.out.append(' & ')

        # multirow, multicolumn
        if 'morerows' in node and 'morecols' in node:
            raise NotImplementedError('Cells that span multiple rows *and* '
                                      'columns currently not supported '
                                      'by the LaTeX writer')
            # TODO: should be possible with LaTeX, see e.g.
            # http://texblog.org/2012/12/21/multi-column-and-multi-row-cells-in-latex-tables/
        # multirow in LaTeX simply will enlarge the cell over several rows
        # (the following n if n is positive, the former if negative).
        if 'morerows' in node:
            self.requirements['multirow'] = r'\usepackage{multirow}'
            mrows = node['morerows'] + 1
            self.active_table.set_rowspan(
                            self.active_table.get_entry_number(), mrows)
            self.out.append('\\multirow{%d}{%s}{' %
                            (mrows, self.active_table.get_column_width()))
            self.context.append('}')
        elif 'morecols' in node:
            # the vertical bar before column is missing if it is the first
            # column. the one after always.
            if self.active_table.get_entry_number() == 0:
                bar1 = self.active_table.get_vertical_bar()
            else:
                bar1 = ''
            mcols = node['morecols'] + 1
            self.out.append('\\multicolumn{%d}{%s%s%s}{' %
                            (mcols,
                             bar1,
                             self.active_table.get_multicolumn_width(
                                 self.active_table.get_entry_number(), mcols),
                             self.active_table.get_vertical_bar()))
            self.context.append('}')
        else:
            self.context.append('')

        # bold header/stub-column
        if len(node) and (isinstance(node.parent.parent, nodes.thead)
                          or self.active_table.is_stub_column()):
            self.out.append('\\textbf{')
            self.context.append('}')
        else:
            self.context.append('')

        # if line ends with '{', mask line break
        if (not self.active_table.colwidths_auto
            and self.out[-1].endswith("{")
            and node.astext()):
            self.out.append("%")

        self.active_table.visit_entry()  # increment cell count

    def depart_entry(self, node):
        self.out.append(self.context.pop())  # header / not header
        self.out.append(self.context.pop())  # multirow/column
        # insert extra "&"s, if following rows are spanned from above:
        self.insert_additional_table_colum_delimiters()

    def visit_row(self, node):
        self.active_table.visit_row()

    def depart_row(self, node):
        self.out.extend(self.active_table.depart_row())

    def visit_enumerated_list(self, node):
        # enumeration styles:
        types = {'': '',
                 'arabic': 'arabic',
                 'loweralpha': 'alph',
                 'upperalpha': 'Alph',
                 'lowerroman': 'roman',
                 'upperroman': 'Roman'}
        # default LaTeX enumeration labels:
        default_labels = [
                          # (präfix, enumtype, suffix)
                          ('',  'arabic', '.'),  # 1.
                          ('(', 'alph',   ')'),  # (a)
                          ('',  'roman',  '.'),  # i.
                          ('',  'Alph',   '.')]  # A.

        prefix = ''
        if self.compound_enumerators:
            if (self.section_prefix_for_enumerators and self.section_level
                and not self._enumeration_counters):
                prefix = '.'.join(str(n) for n in
                                  self._section_number[:self.section_level]
                                  ) + self.section_enumerator_separator
            if self._enumeration_counters:
                prefix += self._enumeration_counters[-1]
        prefix += node.get('prefix', '')
        enumtype = types[node.get('enumtype', 'arabic')]
        suffix = node.get('suffix', '.')

        enum_level = len(self._enumeration_counters)+1
        counter_name = 'enum' + roman.toRoman(enum_level).lower()
        label = r'%s\%s{%s}%s' % (prefix, enumtype, counter_name, suffix)
        self._enumeration_counters.append(label)

        self.duclass_open(node)
        if enum_level <= 4:
            self.out.append('\\begin{enumerate}')
            if (prefix, enumtype, suffix) != default_labels[enum_level-1]:
                self.out.append('\n\\renewcommand{\\label%s}{%s}' %
                                (counter_name, label))
        else:
            self.fallbacks[counter_name] = '\\newcounter{%s}' % counter_name
            self.out.append('\\begin{list}')
            self.out.append('{%s}' % label)
            self.out.append('{\\usecounter{%s}}' % counter_name)
        if 'start' in node:
            self.out.append('\n\\setcounter{%s}{%d}' %
                            (counter_name, node['start']-1))

    def depart_enumerated_list(self, node):
        if len(self._enumeration_counters) <= 4:
            self.out.append('\\end{enumerate}\n')
        else:
            self.out.append('\\end{list}\n')
        self.duclass_close(node)
        self._enumeration_counters.pop()

    def visit_field(self, node):
        # output is done in field_body, field_name
        pass

    def depart_field(self, node):
        pass

    def visit_field_body(self, node):
        if not isinstance(node.parent.parent, nodes.docinfo):
            self.out.append(self.term_postfix(node))

    def depart_field_body(self, node):
        if self.out is self.docinfo:
            self.out.append(r'\\'+'\n')

    def visit_field_list(self, node):
        self.duclass_open(node)
        if self.out is not self.docinfo:
            if not self.fallback_stylesheet:
                self.fallbacks['fieldlist'] = PreambleCmds.fieldlist
            self.out.append('\\begin{DUfieldlist}')

    def depart_field_list(self, node):
        if self.out is not self.docinfo:
            self.out.append('\\end{DUfieldlist}\n')
        self.duclass_close(node)

    def visit_field_name(self, node):
        if self.out is self.docinfo:
            self.out.append('\\textbf{')
        else:
            # Commands with optional args inside an optional arg must be put
            # in a group, e.g. ``\item[{\hyperref[label]{text}}]``.
            self.out.append('\n\\item[{')

    def depart_field_name(self, node):
        if self.out is self.docinfo:
            self.out.append('}: &')
        else:
            self.out.append(':}]')

    def visit_figure(self, node):
        self.requirements['float'] = PreambleCmds.float
        self.duclass_open(node)
        # The 'align' attribute sets the "outer alignment",
        # for "inner alignment" use LaTeX default alignment (similar to HTML)
        alignment = node.attributes.get('align', 'center')
        if alignment != 'center':
            # The LaTeX "figure" environment always uses the full linewidth,
            # so "outer alignment" is ignored. Just write a comment.
            # TODO: use the wrapfigure environment?
            self.out.append('\\begin{figure} %% align = "%s"\n' % alignment)
        else:
            self.out.append('\\begin{figure}\n')
        self.out += self.ids_to_labels(node, newline=True)

    def depart_figure(self, node):
        self.out.append('\\end{figure}\n')
        self.duclass_close(node)

    def visit_footer(self, node):
        self.push_output_collector([])
        self.out.append(r'\newcommand{\DUfooter}{')

    def depart_footer(self, node):
        self.out.append('}')
        self.requirements['~footer'] = ''.join(self.out)
        self.pop_output_collector()

    def visit_footnote(self, node):
        try:
            backref = node['backrefs'][0]
        except IndexError:
            backref = node['ids'][0]  # no backref, use self-ref instead
        if self.docutils_footnotes:
            if not self.fallback_stylesheet:
                self.fallbacks['footnotes'] = PreambleCmds.footnotes
            num = node[0].astext()
            if self.settings.footnote_references == 'brackets':
                num = '[%s]' % num
            self.out.append('%%\n\\DUfootnotetext{%s}{%s}{%s}{' %
                            (node['ids'][0], backref, self.encode(num)))
            if node['ids'] == node['names']:
                self.out += self.ids_to_labels(node)
            # prevent spurious whitespace if footnote starts with paragraph:
            if len(node) > 1 and isinstance(node[1], nodes.paragraph):
                self.out.append('%')
        # TODO: "real" LaTeX \footnote{}s (see visit_footnotes_reference())

    def depart_footnote(self, node):
        self.out.append('}\n')

    def visit_footnote_reference(self, node):
        href = ''
        if 'refid' in node:
            href = node['refid']
        elif 'refname' in node:
            href = self.document.nameids[node['refname']]
        # if not self.docutils_footnotes:
        #     # TODO: insert footnote content at (or near) this place
        #     #       see also docs/dev/todo.txt
        #     try:
        #         referenced_node = self.document.ids[node['refid']]
        #     except (AttributeError, KeyError):
        #         self.document.reporter.error(
        #             'unresolved footnote-reference %s' % node)
        #     print('footnote-ref to %s' % referenced_node)
        format = self.settings.footnote_references
        if format == 'brackets':
            self.append_hypertargets(node)
            self.out.append('\\hyperlink{%s}{[' % href)
            self.context.append(']}')
        else:
            if not self.fallback_stylesheet:
                self.fallbacks['footnotes'] = PreambleCmds.footnotes
            self.out.append(r'\DUfootnotemark{%s}{%s}{' %
                            (node['ids'][0], href))
            self.context.append('}')

    def depart_footnote_reference(self, node):
        self.out.append(self.context.pop())

    # footnote/citation label
    def label_delim(self, node, bracket, superscript):
        if isinstance(node.parent, nodes.footnote):
            raise nodes.SkipNode
        else:
            assert isinstance(node.parent, nodes.citation)
            if not self.use_latex_citations:
                self.out.append(bracket)

    def visit_label(self, node):
        """footnote or citation label: in brackets or as superscript"""
        self.label_delim(node, '[', '\\textsuperscript{')

    def depart_label(self, node):
        self.label_delim(node, ']', '}')

    # elements generated by the framework e.g. section numbers.
    def visit_generated(self, node):
        pass

    def depart_generated(self, node):
        pass

    def visit_header(self, node):
        self.push_output_collector([])
        self.out.append(r'\newcommand{\DUheader}{')

    def depart_header(self, node):
        self.out.append('}')
        self.requirements['~header'] = ''.join(self.out)
        self.pop_output_collector()

    def to_latex_length(self, length_str, pxunit=None):
        """Convert `length_str` with rst length to LaTeX length
        """
        if pxunit is not None:
            warnings.warn(
                'The optional argument `pxunit` '
                'of LaTeXTranslator.to_latex_length() is ignored '
                'and will be removed in Docutils 0.21 or later',
                DeprecationWarning, stacklevel=2)
        match = re.match(r'(\d*\.?\d*)\s*(\S*)', length_str)
        if not match:
            return length_str
        value, unit = match.groups()[:2]
        # no unit or "DTP" points (called 'bp' in TeX):
        if unit in ('', 'pt'):
            length_str = '%sbp' % value
        # percentage: relate to current line width
        elif unit == '%':
            length_str = '%.3f\\linewidth' % (float(value)/100.0)
        elif self.is_xetex and unit == 'px':
            # XeTeX does not know the length unit px.
            # Use \pdfpxdimen, the macro to set the value of 1 px in pdftex.
            # This way, configuring works the same for pdftex and xetex.
            if not self.fallback_stylesheet:
                self.fallbacks['_providelength'] = PreambleCmds.providelength
            self.fallbacks['px'] = '\n\\DUprovidelength{\\pdfpxdimen}{1bp}\n'
            length_str = r'%s\pdfpxdimen' % value
        return length_str

    def visit_image(self, node):
        self.requirements['graphicx'] = self.graphicx_package
        attrs = node.attributes
        # Convert image URI to a local file path
        imagepath = url2pathname(attrs['uri']).replace('\\', '/')
        # alignment defaults:
        if 'align' not in attrs:
            # Set default align of image in a figure to 'center'
            if isinstance(node.parent, nodes.figure):
                attrs['align'] = 'center'
            self.set_align_from_classes(node)
        # pre- and postfix (prefix inserted in reverse order)
        pre = []
        post = []
        include_graphics_options = []
        align_codes = {
            # inline images: by default latex aligns the bottom.
            'bottom': ('', ''),
            'middle': (r'\raisebox{-0.5\height}{', '}'),
            'top':    (r'\raisebox{-\height}{', '}'),
            # block level images:
            'center': (r'\noindent\makebox[\linewidth][c]{', '}'),
            'left':   (r'\noindent{', r'\hfill}'),
            'right':  (r'\noindent{\hfill', '}'),
            }
        if 'align' in attrs:
            # TODO: warn or ignore non-applicable alignment settings?
            try:
                align_code = align_codes[attrs['align']]
                pre.append(align_code[0])
                post.append(align_code[1])
            except KeyError:
                pass                    # TODO: warn?
        if 'height' in attrs:
            include_graphics_options.append(
                'height=%s' % self.to_latex_length(attrs['height']))
        if 'scale' in attrs:
            include_graphics_options.append(
                'scale=%f' % (attrs['scale'] / 100.0))
        if 'width' in attrs:
            include_graphics_options.append(
                'width=%s' % self.to_latex_length(attrs['width']))
        if not (self.is_inline(node)
                or isinstance(node.parent, (nodes.figure, nodes.compound))):
            pre.append('\n')
        if not (self.is_inline(node)
                or isinstance(node.parent, nodes.figure)):
            post.append('\n')
        pre.reverse()
        self.out.extend(pre)
        options = ''
        if include_graphics_options:
            options = '[%s]' % (','.join(include_graphics_options))
        self.out.append('\\includegraphics%s{%s}' % (options, imagepath))
        self.out.extend(post)

    def depart_image(self, node):
        self.out += self.ids_to_labels(node, newline=True)

    def visit_inline(self, node):  # <span>, i.e. custom roles
        for cls in node['classes']:
            if cls.startswith('language-'):
                language = self.babel.language_name(cls[9:])
                if language:
                    self.babel.otherlanguages[language] = True
                    self.out.append(r'\foreignlanguage{%s}{' % language)
            else:
                if not self.fallback_stylesheet:
                    self.fallbacks['inline'] = PreambleCmds.inline
                self.out.append(r'\DUrole{%s}{' % cls)

    def depart_inline(self, node):
        self.out.append('}' * len(node['classes']))

    def visit_legend(self, node):
        if not self.fallback_stylesheet:
            self.fallbacks['legend'] = PreambleCmds.legend
        self.out.append('\\begin{DUlegend}')

    def depart_legend(self, node):
        self.out.append('\\end{DUlegend}\n')

    def visit_line(self, node):
        self.out.append(r'\item[] ')

    def depart_line(self, node):
        self.out.append('\n')

    def visit_line_block(self, node):
        if not self.fallback_stylesheet:
            self.fallbacks['_providelength'] = PreambleCmds.providelength
            self.fallbacks['lineblock'] = PreambleCmds.lineblock
        self.set_align_from_classes(node)
        if isinstance(node.parent, nodes.line_block):
            self.out.append('\\item[]\n'
                            '\\begin{DUlineblock}{\\DUlineblockindent}\n')
            # nested line-blocks cannot be given class arguments
        else:
            self.duclass_open(node)
            self.out.append('\\begin{DUlineblock}{0em}\n')
            self.insert_align_declaration(node)

    def depart_line_block(self, node):
        self.out.append('\\end{DUlineblock}\n')
        self.duclass_close(node)

    def visit_list_item(self, node):
        self.out.append('\n\\item ')

    def depart_list_item(self, node):
        pass

    def visit_literal(self, node):
        self.literal = True
        if ('code' in node['classes']
            and self.settings.syntax_highlight != 'none'):
            self.requirements['color'] = PreambleCmds.color
            if not self.fallback_stylesheet:
                self.fallbacks['code'] = PreambleCmds.highlight_rules
        self.out.append('\\texttt{')
        self.visit_inline(node)

    def depart_literal(self, node):
        self.literal = False
        self.depart_inline(node)
        self.out.append('}')

    # Literal blocks are used for '::'-prefixed literal-indented
    # blocks of text, where the inline markup is not recognized,
    # but are also the product of the "parsed-literal" directive,
    # where the markup is respected.
    #
    # In both cases, we want to use a typewriter/monospaced typeface.
    # For "real" literal-blocks, we can use \verbatim, while for all
    # the others we must use \ttfamily and \raggedright.
    #
    # We can distinguish between the two kinds by the number of
    # siblings that compose this node: if it is composed by a
    # single element, it's either
    # * a real one,
    # * a parsed-literal that does not contain any markup, or
    # * a parsed-literal containing just one markup construct.
    def is_plaintext(self, node):
        """Check whether a node can be typeset verbatim"""
        return (len(node) == 1) and isinstance(node[0], nodes.Text)

    def visit_literal_block(self, node):
        """Render a literal block.

        Corresponding rST elements: literal block, parsed-literal, code.
        """
        packages = {'lstlisting':  r'\usepackage{listings}' '\n'
                                   r'\lstset{xleftmargin=\leftmargin}',
                    'listing': r'\usepackage{moreverb}',
                    'Verbatim': r'\usepackage{fancyvrb}',
                    'verbatimtab': r'\usepackage{moreverb}'}

        literal_env = self.literal_block_env

        # Check, if it is possible to use a literal-block environment
        _plaintext = self.is_plaintext(node)
        _in_table = self.active_table.is_open()
        # TODO: fails if normal text precedes the literal block.
        #       Check parent node instead?
        _autowidth_table = _in_table and self.active_table.colwidths_auto
        _no_env_nodes = (nodes.footnote, nodes.sidebar)
        if self.settings.legacy_class_functions:
            _no_env_nodes += (nodes.admonition, nodes.system_message)
        _use_env = _plaintext and not isinstance(node.parent, _no_env_nodes)
        _use_listings = (literal_env == 'lstlisting') and _use_env

        # Labels and classes:
        self.duclass_open(node)
        self.out += self.ids_to_labels(node, newline=True)
        # Highlight code?
        if (not _plaintext
            and 'code' in node['classes']
            and self.settings.syntax_highlight != 'none'):
            self.requirements['color'] = PreambleCmds.color
            if not self.fallback_stylesheet:
                self.fallbacks['code'] = PreambleCmds.highlight_rules
        # Wrap?
        if _in_table and _use_env and not _autowidth_table:
            # Wrap in minipage to prevent extra vertical space
            # with alltt and verbatim-like environments:
            self.fallbacks['ttem'] = PreambleCmds.ttem
            self.out.append(
                '\\begin{minipage}{%d\\ttemwidth}\n' %
                (max(len(line) for line in node.astext().split('\n'))))
            self.context.append('\n\\end{minipage}\n')
        elif not _in_table and not _use_listings:
            # Wrap in quote to set off vertically and indent
            self.out.append('\\begin{quote}\n')
            self.context.append('\n\\end{quote}\n')
        else:
            self.context.append('\n')

        # Use verbatim-like environment, if defined and possible
        # (in an auto-width table, only listings works):
        if literal_env and _use_env and (not _autowidth_table
                                         or _use_listings):
            try:
                self.requirements['literal_block'] = packages[literal_env]
            except KeyError:
                pass
            self.verbatim = True
            if _in_table and _use_listings:
                self.out.append('\\lstset{xleftmargin=0pt}\n')
            self.out.append('\\begin{%s}%s\n' %
                            (literal_env, self.literal_block_options))
            self.context.append('\n\\end{%s}' % literal_env)
        elif _use_env and not _autowidth_table:
            self.alltt = True
            self.requirements['alltt'] = r'\usepackage{alltt}'
            self.out.append('\\begin{alltt}\n')
            self.context.append('\n\\end{alltt}')
        else:
            self.literal = True
            self.insert_newline = True
            self.insert_non_breaking_blanks = True
            # \raggedright ensures leading blanks are respected but
            # leads to additional leading vspace if the first line
            # of the block is overfull :-(
            self.out.append('\\ttfamily\\raggedright\n')
            self.context.append('')

    def depart_literal_block(self, node):
        self.insert_non_breaking_blanks = False
        self.insert_newline = False
        self.literal = False
        self.verbatim = False
        self.alltt = False
        self.out.append(self.context.pop())
        self.out.append(self.context.pop())
        self.duclass_close(node)

    def visit_meta(self, node):
        name = node.attributes.get('name')
        content = node.attributes.get('content')
        if not name or not content:
            return
        if name in ('author', 'creator', 'keywords', 'subject', 'title'):
            # fields with dedicated hyperref options:
            self.pdfinfo.append('  pdf%s={%s},'%(name, content))
        elif name == 'producer':
            self.pdfinfo.append('  addtopdfproducer={%s},'%content)
        else:
            # generic interface (case sensitive!)
            # TODO: filter irrelevant nodes ("http-equiv", ...)?
            self.pdfinfo.append('  pdfinfo={%s={%s}},'%(name, content))

    def depart_meta(self, node):
        pass

    def visit_math(self, node, math_env='$'):
        """math role"""
        self.visit_inline(node)
        self.requirements['amsmath'] = r'\usepackage{amsmath}'
        math_code = node.astext().translate(unichar2tex.uni2tex_table)
        if math_env == '$':
            if self.alltt:
                wrapper = ['\\(', '\\)']
            else:
                wrapper = ['$', '$']
        else:
            labels = self.ids_to_labels(node, set_anchor=False, newline=True)
            wrapper = ['%%\n\\begin{%s}\n' % math_env,
                       '\n',
                       ''.join(labels),
                       '\\end{%s}' % math_env]
        wrapper.insert(1, math_code)
        self.out.extend(wrapper)
        self.depart_inline(node)
        # Content already processed:
        raise nodes.SkipNode

    def depart_math(self, node):
        pass  # never reached

    def visit_math_block(self, node):
        math_env = pick_math_environment(node.astext())
        self.visit_math(node, math_env=math_env)

    def depart_math_block(self, node):
        pass  # never reached

    def visit_option(self, node):
        if self.context[-1]:
            # this is not the first option
            self.out.append(', ')

    def depart_option(self, node):
        # flag that the first option is done.
        self.context[-1] += 1

    def visit_option_argument(self, node):
        """Append the delimiter between an option and its argument to body."""
        self.out.append(node.get('delimiter', ' '))

    def depart_option_argument(self, node):
        pass

    def visit_option_group(self, node):
        self.out.append('\n\\item[')
        # flag for first option
        self.context.append(0)

    def depart_option_group(self, node):
        self.context.pop()  # the flag
        self.out.append('] ')

    def visit_option_list(self, node):
        if not self.fallback_stylesheet:
            self.fallbacks['_providelength'] = PreambleCmds.providelength
            self.fallbacks['optionlist'] = PreambleCmds.optionlist
        self.duclass_open(node)
        self.out.append('\\begin{DUoptionlist}')

    def depart_option_list(self, node):
        self.out.append('\\end{DUoptionlist}\n')
        self.duclass_close(node)

    def visit_option_list_item(self, node):
        pass

    def depart_option_list_item(self, node):
        pass

    def visit_option_string(self, node):
        ## self.out.append(self.starttag(node, 'span', '', CLASS='option'))
        pass

    def depart_option_string(self, node):
        ## self.out.append('</span>')
        pass

    def visit_organization(self, node):
        self.visit_docinfo_item(node, 'organization')

    def depart_organization(self, node):
        self.depart_docinfo_item(node)

    def visit_paragraph(self, node):
        # insert blank line, unless
        # * the paragraph is first in a list item, compound, or container
        # * follows a non-paragraph node in a compound,
        # * is in a table with auto-width columns
        index = node.parent.index(node)
        if index == 0 and isinstance(node.parent,
                                     (nodes.list_item, nodes.description,
                                      nodes.compound, nodes.container)):
            pass
        elif (index > 0
              and isinstance(node.parent, nodes.compound)
              and not isinstance(node.parent[index - 1],
                                 (nodes.paragraph, nodes.compound))):
            pass
        elif self.active_table.colwidths_auto:
            if index == 1:  # second paragraph
                self.warn('LaTeX merges paragraphs in tables '
                          'with auto-sized columns!', base_node=node)
            if index > 0:
                self.out.append('\n')
        else:
            self.out.append('\n')
        self.out += self.ids_to_labels(node, newline=True)
        self.visit_inline(node)

    def depart_paragraph(self, node):
        self.depart_inline(node)
        if not self.active_table.colwidths_auto:
            self.out.append('\n')

    def visit_problematic(self, node):
        self.requirements['color'] = PreambleCmds.color
        self.out.append('%\n')
        self.append_hypertargets(node)
        self.out.append(r'\hyperlink{%s}{\textbf{\color{red}' % node['refid'])

    def depart_problematic(self, node):
        self.out.append('}}')

    def visit_raw(self, node):
        if 'latex' not in node.get('format', '').split():
            raise nodes.SkipNode
        if not (self.is_inline(node)
                or isinstance(node.parent, nodes.compound)):
            self.out.append('\n')
        self.visit_inline(node)
        # append "as-is" skipping any LaTeX-encoding
        self.verbatim = True

    def depart_raw(self, node):
        self.verbatim = False
        self.depart_inline(node)
        if not self.is_inline(node):
            self.out.append('\n')

    def has_unbalanced_braces(self, string):
        """Test whether there are unmatched '{' or '}' characters."""
        level = 0
        for ch in string:
            if ch == '{':
                level += 1
            if ch == '}':
                level -= 1
            if level < 0:
                return True
        return level != 0

    def visit_reference(self, node):
        # We need to escape #, \, and % if we use the URL in a command.
        special_chars = {ord('#'): '\\#',
                         ord('%'): '\\%',
                         ord('\\'): '\\\\',
                         }
        # external reference (URL)
        if 'refuri' in node:
            href = str(node['refuri']).translate(special_chars)
            # problematic chars double caret and unbalanced braces:
            if href.find('^^') != -1 or self.has_unbalanced_braces(href):
                self.error(
                    'External link "%s" not supported by LaTeX.\n'
                    ' (Must not contain "^^" or unbalanced braces.)' % href)
            if node['refuri'] == node.astext():
                self.out.append(r'\url{%s}' % href)
                raise nodes.SkipNode
            self.out.append(r'\href{%s}{' % href)
            return
        # internal reference
        if 'refid' in node:
            href = node['refid']
        elif 'refname' in node:
            href = self.document.nameids[node['refname']]
        else:
            raise AssertionError('Unknown reference.')
        if not self.is_inline(node):
            self.out.append('\n')
        self.out.append('\\hyperref[%s]{' % href)
        if self._reference_label:
            self.out.append('\\%s{%s}}' %
                            (self._reference_label, href.replace('#', '')))
            raise nodes.SkipNode

    def depart_reference(self, node):
        self.out.append('}')
        if not self.is_inline(node):
            self.out.append('\n')

    def visit_revision(self, node):
        self.visit_docinfo_item(node, 'revision')

    def depart_revision(self, node):
        self.depart_docinfo_item(node)

    def visit_rubric(self, node):
        if not self.fallback_stylesheet:
            self.fallbacks['rubric'] = PreambleCmds.rubric
        # class wrapper would interfere with ``\section*"`` type commands
        # (spacing/indent of first paragraph)
        self.out.append('\n\\DUrubric{')

    def depart_rubric(self, node):
        self.out.append('}\n')

    def visit_section(self, node):
        self.section_level += 1
        # Initialize counter for potential subsections:
        self._section_number.append(0)
        # Counter for this section's level (initialized by parent section):
        self._section_number[self.section_level - 1] += 1

    def depart_section(self, node):
        # Remove counter for potential subsections:
        self._section_number.pop()
        self.section_level -= 1

    def visit_sidebar(self, node):
        self.duclass_open(node)
        self.requirements['color'] = PreambleCmds.color
        if not self.fallback_stylesheet:
            self.fallbacks['sidebar'] = PreambleCmds.sidebar
        self.out.append('\\DUsidebar{')

    def depart_sidebar(self, node):
        self.out.append('}\n')
        self.duclass_close(node)

    attribution_formats = {'dash': ('—', ''),  # EM DASH
                           'parentheses': ('(', ')'),
                           'parens': ('(', ')'),
                           'none': ('', '')}

    def visit_attribution(self, node):
        prefix, suffix = self.attribution_formats[self.settings.attribution]
        self.out.append('\\nopagebreak\n\n\\raggedleft ')
        self.out.append(prefix)
        self.context.append(suffix)

    def depart_attribution(self, node):
        self.out.append(self.context.pop() + '\n')

    def visit_status(self, node):
        self.visit_docinfo_item(node, 'status')

    def depart_status(self, node):
        self.depart_docinfo_item(node)

    def visit_strong(self, node):
        self.out.append('\\textbf{')
        self.visit_inline(node)

    def depart_strong(self, node):
        self.depart_inline(node)
        self.out.append('}')

    def visit_substitution_definition(self, node):
        raise nodes.SkipNode

    def visit_substitution_reference(self, node):
        self.unimplemented_visit(node)

    def visit_subtitle(self, node):
        if isinstance(node.parent, nodes.document):
            self.push_output_collector(self.subtitle)
            if not self.fallback_stylesheet:
                self.fallbacks['documentsubtitle'] = PreambleCmds.documentsubtitle  # noqa:E501
            protect = (self.settings.documentclass == 'memoir')
            self.subtitle_labels += self.ids_to_labels(node, set_anchor=False,
                                                       protect=protect)
        # section subtitle: "starred" (no number, not in ToC)
        elif isinstance(node.parent, nodes.section):
            self.out.append(r'\%s*{' %
                            self.d_class.section(self.section_level + 1))
        else:
            if not self.fallback_stylesheet:
                self.fallbacks['subtitle'] = PreambleCmds.subtitle
            self.out.append('\n\\DUsubtitle{')

    def depart_subtitle(self, node):
        if isinstance(node.parent, nodes.document):
            self.pop_output_collector()
        else:
            self.out.append('}\n')

    def visit_system_message(self, node):
        self.requirements['color'] = PreambleCmds.color
        if not self.fallback_stylesheet:
            self.fallbacks['title'] = PreambleCmds.title
        if self.settings.legacy_class_functions:
            self.fallbacks['title'] = PreambleCmds.title_legacy
        node['classes'] = ['system-message']
        self.visit_admonition(node)
        if self.settings.legacy_class_functions:
            self.out.append('\n\\DUtitle[system-message]{system-message\n')
        else:
            self.out.append('\n\\DUtitle{system-message\n')
        self.append_hypertargets(node)
        try:
            line = ', line~%s' % node['line']
        except KeyError:
            line = ''
        self.out.append('}\n\n{\\color{red}%s/%s} in \\texttt{%s}%s\n' %
                        (node['type'], node['level'],
                         self.encode(node['source']), line))
        if len(node['backrefs']) == 1:
            self.out.append('\n\\hyperlink{%s}{' % node['backrefs'][0])
            self.context.append('}')
        else:
            backrefs = ['\\hyperlink{%s}{%d}' % (href, i+1)
                        for (i, href) in enumerate(node['backrefs'])]
            self.context.append('backrefs: ' + ' '.join(backrefs))

    def depart_system_message(self, node):
        self.out.append(self.context.pop())
        self.depart_admonition(node)

    def visit_table(self, node):
        self.duclass_open(node)
        self.requirements['table'] = PreambleCmds.table
        if not self.settings.legacy_column_widths:
            self.requirements['table1'] = PreambleCmds.table_columnwidth
        if self.active_table.is_open():
            self.table_stack.append(self.active_table)
            # nesting longtable does not work (e.g. 2007-04-18)
            self.active_table = Table(self, 'tabular')
        # A longtable moves before \paragraph and \subparagraph
        # section titles if it immediately follows them:
        if (self.active_table._latex_type == 'longtable'
            and isinstance(node.parent, nodes.section)
            and node.parent.index(node) == 1
            and self.d_class.section(
                   self.section_level).find('paragraph') != -1):
            self.out.append('\\leavevmode')
        self.active_table.open()
        self.active_table.set_table_style(node, self.settings)
        if self.active_table.borders == 'booktabs':
            self.requirements['booktabs'] = r'\usepackage{booktabs}'
        self.push_output_collector([])

    def depart_table(self, node):
        # wrap content in the right environment:
        content = self.out
        self.pop_output_collector()
        try:
            width = self.to_latex_length(node['width'])
        except KeyError:
            width = r'\linewidth'
        # TODO: Don't use a longtable or add \noindent before
        #       the next paragraph, when in a "compound paragraph".
        #       Start a new line or a new paragraph?
        #       if (isinstance(node.parent, nodes.compound)
        #       and self._latex_type != 'longtable')?
        self.out.append(self.active_table.get_opening(width))
        self.out += content
        self.out.append(self.active_table.get_closing() + '\n')
        self.active_table.close()
        if len(self.table_stack) > 0:
            self.active_table = self.table_stack.pop()
        # Insert hyperlabel after (long)table, as
        # other places (beginning, caption) result in LaTeX errors.
        self.out += self.ids_to_labels(node, set_anchor=False, newline=True)
        self.duclass_close(node)

    def visit_target(self, node):
        # Skip indirect targets:
        if ('refuri' in node        # external hyperlink
            or 'refid' in node      # resolved internal link
            or 'refname' in node):  # unresolved internal link
            ## self.out.append('%% %s\n' % node)   # for debugging
            return
        self.out.append('%\n')
        # do we need an anchor (\phantomsection)?
        set_anchor = not isinstance(node.parent, (nodes.caption, nodes.title))
        # TODO: where else can/must we omit the \phantomsection?
        self.out += self.ids_to_labels(node, set_anchor)

    def depart_target(self, node):
        pass

    def visit_tbody(self, node):
        # BUG write preamble if not yet done (colspecs not [])
        # for tables without heads.
        if not self.active_table.get('preamble written'):
            self.visit_thead(node)
            self.depart_thead(None)

    def depart_tbody(self, node):
        pass

    def visit_term(self, node):
        """definition list term"""
        # Commands with optional args inside an optional arg must be put
        # in a group, e.g. ``\item[{\hyperref[label]{text}}]``.
        self.out.append('\\item[{')

    def depart_term(self, node):
        self.out.append('}] ')
        # Do we need a \leavevmode (line break if the field body begins
        # with a list or environment)?
        next_node = node.next_node(descend=False, siblings=True)
        if not isinstance(next_node, nodes.classifier):
            self.out.append(self.term_postfix(next_node))

    def visit_tgroup(self, node):
        pass

    def depart_tgroup(self, node):
        pass

    _thead_depth = 0

    def thead_depth(self):
        return self._thead_depth

    def visit_thead(self, node):
        self._thead_depth += 1
        if 1 == self.thead_depth():
            self.out.append('{%s}\n' % self.active_table.get_colspecs(node))
            self.active_table.set('preamble written', 1)
        self.out.append(self.active_table.get_caption())
        self.out.extend(self.active_table.visit_thead())

    def depart_thead(self, node):
        if node is not None:
            self.out.extend(self.active_table.depart_thead())
            if self.active_table.need_recurse():
                node.walkabout(self)
        self._thead_depth -= 1

    def visit_title(self, node):
        """Append section and other titles."""
        # Document title
        if isinstance(node.parent, nodes.document):
            self.push_output_collector(self.title)
            self.context.append('')
            self.pdfinfo.append('  pdftitle={%s},' %
                                self.encode(node.astext()))
        # Topic titles (topic, admonition, sidebar)
        elif (isinstance(node.parent, nodes.topic)
              or isinstance(node.parent, nodes.admonition)
              or isinstance(node.parent, nodes.sidebar)):
            classes = node.parent['classes'] or [node.parent.tagname]
            if self.settings.legacy_class_functions:
                self.fallbacks['title'] = PreambleCmds.title_legacy
                self.out.append('\n\\DUtitle[%s]{' % ','.join(classes))
            else:
                if not self.fallback_stylesheet:
                    self.fallbacks['title'] = PreambleCmds.title
                self.out.append('\n\\DUtitle{')
            self.context.append('}\n')
        # Table caption
        elif isinstance(node.parent, nodes.table):
            self.push_output_collector(self.active_table.caption)
            self.context.append('')
        # Section title
        else:
            if hasattr(PreambleCmds, 'secnumdepth'):
                self.requirements['secnumdepth'] = PreambleCmds.secnumdepth
            level = self.section_level
            section_name = self.d_class.section(level)
            self.out.append('\n\n')
            if level > len(self.d_class.sections):
                # section level not supported by LaTeX
                if self.settings.legacy_class_functions:
                    self.fallbacks['title'] = PreambleCmds.title_legacy
                    section_name += '[section%s]' % roman.toRoman(level)
                else:
                    if not self.fallback_stylesheet:
                        self.fallbacks['title'] = PreambleCmds.title
                        self.fallbacks['DUclass'] = PreambleCmds.duclass
                    self.out.append('\\begin{DUclass}{section%s}\n'
                                    % roman.toRoman(level))

            # System messages heading in red:
            if 'system-messages' in node.parent['classes']:
                self.requirements['color'] = PreambleCmds.color
                section_title = self.encode(node.astext())
                self.out.append(r'\%s[%s]{\color{red}' % (
                                section_name, section_title))
            else:
                self.out.append(r'\%s{' % section_name)

            # label and ToC entry:
            bookmark = ['']
            # add sections with unsupported level to toc and pdfbookmarks?
            ## if level > len(self.d_class.sections):
            ##     section_title = self.encode(node.astext())
            ##     bookmark.append(r'\addcontentsline{toc}{%s}{%s}' %
            ##               (section_name, section_title))
            bookmark += self.ids_to_labels(node.parent, set_anchor=False)
            self.context.append('%\n  '.join(bookmark) + '%\n}\n')
            if (level > len(self.d_class.sections)
                and not self.settings.legacy_class_functions):
                self.context[-1] += '\\end{DUclass}\n'
            # MAYBE postfix paragraph and subparagraph with \leavevmode to
            # ensure floats stay in the section and text starts on a new line.

    def depart_title(self, node):
        self.out.append(self.context.pop())
        if isinstance(node.parent, (nodes.table, nodes.document)):
            self.pop_output_collector()

    def visit_contents(self, node):
        """Write the table of contents.

        Called from visit_topic() for "contents" topics.
        """
        # requirements/setup for local ToC with package "minitoc",
        if self.use_latex_toc and 'local' in node['classes']:
            section_name = self.d_class.section(self.section_level)
            # minitoc only supports "part" and toplevel sections
            minitoc_names = {'part': 'part',
                             'chapter': 'mini',
                             'section': 'sect'}
            if 'chapter' in self.d_class.sections:
                del minitoc_names['section']
            try:
                mtc_name = minitoc_names[section_name]
            except KeyError:
                self.warn('Skipping local ToC at "%s" level.\n'
                          '  Feature not supported with option "use-latex-toc"'
                          % section_name, base_node=node)
                raise nodes.SkipNode

        # labels and PDF bookmark (sidebar entry)
        self.out.append('\n')  # start new paragraph
        if node['names']:  # don't add labels just for auto-ids
            self.out += self.ids_to_labels(node, newline=True)
        if (isinstance(node.next_node(), nodes.title)
            and 'local' not in node['classes']
            and self.settings.documentclass != 'memoir'):
            self.out.append('\\pdfbookmark[%d]{%s}{%s}\n' %
                            (self.section_level+1,
                             node.next_node().astext(),
                             node.get('ids', ['contents'])[0]))

        # Docutils generated contents list (no page numbers)
        if not self.use_latex_toc:
            self.fallbacks['toc-list'] = PreambleCmds.toc_list
            self.duclass_open(node)
            return

        # ToC by LaTeX
        depth = node.get('depth', 0)
        maxdepth = len(self.d_class.sections)
        if isinstance(node.next_node(), nodes.title):
            title = self.encode(node[0].astext())
        else:
            title = ''
        if 'local' in node['classes']:
            # use the "minitoc" package
            self.requirements['minitoc'] = PreambleCmds.minitoc
            self.requirements['minitoc-'+mtc_name] = r'\do%stoc'%mtc_name
            self.requirements['minitoc-%s-depth' % mtc_name] = (
                r'\mtcsetdepth{%stoc}{%d}' % (mtc_name, maxdepth))
            # "depth" option: Docutils stores a relative depth while
            # minitoc  expects an absolute depth!:
            offset = {'sect': 1, 'mini': 0, 'part': 0}
            if 'chapter' in self.d_class.sections:
                offset['part'] = -1
            if depth:
                self.out.append('\\setcounter{%stocdepth}{%d}' %
                                (mtc_name, depth + offset[mtc_name]))
            # title:
            self.out.append('\\mtcsettitle{%stoc}{%s}\n' % (mtc_name, title))
            # the toc-generating command:
            self.out.append('\\%stoc\n' % mtc_name)
        else:
            if depth:
                self.out.append('\\setcounter{tocdepth}{%d}\n'
                                % self.d_class.latex_section_depth(depth))
            if title != 'Contents':
                self.out.append('\\renewcommand{\\contentsname}{%s}\n' % title)
            self.out.append('\\tableofcontents\n')
            self.has_latex_toc = True
        # ignore rest of node content
        raise nodes.SkipNode

    def visit_topic(self, node):
        # Topic nodes can be generic topic, abstract, dedication, or ToC.
        # table of contents:
        if 'contents' in node['classes']:
            self.visit_contents(node)
        elif ('abstract' in node['classes']
              and self.settings.use_latex_abstract):
            self.push_output_collector(self.abstract)
            self.out.append('\\begin{abstract}')
            if isinstance(node.next_node(), nodes.title):
                node.pop(0)  # LaTeX provides its own title
        else:
            # special topics:
            if 'abstract' in node['classes']:
                if not self.fallback_stylesheet:
                    self.fallbacks['abstract'] = PreambleCmds.abstract
                if self.settings.legacy_class_functions:
                    self.fallbacks['abstract'] = PreambleCmds.abstract_legacy
                self.push_output_collector(self.abstract)
            elif 'dedication' in node['classes']:
                if not self.fallback_stylesheet:
                    self.fallbacks['dedication'] = PreambleCmds.dedication
                self.push_output_collector(self.dedication)
            else:
                node['classes'].insert(0, 'topic')
            self.visit_block_quote(node)

    def depart_topic(self, node):
        if ('abstract' in node['classes']
            and self.settings.use_latex_abstract):
            self.out.append('\\end{abstract}\n')
        elif 'contents' in node['classes']:
            self.duclass_close(node)
        else:
            self.depart_block_quote(node)
        if ('abstract' in node['classes']
            or 'dedication' in node['classes']):
            self.pop_output_collector()

    def visit_transition(self, node):
        if not self.fallback_stylesheet:
            self.fallbacks['transition'] = PreambleCmds.transition
        self.out.append('\n%' + '_' * 75 + '\n')
        self.out.append('\\DUtransition\n')

    def depart_transition(self, node):
        pass

    def visit_version(self, node):
        self.visit_docinfo_item(node, 'version')

    def depart_version(self, node):
        self.depart_docinfo_item(node)

    def unimplemented_visit(self, node):
        raise NotImplementedError('visiting unimplemented node type: %s' %
                                  node.__class__.__name__)

#    def unknown_visit(self, node):
#    def default_visit(self, node):

# vim: set ts=4 et ai :

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/latex2e/__init__.py
NikolaBlagojevic/pyrecodes
# $Id: manpage.py 9206 2022-11-03 10:52:43Z aa-turner $
# Author: Engelbert Gruber <grubert@users.sourceforge.net>
# Copyright: This module is put into the public domain.

"""
Simple man page writer for reStructuredText.

Man pages (short for "manual pages") contain system documentation on unix-like
systems. The pages are grouped in numbered sections:

 1 executable programs and shell commands
 2 system calls
 3 library functions
 4 special files
 5 file formats
 6 games
 7 miscellaneous
 8 system administration

Man pages are written *troff*, a text file formatting system.

See http://www.tldp.org/HOWTO/Man-Page for a start.

Man pages have no subsection only parts.
Standard parts

  NAME ,
  SYNOPSIS ,
  DESCRIPTION ,
  OPTIONS ,
  FILES ,
  SEE ALSO ,
  BUGS ,

and

  AUTHOR .

A unix-like system keeps an index of the DESCRIPTIONs, which is accessible
by the command whatis or apropos.

"""

__docformat__ = 'reStructuredText'

import re

from docutils import nodes, writers, languages
try:
    import roman
except ImportError:
    import docutils.utils.roman as roman

FIELD_LIST_INDENT = 7
DEFINITION_LIST_INDENT = 7
OPTION_LIST_INDENT = 7
BLOCKQOUTE_INDENT = 3.5
LITERAL_BLOCK_INDENT = 3.5

# Define two macros so man/roff can calculate the
# indent/unindent margins by itself
MACRO_DEF = (r""".
.nr rst2man-indent-level 0
.
.de1 rstReportMargin
\\$1 \\n[an-margin]
level \\n[rst2man-indent-level]
level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
-
\\n[rst2man-indent0]
\\n[rst2man-indent1]
\\n[rst2man-indent2]
..
.de1 INDENT
.\" .rstReportMargin pre:
. RS \\$1
. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
. nr rst2man-indent-level +1
.\" .rstReportMargin post:
..
.de UNINDENT
. RE
.\" indent \\n[an-margin]
.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
.nr rst2man-indent-level -1
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
""")


class Writer(writers.Writer):

    supported = ('manpage',)
    """Formats this writer supports."""

    output = None
    """Final translated form of `document`."""

    def __init__(self):
        writers.Writer.__init__(self)
        self.translator_class = Translator

    def translate(self):
        visitor = self.translator_class(self.document)
        self.document.walkabout(visitor)
        self.output = visitor.astext()


class Table:
    def __init__(self):
        self._rows = []
        self._options = ['center']
        self._tab_char = '\t'
        self._coldefs = []

    def new_row(self):
        self._rows.append([])

    def append_separator(self, separator):
        """Append the separator for table head."""
        self._rows.append([separator])

    def append_cell(self, cell_lines):
        """cell_lines is an array of lines"""
        start = 0
        if len(cell_lines) > 0 and cell_lines[0] == '.sp\n':
            start = 1
        self._rows[-1].append(cell_lines[start:])
        if len(self._coldefs) < len(self._rows[-1]):
            self._coldefs.append('l')

    def _minimize_cell(self, cell_lines):
        """Remove leading and trailing blank and ``.sp`` lines"""
        while cell_lines and cell_lines[0] in ('\n', '.sp\n'):
            del cell_lines[0]
        while cell_lines and cell_lines[-1] in ('\n', '.sp\n'):
            del cell_lines[-1]

    def as_list(self):
        text = ['.TS\n']
        text.append(' '.join(self._options) + ';\n')
        text.append('|%s|.\n' % ('|'.join(self._coldefs)))
        for row in self._rows:
            # row = array of cells. cell = array of lines.
            text.append('_\n')       # line above
            text.append('T{\n')
            for i in range(len(row)):
                cell = row[i]
                self._minimize_cell(cell)
                text.extend(cell)
                if not text[-1].endswith('\n'):
                    text[-1] += '\n'
                if i < len(row)-1:
                    text.append('T}'+self._tab_char+'T{\n')
                else:
                    text.append('T}\n')
        text.append('_\n')
        text.append('.TE\n')
        return text


class Translator(nodes.NodeVisitor):
    """"""

    words_and_spaces = re.compile(r'\S+| +|\n')
    possibly_a_roff_command = re.compile(r'\.\w')
    document_start = """Man page generated from reStructuredText."""

    def __init__(self, document):
        nodes.NodeVisitor.__init__(self, document)
        self.settings = settings = document.settings
        lcode = settings.language_code
        self.language = languages.get_language(lcode, document.reporter)
        self.head = []
        self.body = []
        self.foot = []
        self.section_level = 0
        self.context = []
        self.topic_class = ''
        self.colspecs = []
        self.compact_p = 1
        self.compact_simple = None
        # the list style "*" bullet or "#" numbered
        self._list_char = []
        # writing the header .TH and .SH NAME is postboned after
        # docinfo.
        self._docinfo = {
                "title": "", "title_upper": "",
                "subtitle": "",
                "manual_section": "", "manual_group": "",
                "author": [],
                "date": "",
                "copyright": "",
                "version": "",
                    }
        self._docinfo_keys = []     # a list to keep the sequence as in source.
        self._docinfo_names = {}    # to get name from text not normalized.
        self._in_docinfo = None
        self._field_name = None
        self._active_table = None
        self._in_literal = False
        self.header_written = 0
        self._line_block = 0
        self.authors = []
        self.section_level = 0
        self._indent = [0]
        # central definition of simple processing rules
        # what to output on : visit, depart
        # Do not use paragraph requests ``.PP`` because these set indentation.
        # use ``.sp``. Remove superfluous ``.sp`` in ``astext``.
        #
        # Fonts are put on a stack, the top one is used.
        # ``.ft P`` or ``\\fP`` pop from stack.
        # But ``.BI`` seams to fill stack with BIBIBIBIB...
        # ``B`` bold, ``I`` italic, ``R`` roman should be available.
        # Hopefully ``C`` courier too.
        self.defs = {
                'indent': ('.INDENT %.1f\n', '.UNINDENT\n'),
                'definition_list_item': ('.TP', ''),  # par. with hanging tag
                'field_name': ('.TP\n.B ', '\n'),
                'literal': ('\\fB', '\\fP'),
                'literal_block': ('.sp\n.nf\n.ft C\n', '\n.ft P\n.fi\n'),

                'option_list_item': ('.TP\n', ''),

                'reference': (r'\fI\%', r'\fP'),
                'emphasis': ('\\fI', '\\fP'),
                'strong': ('\\fB', '\\fP'),
                'term': ('\n.B ', '\n'),
                'title_reference': ('\\fI', '\\fP'),

                'topic-title': ('.SS ',),
                'sidebar-title': ('.SS ',),

                'problematic': ('\n.nf\n', '\n.fi\n'),
                    }
        # NOTE do not specify the newline before a dot-command, but ensure
        # it is there.

    def comment_begin(self, text):
        """Return commented version of the passed text WITHOUT end of
        line/comment."""
        prefix = '.\\" '
        out_text = ''.join([(prefix + in_line + '\n')
                            for in_line in text.split('\n')])
        return out_text

    def comment(self, text):
        """Return commented version of the passed text."""
        return self.comment_begin(text)+'.\n'

    def ensure_eol(self):
        """Ensure the last line in body is terminated by new line."""
        if len(self.body) > 0 and self.body[-1][-1] != '\n':
            self.body.append('\n')

    def astext(self):
        """Return the final formatted document as a string."""
        if not self.header_written:
            # ensure we get a ".TH" as viewers require it.
            self.append_header()
        # filter body
        for i in range(len(self.body)-1, 0, -1):
            # remove superfluous vertical gaps.
            if self.body[i] == '.sp\n':
                if self.body[i - 1][:4] in ('.BI ', '.IP '):
                    self.body[i] = '.\n'
                elif (self.body[i - 1][:3] == '.B '
                      and self.body[i - 2][:4] == '.TP\n'):
                    self.body[i] = '.\n'
                elif (self.body[i - 1] == '\n'
                      and not self.possibly_a_roff_command.match(
                                  self.body[i - 2])
                      and (self.body[i - 3][:7] == '.TP\n.B '
                           or self.body[i - 3][:4] == '\n.B ')
                      ):
                    self.body[i] = '.\n'
        return ''.join(self.head + self.body + self.foot)

    def deunicode(self, text):
        text = text.replace('\xa0', '\\ ')
        text = text.replace('\u2020', '\\(dg')
        return text

    def visit_Text(self, node):
        text = node.astext()
        text = text.replace('\\', '\\e')
        replace_pairs = [
            ('-', '\\-'),
            ('\'', '\\(aq'),
            ('´', "\\'"),
            ('`', '\\(ga'),
            ('"', '\\(dq'),  # double quotes are a problem on macro lines
            ]
        for (in_char, out_markup) in replace_pairs:
            text = text.replace(in_char, out_markup)
        # unicode
        text = self.deunicode(text)
        # prevent interpretation of "." at line start
        if text.startswith('.'):
            text = '\\&' + text
        if self._in_literal:
            text = text.replace('\n.', '\n\\&.')
        self.body.append(text)

    def depart_Text(self, node):
        pass

    def list_start(self, node):
        class EnumChar:
            enum_style = {
                    'bullet': '\\(bu',
                    'emdash': '\\(em',
                     }

            def __init__(self, style):
                self._style = style
                if 'start' in node:
                    self._cnt = node['start'] - 1
                else:
                    self._cnt = 0
                self._indent = 2
                if style == 'arabic':
                    # indentation depends on number of children
                    # and start value.
                    self._indent = len(str(len(node.children)))
                    self._indent += len(str(self._cnt)) + 1
                elif style == 'loweralpha':
                    self._cnt += ord('a') - 1
                    self._indent = 3
                elif style == 'upperalpha':
                    self._cnt += ord('A') - 1
                    self._indent = 3
                elif style.endswith('roman'):
                    self._indent = 5

            def __next__(self):
                if self._style == 'bullet':
                    return self.enum_style[self._style]
                elif self._style == 'emdash':
                    return self.enum_style[self._style]
                self._cnt += 1
                # TODO add prefix postfix
                if self._style == 'arabic':
                    return "%d." % self._cnt
                elif self._style in ('loweralpha', 'upperalpha'):
                    return "%c." % self._cnt
                elif self._style.endswith('roman'):
                    res = roman.toRoman(self._cnt) + '.'
                    if self._style.startswith('upper'):
                        return res.upper()
                    return res.lower()
                else:
                    return "%d." % self._cnt

            def get_width(self):
                return self._indent

            def __repr__(self):
                return 'enum_style-%s' % list(self._style)

        if 'enumtype' in node:
            self._list_char.append(EnumChar(node['enumtype']))
        else:
            self._list_char.append(EnumChar('bullet'))
        if len(self._list_char) > 1:
            # indent nested lists
            self.indent(self._list_char[-2].get_width())
        else:
            self.indent(self._list_char[-1].get_width())

    def list_end(self):
        self.dedent()
        self._list_char.pop()

    def header(self):
        th = (".TH \"%(title_upper)s\" %(manual_section)s"
              " \"%(date)s\" \"%(version)s\"") % self._docinfo
        if self._docinfo["manual_group"]:
            th += " \"%(manual_group)s\"" % self._docinfo
        th += "\n"
        sh_tmpl = (".SH NAME\n"
                   "%(title)s \\- %(subtitle)s\n")
        return th + sh_tmpl % self._docinfo

    def append_header(self):
        """append header with .TH and .SH NAME"""
        # NOTE before everything
        # .TH title_upper section date source manual
        # BUT macros before .TH for whatis database generators.
        if self.header_written:
            return
        self.head.append(MACRO_DEF)
        self.head.append(self.header())
        self.header_written = 1

    def visit_address(self, node):
        self.visit_docinfo_item(node, 'address')

    def depart_address(self, node):
        pass

    def visit_admonition(self, node, name=None):
        #
        # Make admonitions a simple block quote
        # with a strong heading
        #
        # Using .IP/.RE doesn't preserve indentation
        # when admonitions contain bullets, literal,
        # and/or block quotes.
        #
        if name:
            # .. admonition:: has no name
            self.body.append('.sp\n')
            name = '%s%s:%s\n' % (
                self.defs['strong'][0],
                self.language.labels.get(name, name).upper(),
                self.defs['strong'][1],
                )
            self.body.append(name)
        self.visit_block_quote(node)

    def depart_admonition(self, node):
        self.depart_block_quote(node)

    def visit_attention(self, node):
        self.visit_admonition(node, 'attention')

    depart_attention = depart_admonition

    def visit_docinfo_item(self, node, name):
        if name == 'author':
            self._docinfo[name].append(node.astext())
        else:
            self._docinfo[name] = node.astext()
        self._docinfo_keys.append(name)
        raise nodes.SkipNode

    def depart_docinfo_item(self, node):
        pass

    def visit_author(self, node):
        self.visit_docinfo_item(node, 'author')

    depart_author = depart_docinfo_item

    def visit_authors(self, node):
        # _author is called anyway.
        pass

    def depart_authors(self, node):
        pass

    def visit_block_quote(self, node):
        # BUG/HACK: indent always uses the _last_ indentation,
        # thus we need two of them.
        self.indent(BLOCKQOUTE_INDENT)
        self.indent(0)

    def depart_block_quote(self, node):
        self.dedent()
        self.dedent()

    def visit_bullet_list(self, node):
        self.list_start(node)

    def depart_bullet_list(self, node):
        self.list_end()

    def visit_caption(self, node):
        pass

    def depart_caption(self, node):
        pass

    def visit_caution(self, node):
        self.visit_admonition(node, 'caution')

    depart_caution = depart_admonition

    def visit_citation(self, node):
        num = node.astext().split(None, 1)[0]
        num = num.strip()
        self.body.append('.IP [%s] 5\n' % num)

    def depart_citation(self, node):
        pass

    def visit_citation_reference(self, node):
        self.body.append('['+node.astext()+']')
        raise nodes.SkipNode

    def visit_classifier(self, node):
        pass

    def depart_classifier(self, node):
        pass

    def visit_colspec(self, node):
        self.colspecs.append(node)

    def depart_colspec(self, node):
        pass

    def write_colspecs(self):
        self.body.append("%s.\n" % ('L '*len(self.colspecs)))

    def visit_comment(self, node,
                      sub=re.compile('-(?=-)').sub):
        self.body.append(self.comment(node.astext()))
        raise nodes.SkipNode

    def visit_contact(self, node):
        self.visit_docinfo_item(node, 'contact')

    depart_contact = depart_docinfo_item

    def visit_container(self, node):
        pass

    def depart_container(self, node):
        pass

    def visit_compound(self, node):
        pass

    def depart_compound(self, node):
        pass

    def visit_copyright(self, node):
        self.visit_docinfo_item(node, 'copyright')

    def visit_danger(self, node):
        self.visit_admonition(node, 'danger')

    depart_danger = depart_admonition

    def visit_date(self, node):
        self.visit_docinfo_item(node, 'date')

    def visit_decoration(self, node):
        pass

    def depart_decoration(self, node):
        pass

    def visit_definition(self, node):
        pass

    def depart_definition(self, node):
        pass

    def visit_definition_list(self, node):
        self.indent(DEFINITION_LIST_INDENT)

    def depart_definition_list(self, node):
        self.dedent()

    def visit_definition_list_item(self, node):
        self.body.append(self.defs['definition_list_item'][0])

    def depart_definition_list_item(self, node):
        self.body.append(self.defs['definition_list_item'][1])

    def visit_description(self, node):
        pass

    def depart_description(self, node):
        pass

    def visit_docinfo(self, node):
        self._in_docinfo = 1

    def depart_docinfo(self, node):
        self._in_docinfo = None
        # NOTE nothing should be written before this
        self.append_header()

    def visit_doctest_block(self, node):
        self.body.append(self.defs['literal_block'][0])
        self._in_literal = True

    def depart_doctest_block(self, node):
        self._in_literal = False
        self.body.append(self.defs['literal_block'][1])

    def visit_document(self, node):
        # no blank line between comment and header.
        self.head.append(self.comment(self.document_start).rstrip()+'\n')
        # writing header is postponed
        self.header_written = 0

    def depart_document(self, node):
        if self._docinfo['author']:
            self.body.append('.SH AUTHOR\n%s\n'
                             % ', '.join(self._docinfo['author']))
        skip = ('author', 'copyright', 'date',
                'manual_group', 'manual_section',
                'subtitle',
                'title', 'title_upper', 'version')
        for name in self._docinfo_keys:
            if name == 'address':
                self.body.append("\n%s:\n%s%s.nf\n%s\n.fi\n%s%s" % (
                                    self.language.labels.get(name, name),
                                    self.defs['indent'][0] % 0,
                                    self.defs['indent'][0] % BLOCKQOUTE_INDENT,
                                    self._docinfo[name],
                                    self.defs['indent'][1],
                                    self.defs['indent'][1]))
            elif name not in skip:
                if name in self._docinfo_names:
                    label = self._docinfo_names[name]
                else:
                    label = self.language.labels.get(name, name)
                self.body.append("\n%s: %s\n" % (label, self._docinfo[name]))
        if self._docinfo['copyright']:
            self.body.append('.SH COPYRIGHT\n%s\n'
                             % self._docinfo['copyright'])
        self.body.append(self.comment('Generated by docutils manpage writer.'))

    def visit_emphasis(self, node):
        self.body.append(self.defs['emphasis'][0])

    def depart_emphasis(self, node):
        self.body.append(self.defs['emphasis'][1])

    def visit_entry(self, node):
        # a cell in a table row
        if 'morerows' in node:
            self.document.reporter.warning(
                '"table row spanning" not supported', base_node=node)
        if 'morecols' in node:
            self.document.reporter.warning(
                '"table cell spanning" not supported', base_node=node)
        self.context.append(len(self.body))

    def depart_entry(self, node):
        start = self.context.pop()
        self._active_table.append_cell(self.body[start:])
        del self.body[start:]

    def visit_enumerated_list(self, node):
        self.list_start(node)

    def depart_enumerated_list(self, node):
        self.list_end()

    def visit_error(self, node):
        self.visit_admonition(node, 'error')

    depart_error = depart_admonition

    def visit_field(self, node):
        pass

    def depart_field(self, node):
        pass

    def visit_field_body(self, node):
        if self._in_docinfo:
            name_normalized = self._field_name.lower().replace(" ", "_")
            self._docinfo_names[name_normalized] = self._field_name
            self.visit_docinfo_item(node, name_normalized)
            raise nodes.SkipNode

    def depart_field_body(self, node):
        pass

    def visit_field_list(self, node):
        self.indent(FIELD_LIST_INDENT)

    def depart_field_list(self, node):
        self.dedent()

    def visit_field_name(self, node):
        if self._in_docinfo:
            self._field_name = node.astext()
            raise nodes.SkipNode
        else:
            self.body.append(self.defs['field_name'][0])

    def depart_field_name(self, node):
        self.body.append(self.defs['field_name'][1])

    def visit_figure(self, node):
        self.indent(2.5)
        self.indent(0)

    def depart_figure(self, node):
        self.dedent()
        self.dedent()

    def visit_footer(self, node):
        self.document.reporter.warning('"footer" not supported',
                                       base_node=node)

    def depart_footer(self, node):
        pass

    def visit_footnote(self, node):
        num, text = node.astext().split(None, 1)
        num = num.strip()
        self.body.append('.IP [%s] 5\n' % self.deunicode(num))

    def depart_footnote(self, node):
        pass

    def footnote_backrefs(self, node):
        self.document.reporter.warning('"footnote_backrefs" not supported',
                                       base_node=node)

    def visit_footnote_reference(self, node):
        self.body.append('['+self.deunicode(node.astext())+']')
        raise nodes.SkipNode

    def depart_footnote_reference(self, node):
        pass

    def visit_generated(self, node):
        pass

    def depart_generated(self, node):
        pass

    def visit_header(self, node):
        raise NotImplementedError(node.astext())

    def depart_header(self, node):
        pass

    def visit_hint(self, node):
        self.visit_admonition(node, 'hint')

    depart_hint = depart_admonition

    def visit_subscript(self, node):
        self.body.append('\\s-2\\d')

    def depart_subscript(self, node):
        self.body.append('\\u\\s0')

    def visit_superscript(self, node):
        self.body.append('\\s-2\\u')

    def depart_superscript(self, node):
        self.body.append('\\d\\s0')

    def visit_attribution(self, node):
        self.body.append('\\(em ')

    def depart_attribution(self, node):
        self.body.append('\n')

    def visit_image(self, node):
        self.document.reporter.warning('"image" not supported',
                                       base_node=node)
        text = []
        if 'alt' in node.attributes:
            text.append(node.attributes['alt'])
        if 'uri' in node.attributes:
            text.append(node.attributes['uri'])
        self.body.append('[image: %s]\n' % ('/'.join(text)))
        raise nodes.SkipNode

    def visit_important(self, node):
        self.visit_admonition(node, 'important')

    depart_important = depart_admonition

    def visit_inline(self, node):
        pass

    def depart_inline(self, node):
        pass

    def visit_label(self, node):
        # footnote and citation
        if (isinstance(node.parent, nodes.footnote)
            or isinstance(node.parent, nodes.citation)):
            raise nodes.SkipNode
        self.document.reporter.warning('"unsupported "label"',
                                       base_node=node)
        self.body.append('[')

    def depart_label(self, node):
        self.body.append(']\n')

    def visit_legend(self, node):
        pass

    def depart_legend(self, node):
        pass

    # WHAT should we use .INDENT, .UNINDENT ?
    def visit_line_block(self, node):
        self._line_block += 1
        if self._line_block == 1:
            # TODO: separate inline blocks from previous paragraphs
            # see http://hg.intevation.org/mercurial/crew/rev/9c142ed9c405
            # self.body.append('.sp\n')
            # but it does not work for me.
            self.body.append('.nf\n')
        else:
            self.body.append('.in +2\n')

    def depart_line_block(self, node):
        self._line_block -= 1
        if self._line_block == 0:
            self.body.append('.fi\n')
            self.body.append('.sp\n')
        else:
            self.body.append('.in -2\n')

    def visit_line(self, node):
        pass

    def depart_line(self, node):
        self.body.append('\n')

    def visit_list_item(self, node):
        # man 7 man argues to use ".IP" instead of ".TP"
        self.body.append('.IP %s %d\n' % (
                next(self._list_char[-1]),
                self._list_char[-1].get_width(),))

    def depart_list_item(self, node):
        pass

    def visit_literal(self, node):
        self.body.append(self.defs['literal'][0])

    def depart_literal(self, node):
        self.body.append(self.defs['literal'][1])

    def visit_literal_block(self, node):
        # BUG/HACK: indent always uses the _last_ indentation,
        # thus we need two of them.
        self.indent(LITERAL_BLOCK_INDENT)
        self.indent(0)
        self.body.append(self.defs['literal_block'][0])
        self._in_literal = True

    def depart_literal_block(self, node):
        self._in_literal = False
        self.body.append(self.defs['literal_block'][1])
        self.dedent()
        self.dedent()

    def visit_math(self, node):
        self.document.reporter.warning('"math" role not supported',
                                       base_node=node)
        self.visit_literal(node)

    def depart_math(self, node):
        self.depart_literal(node)

    def visit_math_block(self, node):
        self.document.reporter.warning('"math" directive not supported',
                                       base_node=node)
        self.visit_literal_block(node)

    def depart_math_block(self, node):
        self.depart_literal_block(node)

    # <meta> shall become an optional standard node:
    # def visit_meta(self, node):
    #     raise NotImplementedError(node.astext())

    # def depart_meta(self, node):
    #     pass

    def visit_note(self, node):
        self.visit_admonition(node, 'note')

    depart_note = depart_admonition

    def indent(self, by=0.5):
        # if we are in a section ".SH" there already is a .RS
        step = self._indent[-1]
        self._indent.append(by)
        self.body.append(self.defs['indent'][0] % step)

    def dedent(self):
        self._indent.pop()
        self.body.append(self.defs['indent'][1])

    def visit_option_list(self, node):
        self.indent(OPTION_LIST_INDENT)

    def depart_option_list(self, node):
        self.dedent()

    def visit_option_list_item(self, node):
        # one item of the list
        self.body.append(self.defs['option_list_item'][0])

    def depart_option_list_item(self, node):
        self.body.append(self.defs['option_list_item'][1])

    def visit_option_group(self, node):
        # as one option could have several forms it is a group
        # options without parameter bold only, .B, -v
        # options with parameter bold italic, .BI, -f file
        #
        # we do not know if .B or .BI, blind guess:
        self.context.append('.B ')  # Add blank for sphinx (docutils/bugs/380)
        self.context.append(len(self.body))  # to be able to insert later
        self.context.append(0)               # option counter

    def depart_option_group(self, node):
        self.context.pop()  # the counter
        start_position = self.context.pop()
        text = self.body[start_position:]
        del self.body[start_position:]
        self.body.append('%s%s\n' % (self.context.pop(), ''.join(text)))

    def visit_option(self, node):
        # each form of the option will be presented separately
        if self.context[-1] > 0:
            if self.context[-3] == '.BI':
                self.body.append('\\fR,\\fB ')
            else:
                self.body.append('\\fP,\\fB ')
        if self.context[-3] == '.BI':
            self.body.append('\\')
        self.body.append(' ')

    def depart_option(self, node):
        self.context[-1] += 1

    def visit_option_string(self, node):
        # do not know if .B or .BI
        pass

    def depart_option_string(self, node):
        pass

    def visit_option_argument(self, node):
        self.context[-3] = '.BI'  # bold/italic alternate
        if node['delimiter'] != ' ':
            self.body.append('\\fB%s ' % node['delimiter'])
        elif self.body[len(self.body)-1].endswith('='):
            # a blank only means no blank in output, just changing font
            self.body.append(' ')
        else:
            # blank backslash blank, switch font then a blank
            self.body.append(' \\ ')

    def depart_option_argument(self, node):
        pass

    def visit_organization(self, node):
        self.visit_docinfo_item(node, 'organization')

    def depart_organization(self, node):
        pass

    def first_child(self, node):
        first = isinstance(node.parent[0], nodes.label)  # skip label
        for child in node.parent.children[first:]:
            if isinstance(child, nodes.Invisible):
                continue
            if child is node:
                return 1
            break
        return 0

    def visit_paragraph(self, node):
        # ``.PP`` : Start standard indented paragraph.
        # ``.LP`` : Start block paragraph, all except the first.
        # ``.P [type]``  : Start paragraph type.
        # NOTE do not use paragraph starts because they reset indentation.
        # ``.sp`` is only vertical space
        self.ensure_eol()
        if not self.first_child(node):
            self.body.append('.sp\n')
        # set in literal to escape dots after a new-line-character
        self._in_literal = True

    def depart_paragraph(self, node):
        self._in_literal = False
        self.body.append('\n')

    def visit_problematic(self, node):
        self.body.append(self.defs['problematic'][0])

    def depart_problematic(self, node):
        self.body.append(self.defs['problematic'][1])

    def visit_raw(self, node):
        if node.get('format') == 'manpage':
            self.body.append(node.astext() + "\n")
        # Keep non-manpage raw text out of output:
        raise nodes.SkipNode

    def visit_reference(self, node):
        """E.g. link or email address."""
        self.body.append(self.defs['reference'][0])

    def depart_reference(self, node):
        # TODO check node text is different from refuri
        # self.body.append("\n'UR " + node['refuri'] + "\n'UE\n")
        self.body.append(self.defs['reference'][1])

    def visit_revision(self, node):
        self.visit_docinfo_item(node, 'revision')

    depart_revision = depart_docinfo_item

    def visit_row(self, node):
        self._active_table.new_row()

    def depart_row(self, node):
        pass

    def visit_section(self, node):
        self.section_level += 1

    def depart_section(self, node):
        self.section_level -= 1

    def visit_status(self, node):
        self.visit_docinfo_item(node, 'status')

    depart_status = depart_docinfo_item

    def visit_strong(self, node):
        self.body.append(self.defs['strong'][0])

    def depart_strong(self, node):
        self.body.append(self.defs['strong'][1])

    def visit_substitution_definition(self, node):
        """Internal only."""
        raise nodes.SkipNode

    def visit_substitution_reference(self, node):
        self.document.reporter.warning(
            '"substitution_reference" not supported', base_node=node)

    def visit_subtitle(self, node):
        if isinstance(node.parent, nodes.sidebar):
            self.body.append(self.defs['strong'][0])
        elif isinstance(node.parent, nodes.document):
            self.visit_docinfo_item(node, 'subtitle')
        elif isinstance(node.parent, nodes.section):
            self.body.append(self.defs['strong'][0])

    def depart_subtitle(self, node):
        # document subtitle calls SkipNode
        self.body.append(self.defs['strong'][1]+'\n.PP\n')

    def visit_system_message(self, node):
        # TODO add report_level
        # if node['level'] < self.document.reporter['writer'].report_level:
        #    Level is too low to display:
        #    raise nodes.SkipNode
        attr = {}
        if node.hasattr('id'):
            attr['name'] = node['id']
        if node.hasattr('line'):
            line = ', line %s' % node['line']
        else:
            line = ''
        self.body.append('.IP "System Message: %s/%s (%s:%s)"\n'
                         % (node['type'], node['level'], node['source'], line))

    def depart_system_message(self, node):
        pass

    def visit_table(self, node):
        self._active_table = Table()

    def depart_table(self, node):
        self.ensure_eol()
        self.body.extend(self._active_table.as_list())
        self._active_table = None

    def visit_target(self, node):
        # targets are in-document hyper targets, without any use for man-pages.
        raise nodes.SkipNode

    def visit_tbody(self, node):
        pass

    def depart_tbody(self, node):
        pass

    def visit_term(self, node):
        self.body.append(self.defs['term'][0])

    def depart_term(self, node):
        self.body.append(self.defs['term'][1])

    def visit_tgroup(self, node):
        pass

    def depart_tgroup(self, node):
        pass

    def visit_thead(self, node):
        # MAYBE double line '='
        pass

    def depart_thead(self, node):
        # MAYBE double line '='
        pass

    def visit_tip(self, node):
        self.visit_admonition(node, 'tip')

    depart_tip = depart_admonition

    def visit_title(self, node):
        if isinstance(node.parent, nodes.topic):
            self.body.append(self.defs['topic-title'][0])
        elif isinstance(node.parent, nodes.sidebar):
            self.body.append(self.defs['sidebar-title'][0])
        elif isinstance(node.parent, nodes.admonition):
            self.body.append('.IP "')
        elif self.section_level == 0:
            self._docinfo['title'] = node.astext()
            # document title for .TH
            self._docinfo['title_upper'] = node.astext().upper()
            raise nodes.SkipNode
        elif self.section_level == 1:
            self.body.append('.SH %s\n'%self.deunicode(node.astext().upper()))
            raise nodes.SkipNode
        else:
            self.body.append('.SS ')

    def depart_title(self, node):
        if isinstance(node.parent, nodes.admonition):
            self.body.append('"')
        self.body.append('\n')

    def visit_title_reference(self, node):
        """inline citation reference"""
        self.body.append(self.defs['title_reference'][0])

    def depart_title_reference(self, node):
        self.body.append(self.defs['title_reference'][1])

    def visit_topic(self, node):
        pass

    def depart_topic(self, node):
        pass

    def visit_sidebar(self, node):
        pass

    def depart_sidebar(self, node):
        pass

    def visit_rubric(self, node):
        pass

    def depart_rubric(self, node):
        self.body.append('\n')

    def visit_transition(self, node):
        # .PP      Begin a new paragraph and reset prevailing indent.
        # .sp N    leaves N lines of blank space.
        # .ce      centers the next line
        self.body.append('\n.sp\n.ce\n----\n')

    def depart_transition(self, node):
        self.body.append('\n.ce 0\n.sp\n')

    def visit_version(self, node):
        self.visit_docinfo_item(node, 'version')

    def visit_warning(self, node):
        self.visit_admonition(node, 'warning')

    depart_warning = depart_admonition

    def unimplemented_visit(self, node):
        raise NotImplementedError('visiting unimplemented node type: %s'
                                  % node.__class__.__name__)

# vim: set fileencoding=utf-8 et ts=4 ai :

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/manpage.py
NikolaBlagojevic/pyrecodes
# $Id: null.py 9352 2023-04-17 20:26:41Z milde $
# Author: David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
A do-nothing Writer.

`self.output` will change from ``None`` to the empty string
in Docutils 0.22.
"""

from docutils import writers


class Writer(writers.UnfilteredWriter):

    supported = ('null',)
    """Formats this writer supports."""

    config_section = 'null writer'
    config_section_dependencies = ('writers',)

    def translate(self):
        # output = None   # TODO in 0.22
        pass

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/null.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9258 2022-11-21 14:51:43Z milde $
# Author: Dave Kuhlman <dkuhlman@davekuhlman.org>
# Copyright: This module has been placed in the public domain.

"""
Open Document Format (ODF) Writer.

This module is provisional:
the API is not settled and may change with any minor Docutils version.
"""

__docformat__ = 'reStructuredText'


from configparser import ConfigParser
import copy
from io import StringIO
import itertools
import locale
import os
import os.path
import re
import subprocess
import tempfile
import time
from urllib.request import urlopen
from urllib.error import HTTPError
import weakref
from xml.etree import ElementTree as etree
from xml.dom import minidom
import zipfile

import docutils
from docutils import frontend, nodes, utils, writers, languages
from docutils.parsers.rst.directives.images import PIL  # optional
from docutils.readers import standalone
from docutils.transforms import references

# Import pygments and odtwriter pygments formatters if possible.
try:
    import pygments
    import pygments.lexers
    from .pygmentsformatter import (OdtPygmentsProgFormatter,
                                    OdtPygmentsLaTeXFormatter)
except (ImportError, SyntaxError):
    pygments = None

# import warnings
# warnings.warn('importing IPShellEmbed', UserWarning)
# from IPython.Shell import IPShellEmbed
# args = ['-pdb', '-pi1', 'In <\\#>: ', '-pi2', '   .\\D.: ',
#         '-po', 'Out<\\#>: ', '-nosep']
# ipshell = IPShellEmbed(args,
#                        banner = 'Entering IPython.  Press Ctrl-D to exit.',
#                        exit_msg = 'Leaving Interpreter, back to program.')

VERSION = '1.0a'

IMAGE_NAME_COUNTER = itertools.count()


#
# ElementTree does not support getparent method (lxml does).
# This wrapper class and the following support functions provide
#   that support for the ability to get the parent of an element.
#
_parents = weakref.WeakKeyDictionary()
if isinstance(etree.Element, type):
    _ElementInterface = etree.Element
else:
    _ElementInterface = etree._ElementInterface


class _ElementInterfaceWrapper(_ElementInterface):
    def __init__(self, tag, attrib=None):
        _ElementInterface.__init__(self, tag, attrib)
        _parents[self] = None

    def setparent(self, parent):
        _parents[self] = parent

    def getparent(self):
        return _parents[self]


#
# Constants and globals

SPACES_PATTERN = re.compile(r'( +)')
TABS_PATTERN = re.compile(r'(\t+)')
FILL_PAT1 = re.compile(r'^ +')
FILL_PAT2 = re.compile(r' {2,}')

TABLESTYLEPREFIX = 'rststyle-table-'
TABLENAMEDEFAULT = '%s0' % TABLESTYLEPREFIX
TABLEPROPERTYNAMES = (
    'border', 'border-top', 'border-left',
    'border-right', 'border-bottom', )

GENERATOR_DESC = 'Docutils.org/odf_odt'

NAME_SPACE_1 = 'urn:oasis:names:tc:opendocument:xmlns:office:1.0'

CONTENT_NAMESPACE_DICT = CNSD = {
    # 'office:version': '1.0',
    'chart': 'urn:oasis:names:tc:opendocument:xmlns:chart:1.0',
    'dc': 'http://purl.org/dc/elements/1.1/',
    'dom': 'http://www.w3.org/2001/xml-events',
    'dr3d': 'urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0',
    'draw': 'urn:oasis:names:tc:opendocument:xmlns:drawing:1.0',
    'fo': 'urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0',
    'form': 'urn:oasis:names:tc:opendocument:xmlns:form:1.0',
    'math': 'http://www.w3.org/1998/Math/MathML',
    'meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
    'number': 'urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0',
    'office': NAME_SPACE_1,
    'ooo': 'http://openoffice.org/2004/office',
    'oooc': 'http://openoffice.org/2004/calc',
    'ooow': 'http://openoffice.org/2004/writer',
    'presentation': 'urn:oasis:names:tc:opendocument:xmlns:presentation:1.0',

    'script': 'urn:oasis:names:tc:opendocument:xmlns:script:1.0',
    'style': 'urn:oasis:names:tc:opendocument:xmlns:style:1.0',
    'svg': 'urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0',
    'table': 'urn:oasis:names:tc:opendocument:xmlns:table:1.0',
    'text': 'urn:oasis:names:tc:opendocument:xmlns:text:1.0',
    'xforms': 'http://www.w3.org/2002/xforms',
    'xlink': 'http://www.w3.org/1999/xlink',
    'xsd': 'http://www.w3.org/2001/XMLSchema',
    'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
}

STYLES_NAMESPACE_DICT = SNSD = {
    # 'office:version': '1.0',
    'chart': 'urn:oasis:names:tc:opendocument:xmlns:chart:1.0',
    'dc': 'http://purl.org/dc/elements/1.1/',
    'dom': 'http://www.w3.org/2001/xml-events',
    'dr3d': 'urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0',
    'draw': 'urn:oasis:names:tc:opendocument:xmlns:drawing:1.0',
    'fo': 'urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0',
    'form': 'urn:oasis:names:tc:opendocument:xmlns:form:1.0',
    'math': 'http://www.w3.org/1998/Math/MathML',
    'meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
    'number': 'urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0',
    'office': NAME_SPACE_1,
    'presentation': 'urn:oasis:names:tc:opendocument:xmlns:presentation:1.0',
    'ooo': 'http://openoffice.org/2004/office',
    'oooc': 'http://openoffice.org/2004/calc',
    'ooow': 'http://openoffice.org/2004/writer',
    'script': 'urn:oasis:names:tc:opendocument:xmlns:script:1.0',
    'style': 'urn:oasis:names:tc:opendocument:xmlns:style:1.0',
    'svg': 'urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0',
    'table': 'urn:oasis:names:tc:opendocument:xmlns:table:1.0',
    'text': 'urn:oasis:names:tc:opendocument:xmlns:text:1.0',
    'xlink': 'http://www.w3.org/1999/xlink',
}

MANIFEST_NAMESPACE_DICT = MANNSD = {
    'manifest': 'urn:oasis:names:tc:opendocument:xmlns:manifest:1.0',
}

META_NAMESPACE_DICT = METNSD = {
    # 'office:version': '1.0',
    'dc': 'http://purl.org/dc/elements/1.1/',
    'meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
    'office': NAME_SPACE_1,
    'ooo': 'http://openoffice.org/2004/office',
    'xlink': 'http://www.w3.org/1999/xlink',
}

# Attribute dictionaries for use with ElementTree, which
# does not support use of nsmap parameter on Element() and SubElement().

CONTENT_NAMESPACE_ATTRIB = {
    # 'office:version': '1.0',
    'xmlns:chart': 'urn:oasis:names:tc:opendocument:xmlns:chart:1.0',
    'xmlns:dc': 'http://purl.org/dc/elements/1.1/',
    'xmlns:dom': 'http://www.w3.org/2001/xml-events',
    'xmlns:dr3d': 'urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0',
    'xmlns:draw': 'urn:oasis:names:tc:opendocument:xmlns:drawing:1.0',
    'xmlns:fo': 'urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0',
    'xmlns:form': 'urn:oasis:names:tc:opendocument:xmlns:form:1.0',
    'xmlns:math': 'http://www.w3.org/1998/Math/MathML',
    'xmlns:meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
    'xmlns:number': 'urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0',
    'xmlns:office': NAME_SPACE_1,
    'xmlns:presentation':
        'urn:oasis:names:tc:opendocument:xmlns:presentation:1.0',
    'xmlns:ooo': 'http://openoffice.org/2004/office',
    'xmlns:oooc': 'http://openoffice.org/2004/calc',
    'xmlns:ooow': 'http://openoffice.org/2004/writer',
    'xmlns:script': 'urn:oasis:names:tc:opendocument:xmlns:script:1.0',
    'xmlns:style': 'urn:oasis:names:tc:opendocument:xmlns:style:1.0',
    'xmlns:svg': 'urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0',
    'xmlns:table': 'urn:oasis:names:tc:opendocument:xmlns:table:1.0',
    'xmlns:text': 'urn:oasis:names:tc:opendocument:xmlns:text:1.0',
    'xmlns:xforms': 'http://www.w3.org/2002/xforms',
    'xmlns:xlink': 'http://www.w3.org/1999/xlink',
    'xmlns:xsd': 'http://www.w3.org/2001/XMLSchema',
    'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
}

STYLES_NAMESPACE_ATTRIB = {
    # 'office:version': '1.0',
    'xmlns:chart': 'urn:oasis:names:tc:opendocument:xmlns:chart:1.0',
    'xmlns:dc': 'http://purl.org/dc/elements/1.1/',
    'xmlns:dom': 'http://www.w3.org/2001/xml-events',
    'xmlns:dr3d': 'urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0',
    'xmlns:draw': 'urn:oasis:names:tc:opendocument:xmlns:drawing:1.0',
    'xmlns:fo': 'urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0',
    'xmlns:form': 'urn:oasis:names:tc:opendocument:xmlns:form:1.0',
    'xmlns:math': 'http://www.w3.org/1998/Math/MathML',
    'xmlns:meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
    'xmlns:number': 'urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0',
    'xmlns:office': NAME_SPACE_1,
    'xmlns:presentation':
        'urn:oasis:names:tc:opendocument:xmlns:presentation:1.0',
    'xmlns:ooo': 'http://openoffice.org/2004/office',
    'xmlns:oooc': 'http://openoffice.org/2004/calc',
    'xmlns:ooow': 'http://openoffice.org/2004/writer',
    'xmlns:script': 'urn:oasis:names:tc:opendocument:xmlns:script:1.0',
    'xmlns:style': 'urn:oasis:names:tc:opendocument:xmlns:style:1.0',
    'xmlns:svg': 'urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0',
    'xmlns:table': 'urn:oasis:names:tc:opendocument:xmlns:table:1.0',
    'xmlns:text': 'urn:oasis:names:tc:opendocument:xmlns:text:1.0',
    'xmlns:xlink': 'http://www.w3.org/1999/xlink',
}

MANIFEST_NAMESPACE_ATTRIB = {
    'xmlns:manifest': 'urn:oasis:names:tc:opendocument:xmlns:manifest:1.0',
}

META_NAMESPACE_ATTRIB = {
    # 'office:version': '1.0',
    'xmlns:dc': 'http://purl.org/dc/elements/1.1/',
    'xmlns:meta': 'urn:oasis:names:tc:opendocument:xmlns:meta:1.0',
    'xmlns:office': NAME_SPACE_1,
    'xmlns:ooo': 'http://openoffice.org/2004/office',
    'xmlns:xlink': 'http://www.w3.org/1999/xlink',
}


#
# Functions
#

#
# ElementTree support functions.
# In order to be able to get the parent of elements, must use these
#   instead of the functions with same name provided by ElementTree.
#
def Element(tag, attrib=None, nsmap=None, nsdict=CNSD):
    if attrib is None:
        attrib = {}
    tag, attrib = fix_ns(tag, attrib, nsdict)
    return _ElementInterfaceWrapper(tag, attrib)


def SubElement(parent, tag, attrib=None, nsmap=None, nsdict=CNSD):
    if attrib is None:
        attrib = {}
    tag, attrib = fix_ns(tag, attrib, nsdict)
    el = _ElementInterfaceWrapper(tag, attrib)
    parent.append(el)
    el.setparent(parent)
    return el


def fix_ns(tag, attrib, nsdict):
    nstag = add_ns(tag, nsdict)
    nsattrib = {}
    for key, val in list(attrib.items()):
        nskey = add_ns(key, nsdict)
        nsattrib[nskey] = val
    return nstag, nsattrib


def add_ns(tag, nsdict=CNSD):
    return tag


def ToString(et):
    outstream = StringIO()
    et.write(outstream, encoding="unicode")
    s1 = outstream.getvalue()
    outstream.close()
    return s1


def escape_cdata(text):
    text = text.replace("&", "&amp;")
    text = text.replace("<", "&lt;")
    text = text.replace(">", "&gt;")
    ascii = ''
    for char in text:
        if ord(char) >= ord("\x7f"):
            ascii += "&#x%X;" % (ord(char), )
        else:
            ascii += char
    return ascii


#
# Classes
#


class TableStyle:
    def __init__(self, border=None, backgroundcolor=None):
        self.border = border
        self.backgroundcolor = backgroundcolor

    def get_border_(self):
        return self.border_

    def set_border_(self, border):
        self.border_ = border

    border = property(get_border_, set_border_)

    def get_backgroundcolor_(self):
        return self.backgroundcolor_

    def set_backgroundcolor_(self, backgroundcolor):
        self.backgroundcolor_ = backgroundcolor
    backgroundcolor = property(get_backgroundcolor_, set_backgroundcolor_)


BUILTIN_DEFAULT_TABLE_STYLE = TableStyle(
    border='0.0007in solid #000000')


#
# Information about the indentation level for lists nested inside
#   other contexts, e.g. dictionary lists.
class ListLevel:
    def __init__(self, level, sibling_level=True, nested_level=True):
        self.level = level
        self.sibling_level = sibling_level
        self.nested_level = nested_level

    def set_sibling(self, sibling_level):
        self.sibling_level = sibling_level

    def get_sibling(self):
        return self.sibling_level

    def set_nested(self, nested_level):
        self.nested_level = nested_level

    def get_nested(self):
        return self.nested_level

    def set_level(self, level):
        self.level = level

    def get_level(self):
        return self.level


class Writer(writers.Writer):

    MIME_TYPE = 'application/vnd.oasis.opendocument.text'
    EXTENSION = '.odt'

    supported = ('odt', )
    """Formats this writer supports."""

    default_stylesheet = 'styles' + EXTENSION

    default_stylesheet_path = utils.relative_path(
        os.path.join(os.getcwd(), 'dummy'),
        os.path.join(os.path.dirname(__file__), default_stylesheet))

    default_template = 'template.txt'

    default_template_path = utils.relative_path(
        os.path.join(os.getcwd(), 'dummy'),
        os.path.join(os.path.dirname(__file__), default_template))

    settings_spec = (
        'ODF-Specific Options',
        None,
        (
            ('Specify a stylesheet.  '
                'Default: "%s"' % default_stylesheet_path,
                ['--stylesheet'],
                {
                    'default': default_stylesheet_path,
                    'dest': 'stylesheet'
                }),
            ('Specify a configuration/mapping file relative to the '
                'current working '
                'directory for additional ODF options.  '
                'In particular, this file may contain a section named '
                '"Formats" that maps default style names to '
                'names to be used in the resulting output file allowing for '
                'adhering to external standards. '
                'For more info and the format of the '
                'configuration/mapping file, '
                'see the odtwriter doc.',
                ['--odf-config-file'],
                {'metavar': '<file>'}),
            ('Obfuscate email addresses to confuse harvesters while still '
                'keeping email links usable with '
                'standards-compliant browsers.',
                ['--cloak-email-addresses'],
                {'default': False,
                    'action': 'store_true',
                    'dest': 'cloak_email_addresses',
                    'validator': frontend.validate_boolean}),
            ('Do not obfuscate email addresses.',
                ['--no-cloak-email-addresses'],
                {'default': False,
                    'action': 'store_false',
                    'dest': 'cloak_email_addresses',
                    'validator': frontend.validate_boolean}),
            ('Specify the thickness of table borders in thousands of a cm.  '
                'Default is 35.',
                ['--table-border-thickness'],
                {'default': None,
                    'validator': frontend.validate_nonnegative_int}),
            ('Add syntax highlighting in literal code blocks.',
                ['--add-syntax-highlighting'],
                {'default': False,
                    'action': 'store_true',
                    'dest': 'add_syntax_highlighting',
                    'validator': frontend.validate_boolean}),
            ('Do not add syntax highlighting in '
             'literal code blocks. (default)',
                ['--no-syntax-highlighting'],
                {'default': False,
                    'action': 'store_false',
                    'dest': 'add_syntax_highlighting',
                    'validator': frontend.validate_boolean}),
            ('Create sections for headers.  (default)',
                ['--create-sections'],
                {'default': True,
                    'action': 'store_true',
                    'dest': 'create_sections',
                    'validator': frontend.validate_boolean}),
            ('Do not create sections for headers.',
                ['--no-sections'],
                {'default': True,
                    'action': 'store_false',
                    'dest': 'create_sections',
                    'validator': frontend.validate_boolean}),
            ('Create links.',
                ['--create-links'],
                {'default': False,
                    'action': 'store_true',
                    'dest': 'create_links',
                    'validator': frontend.validate_boolean}),
            ('Do not create links.  (default)',
                ['--no-links'],
                {'default': False,
                    'action': 'store_false',
                    'dest': 'create_links',
                    'validator': frontend.validate_boolean}),
            ('Generate endnotes at end of document, not footnotes '
                'at bottom of page.',
                ['--endnotes-end-doc'],
                {'default': False,
                    'action': 'store_true',
                    'dest': 'endnotes_end_doc',
                    'validator': frontend.validate_boolean}),
            ('Generate footnotes at bottom of page, not endnotes '
                'at end of document. (default)',
                ['--no-endnotes-end-doc'],
                {'default': False,
                    'action': 'store_false',
                    'dest': 'endnotes_end_doc',
                    'validator': frontend.validate_boolean}),
            ('Generate a bullet list table of contents, not '
                'an ODF/oowriter table of contents.',
                ['--generate-list-toc'],
                {'default': True,
                    'action': 'store_false',
                    'dest': 'generate_oowriter_toc',
                    'validator': frontend.validate_boolean}),
            ('Generate an ODF/oowriter table of contents, not '
                'a bullet list. (default)',
                ['--generate-oowriter-toc'],
                {'default': True,
                    'action': 'store_true',
                    'dest': 'generate_oowriter_toc',
                    'validator': frontend.validate_boolean}),
            ('Specify the contents of an custom header line.  '
                'See odf_odt writer documentation for details '
                'about special field character sequences.',
                ['--custom-odt-header'],
                {'default': '',
                    'dest': 'custom_header', }),
            ('Specify the contents of an custom footer line.  '
                'See odf_odt writer documentation for details '
                'about special field character sequences.',
                ['--custom-odt-footer'],
                {'default': '',
                    'dest': 'custom_footer', }),
        )
    )

    settings_defaults = {
        'output_encoding_error_handler': 'xmlcharrefreplace',
    }

    relative_path_settings = (
        'stylesheet_path',
    )

    config_section = 'odf_odt writer'
    config_section_dependencies = ('writers',)

    def __init__(self):
        writers.Writer.__init__(self)
        self.translator_class = ODFTranslator

    def translate(self):
        self.settings = self.document.settings
        self.visitor = self.translator_class(self.document)
        self.visitor.retrieve_styles(self.EXTENSION)
        self.document.walkabout(self.visitor)
        self.visitor.add_doc_title()
        self.assemble_my_parts()
        self.output = self.parts['whole']

    def assemble_my_parts(self):
        """Assemble the `self.parts` dictionary.  Extend in subclasses.
        """
        writers.Writer.assemble_parts(self)
        f = tempfile.NamedTemporaryFile()
        zfile = zipfile.ZipFile(f, 'w', zipfile.ZIP_DEFLATED)
        self.write_zip_str(
            zfile, 'mimetype', self.MIME_TYPE,
            compress_type=zipfile.ZIP_STORED)
        content = self.visitor.content_astext()
        self.write_zip_str(zfile, 'content.xml', content)
        s1 = self.create_manifest()
        self.write_zip_str(zfile, 'META-INF/manifest.xml', s1)
        s1 = self.create_meta()
        self.write_zip_str(zfile, 'meta.xml', s1)
        s1 = self.get_stylesheet()
        # Set default language in document to be generated.
        # Language is specified by the -l/--language command line option.
        # The format is described in BCP 47.  If region is omitted, we use
        # local.normalize(ll) to obtain a region.
        language_code = None
        region_code = None
        if self.visitor.language_code:
            language_ids = self.visitor.language_code.replace('_', '-')
            language_ids = language_ids.split('-')
            # first tag is primary language tag
            language_code = language_ids[0].lower()
            # 2-letter region subtag may follow in 2nd or 3rd position
            for subtag in language_ids[1:]:
                if len(subtag) == 2 and subtag.isalpha():
                    region_code = subtag.upper()
                    break
                elif len(subtag) == 1:
                    break   # 1-letter tag is never before valid region tag
            if region_code is None:
                try:
                    rcode = locale.normalize(language_code)
                except NameError:
                    rcode = language_code
                rcode = rcode.split('_')
                if len(rcode) > 1:
                    rcode = rcode[1].split('.')
                    region_code = rcode[0]
                if region_code is None:
                    self.document.reporter.warning(
                        'invalid language-region.\n'
                        '  Could not find region with locale.normalize().\n'
                        '  Please specify both language and region (ll-RR).\n'
                        '  Examples: es-MX (Spanish, Mexico),\n'
                        '  en-AU (English, Australia).')
        # Update the style ElementTree with the language and region.
        # Note that we keep a reference to the modified node because
        # it is possible that ElementTree will throw away the Python
        # representation of the updated node if we do not.
        updated, new_dom_styles, updated_node = self.update_stylesheet(
            self.visitor.get_dom_stylesheet(), language_code, region_code)
        if updated:
            s1 = etree.tostring(new_dom_styles)
        self.write_zip_str(zfile, 'styles.xml', s1)
        self.store_embedded_files(zfile)
        self.copy_from_stylesheet(zfile)
        zfile.close()
        f.seek(0)
        whole = f.read()
        f.close()
        self.parts['whole'] = whole
        self.parts['encoding'] = self.document.settings.output_encoding
        self.parts['version'] = docutils.__version__

    def update_stylesheet(self, stylesheet_root, language_code, region_code):
        """Update xml style sheet element with language and region/country."""
        updated = False
        modified_nodes = set()
        if language_code is not None or region_code is not None:
            n1 = stylesheet_root.find(
                '{urn:oasis:names:tc:opendocument:xmlns:office:1.0}'
                'styles')
            if n1 is None:
                raise RuntimeError(
                    "Cannot find 'styles' element in styles.odt/styles.xml")
            n2_nodes = n1.findall(
                '{urn:oasis:names:tc:opendocument:xmlns:style:1.0}'
                'default-style')
            if not n2_nodes:
                raise RuntimeError(
                    "Cannot find 'default-style' "
                    "element in styles.xml")
            for node in n2_nodes:
                family = node.attrib.get(
                    '{urn:oasis:names:tc:opendocument:xmlns:style:1.0}'
                    'family')
                if family == 'paragraph' or family == 'graphic':
                    n3 = node.find(
                        '{urn:oasis:names:tc:opendocument:xmlns:style:1.0}'
                        'text-properties')
                    if n3 is None:
                        raise RuntimeError(
                            "Cannot find 'text-properties' "
                            "element in styles.xml")
                    if language_code is not None:
                        n3.attrib[
                            '{urn:oasis:names:tc:opendocument:xmlns:'
                            'xsl-fo-compatible:1.0}language'] = language_code
                        n3.attrib[
                            '{urn:oasis:names:tc:opendocument:xmlns:'
                            'style:1.0}language-complex'] = language_code
                        updated = True
                        modified_nodes.add(n3)
                    if region_code is not None:
                        n3.attrib[
                            '{urn:oasis:names:tc:opendocument:xmlns:'
                            'xsl-fo-compatible:1.0}country'] = region_code
                        n3.attrib[
                            '{urn:oasis:names:tc:opendocument:xmlns:'
                            'style:1.0}country-complex'] = region_code
                        updated = True
                        modified_nodes.add(n3)
        return updated, stylesheet_root, modified_nodes

    def write_zip_str(
            self, zfile, name, bytes, compress_type=zipfile.ZIP_DEFLATED):
        localtime = time.localtime(time.time())
        zinfo = zipfile.ZipInfo(name, localtime)
        # Add some standard UNIX file access permissions (-rw-r--r--).
        zinfo.external_attr = (0x81a4 & 0xFFFF) << 16
        zinfo.compress_type = compress_type
        zfile.writestr(zinfo, bytes)

    def store_embedded_files(self, zfile):
        embedded_files = self.visitor.get_embedded_file_list()
        for source, destination in embedded_files:
            if source is None:
                continue
            try:
                zfile.write(source, destination)
            except OSError:
                self.document.reporter.warning(
                    "Can't open file %s." % (source, ))

    def get_settings(self):
        """
        modeled after get_stylesheet
        """
        stylespath = self.settings.stylesheet
        zfile = zipfile.ZipFile(stylespath, 'r')
        s1 = zfile.read('settings.xml')
        zfile.close()
        return s1

    def get_stylesheet(self):
        """Get the stylesheet from the visitor.
        Ask the visitor to setup the page.
        """
        return self.visitor.setup_page()

    def copy_from_stylesheet(self, outzipfile):
        """Copy images, settings, etc from the stylesheet doc into target doc.
        """
        stylespath = self.settings.stylesheet
        inzipfile = zipfile.ZipFile(stylespath, 'r')
        # Copy the styles.
        s1 = inzipfile.read('settings.xml')
        self.write_zip_str(outzipfile, 'settings.xml', s1)
        # Copy the images.
        namelist = inzipfile.namelist()
        for name in namelist:
            if name.startswith('Pictures/'):
                imageobj = inzipfile.read(name)
                outzipfile.writestr(name, imageobj)
        inzipfile.close()

    def assemble_parts(self):
        pass

    def create_manifest(self):
        root = Element(
            'manifest:manifest',
            attrib=MANIFEST_NAMESPACE_ATTRIB,
            nsdict=MANIFEST_NAMESPACE_DICT,
        )
        doc = etree.ElementTree(root)
        SubElement(root, 'manifest:file-entry', attrib={
            'manifest:media-type': self.MIME_TYPE,
            'manifest:full-path': '/',
        }, nsdict=MANNSD)
        SubElement(root, 'manifest:file-entry', attrib={
            'manifest:media-type': 'text/xml',
            'manifest:full-path': 'content.xml',
        }, nsdict=MANNSD)
        SubElement(root, 'manifest:file-entry', attrib={
            'manifest:media-type': 'text/xml',
            'manifest:full-path': 'styles.xml',
        }, nsdict=MANNSD)
        SubElement(root, 'manifest:file-entry', attrib={
            'manifest:media-type': 'text/xml',
            'manifest:full-path': 'settings.xml',
        }, nsdict=MANNSD)
        SubElement(root, 'manifest:file-entry', attrib={
            'manifest:media-type': 'text/xml',
            'manifest:full-path': 'meta.xml',
        }, nsdict=MANNSD)
        s1 = ToString(doc)
        doc = minidom.parseString(s1)
        return doc.toprettyxml('  ')

    def create_meta(self):
        root = Element(
            'office:document-meta',
            attrib=META_NAMESPACE_ATTRIB,
            nsdict=META_NAMESPACE_DICT,
        )
        doc = etree.ElementTree(root)
        root = SubElement(root, 'office:meta', nsdict=METNSD)
        el1 = SubElement(root, 'meta:generator', nsdict=METNSD)
        el1.text = 'Docutils/rst2odf.py/%s' % (VERSION, )
        s1 = os.environ.get('USER', '')
        el1 = SubElement(root, 'meta:initial-creator', nsdict=METNSD)
        el1.text = s1
        s2 = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime())
        el1 = SubElement(root, 'meta:creation-date', nsdict=METNSD)
        el1.text = s2
        el1 = SubElement(root, 'dc:creator', nsdict=METNSD)
        el1.text = s1
        el1 = SubElement(root, 'dc:date', nsdict=METNSD)
        el1.text = s2
        el1 = SubElement(root, 'dc:language', nsdict=METNSD)
        el1.text = 'en-US'
        el1 = SubElement(root, 'meta:editing-cycles', nsdict=METNSD)
        el1.text = '1'
        el1 = SubElement(root, 'meta:editing-duration', nsdict=METNSD)
        el1.text = 'PT00M01S'
        title = self.visitor.get_title()
        el1 = SubElement(root, 'dc:title', nsdict=METNSD)
        if title:
            el1.text = title
        else:
            el1.text = '[no title]'
        for prop, value in self.visitor.get_meta_dict().items():
            # 'keywords', 'description', and 'subject' have their own fields:
            if prop == 'keywords':
                keywords = re.split(', *', value)
                for keyword in keywords:
                    el1 = SubElement(root, 'meta:keyword', nsdict=METNSD)
                    el1.text = keyword
            elif prop == 'description':
                el1 = SubElement(root, 'dc:description', nsdict=METNSD)
                el1.text = value
            elif prop == 'subject':
                el1 = SubElement(root, 'dc:subject', nsdict=METNSD)
                el1.text = value
            else:  # Store remaining properties as custom/user-defined
                el1 = SubElement(root, 'meta:user-defined',
                                 attrib={'meta:name': prop}, nsdict=METNSD)
                el1.text = value
        s1 = ToString(doc)
        # doc = minidom.parseString(s1)
        # s1 = doc.toprettyxml('  ')
        return s1


# class ODFTranslator(nodes.SparseNodeVisitor):
class ODFTranslator(nodes.GenericNodeVisitor):

    used_styles = (
        'attribution', 'blockindent', 'blockquote', 'blockquote-bulletitem',
        'blockquote-bulletlist', 'blockquote-enumitem', 'blockquote-enumlist',
        'bulletitem', 'bulletlist',
        'caption', 'legend',
        'centeredtextbody', 'codeblock', 'codeblock-indented',
        'codeblock-classname', 'codeblock-comment', 'codeblock-functionname',
        'codeblock-keyword', 'codeblock-name', 'codeblock-number',
        'codeblock-operator', 'codeblock-string', 'emphasis', 'enumitem',
        'enumlist', 'epigraph', 'epigraph-bulletitem', 'epigraph-bulletlist',
        'epigraph-enumitem', 'epigraph-enumlist', 'footer',
        'footnote', 'citation',
        'header', 'highlights', 'highlights-bulletitem',
        'highlights-bulletlist', 'highlights-enumitem', 'highlights-enumlist',
        'horizontalline', 'inlineliteral', 'quotation', 'rubric',
        'strong', 'table-title', 'textbody', 'tocbulletlist', 'tocenumlist',
        'title',
        'subtitle',
        'heading1',
        'heading2',
        'heading3',
        'heading4',
        'heading5',
        'heading6',
        'heading7',
        'admon-attention-hdr',
        'admon-attention-body',
        'admon-caution-hdr',
        'admon-caution-body',
        'admon-danger-hdr',
        'admon-danger-body',
        'admon-error-hdr',
        'admon-error-body',
        'admon-generic-hdr',
        'admon-generic-body',
        'admon-hint-hdr',
        'admon-hint-body',
        'admon-important-hdr',
        'admon-important-body',
        'admon-note-hdr',
        'admon-note-body',
        'admon-tip-hdr',
        'admon-tip-body',
        'admon-warning-hdr',
        'admon-warning-body',
        'tableoption',
        'tableoption.%c', 'tableoption.%c%d', 'Table%d', 'Table%d.%c',
        'Table%d.%c%d',
        'lineblock1',
        'lineblock2',
        'lineblock3',
        'lineblock4',
        'lineblock5',
        'lineblock6',
        'image', 'figureframe',
    )

    def __init__(self, document):
        # nodes.SparseNodeVisitor.__init__(self, document)
        nodes.GenericNodeVisitor.__init__(self, document)
        self.settings = document.settings
        self.language_code = self.settings.language_code
        self.language = languages.get_language(
            self.language_code,
            document.reporter)
        self.format_map = {}
        if self.settings.odf_config_file:
            parser = ConfigParser()
            parser.read(self.settings.odf_config_file)
            for rststyle, format in parser.items("Formats"):
                if rststyle not in self.used_styles:
                    self.document.reporter.warning(
                        'Style "%s" is not a style used by odtwriter.' % (
                            rststyle, ))
                self.format_map[rststyle] = format
        self.section_level = 0
        self.section_count = 0
        # Create ElementTree content and styles documents.
        root = Element(
            'office:document-content',
            attrib=CONTENT_NAMESPACE_ATTRIB,
        )
        self.content_tree = etree.ElementTree(element=root)
        self.current_element = root
        SubElement(root, 'office:scripts')
        SubElement(root, 'office:font-face-decls')
        el = SubElement(root, 'office:automatic-styles')
        self.automatic_styles = el
        el = SubElement(root, 'office:body')
        el = self.generate_content_element(el)
        self.current_element = el
        self.body_text_element = el
        self.paragraph_style_stack = [self.rststyle('textbody'), ]
        self.list_style_stack = []
        self.table_count = 0
        self.column_count = ord('A') - 1
        self.trace_level = -1
        self.optiontablestyles_generated = False
        self.field_name = None
        self.field_element = None
        self.title = None
        self.image_count = 0
        self.image_style_count = 0
        self.image_dict = {}
        self.embedded_file_list = []
        self.syntaxhighlighting = 1
        self.syntaxhighlight_lexer = 'python'
        self.header_content = []
        self.footer_content = []
        self.in_header = False
        self.in_footer = False
        self.blockstyle = ''
        self.in_table_of_contents = False
        self.table_of_content_index_body = None
        self.list_level = 0
        self.def_list_level = 0
        self.footnote_ref_dict = {}
        self.footnote_list = []
        self.footnote_chars_idx = 0
        self.footnote_level = 0
        self.pending_ids = []
        self.in_paragraph = False
        self.found_doc_title = False
        self.bumped_list_level_stack = []
        self.meta_dict = {}
        self.line_block_level = 0
        self.line_indent_level = 0
        self.citation_id = None
        self.style_index = 0        # use to form unique style names
        self.str_stylesheet = ''
        self.str_stylesheetcontent = ''
        self.dom_stylesheet = None
        self.table_styles = None
        self.in_citation = False

        # Keep track of nested styling classes
        self.inline_style_count_stack = []

    def get_str_stylesheet(self):
        return self.str_stylesheet

    def retrieve_styles(self, extension):
        """Retrieve the stylesheet from either a .xml file or from
        a .odt (zip) file.  Return the content as a string.
        """
        s2 = None
        stylespath = self.settings.stylesheet
        ext = os.path.splitext(stylespath)[1]
        if ext == '.xml':
            with open(stylespath, 'r', encoding='utf-8') as stylesfile:
                s1 = stylesfile.read()
        elif ext == extension:
            zfile = zipfile.ZipFile(stylespath, 'r')
            s1 = zfile.read('styles.xml')
            s2 = zfile.read('content.xml')
            zfile.close()
        else:
            raise RuntimeError('stylesheet path (%s) must be %s or '
                               '.xml file' % (stylespath, extension))
        self.str_stylesheet = s1
        self.str_stylesheetcontent = s2
        self.dom_stylesheet = etree.fromstring(self.str_stylesheet)
        self.dom_stylesheetcontent = etree.fromstring(
            self.str_stylesheetcontent)
        self.table_styles = self.extract_table_styles(s2)

    def extract_table_styles(self, styles_str):
        root = etree.fromstring(styles_str)
        table_styles = {}
        auto_styles = root.find(
            '{%s}automatic-styles' % (CNSD['office'], ))
        for stylenode in auto_styles:
            name = stylenode.get('{%s}name' % (CNSD['style'], ))
            tablename = name.split('.')[0]
            family = stylenode.get('{%s}family' % (CNSD['style'], ))
            if name.startswith(TABLESTYLEPREFIX):
                tablestyle = table_styles.get(tablename)
                if tablestyle is None:
                    tablestyle = TableStyle()
                    table_styles[tablename] = tablestyle
                if family == 'table':
                    properties = stylenode.find(
                        '{%s}table-properties' % (CNSD['style'], ))
                    property = properties.get(
                        '{%s}%s' % (CNSD['fo'], 'background-color', ))
                    if property is not None and property != 'none':
                        tablestyle.backgroundcolor = property
                elif family == 'table-cell':
                    properties = stylenode.find(
                        '{%s}table-cell-properties' % (CNSD['style'], ))
                    if properties is not None:
                        border = self.get_property(properties)
                        if border is not None:
                            tablestyle.border = border
        return table_styles

    def get_property(self, stylenode):
        border = None
        for propertyname in TABLEPROPERTYNAMES:
            border = stylenode.get('{%s}%s' % (CNSD['fo'], propertyname, ))
            if border is not None and border != 'none':
                return border
        return border

    def add_doc_title(self):
        text = self.settings.title
        if text:
            self.title = text
            if not self.found_doc_title:
                el = Element('text:p', attrib={
                    'text:style-name': self.rststyle('title'),
                })
                el.text = text
                self.body_text_element.insert(0, el)
        el = self.find_first_text_p(self.body_text_element)
        if el is not None:
            self.attach_page_style(el)

    def find_first_text_p(self, el):
        """Search the generated doc and return the first <text:p> element.
        """
        if el.tag == 'text:p' or el.tag == 'text:h':
            return el
        else:
            for child in el:
                el1 = self.find_first_text_p(child)
                if el1 is not None:
                    return el1
            return None

    def attach_page_style(self, el):
        """Attach the default page style.

        Create an automatic-style that refers to the current style
        of this element and that refers to the default page style.
        """
        current_style = el.get('text:style-name')
        style_name = 'P1003'
        el1 = SubElement(
            self.automatic_styles, 'style:style', attrib={
                'style:name': style_name,
                'style:master-page-name': "rststyle-pagedefault",
                'style:family': "paragraph",
            }, nsdict=SNSD)
        if current_style:
            el1.set('style:parent-style-name', current_style)
        el.set('text:style-name', style_name)

    def rststyle(self, name, parameters=()):
        """
        Returns the style name to use for the given style.

        If `parameters` is given `name` must contain a matching number of
        ``%`` and is used as a format expression with `parameters` as
        the value.
        """
        name1 = name % parameters
        return self.format_map.get(name1, 'rststyle-%s' % name1)

    def generate_content_element(self, root):
        return SubElement(root, 'office:text')

    def setup_page(self):
        self.setup_paper(self.dom_stylesheet)
        if (len(self.header_content) > 0
            or len(self.footer_content) > 0
            or self.settings.custom_header
            or self.settings.custom_footer):
            self.add_header_footer(self.dom_stylesheet)
        return etree.tostring(self.dom_stylesheet)

    def get_dom_stylesheet(self):
        return self.dom_stylesheet

    def setup_paper(self, root_el):
        try:
            dimensions = subprocess.check_output(('paperconf', '-s'),
                                                 stderr=subprocess.STDOUT)
            w, h = (float(s) for s in dimensions.split())
        except (subprocess.CalledProcessError, FileNotFoundError, ValueError):
            self.document.reporter.info(
                'Cannot use `paperconf`, defaulting to Letter.')
            w, h = 612, 792     # default to Letter

        def walk(el):
            if el.tag == "{%s}page-layout-properties" % SNSD["style"] and \
                    "{%s}page-width" % SNSD["fo"] not in el.attrib:
                el.attrib["{%s}page-width" % SNSD["fo"]] = "%.3fpt" % w
                el.attrib["{%s}page-height" % SNSD["fo"]] = "%.3fpt" % h
                el.attrib["{%s}margin-left" % SNSD["fo"]] = \
                    el.attrib["{%s}margin-right" % SNSD["fo"]] = \
                    "%.3fpt" % (.1 * w)
                el.attrib["{%s}margin-top" % SNSD["fo"]] = \
                    el.attrib["{%s}margin-bottom" % SNSD["fo"]] = \
                    "%.3fpt" % (.1 * h)
            else:
                for subel in el:
                    walk(subel)
        walk(root_el)

    def add_header_footer(self, root_el):
        automatic_styles = root_el.find(
            '{%s}automatic-styles' % SNSD['office'])
        path = '{%s}master-styles' % (NAME_SPACE_1, )
        master_el = root_el.find(path)
        if master_el is None:
            return
        path = '{%s}master-page' % (SNSD['style'], )
        master_el_container = master_el.findall(path)
        master_el = None
        target_attrib = '{%s}name' % (SNSD['style'], )
        target_name = self.rststyle('pagedefault')
        for el in master_el_container:
            if el.get(target_attrib) == target_name:
                master_el = el
                break
        if master_el is None:
            return
        el1 = master_el
        if self.header_content or self.settings.custom_header:
            el2 = SubElement(
                el1, 'style:header',
                attrib=STYLES_NAMESPACE_ATTRIB,
                nsdict=STYLES_NAMESPACE_DICT,
            )
            for el in self.header_content:
                attrkey = add_ns('text:style-name', nsdict=SNSD)
                el.attrib[attrkey] = self.rststyle('header')
                el2.append(el)
            if self.settings.custom_header:
                self.create_custom_headfoot(
                    el2,
                    self.settings.custom_header, 'header', automatic_styles)
        if self.footer_content or self.settings.custom_footer:
            el2 = SubElement(
                el1, 'style:footer',
                attrib=STYLES_NAMESPACE_ATTRIB,
                nsdict=STYLES_NAMESPACE_DICT,
            )
            for el in self.footer_content:
                attrkey = add_ns('text:style-name', nsdict=SNSD)
                el.attrib[attrkey] = self.rststyle('footer')
                el2.append(el)
            if self.settings.custom_footer:
                self.create_custom_headfoot(
                    el2,
                    self.settings.custom_footer, 'footer', automatic_styles)

    code_none, code_field, code_text = list(range(3))
    field_pat = re.compile(r'%(..?)%')

    def create_custom_headfoot(
            self, parent, text, style_name, automatic_styles):
        parent = SubElement(parent, 'text:p', attrib={
            'text:style-name': self.rststyle(style_name),
        })
        current_element = None
        field_iter = self.split_field_specifiers_iter(text)
        for item in field_iter:
            if item[0] == ODFTranslator.code_field:
                if item[1] not in (
                        'p', 'P',
                        't1', 't2', 't3', 't4',
                        'd1', 'd2', 'd3', 'd4', 'd5',
                        's', 't', 'a'):
                    msg = 'bad field spec: %%%s%%' % (item[1], )
                    raise RuntimeError(msg)
                el1 = self.make_field_element(
                    parent,
                    item[1], style_name, automatic_styles)
                if el1 is None:
                    msg = 'bad field spec: %%%s%%' % (item[1], )
                    raise RuntimeError(msg)
                else:
                    current_element = el1
            else:
                if current_element is None:
                    parent.text = item[1]
                else:
                    current_element.tail = item[1]

    def make_field_element(self, parent, text, style_name, automatic_styles):
        if text == 'p':
            el1 = SubElement(parent, 'text:page-number', attrib={
                # 'text:style-name': self.rststyle(style_name),
                'text:select-page': 'current',
            })
        elif text == 'P':
            el1 = SubElement(parent, 'text:page-count', attrib={
                # 'text:style-name': self.rststyle(style_name),
            })
        elif text == 't1':
            self.style_index += 1
            el1 = SubElement(parent, 'text:time', attrib={
                'text:style-name': self.rststyle(style_name),
                'text:fixed': 'true',
                'style:data-style-name':
                    'rst-time-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:time-style', attrib={
                'style:name': 'rst-time-style-%d' % self.style_index,
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:hours', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ':'
            el3 = SubElement(el2, 'number:minutes', attrib={
                'number:style': 'long',
            })
        elif text == 't2':
            self.style_index += 1
            el1 = SubElement(parent, 'text:time', attrib={
                'text:style-name': self.rststyle(style_name),
                'text:fixed': 'true',
                'style:data-style-name':
                    'rst-time-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:time-style', attrib={
                'style:name': 'rst-time-style-%d' % self.style_index,
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:hours', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ':'
            el3 = SubElement(el2, 'number:minutes', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ':'
            el3 = SubElement(el2, 'number:seconds', attrib={
                'number:style': 'long',
            })
        elif text == 't3':
            self.style_index += 1
            el1 = SubElement(parent, 'text:time', attrib={
                'text:style-name': self.rststyle(style_name),
                'text:fixed': 'true',
                'style:data-style-name':
                    'rst-time-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:time-style', attrib={
                'style:name': 'rst-time-style-%d' % self.style_index,
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:hours', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ':'
            el3 = SubElement(el2, 'number:minutes', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ' '
            el3 = SubElement(el2, 'number:am-pm')
        elif text == 't4':
            self.style_index += 1
            el1 = SubElement(parent, 'text:time', attrib={
                'text:style-name': self.rststyle(style_name),
                'text:fixed': 'true',
                'style:data-style-name':
                    'rst-time-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:time-style', attrib={
                'style:name': 'rst-time-style-%d' % self.style_index,
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:hours', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ':'
            el3 = SubElement(el2, 'number:minutes', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ':'
            el3 = SubElement(el2, 'number:seconds', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ' '
            el3 = SubElement(el2, 'number:am-pm')
        elif text == 'd1':
            self.style_index += 1
            el1 = SubElement(parent, 'text:date', attrib={
                'text:style-name': self.rststyle(style_name),
                'style:data-style-name':
                    'rst-date-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:date-style', attrib={
                'style:name': 'rst-date-style-%d' % self.style_index,
                'number:automatic-order': 'true',
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:month', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = '/'
            el3 = SubElement(el2, 'number:day', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = '/'
            el3 = SubElement(el2, 'number:year')
        elif text == 'd2':
            self.style_index += 1
            el1 = SubElement(parent, 'text:date', attrib={
                'text:style-name': self.rststyle(style_name),
                'style:data-style-name':
                    'rst-date-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:date-style', attrib={
                'style:name': 'rst-date-style-%d' % self.style_index,
                'number:automatic-order': 'true',
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:month', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = '/'
            el3 = SubElement(el2, 'number:day', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = '/'
            el3 = SubElement(el2, 'number:year', attrib={
                'number:style': 'long',
            })
        elif text == 'd3':
            self.style_index += 1
            el1 = SubElement(parent, 'text:date', attrib={
                'text:style-name': self.rststyle(style_name),
                'style:data-style-name':
                    'rst-date-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:date-style', attrib={
                'style:name': 'rst-date-style-%d' % self.style_index,
                'number:automatic-order': 'true',
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:month', attrib={
                'number:textual': 'true',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ' '
            el3 = SubElement(el2, 'number:day', attrib={})
            el3 = SubElement(el2, 'number:text')
            el3.text = ', '
            el3 = SubElement(el2, 'number:year', attrib={
                'number:style': 'long',
            })
        elif text == 'd4':
            self.style_index += 1
            el1 = SubElement(parent, 'text:date', attrib={
                'text:style-name': self.rststyle(style_name),
                'style:data-style-name':
                    'rst-date-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:date-style', attrib={
                'style:name': 'rst-date-style-%d' % self.style_index,
                'number:automatic-order': 'true',
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:month', attrib={
                'number:textual': 'true',
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = ' '
            el3 = SubElement(el2, 'number:day', attrib={})
            el3 = SubElement(el2, 'number:text')
            el3.text = ', '
            el3 = SubElement(el2, 'number:year', attrib={
                'number:style': 'long',
            })
        elif text == 'd5':
            self.style_index += 1
            el1 = SubElement(parent, 'text:date', attrib={
                'text:style-name': self.rststyle(style_name),
                'style:data-style-name':
                    'rst-date-style-%d' % self.style_index,
            })
            el2 = SubElement(automatic_styles, 'number:date-style', attrib={
                'style:name': 'rst-date-style-%d' % self.style_index,
                'xmlns:number': SNSD['number'],
                'xmlns:style': SNSD['style'],
            })
            el3 = SubElement(el2, 'number:year', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = '-'
            el3 = SubElement(el2, 'number:month', attrib={
                'number:style': 'long',
            })
            el3 = SubElement(el2, 'number:text')
            el3.text = '-'
            el3 = SubElement(el2, 'number:day', attrib={
                'number:style': 'long',
            })
        elif text == 's':
            el1 = SubElement(parent, 'text:subject', attrib={
                'text:style-name': self.rststyle(style_name),
            })
        elif text == 't':
            el1 = SubElement(parent, 'text:title', attrib={
                'text:style-name': self.rststyle(style_name),
            })
        elif text == 'a':
            el1 = SubElement(parent, 'text:author-name', attrib={
                'text:fixed': 'false',
            })
        else:
            el1 = None
        return el1

    def split_field_specifiers_iter(self, text):
        pos1 = 0
        while True:
            mo = ODFTranslator.field_pat.search(text, pos1)
            if mo:
                pos2 = mo.start()
                if pos2 > pos1:
                    yield ODFTranslator.code_text, text[pos1:pos2]
                yield ODFTranslator.code_field, mo.group(1)
                pos1 = mo.end()
            else:
                break
        trailing = text[pos1:]
        if trailing:
            yield ODFTranslator.code_text, trailing

    def astext(self):
        root = self.content_tree.getroot()
        et = etree.ElementTree(root)
        return ToString(et)

    def content_astext(self):
        return self.astext()

    def set_title(self, title):
        self.title = title

    def get_title(self):
        return self.title

    def set_embedded_file_list(self, embedded_file_list):
        self.embedded_file_list = embedded_file_list

    def get_embedded_file_list(self):
        return self.embedded_file_list

    def get_meta_dict(self):
        return self.meta_dict

    def process_footnotes(self):
        for node, el1 in self.footnote_list:
            backrefs = node.attributes.get('backrefs', [])
            first = True
            for ref in backrefs:
                el2 = self.footnote_ref_dict.get(ref)
                if el2 is not None:
                    if first:
                        first = False
                        el3 = copy.deepcopy(el1)
                        el2.append(el3)
                    else:
                        if len(el2) > 0:   # and 'id' in el2.attrib:
                            child = el2[0]
                            ref1 = child.text
                            attribkey = add_ns('text:id', nsdict=SNSD)
                            id1 = el2.get(attribkey, 'footnote-error')
                            if id1 is None:
                                id1 = ''
                            tag = add_ns('text:note-ref', nsdict=SNSD)
                            el2.tag = tag
                            if self.settings.endnotes_end_doc:
                                note_class = 'endnote'
                            else:
                                note_class = 'footnote'
                            el2.attrib.clear()
                            attribkey = add_ns('text:note-class', nsdict=SNSD)
                            el2.attrib[attribkey] = note_class
                            attribkey = add_ns('text:ref-name', nsdict=SNSD)
                            el2.attrib[attribkey] = id1
                            attribkey = add_ns(
                                'text:reference-format', nsdict=SNSD)
                            el2.attrib[attribkey] = 'page'
                            el2.text = ref1

    #
    # Utility methods

    def append_child(self, tag, attrib=None, parent=None):
        if parent is None:
            parent = self.current_element
        return SubElement(parent, tag, attrib)

    def append_p(self, style, text=None):
        result = self.append_child('text:p', attrib={
            'text:style-name': self.rststyle(style)})
        self.append_pending_ids(result)
        if text is not None:
            result.text = text
        return result

    def append_pending_ids(self, el):
        if self.settings.create_links:
            for id in self.pending_ids:
                SubElement(el, 'text:reference-mark', attrib={
                    'text:name': id})
        self.pending_ids = []

    def set_current_element(self, el):
        self.current_element = el

    def set_to_parent(self):
        self.current_element = self.current_element.getparent()

    def generate_labeled_block(self, node, label):
        label = '%s:' % (self.language.labels[label], )
        el = self.append_p('textbody')
        el1 = SubElement(
            el, 'text:span',
            attrib={'text:style-name': self.rststyle('strong')})
        el1.text = label
        return self.append_p('blockindent')

    def generate_labeled_line(self, node, label):
        label = '%s:' % (self.language.labels[label], )
        el = self.append_p('textbody')
        el1 = SubElement(
            el, 'text:span',
            attrib={'text:style-name': self.rststyle('strong')})
        el1.text = label
        el1.tail = node.astext()
        return el

    def encode(self, text):
        return text.replace('\n', " ")

    #
    # Visitor functions
    #
    # In alphabetic order, more or less.
    #   See docutils.docutils.nodes.node_class_names.
    #

    def dispatch_visit(self, node):
        """Override to catch basic attributes which many nodes have."""
        self.handle_basic_atts(node)
        nodes.GenericNodeVisitor.dispatch_visit(self, node)

    def handle_basic_atts(self, node):
        if isinstance(node, nodes.Element) and node['ids']:
            self.pending_ids += node['ids']

    def default_visit(self, node):
        self.document.reporter.warning('missing visit_%s' % (node.tagname, ))

    def default_departure(self, node):
        self.document.reporter.warning('missing depart_%s' % (node.tagname, ))

    def visit_Text(self, node):
        # Skip nodes whose text has been processed in parent nodes.
        if isinstance(node.parent, docutils.nodes.literal_block):
            return
        text = node.astext()
        # Are we in mixed content?  If so, add the text to the
        #   etree tail of the previous sibling element.
        if len(self.current_element) > 0:
            if self.current_element[-1].tail:
                self.current_element[-1].tail += text
            else:
                self.current_element[-1].tail = text
        else:
            if self.current_element.text:
                self.current_element.text += text
            else:
                self.current_element.text = text

    def depart_Text(self, node):
        pass

    #
    # Pre-defined fields
    #

    def visit_address(self, node):
        el = self.generate_labeled_block(node, 'address')
        self.set_current_element(el)

    def depart_address(self, node):
        self.set_to_parent()

    def visit_author(self, node):
        if isinstance(node.parent, nodes.authors):
            el = self.append_p('blockindent')
        else:
            el = self.generate_labeled_block(node, 'author')
        self.set_current_element(el)

    def depart_author(self, node):
        self.set_to_parent()

    def visit_authors(self, node):
        label = '%s:' % (self.language.labels['authors'], )
        el = self.append_p('textbody')
        el1 = SubElement(
            el, 'text:span',
            attrib={'text:style-name': self.rststyle('strong')})
        el1.text = label

    def depart_authors(self, node):
        pass

    def visit_contact(self, node):
        el = self.generate_labeled_block(node, 'contact')
        self.set_current_element(el)

    def depart_contact(self, node):
        self.set_to_parent()

    def visit_copyright(self, node):
        el = self.generate_labeled_block(node, 'copyright')
        self.set_current_element(el)

    def depart_copyright(self, node):
        self.set_to_parent()

    def visit_date(self, node):
        self.generate_labeled_line(node, 'date')

    def depart_date(self, node):
        pass

    def visit_organization(self, node):
        el = self.generate_labeled_block(node, 'organization')
        self.set_current_element(el)

    def depart_organization(self, node):
        self.set_to_parent()

    def visit_status(self, node):
        el = self.generate_labeled_block(node, 'status')
        self.set_current_element(el)

    def depart_status(self, node):
        self.set_to_parent()

    def visit_revision(self, node):
        self.generate_labeled_line(node, 'revision')

    def depart_revision(self, node):
        pass

    def visit_version(self, node):
        self.generate_labeled_line(node, 'version')
        # self.set_current_element(el)

    def depart_version(self, node):
        # self.set_to_parent()
        pass

    def visit_attribution(self, node):
        self.append_p('attribution', node.astext())

    def depart_attribution(self, node):
        pass

    def visit_block_quote(self, node):
        if 'epigraph' in node.attributes['classes']:
            self.paragraph_style_stack.append(self.rststyle('epigraph'))
            self.blockstyle = self.rststyle('epigraph')
        elif 'highlights' in node.attributes['classes']:
            self.paragraph_style_stack.append(self.rststyle('highlights'))
            self.blockstyle = self.rststyle('highlights')
        else:
            self.paragraph_style_stack.append(self.rststyle('blockquote'))
            self.blockstyle = self.rststyle('blockquote')
        self.line_indent_level += 1

    def depart_block_quote(self, node):
        self.paragraph_style_stack.pop()
        self.blockstyle = ''
        self.line_indent_level -= 1

    def visit_bullet_list(self, node):
        self.list_level += 1
        if self.in_table_of_contents:
            if self.settings.generate_oowriter_toc:
                pass
            else:
                if 'classes' in node and \
                        'auto-toc' in node.attributes['classes']:
                    el = SubElement(self.current_element, 'text:list', attrib={
                        'text:style-name': self.rststyle('tocenumlist'),
                    })
                    self.list_style_stack.append(self.rststyle('enumitem'))
                else:
                    el = SubElement(self.current_element, 'text:list', attrib={
                        'text:style-name': self.rststyle('tocbulletlist'),
                    })
                    self.list_style_stack.append(self.rststyle('bulletitem'))
                self.set_current_element(el)
        else:
            if self.blockstyle == self.rststyle('blockquote'):
                el = SubElement(self.current_element, 'text:list', attrib={
                    'text:style-name': self.rststyle('blockquote-bulletlist'),
                })
                self.list_style_stack.append(
                    self.rststyle('blockquote-bulletitem'))
            elif self.blockstyle == self.rststyle('highlights'):
                el = SubElement(self.current_element, 'text:list', attrib={
                    'text:style-name': self.rststyle('highlights-bulletlist'),
                })
                self.list_style_stack.append(
                    self.rststyle('highlights-bulletitem'))
            elif self.blockstyle == self.rststyle('epigraph'):
                el = SubElement(self.current_element, 'text:list', attrib={
                    'text:style-name': self.rststyle('epigraph-bulletlist'),
                })
                self.list_style_stack.append(
                    self.rststyle('epigraph-bulletitem'))
            else:
                el = SubElement(self.current_element, 'text:list', attrib={
                    'text:style-name': self.rststyle('bulletlist'),
                })
                self.list_style_stack.append(self.rststyle('bulletitem'))
            self.set_current_element(el)

    def depart_bullet_list(self, node):
        if self.in_table_of_contents:
            if self.settings.generate_oowriter_toc:
                pass
            else:
                self.set_to_parent()
                self.list_style_stack.pop()
        else:
            self.set_to_parent()
            self.list_style_stack.pop()
        self.list_level -= 1

    def visit_caption(self, node):
        raise nodes.SkipChildren()

    def depart_caption(self, node):
        pass

    def visit_comment(self, node):
        el = self.append_p('textbody')
        el1 = SubElement(el, 'office:annotation', attrib={})
        el2 = SubElement(el1, 'dc:creator', attrib={})
        s1 = os.environ.get('USER', '')
        el2.text = s1
        el2 = SubElement(el1, 'text:p', attrib={})
        el2.text = node.astext()

    def depart_comment(self, node):
        pass

    def visit_compound(self, node):
        # The compound directive currently receives no special treatment.
        pass

    def depart_compound(self, node):
        pass

    def visit_container(self, node):
        styles = node.attributes.get('classes', ())
        if len(styles) > 0:
            self.paragraph_style_stack.append(self.rststyle(styles[0]))

    def depart_container(self, node):
        styles = node.attributes.get('classes', ())
        if len(styles) > 0:
            self.paragraph_style_stack.pop()

    def visit_decoration(self, node):
        pass

    def depart_decoration(self, node):
        pass

    def visit_definition_list(self, node):
        self.def_list_level += 1
        if self.list_level > 5:
            raise RuntimeError(
                'max definition list nesting level exceeded')

    def depart_definition_list(self, node):
        self.def_list_level -= 1

    def visit_definition_list_item(self, node):
        pass

    def depart_definition_list_item(self, node):
        pass

    def visit_term(self, node):
        el = self.append_p('deflist-term-%d' % self.def_list_level)
        el.text = node.astext()
        self.set_current_element(el)
        raise nodes.SkipChildren()

    def depart_term(self, node):
        self.set_to_parent()

    def visit_definition(self, node):
        self.paragraph_style_stack.append(
            self.rststyle('deflist-def-%d' % self.def_list_level))
        self.bumped_list_level_stack.append(ListLevel(1))

    def depart_definition(self, node):
        self.paragraph_style_stack.pop()
        self.bumped_list_level_stack.pop()

    def visit_classifier(self, node):
        if len(self.current_element) > 0:
            el = self.current_element[-1]
            el1 = SubElement(
                el, 'text:span',
                attrib={'text:style-name': self.rststyle('emphasis')})
            el1.text = ' (%s)' % (node.astext(), )

    def depart_classifier(self, node):
        pass

    def visit_document(self, node):
        pass

    def depart_document(self, node):
        self.process_footnotes()

    def visit_docinfo(self, node):
        self.section_level += 1
        self.section_count += 1
        if self.settings.create_sections:
            el = self.append_child(
                'text:section', attrib={
                    'text:name': 'Section%d' % self.section_count,
                    'text:style-name': 'Sect%d' % self.section_level,
                }
            )
            self.set_current_element(el)

    def depart_docinfo(self, node):
        self.section_level -= 1
        if self.settings.create_sections:
            self.set_to_parent()

    def visit_emphasis(self, node):
        el = SubElement(
            self.current_element, 'text:span',
            attrib={'text:style-name': self.rststyle('emphasis')})
        self.set_current_element(el)

    def depart_emphasis(self, node):
        self.set_to_parent()

    def visit_enumerated_list(self, node):
        el1 = self.current_element
        if self.blockstyle == self.rststyle('blockquote'):
            el2 = SubElement(el1, 'text:list', attrib={
                'text:style-name': self.rststyle('blockquote-enumlist'),
            })
            self.list_style_stack.append(self.rststyle('blockquote-enumitem'))
        elif self.blockstyle == self.rststyle('highlights'):
            el2 = SubElement(el1, 'text:list', attrib={
                'text:style-name': self.rststyle('highlights-enumlist'),
            })
            self.list_style_stack.append(self.rststyle('highlights-enumitem'))
        elif self.blockstyle == self.rststyle('epigraph'):
            el2 = SubElement(el1, 'text:list', attrib={
                'text:style-name': self.rststyle('epigraph-enumlist'),
            })
            self.list_style_stack.append(self.rststyle('epigraph-enumitem'))
        else:
            liststylename = 'enumlist-%s' % (node.get('enumtype', 'arabic'), )
            el2 = SubElement(el1, 'text:list', attrib={
                'text:style-name': self.rststyle(liststylename),
            })
            self.list_style_stack.append(self.rststyle('enumitem'))
        self.set_current_element(el2)

    def depart_enumerated_list(self, node):
        self.set_to_parent()
        self.list_style_stack.pop()

    def visit_list_item(self, node):
        # If we are in a "bumped" list level, then wrap this
        #   list in an outer lists in order to increase the
        #   indentation level.
        if self.in_table_of_contents:
            if self.settings.generate_oowriter_toc:
                self.paragraph_style_stack.append(
                    self.rststyle('contents-%d' % (self.list_level, )))
            else:
                el1 = self.append_child('text:list-item')
                self.set_current_element(el1)
        else:
            el1 = self.append_child('text:list-item')
            el3 = el1
            if len(self.bumped_list_level_stack) > 0:
                level_obj = self.bumped_list_level_stack[-1]
                if level_obj.get_sibling():
                    level_obj.set_nested(False)
                    for level_obj1 in self.bumped_list_level_stack:
                        for idx in range(level_obj1.get_level()):
                            el2 = self.append_child('text:list', parent=el3)
                            el3 = self.append_child(
                                'text:list-item', parent=el2)
            self.paragraph_style_stack.append(self.list_style_stack[-1])
            self.set_current_element(el3)

    def depart_list_item(self, node):
        if self.in_table_of_contents:
            if self.settings.generate_oowriter_toc:
                self.paragraph_style_stack.pop()
            else:
                self.set_to_parent()
        else:
            if len(self.bumped_list_level_stack) > 0:
                level_obj = self.bumped_list_level_stack[-1]
                if level_obj.get_sibling():
                    level_obj.set_nested(True)
                    for level_obj1 in self.bumped_list_level_stack:
                        for idx in range(level_obj1.get_level()):
                            self.set_to_parent()
                            self.set_to_parent()
            self.paragraph_style_stack.pop()
            self.set_to_parent()

    def visit_header(self, node):
        self.in_header = True

    def depart_header(self, node):
        self.in_header = False

    def visit_footer(self, node):
        self.in_footer = True

    def depart_footer(self, node):
        self.in_footer = False

    def visit_field(self, node):
        pass

    def depart_field(self, node):
        pass

    def visit_field_list(self, node):
        pass

    def depart_field_list(self, node):
        pass

    def visit_field_name(self, node):
        el = self.append_p('textbody')
        el1 = SubElement(
            el, 'text:span',
            attrib={'text:style-name': self.rststyle('strong')})
        el1.text = node.astext()

    def depart_field_name(self, node):
        pass

    def visit_field_body(self, node):
        self.paragraph_style_stack.append(self.rststyle('blockindent'))

    def depart_field_body(self, node):
        self.paragraph_style_stack.pop()

    def visit_figure(self, node):
        pass

    def depart_figure(self, node):
        pass

    def visit_footnote(self, node):
        self.footnote_level += 1
        self.save_footnote_current = self.current_element
        el1 = Element('text:note-body')
        self.current_element = el1
        self.footnote_list.append((node, el1))
        if isinstance(node, docutils.nodes.citation):
            self.paragraph_style_stack.append(self.rststyle('citation'))
        else:
            self.paragraph_style_stack.append(self.rststyle('footnote'))

    def depart_footnote(self, node):
        self.paragraph_style_stack.pop()
        self.current_element = self.save_footnote_current
        self.footnote_level -= 1

    footnote_chars = [
        '*', '**', '***',
        '++', '+++',
        '##', '###',
        '@@', '@@@',
    ]

    def visit_footnote_reference(self, node):
        if self.footnote_level <= 0:
            id = node.attributes['ids'][0]
            refid = node.attributes.get('refid')
            if refid is None:
                refid = ''
            if self.settings.endnotes_end_doc:
                note_class = 'endnote'
            else:
                note_class = 'footnote'
            el1 = self.append_child('text:note', attrib={
                'text:id': '%s' % (refid, ),
                'text:note-class': note_class,
            })
            note_auto = str(node.attributes.get('auto', 1))
            if isinstance(node, docutils.nodes.citation_reference):
                citation = '[%s]' % node.astext()
                el2 = SubElement(el1, 'text:note-citation', attrib={
                    'text:label': citation,
                })
                el2.text = citation
            elif note_auto == '1':
                el2 = SubElement(el1, 'text:note-citation', attrib={
                    'text:label': node.astext(),
                })
                el2.text = node.astext()
            elif note_auto == '*':
                if self.footnote_chars_idx >= len(
                        ODFTranslator.footnote_chars):
                    self.footnote_chars_idx = 0
                footnote_char = ODFTranslator.footnote_chars[
                    self.footnote_chars_idx]
                self.footnote_chars_idx += 1
                el2 = SubElement(el1, 'text:note-citation', attrib={
                    'text:label': footnote_char,
                })
                el2.text = footnote_char
            self.footnote_ref_dict[id] = el1
        raise nodes.SkipChildren()

    def depart_footnote_reference(self, node):
        pass

    def visit_citation(self, node):
        self.in_citation = True
        for id in node.attributes['ids']:
            self.citation_id = id
            break
        self.paragraph_style_stack.append(self.rststyle('blockindent'))
        self.bumped_list_level_stack.append(ListLevel(1))

    def depart_citation(self, node):
        self.citation_id = None
        self.paragraph_style_stack.pop()
        self.bumped_list_level_stack.pop()
        self.in_citation = False

    def visit_citation_reference(self, node):
        if self.settings.create_links:
            id = node.attributes['refid']
            el = self.append_child('text:reference-ref', attrib={
                'text:ref-name': '%s' % (id, ),
                'text:reference-format': 'text',
            })
            el.text = '['
            self.set_current_element(el)
        elif self.current_element.text is None:
            self.current_element.text = '['
        else:
            self.current_element.text += '['

    def depart_citation_reference(self, node):
        self.current_element.text += ']'
        if self.settings.create_links:
            self.set_to_parent()

    def visit_label(self, node):
        if isinstance(node.parent, docutils.nodes.footnote):
            raise nodes.SkipChildren()
        elif self.citation_id is not None:
            el = self.append_p('textbody')
            self.set_current_element(el)
            if self.settings.create_links:
                el0 = SubElement(el, 'text:span')
                el0.text = '['
                self.append_child('text:reference-mark-start', attrib={
                    'text:name': '%s' % (self.citation_id, ),
                })
            else:
                el.text = '['

    def depart_label(self, node):
        if isinstance(node.parent, docutils.nodes.footnote):
            pass
        elif self.citation_id is not None:
            if self.settings.create_links:
                self.append_child('text:reference-mark-end', attrib={
                    'text:name': '%s' % (self.citation_id, ),
                })
                el0 = SubElement(self.current_element, 'text:span')
                el0.text = ']'
            else:
                self.current_element.text += ']'
            self.set_to_parent()

    def visit_generated(self, node):
        pass

    def depart_generated(self, node):
        pass

    def check_file_exists(self, path):
        if os.path.exists(path):
            return 1
        else:
            return 0

    def visit_image(self, node):
        # Capture the image file.
        if 'uri' in node.attributes:
            source = node.attributes['uri']
            if not (source.startswith('http:') or source.startswith('https:')):
                if not source.startswith(os.sep):
                    docsource, line = utils.get_source_line(node)
                    if docsource:
                        dirname = os.path.dirname(docsource)
                        if dirname:
                            source = '%s%s%s' % (dirname, os.sep, source, )
                if not self.check_file_exists(source):
                    self.document.reporter.warning(
                        'Cannot find image file %s.' % (source, ))
                    return
        else:
            return
        if source in self.image_dict:
            filename, destination = self.image_dict[source]
        else:
            self.image_count += 1
            filename = os.path.split(source)[1]
            destination = 'Pictures/1%08x%s' % (self.image_count, filename, )
            if source.startswith('http:') or source.startswith('https:'):
                try:
                    imgfile = urlopen(source)
                    content = imgfile.read()
                    imgfile.close()
                    imgfile2 = tempfile.NamedTemporaryFile('wb', delete=False)
                    imgfile2.write(content)
                    imgfile2.close()
                    imgfilename = imgfile2.name
                    source = imgfilename
                except HTTPError:
                    self.document.reporter.warning(
                        "Can't open image url %s." % (source, ))
                spec = (source, destination,)
            else:
                spec = (os.path.abspath(source), destination,)
            self.embedded_file_list.append(spec)
            self.image_dict[source] = (source, destination,)
        # Is this a figure (containing an image) or just a plain image?
        if self.in_paragraph:
            el1 = self.current_element
        else:
            el1 = SubElement(
                self.current_element, 'text:p',
                attrib={'text:style-name': self.rststyle('textbody')})
        el2 = el1
        if isinstance(node.parent, docutils.nodes.figure):
            el3, el4, el5, caption = self.generate_figure(
                node, source,
                destination, el2)
            attrib = {}
            el6, width = self.generate_image(
                node, source, destination,
                el5, attrib)
            if caption is not None:
                el6.tail = caption
        else:   # if isinstance(node.parent, docutils.nodes.image):
            self.generate_image(node, source, destination, el2)

    def depart_image(self, node):
        pass

    def get_image_width_height(self, node, attr):
        size = None
        unit = None
        if attr in node.attributes:
            size = node.attributes[attr]
            size = size.strip()
            # For conversion factors, see:
            # http://www.unitconversion.org/unit_converter/typography-ex.html
            try:
                if size.endswith('%'):
                    if attr == 'height':
                        # Percentage allowed for width but not height.
                        raise ValueError('percentage not allowed for height')
                    size = size.rstrip(' %')
                    size = float(size) / 100.0
                    unit = '%'
                else:
                    size, unit = self.convert_to_cm(size)
            except ValueError as exp:
                self.document.reporter.warning(
                    'Invalid %s for image: "%s".  '
                    'Error: "%s".' % (
                        attr, node.attributes[attr], exp))
        return size, unit

    def convert_to_cm(self, size):
        """Convert various units to centimeters.

        Note that a call to this method should be wrapped in:
            try: except ValueError:
        """
        size = size.strip()
        if size.endswith('px'):
            size = float(size[:-2]) * 0.026     # convert px to cm
        elif size.endswith('in'):
            size = float(size[:-2]) * 2.54      # convert in to cm
        elif size.endswith('pt'):
            size = float(size[:-2]) * 0.035     # convert pt to cm
        elif size.endswith('pc'):
            size = float(size[:-2]) * 2.371     # convert pc to cm
        elif size.endswith('mm'):
            size = float(size[:-2]) * 0.1       # convert mm to cm
        elif size.endswith('cm'):
            size = float(size[:-2])
        else:
            raise ValueError('unknown unit type')
        unit = 'cm'
        return size, unit

    def get_image_scale(self, node):
        if 'scale' in node.attributes:
            scale = node.attributes['scale']
            try:
                scale = int(scale)
            except ValueError:
                self.document.reporter.warning(
                    'Invalid scale for image: "%s"' % (
                        node.attributes['scale'], ))
            if scale < 1:       # or scale > 100:
                self.document.reporter.warning(
                    'scale out of range (%s), using 1.' % (scale, ))
                scale = 1
            scale = scale * 0.01
        else:
            scale = 1.0
        return scale

    def get_image_scaled_width_height(self, node, source):
        """Return the image size in centimeters adjusted by image attrs."""
        scale = self.get_image_scale(node)
        width, width_unit = self.get_image_width_height(node, 'width')
        height, _ = self.get_image_width_height(node, 'height')
        dpi = (72, 72)
        if PIL is not None and source in self.image_dict:
            filename, destination = self.image_dict[source]
            with PIL.Image.open(filename, 'r') as img:
                img_size = img.size
                dpi = img.info.get('dpi', dpi)
            # dpi information can be (xdpi, ydpi) or xydpi
            try:
                iter(dpi)
            except TypeError:
                dpi = (dpi, dpi)
        else:
            img_size = None
        if width is None or height is None:
            if img_size is None:
                raise RuntimeError(
                    'image size not fully specified and PIL not installed')
            if width is None:
                width = img_size[0]
                width = float(width) * 0.026        # convert px to cm
            if height is None:
                height = img_size[1]
                height = float(height) * 0.026      # convert px to cm
            if width_unit == '%':
                factor = width
                image_width = img_size[0]
                image_width = float(image_width) * 0.026    # convert px to cm
                image_height = img_size[1]
                image_height = float(image_height) * 0.026  # convert px to cm
                line_width = self.get_page_width()
                width = factor * line_width
                factor = (factor * line_width) / image_width
                height = factor * image_height
        width *= scale
        height *= scale
        width = '%.2fcm' % width
        height = '%.2fcm' % height
        return width, height

    def get_page_width(self):
        """Return the document's page width in centimeters."""
        root = self.get_dom_stylesheet()
        nodes = root.iterfind(
            './/{urn:oasis:names:tc:opendocument:xmlns:style:1.0}'
            'page-layout/'
            '{urn:oasis:names:tc:opendocument:xmlns:style:1.0}'
            'page-layout-properties')
        width = None
        for node in nodes:
            page_width = node.get(
                '{urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0}'
                'page-width')
            margin_left = node.get(
                '{urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0}'
                'margin-left')
            margin_right = node.get(
                '{urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0}'
                'margin-right')
            if (page_width is None
                or margin_left is None
                or margin_right is None):
                continue
            try:
                page_width, _ = self.convert_to_cm(page_width)
                margin_left, _ = self.convert_to_cm(margin_left)
                margin_right, _ = self.convert_to_cm(margin_right)
            except ValueError:
                self.document.reporter.warning(
                    'Stylesheet file contains invalid page width '
                    'or margin size.')
            width = page_width - margin_left - margin_right
        if width is None:
            # We can't find the width in styles, so we make a guess.
            # Use a width of 6 in = 15.24 cm.
            width = 15.24
        return width

    def generate_figure(self, node, source, destination, current_element):
        caption = None
        width, height = self.get_image_scaled_width_height(node, source)
        for node1 in node.parent.children:
            if node1.tagname == 'caption':
                caption = node1.astext()
        self.image_style_count += 1
        #
        # Add the style for the caption.
        if caption is not None:
            attrib = {
                'style:class': 'extra',
                'style:family': 'paragraph',
                'style:name': 'Caption',
                'style:parent-style-name': 'Standard',
            }
            el1 = SubElement(self.automatic_styles, 'style:style',
                             attrib=attrib, nsdict=SNSD)
            attrib = {
                'fo:margin-bottom': '0.0835in',
                'fo:margin-top': '0.0835in',
                'text:line-number': '0',
                'text:number-lines': 'false',
            }
            SubElement(el1, 'style:paragraph-properties',
                       attrib=attrib, nsdict=SNSD)
            attrib = {
                'fo:font-size': '12pt',
                'fo:font-style': 'italic',
                'style:font-name': 'Times',
                'style:font-name-complex': 'Lucidasans1',
                'style:font-size-asian': '12pt',
                'style:font-size-complex': '12pt',
                'style:font-style-asian': 'italic',
                'style:font-style-complex': 'italic',
            }
            SubElement(el1, 'style:text-properties',
                       attrib=attrib, nsdict=SNSD)
        style_name = 'rstframestyle%d' % self.image_style_count
        draw_name = 'graphics%d' % next(IMAGE_NAME_COUNTER)
        # Add the styles
        attrib = {
            'style:name': style_name,
            'style:family': 'graphic',
            'style:parent-style-name': self.rststyle('figureframe'),
        }
        el1 = SubElement(self.automatic_styles,
                         'style:style', attrib=attrib, nsdict=SNSD)
        attrib = {}
        wrap = False
        classes = node.parent.attributes.get('classes')
        if classes and 'wrap' in classes:
            wrap = True
        if wrap:
            attrib['style:wrap'] = 'dynamic'
        else:
            attrib['style:wrap'] = 'none'
        SubElement(el1, 'style:graphic-properties',
                   attrib=attrib, nsdict=SNSD)
        attrib = {
            'draw:style-name': style_name,
            'draw:name': draw_name,
            'text:anchor-type': 'paragraph',
            'draw:z-index': '0',
        }
        attrib['svg:width'] = width
        el3 = SubElement(current_element, 'draw:frame', attrib=attrib)
        attrib = {}
        el4 = SubElement(el3, 'draw:text-box', attrib=attrib)
        attrib = {
            'text:style-name': self.rststyle('caption'),
        }
        el5 = SubElement(el4, 'text:p', attrib=attrib)
        return el3, el4, el5, caption

    def generate_image(self, node, source, destination, current_element,
                       frame_attrs=None):
        width, height = self.get_image_scaled_width_height(node, source)
        self.image_style_count += 1
        style_name = 'rstframestyle%d' % self.image_style_count
        # Add the style.
        attrib = {
            'style:name': style_name,
            'style:family': 'graphic',
            'style:parent-style-name': self.rststyle('image'),
        }
        el1 = SubElement(self.automatic_styles,
                         'style:style', attrib=attrib, nsdict=SNSD)
        halign = None
        valign = None
        if 'align' in node.attributes:
            align = node.attributes['align'].split()
            for val in align:
                if val in ('left', 'center', 'right'):
                    halign = val
                elif val in ('top', 'middle', 'bottom'):
                    valign = val
        if frame_attrs is None:
            attrib = {
                'style:vertical-pos': 'top',
                'style:vertical-rel': 'paragraph',
                'style:horizontal-rel': 'paragraph',
                'style:mirror': 'none',
                'fo:clip': 'rect(0cm 0cm 0cm 0cm)',
                'draw:luminance': '0%',
                'draw:contrast': '0%',
                'draw:red': '0%',
                'draw:green': '0%',
                'draw:blue': '0%',
                'draw:gamma': '100%',
                'draw:color-inversion': 'false',
                'draw:image-opacity': '100%',
                'draw:color-mode': 'standard',
            }
        else:
            attrib = frame_attrs
        if halign is not None:
            attrib['style:horizontal-pos'] = halign
        if valign is not None:
            attrib['style:vertical-pos'] = valign
        # If there is a classes/wrap directive or we are
        #   inside a table, add a no-wrap style.
        wrap = False
        classes = node.attributes.get('classes')
        if classes and 'wrap' in classes:
            wrap = True
        if wrap:
            attrib['style:wrap'] = 'dynamic'
        else:
            attrib['style:wrap'] = 'none'
        # If we are inside a table, add a no-wrap style.
        if self.is_in_table(node):
            attrib['style:wrap'] = 'none'
        SubElement(el1, 'style:graphic-properties',
                   attrib=attrib, nsdict=SNSD)
        draw_name = 'graphics%d' % next(IMAGE_NAME_COUNTER)
        # Add the content.
        # el = SubElement(current_element, 'text:p',
        #     attrib={'text:style-name': self.rststyle('textbody')})
        attrib = {
            'draw:style-name': style_name,
            'draw:name': draw_name,
            'draw:z-index': '1',
        }
        if isinstance(node.parent, nodes.TextElement):
            attrib['text:anchor-type'] = 'as-char'      # vds
        else:
            attrib['text:anchor-type'] = 'paragraph'
        attrib['svg:width'] = width
        attrib['svg:height'] = height
        el1 = SubElement(current_element, 'draw:frame', attrib=attrib)
        SubElement(el1, 'draw:image', attrib={
            'xlink:href': '%s' % (destination, ),
            'xlink:type': 'simple',
            'xlink:show': 'embed',
            'xlink:actuate': 'onLoad',
        })
        return el1, width

    def is_in_table(self, node):
        node1 = node.parent
        while node1:
            if isinstance(node1, docutils.nodes.entry):
                return True
            node1 = node1.parent
        return False

    def visit_legend(self, node):
        if isinstance(node.parent, docutils.nodes.figure):
            el1 = self.current_element[-1]
            el1 = el1[0][0]
            self.current_element = el1
            self.paragraph_style_stack.append(self.rststyle('legend'))

    def depart_legend(self, node):
        if isinstance(node.parent, docutils.nodes.figure):
            self.paragraph_style_stack.pop()
            self.set_to_parent()
            self.set_to_parent()
            self.set_to_parent()

    def visit_line_block(self, node):
        self.line_indent_level += 1
        self.line_block_level += 1

    def depart_line_block(self, node):
        self.line_indent_level -= 1
        self.line_block_level -= 1

    def visit_line(self, node):
        style = 'lineblock%d' % self.line_indent_level
        el1 = SubElement(self.current_element, 'text:p',
                         attrib={'text:style-name': self.rststyle(style), })
        self.current_element = el1

    def depart_line(self, node):
        self.set_to_parent()

    def visit_literal(self, node):
        el = SubElement(
            self.current_element, 'text:span',
            attrib={'text:style-name': self.rststyle('inlineliteral')})
        self.set_current_element(el)

    def depart_literal(self, node):
        self.set_to_parent()

    def visit_inline(self, node):
        styles = node.attributes.get('classes', ())
        if styles:
            el = self.current_element
            for inline_style in styles:
                el = SubElement(el, 'text:span',
                                attrib={'text:style-name':
                                        self.rststyle(inline_style)})
            count = len(styles)
        else:
            # No style was specified so use a default style (old code
            # crashed if no style was given)
            el = SubElement(self.current_element, 'text:span')
            count = 1

        self.set_current_element(el)
        self.inline_style_count_stack.append(count)

    def depart_inline(self, node):
        count = self.inline_style_count_stack.pop()
        for x in range(count):
            self.set_to_parent()

    def _calculate_code_block_padding(self, line):
        count = 0
        matchobj = SPACES_PATTERN.match(line)
        if matchobj:
            pad = matchobj.group()
            count = len(pad)
        else:
            matchobj = TABS_PATTERN.match(line)
            if matchobj:
                pad = matchobj.group()
                count = len(pad) * 8
        return count

    def _add_syntax_highlighting(self, insource, language):
        lexer = pygments.lexers.get_lexer_by_name(language, stripall=True)
        if language in ('latex', 'tex'):
            fmtr = OdtPygmentsLaTeXFormatter(
                lambda name, parameters=():
                self.rststyle(name, parameters),
                escape_function=escape_cdata)
        else:
            fmtr = OdtPygmentsProgFormatter(
                lambda name, parameters=():
                self.rststyle(name, parameters),
                escape_function=escape_cdata)
        return pygments.highlight(insource, lexer, fmtr)

    def fill_line(self, line):
        line = FILL_PAT1.sub(self.fill_func1, line)
        return FILL_PAT2.sub(self.fill_func2, line)

    def fill_func1(self, matchobj):
        spaces = matchobj.group(0)
        return '<text:s text:c="%d"/>' % (len(spaces), )

    def fill_func2(self, matchobj):
        spaces = matchobj.group(0)
        return ' <text:s text:c="%d"/>' % (len(spaces) - 1, )

    def visit_literal_block(self, node):
        if len(self.paragraph_style_stack) > 1:
            wrapper1 = '<text:p text:style-name="%s">%%s</text:p>' % (
                self.rststyle('codeblock-indented'), )
        else:
            wrapper1 = '<text:p text:style-name="%s">%%s</text:p>' % (
                self.rststyle('codeblock'), )
        source = node.astext()
        if (pygments and self.settings.add_syntax_highlighting):
            language = node.get('language', 'python')
            source = self._add_syntax_highlighting(source, language)
        else:
            source = escape_cdata(source)
        lines = source.split('\n')
        # If there is an empty last line, remove it.
        if lines[-1] == '':
            del lines[-1]
        lines1 = ['<wrappertag1 xmlns:text="urn:oasis:names:tc:'
                  'opendocument:xmlns:text:1.0">']
        my_lines = []
        for my_line in lines:
            my_line = self.fill_line(my_line)
            my_line = my_line.replace("&#10;", "\n")
            my_lines.append(my_line)
        my_lines_str = '<text:line-break/>'.join(my_lines)
        my_lines_str2 = wrapper1 % (my_lines_str, )
        lines1.append(my_lines_str2)
        lines1.append('</wrappertag1>')
        s1 = ''.join(lines1)
        s1 = s1.encode("utf-8")
        el1 = etree.fromstring(s1)
        for child in el1:
            self.current_element.append(child)

    def depart_literal_block(self, node):
        pass

    visit_doctest_block = visit_literal_block
    depart_doctest_block = depart_literal_block

    # placeholder for math (see docs/dev/todo.txt)
    def visit_math(self, node):
        self.document.reporter.warning('"math" role not supported',
                                       base_node=node)
        self.visit_literal(node)

    def depart_math(self, node):
        self.depart_literal(node)

    def visit_math_block(self, node):
        self.document.reporter.warning('"math" directive not supported',
                                       base_node=node)
        self.visit_literal_block(node)

    def depart_math_block(self, node):
        self.depart_literal_block(node)

    def visit_meta(self, node):
        name = node.attributes.get('name')
        content = node.attributes.get('content')
        if name is not None and content is not None:
            self.meta_dict[name] = content

    def depart_meta(self, node):
        pass

    def visit_option_list(self, node):
        table_name = 'tableoption'
        #
        # Generate automatic styles
        if not self.optiontablestyles_generated:
            self.optiontablestyles_generated = True
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle(table_name),
                'style:family': 'table'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-properties', attrib={
                'style:width': '17.59cm',
                'table:align': 'left',
                'style:shadow': 'none'}, nsdict=SNSD)
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle('%s.%%c' % table_name, ('A', )),
                'style:family': 'table-column'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-column-properties', attrib={
                'style:column-width': '4.999cm'}, nsdict=SNSD)
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle('%s.%%c' % table_name, ('B', )),
                'style:family': 'table-column'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-column-properties', attrib={
                'style:column-width': '12.587cm'}, nsdict=SNSD)
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle(
                    '%s.%%c%%d' % table_name, ('A', 1, )),
                'style:family': 'table-cell'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-cell-properties', attrib={
                'fo:background-color': 'transparent',
                'fo:padding': '0.097cm',
                'fo:border-left': '0.035cm solid #000000',
                'fo:border-right': 'none',
                'fo:border-top': '0.035cm solid #000000',
                'fo:border-bottom': '0.035cm solid #000000'}, nsdict=SNSD)
            el2 = SubElement(el1, 'style:background-image', nsdict=SNSD)
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle(
                    '%s.%%c%%d' % table_name, ('B', 1, )),
                'style:family': 'table-cell'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-cell-properties', attrib={
                'fo:padding': '0.097cm',
                'fo:border': '0.035cm solid #000000'}, nsdict=SNSD)
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle(
                    '%s.%%c%%d' % table_name, ('A', 2, )),
                'style:family': 'table-cell'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-cell-properties', attrib={
                'fo:padding': '0.097cm',
                'fo:border-left': '0.035cm solid #000000',
                'fo:border-right': 'none',
                'fo:border-top': 'none',
                'fo:border-bottom': '0.035cm solid #000000'}, nsdict=SNSD)
            el = SubElement(self.automatic_styles, 'style:style', attrib={
                'style:name': self.rststyle(
                    '%s.%%c%%d' % table_name, ('B', 2, )),
                'style:family': 'table-cell'}, nsdict=SNSD)
            el1 = SubElement(el, 'style:table-cell-properties', attrib={
                'fo:padding': '0.097cm',
                'fo:border-left': '0.035cm solid #000000',
                'fo:border-right': '0.035cm solid #000000',
                'fo:border-top': 'none',
                'fo:border-bottom': '0.035cm solid #000000'}, nsdict=SNSD)
        #
        # Generate table data
        el = self.append_child('table:table', attrib={
            'table:name': self.rststyle(table_name),
            'table:style-name': self.rststyle(table_name),
        })
        el1 = SubElement(el, 'table:table-column', attrib={
            'table:style-name': self.rststyle(
                '%s.%%c' % table_name, ('A', ))})
        el1 = SubElement(el, 'table:table-column', attrib={
            'table:style-name': self.rststyle(
                '%s.%%c' % table_name, ('B', ))})
        el1 = SubElement(el, 'table:table-header-rows')
        el2 = SubElement(el1, 'table:table-row')
        el3 = SubElement(el2, 'table:table-cell', attrib={
            'table:style-name': self.rststyle(
                '%s.%%c%%d' % table_name, ('A', 1, )),
            'office:value-type': 'string'})
        el4 = SubElement(el3, 'text:p', attrib={
            'text:style-name': 'Table_20_Heading'})
        el4.text = 'Option'
        el3 = SubElement(el2, 'table:table-cell', attrib={
            'table:style-name': self.rststyle(
                '%s.%%c%%d' % table_name, ('B', 1, )),
            'office:value-type': 'string'})
        el4 = SubElement(el3, 'text:p', attrib={
            'text:style-name': 'Table_20_Heading'})
        el4.text = 'Description'
        self.set_current_element(el)

    def depart_option_list(self, node):
        self.set_to_parent()

    def visit_option_list_item(self, node):
        el = self.append_child('table:table-row')
        self.set_current_element(el)

    def depart_option_list_item(self, node):
        self.set_to_parent()

    def visit_option_group(self, node):
        el = self.append_child('table:table-cell', attrib={
            'table:style-name': 'Table%d.A2' % self.table_count,
            'office:value-type': 'string',
        })
        self.set_current_element(el)

    def depart_option_group(self, node):
        self.set_to_parent()

    def visit_option(self, node):
        el = self.append_child('text:p', attrib={
            'text:style-name': 'Table_20_Contents'})
        el.text = node.astext()

    def depart_option(self, node):
        pass

    def visit_option_string(self, node):
        pass

    def depart_option_string(self, node):
        pass

    def visit_option_argument(self, node):
        pass

    def depart_option_argument(self, node):
        pass

    def visit_description(self, node):
        el = self.append_child('table:table-cell', attrib={
            'table:style-name': 'Table%d.B2' % self.table_count,
            'office:value-type': 'string',
        })
        el1 = SubElement(el, 'text:p', attrib={
            'text:style-name': 'Table_20_Contents'})
        el1.text = node.astext()
        raise nodes.SkipChildren()

    def depart_description(self, node):
        pass

    def visit_paragraph(self, node):
        self.in_paragraph = True
        if self.in_header:
            el = self.append_p('header')
        elif self.in_footer:
            el = self.append_p('footer')
        else:
            style_name = self.paragraph_style_stack[-1]
            el = self.append_child(
                'text:p',
                attrib={'text:style-name': style_name})
            self.append_pending_ids(el)
        self.set_current_element(el)

    def depart_paragraph(self, node):
        self.in_paragraph = False
        self.set_to_parent()
        if self.in_header:
            self.header_content.append(self.current_element[-1])
            self.current_element.remove(self.current_element[-1])
        elif self.in_footer:
            self.footer_content.append(self.current_element[-1])
            self.current_element.remove(self.current_element[-1])

    def visit_problematic(self, node):
        pass

    def depart_problematic(self, node):
        pass

    def visit_raw(self, node):
        if 'format' in node.attributes:
            formats = node.attributes['format']
            formatlist = formats.split()
            if 'odt' in formatlist:
                rawstr = node.astext()
                attrstr = ' '.join(
                    '%s="%s"' % (k, v, )
                    for k, v in list(CONTENT_NAMESPACE_ATTRIB.items()))
                contentstr = '<stuff %s>%s</stuff>' % (attrstr, rawstr, )
                contentstr = contentstr.encode("utf-8")
                content = etree.fromstring(contentstr)
                if len(content) > 0:
                    el1 = content[0]
                    if self.in_header:
                        pass
                    elif self.in_footer:
                        pass
                    else:
                        self.current_element.append(el1)
        raise nodes.SkipChildren()

    def depart_raw(self, node):
        if self.in_header:
            pass
        elif self.in_footer:
            pass
        else:
            pass

    def visit_reference(self, node):
        # text = node.astext()
        if self.settings.create_links:
            if 'refuri' in node:
                href = node['refuri']
                if (self.settings.cloak_email_addresses
                    and href.startswith('mailto:')):
                    href = self.cloak_mailto(href)
                el = self.append_child('text:a', attrib={
                    'xlink:href': '%s' % href,
                    'xlink:type': 'simple',
                })
                self.set_current_element(el)
            elif 'refid' in node:
                if self.settings.create_links:
                    href = node['refid']
                    el = self.append_child('text:reference-ref', attrib={
                        'text:ref-name': '%s' % href,
                        'text:reference-format': 'text',
                    })
            else:
                self.document.reporter.warning(
                    'References must have "refuri" or "refid" attribute.')
        if (self.in_table_of_contents
            and len(node.children) >= 1
            and isinstance(node.children[0], docutils.nodes.generated)):
            node.remove(node.children[0])

    def depart_reference(self, node):
        if self.settings.create_links:
            if 'refuri' in node:
                self.set_to_parent()

    def visit_rubric(self, node):
        style_name = self.rststyle('rubric')
        classes = node.get('classes')
        if classes:
            class1 = classes[0]
            if class1:
                style_name = class1
        el = SubElement(self.current_element, 'text:h', attrib={
            # 'text:outline-level': '%d' % section_level,
            # 'text:style-name': 'Heading_20_%d' % section_level,
            'text:style-name': style_name,
        })
        text = node.astext()
        el.text = self.encode(text)

    def depart_rubric(self, node):
        pass

    def visit_section(self, node, move_ids=1):
        self.section_level += 1
        self.section_count += 1
        if self.settings.create_sections:
            el = self.append_child('text:section', attrib={
                'text:name': 'Section%d' % self.section_count,
                'text:style-name': 'Sect%d' % self.section_level,
            })
            self.set_current_element(el)

    def depart_section(self, node):
        self.section_level -= 1
        if self.settings.create_sections:
            self.set_to_parent()

    def visit_strong(self, node):
        el = SubElement(self.current_element, 'text:span',
                        attrib={'text:style-name': self.rststyle('strong')})
        self.set_current_element(el)

    def depart_strong(self, node):
        self.set_to_parent()

    def visit_substitution_definition(self, node):
        raise nodes.SkipChildren()

    def depart_substitution_definition(self, node):
        pass

    def visit_system_message(self, node):
        pass

    def depart_system_message(self, node):
        pass

    def get_table_style(self, node):
        table_style = None
        table_name = None
        str_classes = node.get('classes')
        if str_classes is not None:
            for str_class in str_classes:
                if str_class.startswith(TABLESTYLEPREFIX):
                    table_name = str_class
                    break
        if table_name is not None:
            table_style = self.table_styles.get(table_name)
            if table_style is None:
                # If we can't find the table style, issue warning
                #   and use the default table style.
                self.document.reporter.warning(
                    'Can\'t find table style "%s".  Using default.' % (
                        table_name, ))
                table_name = TABLENAMEDEFAULT
                table_style = self.table_styles.get(table_name)
                if table_style is None:
                    # If we can't find the default table style, issue a warning
                    #   and use a built-in default style.
                    self.document.reporter.warning(
                        'Can\'t find default table style "%s".  '
                        'Using built-in default.' % (
                            table_name, ))
                    table_style = BUILTIN_DEFAULT_TABLE_STYLE
        else:
            table_name = TABLENAMEDEFAULT
            table_style = self.table_styles.get(table_name)
            if table_style is None:
                # If we can't find the default table style, issue a warning
                #   and use a built-in default style.
                self.document.reporter.warning(
                    'Can\'t find default table style "%s".  '
                    'Using built-in default.' % (
                        table_name, ))
                table_style = BUILTIN_DEFAULT_TABLE_STYLE
        return table_style

    def visit_table(self, node):
        self.table_count += 1
        table_style = self.get_table_style(node)
        table_name = '%s%%d' % TABLESTYLEPREFIX
        el1 = SubElement(self.automatic_styles, 'style:style', attrib={
            'style:name': self.rststyle(
                '%s' % table_name, (self.table_count, )),
            'style:family': 'table',
        }, nsdict=SNSD)
        if table_style.backgroundcolor is None:
            SubElement(el1, 'style:table-properties', attrib={
                # 'style:width': '17.59cm',
                # 'table:align': 'margins',
                'table:align': 'left',
                'fo:margin-top': '0in',
                'fo:margin-bottom': '0.10in',
            }, nsdict=SNSD)
        else:
            SubElement(el1, 'style:table-properties', attrib={
                # 'style:width': '17.59cm',
                'table:align': 'margins',
                'fo:margin-top': '0in',
                'fo:margin-bottom': '0.10in',
                'fo:background-color': table_style.backgroundcolor,
            }, nsdict=SNSD)
        # We use a single cell style for all cells in this table.
        # That's probably not correct, but seems to work.
        el2 = SubElement(self.automatic_styles, 'style:style', attrib={
            'style:name': self.rststyle(
                '%s.%%c%%d' % table_name, (self.table_count, 'A', 1, )),
            'style:family': 'table-cell',
        }, nsdict=SNSD)
        thickness = self.settings.table_border_thickness
        if thickness is None:
            line_style1 = table_style.border
        else:
            line_style1 = '0.%03dcm solid #000000' % (thickness, )
        SubElement(el2, 'style:table-cell-properties', attrib={
            'fo:padding': '0.049cm',
            'fo:border-left': line_style1,
            'fo:border-right': line_style1,
            'fo:border-top': line_style1,
            'fo:border-bottom': line_style1,
        }, nsdict=SNSD)
        title = None
        for child in node.children:
            if child.tagname == 'title':
                title = child.astext()
                break
        if title is not None:
            self.append_p('table-title', title)
        else:
            pass
        el4 = SubElement(self.current_element, 'table:table', attrib={
            'table:name': self.rststyle(
                '%s' % table_name, (self.table_count, )),
            'table:style-name': self.rststyle(
                '%s' % table_name, (self.table_count, )),
        })
        self.set_current_element(el4)
        self.current_table_style = el1
        self.table_width = 0.0

    def depart_table(self, node):
        attribkey = add_ns('style:width', nsdict=SNSD)
        attribval = '%.4fin' % (self.table_width, )
        el1 = self.current_table_style
        el2 = el1[0]
        el2.attrib[attribkey] = attribval
        self.set_to_parent()

    def visit_tgroup(self, node):
        self.column_count = ord('A') - 1

    def depart_tgroup(self, node):
        pass

    def visit_colspec(self, node):
        self.column_count += 1
        colspec_name = self.rststyle(
            '%s%%d.%%s' % TABLESTYLEPREFIX,
            (self.table_count, chr(self.column_count), )
        )
        colwidth = node['colwidth'] / 12.0
        el1 = SubElement(self.automatic_styles, 'style:style', attrib={
            'style:name': colspec_name,
            'style:family': 'table-column',
        }, nsdict=SNSD)
        SubElement(el1, 'style:table-column-properties',
                   attrib={'style:column-width': '%.4fin' % colwidth},
                   nsdict=SNSD)
        self.append_child('table:table-column',
                          attrib={'table:style-name': colspec_name, })
        self.table_width += colwidth

    def depart_colspec(self, node):
        pass

    def visit_thead(self, node):
        el = self.append_child('table:table-header-rows')
        self.set_current_element(el)
        self.in_thead = True
        self.paragraph_style_stack.append('Table_20_Heading')

    def depart_thead(self, node):
        self.set_to_parent()
        self.in_thead = False
        self.paragraph_style_stack.pop()

    def visit_row(self, node):
        self.column_count = ord('A') - 1
        el = self.append_child('table:table-row')
        self.set_current_element(el)

    def depart_row(self, node):
        self.set_to_parent()

    def visit_entry(self, node):
        self.column_count += 1
        cellspec_name = self.rststyle(
            '%s%%d.%%c%%d' % TABLESTYLEPREFIX,
            (self.table_count, 'A', 1, )
        )
        attrib = {
            'table:style-name': cellspec_name,
            'office:value-type': 'string',
        }
        morecols = node.get('morecols', 0)
        if morecols > 0:
            attrib['table:number-columns-spanned'] = '%d' % (morecols + 1,)
            self.column_count += morecols
        morerows = node.get('morerows', 0)
        if morerows > 0:
            attrib['table:number-rows-spanned'] = '%d' % (morerows + 1,)
        el1 = self.append_child('table:table-cell', attrib=attrib)
        self.set_current_element(el1)

    def depart_entry(self, node):
        self.set_to_parent()

    def visit_tbody(self, node):
        pass

    def depart_tbody(self, node):
        pass

    def visit_target(self, node):
        #
        # I don't know how to implement targets in ODF.
        # How do we create a target in oowriter?  A cross-reference?
        if ('refuri' not in node
                and 'refid' not in node
                and 'refname' not in node):
            pass
        else:
            pass

    def depart_target(self, node):
        pass

    def visit_title(self, node, move_ids=1, title_type='title'):
        if isinstance(node.parent, docutils.nodes.section):
            section_level = self.section_level
            if section_level > 7:
                self.document.reporter.warning(
                    'Heading/section levels greater than 7 not supported.')
                self.document.reporter.warning(
                    '    Reducing to heading level 7 for heading: "%s"' % (
                        node.astext(), ))
                section_level = 7
            el1 = self.append_child(
                'text:h', attrib={
                    'text:outline-level': '%d' % section_level,
                    # 'text:style-name': 'Heading_20_%d' % section_level,
                    'text:style-name': self.rststyle(
                        'heading%d', (section_level, )),
                })
            self.append_pending_ids(el1)
            self.set_current_element(el1)
        elif isinstance(node.parent, docutils.nodes.document):
            # text = self.settings.title
            # else:
            #     text = node.astext()
            el1 = SubElement(self.current_element, 'text:p', attrib={
                'text:style-name': self.rststyle(title_type),
            })
            self.append_pending_ids(el1)
            text = node.astext()
            self.title = text
            self.found_doc_title = True
            self.set_current_element(el1)

    def depart_title(self, node):
        if (isinstance(node.parent, docutils.nodes.section)
            or isinstance(node.parent, docutils.nodes.document)):
            self.set_to_parent()

    def visit_subtitle(self, node, move_ids=1):
        self.visit_title(node, move_ids, title_type='subtitle')

    def depart_subtitle(self, node):
        self.depart_title(node)

    def visit_title_reference(self, node):
        el = self.append_child('text:span', attrib={
            'text:style-name': self.rststyle('quotation')})
        el.text = self.encode(node.astext())
        raise nodes.SkipChildren()

    def depart_title_reference(self, node):
        pass

    def generate_table_of_content_entry_template(self, el1):
        for idx in range(1, 11):
            el2 = SubElement(
                el1,
                'text:table-of-content-entry-template',
                attrib={
                    'text:outline-level': "%d" % (idx, ),
                    'text:style-name': self.rststyle('contents-%d' % (idx, )),
                })
            SubElement(el2, 'text:index-entry-chapter')
            SubElement(el2, 'text:index-entry-text')
            SubElement(el2, 'text:index-entry-tab-stop', attrib={
                'style:leader-char': ".",
                'style:type': "right",
            })
            SubElement(el2, 'text:index-entry-page-number')

    def find_title_label(self, node, class_type, label_key):
        label = ''
        title_node = None
        for child in node.children:
            if isinstance(child, class_type):
                title_node = child
                break
        if title_node is not None:
            label = title_node.astext()
        else:
            label = self.language.labels[label_key]
        return label

    def visit_topic(self, node):
        if 'classes' in node.attributes:
            if 'contents' in node.attributes['classes']:
                label = self.find_title_label(
                    node, docutils.nodes.title, 'contents')
                if self.settings.generate_oowriter_toc:
                    el1 = self.append_child('text:table-of-content', attrib={
                        'text:name': 'Table of Contents1',
                        'text:protected': 'true',
                        'text:style-name': 'Sect1',
                    })
                    el2 = SubElement(
                        el1,
                        'text:table-of-content-source',
                        attrib={
                            'text:outline-level': '10',
                        })
                    el3 = SubElement(el2, 'text:index-title-template', attrib={
                        'text:style-name': 'Contents_20_Heading',
                    })
                    el3.text = label
                    self.generate_table_of_content_entry_template(el2)
                    el4 = SubElement(el1, 'text:index-body')
                    el5 = SubElement(el4, 'text:index-title')
                    el6 = SubElement(el5, 'text:p', attrib={
                        'text:style-name': self.rststyle('contents-heading'),
                    })
                    el6.text = label
                    self.save_current_element = self.current_element
                    self.table_of_content_index_body = el4
                    self.set_current_element(el4)
                else:
                    el = self.append_p('horizontalline')
                    el = self.append_p('centeredtextbody')
                    el1 = SubElement(
                        el, 'text:span',
                        attrib={'text:style-name': self.rststyle('strong')})
                    el1.text = label
                self.in_table_of_contents = True
            elif 'abstract' in node.attributes['classes']:
                el = self.append_p('horizontalline')
                el = self.append_p('centeredtextbody')
                el1 = SubElement(
                    el, 'text:span',
                    attrib={'text:style-name': self.rststyle('strong')})
                label = self.find_title_label(
                    node, docutils.nodes.title,
                    'abstract')
                el1.text = label
            elif 'dedication' in node.attributes['classes']:
                el = self.append_p('horizontalline')
                el = self.append_p('centeredtextbody')
                el1 = SubElement(
                    el, 'text:span',
                    attrib={'text:style-name': self.rststyle('strong')})
                label = self.find_title_label(
                    node, docutils.nodes.title,
                    'dedication')
                el1.text = label

    def depart_topic(self, node):
        if 'classes' in node.attributes:
            if 'contents' in node.attributes['classes']:
                if self.settings.generate_oowriter_toc:
                    self.update_toc_page_numbers(
                        self.table_of_content_index_body)
                    self.set_current_element(self.save_current_element)
                else:
                    self.append_p('horizontalline')
                self.in_table_of_contents = False

    def update_toc_page_numbers(self, el):
        collection = []
        self.update_toc_collect(el, 0, collection)
        self.update_toc_add_numbers(collection)

    def update_toc_collect(self, el, level, collection):
        collection.append((level, el))
        level += 1
        for child_el in el:
            if child_el.tag != 'text:index-body':
                self.update_toc_collect(child_el, level, collection)

    def update_toc_add_numbers(self, collection):
        for level, el1 in collection:
            if (el1.tag == 'text:p'
                and el1.text != 'Table of Contents'):
                el2 = SubElement(el1, 'text:tab')
                el2.tail = '9999'

    def visit_transition(self, node):
        self.append_p('horizontalline')

    def depart_transition(self, node):
        pass

    #
    # Admonitions
    #
    def visit_warning(self, node):
        self.generate_admonition(node, 'warning')

    def depart_warning(self, node):
        self.paragraph_style_stack.pop()

    def visit_attention(self, node):
        self.generate_admonition(node, 'attention')

    depart_attention = depart_warning

    def visit_caution(self, node):
        self.generate_admonition(node, 'caution')

    depart_caution = depart_warning

    def visit_danger(self, node):
        self.generate_admonition(node, 'danger')

    depart_danger = depart_warning

    def visit_error(self, node):
        self.generate_admonition(node, 'error')

    depart_error = depart_warning

    def visit_hint(self, node):
        self.generate_admonition(node, 'hint')

    depart_hint = depart_warning

    def visit_important(self, node):
        self.generate_admonition(node, 'important')

    depart_important = depart_warning

    def visit_note(self, node):
        self.generate_admonition(node, 'note')

    depart_note = depart_warning

    def visit_tip(self, node):
        self.generate_admonition(node, 'tip')

    depart_tip = depart_warning

    def visit_admonition(self, node):
        title = None
        for child in node.children:
            if child.tagname == 'title':
                title = child.astext()
        if title is None:
            classes1 = node.get('classes')
            if classes1:
                title = classes1[0]
        self.generate_admonition(node, 'generic', title)

    depart_admonition = depart_warning

    def generate_admonition(self, node, label, title=None):
        if hasattr(self.language, 'labels'):
            translated_label = self.language.labels.get(label, label)
        else:
            translated_label = label
        el1 = SubElement(self.current_element, 'text:p', attrib={
            'text:style-name': self.rststyle(
                'admon-%s-hdr', (label, )),
        })
        if title:
            el1.text = title
        else:
            el1.text = '%s!' % (translated_label.capitalize(), )
        s1 = self.rststyle('admon-%s-body', (label, ))
        self.paragraph_style_stack.append(s1)

    #
    # Roles (e.g. subscript, superscript, strong, ...
    #
    def visit_subscript(self, node):
        el = self.append_child('text:span', attrib={
            'text:style-name': 'rststyle-subscript',
        })
        self.set_current_element(el)

    def depart_subscript(self, node):
        self.set_to_parent()

    def visit_superscript(self, node):
        el = self.append_child('text:span', attrib={
            'text:style-name': 'rststyle-superscript',
        })
        self.set_current_element(el)

    def depart_superscript(self, node):
        self.set_to_parent()

    def visit_abbreviation(self, node):
        pass

    def depart_abbreviation(self, node):
        pass

    def visit_acronym(self, node):
        pass

    def depart_acronym(self, node):
        pass

    def visit_sidebar(self, node):
        pass

    def depart_sidebar(self, node):
        pass


# Use an own reader to modify transformations done.
class Reader(standalone.Reader):

    def get_transforms(self):
        transforms = super().get_transforms()
        if not self.settings.create_links:
            transforms.remove(references.DanglingReferences)
        return transforms

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/odf_odt/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3

# $Id: prepstyles.py 9386 2023-05-16 14:49:31Z milde $
# Author: Dave Kuhlman <dkuhlman@rexx.com>
# Copyright: This module has been placed in the public domain.

"""
Adapt a word-processor-generated styles.odt for odtwriter use:

Drop page size specifications from styles.xml in STYLE_FILE.odt.
See https://docutils.sourceforge.io/docs/user/odt.html#page-size
"""

# Author: Michael Schutte <michi@uiae.at>

from xml.etree import ElementTree as etree

import sys
import zipfile
from tempfile import mkstemp
import shutil
import os

NAMESPACES = {
    "style": "urn:oasis:names:tc:opendocument:xmlns:style:1.0",
    "fo": "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"
}


def prepstyle(filename):

    zin = zipfile.ZipFile(filename)
    styles = zin.open("styles.xml")

    root = None
    # some extra effort to preserve namespace prefixes
    for event, elem in etree.iterparse(styles, events=("start", "start-ns")):
        if event == "start-ns":
            etree.register_namespace(elem[0], elem[1])
        elif event == "start":
            if root is None:
                root = elem

    styles.close()

    for el in root.findall(".//style:page-layout-properties",
                           namespaces=NAMESPACES):
        for attr in list(el.attrib):
            if attr.startswith("{%s}" % NAMESPACES["fo"]):
                del el.attrib[attr]

    tempname = mkstemp()
    zout = zipfile.ZipFile(os.fdopen(tempname[0], "wb"), "w",
                           zipfile.ZIP_DEFLATED)

    for item in zin.infolist():
        if item.filename == "styles.xml":
            zout.writestr(item, etree.tostring(root, encoding="UTF-8"))
        else:
            zout.writestr(item, zin.read(item.filename))

    zout.close()
    zin.close()
    shutil.move(tempname[1], filename)


def main():
    args = sys.argv[1:]
    if len(args) != 1 or args[0] in ('-h', '--help'):
        print(__doc__, file=sys.stderr)
        print("Usage: %s STYLE_FILE.odt\n" % sys.argv[0], file=sys.stderr)
        sys.exit(1)
    filename = args[0]
    prepstyle(filename)


if __name__ == '__main__':
    main()

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/odf_odt/prepstyles.py
NikolaBlagojevic/pyrecodes
# $Id: __init__.py 9240 2022-11-13 16:15:23Z milde $
# Authors: Chris Liechti <cliechti@gmx.net>;
#          David Goodger <goodger@python.org>
# Copyright: This module has been placed in the public domain.

"""
S5/HTML Slideshow Writer.
"""

__docformat__ = 'reStructuredText'


import sys
import os
import re
import docutils
from docutils import frontend, nodes, utils
from docutils.writers import html4css1

themes_dir_path = utils.relative_path(
    os.path.join(os.getcwd(), 'dummy'),
    os.path.join(os.path.dirname(__file__), 'themes'))


def find_theme(name):
    # Where else to look for a theme?
    # Check working dir?  Destination dir?  Config dir?  Plugins dir?
    path = os.path.join(themes_dir_path, name)
    if not os.path.isdir(path):
        raise docutils.ApplicationError(
            'Theme directory not found: %r (path: %r)' % (name, path))
    return path


class Writer(html4css1.Writer):

    settings_spec = html4css1.Writer.settings_spec + (
        'S5 Slideshow Specific Options',
        'For the S5/HTML writer, the --no-toc-backlinks option '
        '(defined in General Docutils Options above) is the default, '
        'and should not be changed.',
        (('Specify an installed S5 theme by name.  Overrides --theme-url.  '
          'The default theme name is "default".  The theme files will be '
          'copied into a "ui/<theme>" directory, in the same directory as the '
          'destination file (output HTML).  Note that existing theme files '
          'will not be overwritten (unless --overwrite-theme-files is used).',
          ['--theme'],
          {'default': 'default', 'metavar': '<name>',
           'overrides': 'theme_url'}),
         ('Specify an S5 theme URL.  The destination file (output HTML) will '
          'link to this theme; nothing will be copied.  Overrides --theme.',
          ['--theme-url'],
          {'metavar': '<URL>', 'overrides': 'theme'}),
         ('Allow existing theme files in the ``ui/<theme>`` directory to be '
          'overwritten.  The default is not to overwrite theme files.',
          ['--overwrite-theme-files'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Keep existing theme files in the ``ui/<theme>`` directory; do not '
          'overwrite any.  This is the default.',
          ['--keep-theme-files'],
          {'dest': 'overwrite_theme_files', 'action': 'store_false'}),
         ('Set the initial view mode to "slideshow" [default] or "outline".',
          ['--view-mode'],
          {'choices': ['slideshow', 'outline'], 'default': 'slideshow',
           'metavar': '<mode>'}),
         ('Normally hide the presentation controls in slideshow mode. '
          'This is the default.',
          ['--hidden-controls'],
          {'action': 'store_true', 'default': True,
           'validator': frontend.validate_boolean}),
         ('Always show the presentation controls in slideshow mode.  '
          'The default is to hide the controls.',
          ['--visible-controls'],
          {'dest': 'hidden_controls', 'action': 'store_false'}),
         ('Enable the current slide indicator ("1 / 15").  '
          'The default is to disable it.',
          ['--current-slide'],
          {'action': 'store_true', 'validator': frontend.validate_boolean}),
         ('Disable the current slide indicator.  This is the default.',
          ['--no-current-slide'],
          {'dest': 'current_slide', 'action': 'store_false'}),))

    settings_default_overrides = {'toc_backlinks': 0}

    config_section = 's5_html writer'
    config_section_dependencies = ('writers', 'html writers',
                                   'html4css1 writer')

    def __init__(self):
        html4css1.Writer.__init__(self)
        self.translator_class = S5HTMLTranslator


class S5HTMLTranslator(html4css1.HTMLTranslator):

    s5_stylesheet_template = """\
<!-- configuration parameters -->
<meta name="defaultView" content="%(view_mode)s" />
<meta name="controlVis" content="%(control_visibility)s" />
<!-- style sheet links -->
<script src="%(path)s/slides.js" type="text/javascript"></script>
<link rel="stylesheet" href="%(path)s/slides.css"
      type="text/css" media="projection" id="slideProj" />
<link rel="stylesheet" href="%(path)s/outline.css"
      type="text/css" media="screen" id="outlineStyle" />
<link rel="stylesheet" href="%(path)s/print.css"
      type="text/css" media="print" id="slidePrint" />
<link rel="stylesheet" href="%(path)s/opera.css"
      type="text/css" media="projection" id="operaFix" />\n"""
    # The script element must go in front of the link elements to
    # avoid a flash of unstyled content (FOUC), reproducible with
    # Firefox.

    disable_current_slide = """
<style type="text/css">
#currentSlide {display: none;}
</style>\n"""

    layout_template = """\
<div class="layout">
<div id="controls"></div>
<div id="currentSlide"></div>
<div id="header">
%(header)s
</div>
<div id="footer">
%(title)s%(footer)s
</div>
</div>\n"""
# <div class="topleft"></div>
# <div class="topright"></div>
# <div class="bottomleft"></div>
# <div class="bottomright"></div>

    default_theme = 'default'
    """Name of the default theme."""

    base_theme_file = '__base__'
    """Name of the file containing the name of the base theme."""

    direct_theme_files = (
        'slides.css', 'outline.css', 'print.css', 'opera.css', 'slides.js')
    """Names of theme files directly linked to in the output HTML"""

    indirect_theme_files = (
        's5-core.css', 'framing.css', 'pretty.css')
    """Names of files used indirectly; imported or used by files in
    `direct_theme_files`."""

    required_theme_files = indirect_theme_files + direct_theme_files
    """Names of mandatory theme files."""

    def __init__(self, *args):
        html4css1.HTMLTranslator.__init__(self, *args)
        # insert S5-specific stylesheet and script stuff:
        self.theme_file_path = None
        self.setup_theme()
        view_mode = self.document.settings.view_mode
        control_visibility = ('visible', 'hidden')[self.document.settings
                                                   .hidden_controls]
        self.stylesheet.append(self.s5_stylesheet_template
                               % {'path': self.theme_file_path,
                                  'view_mode': view_mode,
                                  'control_visibility': control_visibility})
        if not self.document.settings.current_slide:
            self.stylesheet.append(self.disable_current_slide)
        self.meta.append('<meta name="version" content="S5 1.1" />\n')
        self.s5_footer = []
        self.s5_header = []
        self.section_count = 0
        self.theme_files_copied = None

    def setup_theme(self):
        if self.document.settings.theme:
            self.copy_theme()
        elif self.document.settings.theme_url:
            self.theme_file_path = self.document.settings.theme_url
        else:
            raise docutils.ApplicationError(
                'No theme specified for S5/HTML writer.')

    def copy_theme(self):
        """
        Locate & copy theme files.

        A theme may be explicitly based on another theme via a '__base__'
        file.  The default base theme is 'default'.  Files are accumulated
        from the specified theme, any base themes, and 'default'.
        """
        settings = self.document.settings
        path = find_theme(settings.theme)
        theme_paths = [path]
        self.theme_files_copied = {}
        required_files_copied = {}
        # This is a link (URL) in HTML, so we use "/", not os.sep:
        self.theme_file_path = '%s/%s' % ('ui', settings.theme)
        if settings._destination:
            dest = os.path.join(
                os.path.dirname(settings._destination), 'ui', settings.theme)
            if not os.path.isdir(dest):
                os.makedirs(dest)
        else:
            # no destination, so we can't copy the theme
            return
        default = False
        while path:
            for f in os.listdir(path):  # copy all files from each theme
                if f == self.base_theme_file:
                    continue            # ... except the "__base__" file
                if (self.copy_file(f, path, dest)
                    and f in self.required_theme_files):
                    required_files_copied[f] = 1
            if default:
                break                   # "default" theme has no base theme
            # Find the "__base__" file in theme directory:
            base_theme_file = os.path.join(path, self.base_theme_file)
            # If it exists, read it and record the theme path:
            if os.path.isfile(base_theme_file):
                with open(base_theme_file, encoding='utf-8') as f:
                    lines = f.readlines()
                for line in lines:
                    line = line.strip()
                    if line and not line.startswith('#'):
                        path = find_theme(line)
                        if path in theme_paths:  # check for duplicates/cycles
                            path = None          # if found, use default base
                        else:
                            theme_paths.append(path)
                        break
                else:                   # no theme name found
                    path = None         # use default base
            else:                       # no base theme file found
                path = None             # use default base
            if not path:
                path = find_theme(self.default_theme)
                theme_paths.append(path)
                default = True
        if len(required_files_copied) != len(self.required_theme_files):
            # Some required files weren't found & couldn't be copied.
            required = list(self.required_theme_files)
            for f in required_files_copied.keys():
                required.remove(f)
            raise docutils.ApplicationError(
                'Theme files not found: %s'
                % ', '.join('%r' % f for f in required))

    files_to_skip_pattern = re.compile(r'~$|\.bak$|#$|\.cvsignore$')

    def copy_file(self, name, source_dir, dest_dir):
        """
        Copy file `name` from `source_dir` to `dest_dir`.
        Return 1 if the file exists in either `source_dir` or `dest_dir`.
        """
        source = os.path.join(source_dir, name)
        dest = os.path.join(dest_dir, name)
        if dest in self.theme_files_copied:
            return 1
        else:
            self.theme_files_copied[dest] = 1
        if os.path.isfile(source):
            if self.files_to_skip_pattern.search(source):
                return None
            settings = self.document.settings
            if os.path.exists(dest) and not settings.overwrite_theme_files:
                settings.record_dependencies.add(dest)
            else:
                with open(source, 'rb') as src_file:
                    src_data = src_file.read()
                with open(dest, 'wb') as dest_file:
                    dest_dir = dest_dir.replace(os.sep, '/')
                    dest_file.write(src_data.replace(
                        b'ui/default',
                        dest_dir[dest_dir.rfind('ui/'):].encode(
                            sys.getfilesystemencoding())))
                settings.record_dependencies.add(source)
            return 1
        if os.path.isfile(dest):
            return 1

    def depart_document(self, node):
        self.head_prefix.extend([self.doctype,
                                 self.head_prefix_template %
                                 {'lang': self.settings.language_code}])
        self.html_prolog.append(self.doctype)
        self.head = self.meta[:] + self.head
        if self.math_header:
            if self.math_output == 'mathjax':
                self.head.extend(self.math_header)
            else:
                self.stylesheet.extend(self.math_header)
        # skip content-type meta tag with interpolated charset value:
        self.html_head.extend(self.head[1:])
        self.fragment.extend(self.body)
        # special S5 code up to the next comment line
        header = ''.join(self.s5_header)
        footer = ''.join(self.s5_footer)
        title = ''.join(self.html_title).replace('<h1 class="title">', '<h1>')
        layout = self.layout_template % {'header': header,
                                         'title': title,
                                         'footer': footer}
        self.body_prefix.extend(layout)
        self.body_prefix.append('<div class="presentation">\n')
        self.body_prefix.append(
            self.starttag({'classes': ['slide'], 'ids': ['slide0']}, 'div'))
        if not self.section_count:
            self.body.append('</div>\n')
        #
        self.body_suffix.insert(0, '</div>\n')
        self.html_body.extend(self.body_prefix[1:] + self.body_pre_docinfo
                              + self.docinfo + self.body
                              + self.body_suffix[:-1])

    def depart_footer(self, node):
        start = self.context.pop()
        self.s5_footer.append('<h2>')
        self.s5_footer.extend(self.body[start:])
        self.s5_footer.append('</h2>')
        del self.body[start:]

    def depart_header(self, node):
        start = self.context.pop()
        header = ['<div id="header">\n']
        header.extend(self.body[start:])
        header.append('\n</div>\n')
        del self.body[start:]
        self.s5_header.extend(header)

    def visit_section(self, node):
        if not self.section_count:
            self.body.append('\n</div>\n')
        self.section_count += 1
        self.section_level += 1
        if self.section_level > 1:
            # dummy for matching div's
            self.body.append(self.starttag(node, 'div', CLASS='section'))
        else:
            self.body.append(self.starttag(node, 'div', CLASS='slide'))

    def visit_subtitle(self, node):
        if isinstance(node.parent, nodes.section):
            level = self.section_level + self.initial_header_level - 1
            if level == 1:
                level = 2
            tag = 'h%s' % level
            self.body.append(self.starttag(node, tag, ''))
            self.context.append('</%s>\n' % tag)
        else:
            html4css1.HTMLTranslator.visit_subtitle(self, node)

    def visit_title(self, node):
        html4css1.HTMLTranslator.visit_title(self, node)

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/s5_html/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
# :Author: Günter Milde <milde@users.sf.net>
# :Revision: $Revision: 9293 $
# :Date: $Date: 2022-12-01 22:13:54 +0100 (Do, 01. Dez 2022) $
# :Copyright: © 2010 Günter Milde.
# :License: Released under the terms of the `2-Clause BSD license`_, in short:
#
#    Copying and distribution of this file, with or without modification,
#    are permitted in any medium without royalty provided the copyright
#    notice and this notice are preserved.
#    This file is offered as-is, without any warranty.
#
# .. _2-Clause BSD license: https://opensource.org/licenses/BSD-2-Clause

"""
XeLaTeX document tree Writer.

A variant of Docutils' standard 'latex2e' writer producing LaTeX output
suited for processing with the Unicode-aware TeX engines
LuaTeX and XeTeX.
"""

__docformat__ = 'reStructuredText'

from docutils import frontend
from docutils.writers import latex2e


class Writer(latex2e.Writer):
    """A writer for Unicode-aware LaTeX variants (XeTeX, LuaTeX)"""

    supported = ('latex', 'tex', 'xetex', 'xelatex', 'luatex', 'lualatex')
    """Formats this writer supports."""

    default_template = 'xelatex.tex'
    default_preamble = """\
% Linux Libertine (free, wide coverage, not only for Linux)
\\setmainfont{Linux Libertine O}
\\setsansfont{Linux Biolinum O}
\\setmonofont[HyphenChar=None,Scale=MatchLowercase]{DejaVu Sans Mono}"""

    config_section = 'xetex writer'
    config_section_dependencies = ('writers', 'latex writers')

    # use a copy of the parent spec with some modifications:
    settings_spec = frontend.filter_settings_spec(
        latex2e.Writer.settings_spec,
        # removed settings
        'font_encoding',
        # changed settings:
        template=('Template file. Default: "%s".' % default_template,
                  ['--template'],
                  {'default': default_template, 'metavar': '<file>'}),
        latex_preamble=('Customization by LaTeX code in the preamble. '
                        'Default: select "Linux Libertine" fonts.',
                        ['--latex-preamble'],
                        {'default': default_preamble}),
        )

    def __init__(self):
        latex2e.Writer.__init__(self)
        self.settings_defaults.update({'fontencoding': ''})  # use default (TU)
        self.translator_class = XeLaTeXTranslator


class Babel(latex2e.Babel):
    """Language specifics for XeTeX.

    Use `polyglossia` instead of `babel` and adapt settings.
    """
    language_codes = latex2e.Babel.language_codes.copy()
    # Additionally supported or differently named languages:
    language_codes.update({
        # code        Polyglossia-name  comment
        'cop':        'coptic',
        'de':         'german',         # new spelling (de_1996)
        'de-1901':    'ogerman',        # old spelling
        'dv':         'divehi',         # Maldivian
        'dsb':        'lsorbian',
        'el-polyton': 'polygreek',
        'fa':         'farsi',
        'grc':        'ancientgreek',
        'ko':         'korean',
        'hsb':        'usorbian',
        'sh-Cyrl':    'serbian',        # Serbo-Croatian, Cyrillic script
        'sh-Latn':    'croatian',       # Serbo-Croatian, Latin script
        'sq':         'albanian',
        'sr':         'serbian',        # Cyrillic script (sr-Cyrl)
        'th':         'thai',
        'vi':         'vietnamese',
        # zh-Latn:    ???               # Chinese Pinyin
        })
    # normalize (downcase) keys
    language_codes = {k.lower(): v for k, v in language_codes.items()}

    # Languages without Polyglossia support:
    for key in ('af',           # 'afrikaans',
                'de-AT',        # 'naustrian',
                'de-AT-1901',   # 'austrian',
                # TODO: use variant=... for English variants
                'en-CA',        # 'canadian',
                'en-GB',        # 'british',
                'en-NZ',        # 'newzealand',
                'en-US',        # 'american',
                'fr-CA',        # 'canadien',
                'grc-ibycus',   # 'ibycus', (Greek Ibycus encoding)
                'sr-Latn',      # 'serbian script=latin'
                ):
        del language_codes[key.lower()]

    def __init__(self, language_code, reporter):
        self.language_code = language_code
        self.reporter = reporter
        self.language = self.language_name(language_code)
        self.otherlanguages = {}
        self.warn_msg = 'Language "%s" not supported by Polyglossia.'
        self.quote_index = 0
        self.quotes = ('"', '"')
        # language dependent configuration:
        # double quotes are "active" in some languages (e.g. German).
        self.literal_double_quote = '"'  # TODO: use \textquotedbl ?

    def __call__(self):
        setup = [r'\usepackage{polyglossia}',
                 r'\setdefaultlanguage{%s}' % self.language]
        if self.otherlanguages:
            setup.append(r'\setotherlanguages{%s}' %
                         ','.join(sorted(self.otherlanguages.keys())))
        return '\n'.join(setup)


class XeLaTeXTranslator(latex2e.LaTeXTranslator):
    """
    Generate code for LaTeX using Unicode fonts (XeLaTex or LuaLaTeX).

    See the docstring of docutils.writers._html_base.HTMLTranslator for
    notes on and examples of safe subclassing.
    """

    def __init__(self, document):
        self.is_xetex = True  # typeset with XeTeX or LuaTeX engine
        latex2e.LaTeXTranslator.__init__(self, document, Babel)
        if self.latex_encoding == 'utf8':
            self.requirements.pop('_inputenc', None)
        else:
            self.requirements['_inputenc'] = (r'\XeTeXinputencoding %s '
                                              % self.latex_encoding)

env_pyrecodes/lib/python3.9/site-packages/docutils/writers/xetex/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  __init__.py
"""
List of common pagesizes and some tools for working with them.

This module defines a few common page sizes in points (1/72 inch).
"""
#
#  Copyright © 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Based on reportlab.lib.pagesizes and reportlab.lib.units
#    www.reportlab.co.uk
#    Copyright ReportLab Europe Ltd. 2000-2017
#    Copyright (c) 2000-2018, ReportLab Inc.
#    All rights reserved.
#    Licensed under the BSD License
#
#  Includes data from en.wikipedia.org.
#  Licensed under the Creative Commons Attribution-ShareAlike License
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# this package
from .classes import *
from .sizes import *
from .units import *
from .utils import *

env_pyrecodes/lib/python3.9/site-packages/domdf_python_tools/pagesizes/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python
#
#  stringlist.py
"""
A list of strings that represent lines in a multiline string.

.. versionchanged:: 1.0.0

	:class:`~domdf_python_tools.typing.String` should now be imported from :mod:`domdf_python_tools.typing`.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from contextlib import contextmanager
from itertools import chain
from typing import Any, Iterable, Iterator, List, Reversible, Tuple, TypeVar, Union, cast, overload

# this package
from domdf_python_tools.doctools import prettify_docstrings
from domdf_python_tools.typing import String, SupportsIndex
from domdf_python_tools.utils import convert_indents

__all__ = ["Indent", "StringList", "DelimitedList", "_SL", "splitlines", "joinlines"]

_S = TypeVar("_S")
_SL = TypeVar("_SL", bound="StringList")


@prettify_docstrings
class Indent:
	"""
	Represents an indent, having a symbol/type and a size.

	:param size: The indent size.
	:param type: The indent character.
	"""

	def __init__(self, size: int = 0, type: str = '\t'):  # noqa: A002  # pylint: disable=redefined-builtin
		self.size = int(size)
		self.type = str(type)

	def __iter__(self) -> Iterator[Union[str, Any]]:
		"""
		Returns the size and type of the :class:`~domdf_python_tools.stringlist.Indent`.
		"""

		yield self.size
		yield self.type

	@property
	def size(self) -> int:
		"""
		The indent size.
		"""

		return self._size

	@size.setter
	def size(self, size: int) -> None:
		self._size = int(size)

	@property  # noqa: A003  # pylint: disable=redefined-builtin
	def type(self) -> str:
		"""
		The indent character.
		"""

		return self._type

	@type.setter  # noqa: A003  # pylint: disable=redefined-builtin
	def type(self, type: str) -> None:  # noqa: A002  # pylint: disable=redefined-builtin
		if not str(type):
			raise ValueError("'type' cannot an empty string.")

		self._type = str(type)

	def __str__(self) -> str:
		"""
		Returns the :class:`~domdf_python_tools.stringlist.Indent` as a string.
		"""

		return self.type * self.size

	def __repr__(self) -> str:
		"""
		Returns the string representation of the :class:`~domdf_python_tools.stringlist.Indent`.
		"""

		return f"{type(self).__name__}(size={self.size}, type={self.type!r})"

	def __eq__(self, other):
		if isinstance(other, Indent):
			return other.size == self.size and other.type == self.type
		elif isinstance(other, str):
			return str(self) == other
		elif isinstance(other, tuple):
			return tuple(self) == other
		else:
			return NotImplemented


class StringList(List[str]):
	"""
	A list of strings that represent lines in a multiline string.

	:param iterable: Content to populate the StringList with.
	:param convert_indents: Whether indents at the start of lines should be converted.
	"""

	#: The indent to insert at the beginning of new lines.
	indent: Indent

	convert_indents: bool
	"""
	Whether indents at the start of lines should be converted.

	Only applies to lines added after this is enabled/disabled.

	Can only be used when the indent is ``'\\t'`` or ``'␣'``.
	"""

	def __init__(
			self,
			iterable: Iterable[String] = (),
			convert_indents: bool = False,
			) -> None:

		if isinstance(iterable, str):
			iterable = iterable.split('\n')

		self.indent = Indent()
		self.convert_indents = convert_indents
		super().__init__([self._make_line(str(x)) for x in iterable])

	def _make_line(self, line: str) -> str:
		if not str(self.indent_type).strip(" \t") and self.convert_indents:
			if self.indent_type == '\t':
				line = convert_indents(line, tab_width=1, from_="    ", to='\t')
			else:  # pragma: no cover
				line = convert_indents(line, tab_width=1, from_='\t', to=self.indent_type)

		return f"{self.indent}{line}".rstrip()

	def append(self, line: String) -> None:
		"""
		Append a line to the end of the :class:`~domdf_python_tools.stringlist.StringList`.

		:param line:
		"""

		for inner_line in str(line).split('\n'):
			super().append(self._make_line(inner_line))

	def extend(self, iterable: Iterable[String]) -> None:
		"""
		Extend the :class:`~domdf_python_tools.stringlist.StringList` with lines from ``iterable``.

		:param iterable: An iterable of string-like objects to add to the end of the
			:class:`~domdf_python_tools.stringlist.StringList`.
		"""

		for line in iterable:
			self.append(line)

	def copy(self: _SL) -> _SL:
		"""
		Returns a shallow copy of the :class:`~domdf_python_tools.stringlist.StringList`.

		Equivalent to ``a[:]``.

		:rtype: :class:`~domdf_python_tools.stringlist.StringList`
		"""

		return self.__class__(super().copy())

	def count_blanklines(self) -> int:
		"""
		Returns a count of the blank lines in the :class:`~domdf_python_tools.stringlist.StringList`.

		.. versionadded:: 0.7.1
		"""

		return self.count('')

	def insert(self, index: SupportsIndex, line: String) -> None:
		"""
		Insert a line into the :class:`~domdf_python_tools.stringlist.StringList` at the given position.

		:param index:
		:param line:

		.. versionchanged:: 3.2.0  Changed :class:`int` in the type annotation to :protocol:`~.SupportsIndex`.
		"""

		lines: List[str]

		index = index.__index__()

		if index < 0 or index > len(self):
			lines = str(line).split('\n')
		else:
			lines = cast(list, reversed(str(line).split('\n')))

		for inner_line in lines:
			super().insert(index, self._make_line(inner_line))

	@overload
	def __setitem__(self, index: SupportsIndex, line: String) -> None: ...

	@overload
	def __setitem__(self, index: slice, line: Iterable[String]) -> None: ...

	def __setitem__(self, index: Union[SupportsIndex, slice], line: Union[String, Iterable[String]]):
		"""
		Replaces the given line with new content.

		If the new content consists of multiple lines subsequent content in the
		:class:`~domdf_python_tools.stringlist.StringList` will be shifted down.

		:param index:
		:param line:

		.. versionchanged:: 3.2.0  Changed :class:`int` in the type annotation to :protocol:`~.SupportsIndex`.
		"""

		if isinstance(index, slice):
			line = cast(Iterable[String], line)

			if not isinstance(line, Reversible):
				line = tuple(line)

			for lline, idx in zip(
				reversed(line),
				reversed(range(index.start or 0, index.stop + 1, index.step or 1)),
				):
				self[idx] = lline
		else:
			line = cast(String, line)
			index = index.__index__()

			if self and index < len(self):
				self.pop(index)
			if index < 0:
				index = len(self) + index + 1

			self.insert(index, line)

	@overload
	def __getitem__(self, index: SupportsIndex) -> str: ...

	@overload
	def __getitem__(self: _SL, index: slice) -> _SL: ...

	def __getitem__(self: _SL, index: Union[SupportsIndex, slice]) -> Union[str, _SL]:
		r"""
		Returns the line with the given index.

		:param index:

		:rtype: :py:obj:`~typing.Union`\[:class:`str`, :class:`~domdf_python_tools.stringlist.StringList`\]

		.. versionchanged:: 1.8.0

			Now returns a :class:`~domdf_python_tools.stringlist.StringList` when ``index`` is a :class:`slice`.

		.. versionchanged:: 3.2.0  Changed :class:`int` in the type annotation to :protocol:`~.SupportsIndex`.
		"""

		if isinstance(index, slice):
			return self.__class__(super().__getitem__(index))
		else:
			return super().__getitem__(index)

	def blankline(self, ensure_single: bool = False):
		"""
		Append a blank line to the end of the :class:`~domdf_python_tools.stringlist.StringList`.

		:param ensure_single: Ensure only a single blank line exists after the previous line of text.
		"""

		if ensure_single:
			while self and not self[-1]:
				self.pop(-1)

		self.append('')

	def set_indent_size(self, size: int = 0):
		"""
		Sets the size of the indent to insert at the beginning of new lines.

		:param size: The indent size to use for new lines.
		"""

		self.indent.size = int(size)

	def set_indent_type(self, indent_type: str = '\t'):
		"""
		Sets the type of the indent to insert at the beginning of new lines.

		:param indent_type: The type of indent to use for new lines.
		"""

		self.indent.type = str(indent_type)

	def set_indent(self, indent: Union[String, Indent], size: int = 0):
		"""
		Sets the indent to insert at the beginning of new lines.

		:param indent: The :class:`~.Indent` to use for new lines, or the indent type.
		:param size: If ``indent`` is an indent type, the indent size to use for new lines.
		"""

		if isinstance(indent, Indent):
			if size:
				raise TypeError("'size' argument cannot be used when providing an 'Indent' object.")

			self.indent = indent
		else:
			self.indent = Indent(int(size), str(indent))

	@property
	def indent_size(self) -> int:
		"""
		The current indent size.
		"""

		return int(self.indent.size)

	@indent_size.setter
	def indent_size(self, size: int) -> None:
		"""
		Sets the indent size.
		"""

		self.indent.size = int(size)

	@property
	def indent_type(self) -> str:
		"""
		The current indent type.
		"""

		return str(self.indent.type)

	@indent_type.setter
	def indent_type(self, type: str) -> None:  # noqa: A002  # pylint: disable=redefined-builtin
		"""
		Sets the indent type.
		"""

		self.indent.type = str(type)

	def __str__(self) -> str:
		"""
		Returns the :class:`~domdf_python_tools.stringlist.StringList` as a string.
		"""

		return '\n'.join(self)

	def __bytes__(self) -> bytes:
		"""
		Returns the :class:`~domdf_python_tools.stringlist.StringList` as bytes.

		.. versionadded:: 2.1.0
		"""

		return str(self).encode("UTF-8")

	def __eq__(self, other) -> bool:
		"""
		Returns whether the other object is equal to this :class:`~domdf_python_tools.stringlist.StringList`.
		"""

		if isinstance(other, str):
			return str(self) == other
		else:
			return super().__eq__(other)

	@contextmanager
	def with_indent(self, indent: Union[String, Indent], size: int = 0):
		"""
		Context manager to temporarily use a different indent.

		.. code-block:: python

			>>> sl = StringList()
			>>> with sl.with_indent("    ", 1):
			...     sl.append("Hello World")

		:param indent: The :class:`~.Indent` to use within the ``with`` block, or the indent type.
		:param size: If ``indent`` is an indent type, the indent size to use within the ``with`` block.
		"""

		original_indent: Tuple[int, str] = tuple(self.indent)  # type: ignore

		try:
			self.set_indent(indent, size)
			yield
		finally:
			self.indent = Indent(*original_indent)

	@contextmanager
	def with_indent_size(self, size: int = 0):
		"""
		Context manager to temporarily use a different indent size.

		.. code-block:: python

			>>> sl = StringList()
			>>> with sl.with_indent_size(1):
			...     sl.append("Hello World")

		:param size: The indent size to use within the ``with`` block.
		"""

		original_indent_size = self.indent_size

		try:
			self.indent_size = size
			yield
		finally:
			self.indent_size = original_indent_size

	@contextmanager
	def with_indent_type(self, indent_type: str = '\t'):
		"""
		Context manager to temporarily use a different indent type.

		.. code-block:: python

			>>> sl = StringList()
			>>> with sl.with_indent_type("    "):
			...     sl.append("Hello World")

		:param indent_type: The type of indent to use within the ``with`` block.
		"""

		original_indent_type = self.indent_type

		try:
			self.indent_type = indent_type
			yield
		finally:
			self.indent_type = original_indent_type


class DelimitedList(List[_S]):
	"""
	Subclass of :class:`list` that supports custom delimiters in format strings.

	**Example:**

	.. code-block:: python

		>>> l = DelimitedList([1, 2, 3, 4, 5])
		>>> format(l, ", ")
		'1, 2, 3, 4, 5'
		>>> f"Numbers: {l:, }"
		'Numbers: 1, 2, 3, 4, 5'

	.. autoclasssumm:: DelimitedList
		:autosummary-sections: ;;

	.. versionadded:: 1.1.0
	"""

	def __format__(self, format_spec: str) -> str:
		return format_spec.join([str(x) for x in self])  # pylint: disable=not-an-iterable


def splitlines(string: str) -> List[Tuple[str, str]]:
	"""
	Split ``string`` into a list of two-element tuples,
	containing the line content and the newline character(s), if any.

	.. versionadded:: 3.2.0

	:param string:

	:rtype:

	.. seealso:: :meth:`str.splitlines` and :func:`~.stringlist.joinlines`
	"""  # noqa: D400

	# Translated and adapted from https://github.com/python/cpython/blob/main/Objects/stringlib/split.h

	str_len: int = len(string)
	i: int = 0
	j: int = 0
	eol: int
	the_list: List[Tuple[str, str]] = []

	while i < str_len:

		# Find a line and append it
		while i < str_len and not string[i] in "\n\r":
			i += 1

		# Skip the line break reading CRLF as one line break
		eol = i
		if i < str_len:
			if (string[i] == '\r') and (i + 1 < str_len) and (string[i + 1] == '\n'):
				i += 2
			else:
				i += 1

		if j == 0 and eol == str_len and type(string) is str:  # pylint: disable=unidiomatic-typecheck
			# No whitespace in string, so just use it as the_list[0]
			the_list.append((string, ''))
			break

		the_list.append((string[j:eol], string[eol:i]))
		j = i

	return the_list


def joinlines(lines: List[Tuple[str, str]]) -> str:
	"""
	Given a list of two-element tuples, each containing a line and a newline character (or empty string),
	return a single string.

	.. versionadded:: 3.2.0

	:param lines:

	:rtype:

	.. seealso:: :func:`~.stringlist.splitlines`
	"""  # noqa: D400

	return ''.join(chain.from_iterable(lines))

env_pyrecodes/lib/python3.9/site-packages/domdf_python_tools/stringlist.py
NikolaBlagojevic/pyrecodes
"""encutils - encoding detection collection for Python

A collection of helper functions to detect encodings of text files
(like HTML, XHTML, XML, CSS, etc.) retrieved via HTTP, file or string.

:func:`getEncodingInfo` is probably the main function of interest which uses
other supplied functions itself and gathers all information together and
supplies an :class:`EncodingInfo` object.

example::

    >>> import encutils
    >>> info = encutils.getEncodingInfo(url='http://jaraco.com')

    >>> print(info)
    utf-8

    >>> info # doctest:+ELLIPSIS
    <encutils.EncodingInfo object encoding='utf-8' mismatch=False at...>

    >>> print(info.logtext)
    HTTP media_type: text/html
    HTTP encoding: utf-8
    Encoding (probably): utf-8 (Mismatch: False)
    <BLANKLINE>

references
    XML
        RFC 3023 (http://www.ietf.org/rfc/rfc3023.txt)

        easier explained in
            - http://feedparser.org/docs/advanced.html
            - http://www.xml.com/pub/a/2004/07/21/dive.html

    HTML
        http://www.w3.org/TR/REC-html40/charset.html#h-5.2.2

TODO
    - parse @charset of HTML elements?
    - check for more texttypes if only text given
"""
__all__ = [
    'buildlog',
    'encodingByMediaType',
    'getHTTPInfo',
    'getMetaInfo',
    'detectXMLEncoding',
    'getEncodingInfo',
    'tryEncodings',
    'EncodingInfo',
]

from email.message import Message
import html.parser
import io
import re
import sys
import urllib.request
import urllib.parse
import urllib.error


class _MetaHTMLParser(html.parser.HTMLParser):
    """Parse given data for <meta http-equiv="content-type">."""

    content_type = None

    def handle_starttag(self, tag, attrs):
        if tag == 'meta' and not self.content_type:
            atts = {a.lower(): v.lower() for a, v in attrs}
            if atts.get('http-equiv', '').strip() == 'content-type':
                self.content_type = atts.get('content')


# application/xml, application/xml-dtd,
# application/xml-external-parsed-entity,
# or a subtype like application/rss+xml.
_XML_APPLICATION_TYPE = 0

# text/xml, text/xml-external-parsed-entity, or a subtype like text/AnythingAtAll+xml
_XML_TEXT_TYPE = 1

# text/html
_HTML_TEXT_TYPE = 2

# any other of text/* like text/plain, ...
_TEXT_TYPE = 3

# any text/* like which defaults to UTF-8 encoding, for now only text/css
_TEXT_UTF8 = 5

# types not fitting in above types
_OTHER_TYPE = 4


class EncodingInfo:
    """
    All encoding related information, returned by :func:`getEncodingInfo`.

    Attributes filled:
        - ``encoding``: The guessed encoding
            Encoding is the explicit or implicit encoding or None and
            always lowercase.

        - from HTTP response
            * ``http_encoding``
            * ``http_media_type``

        - from HTML <meta> element
            * ``meta_encoding``
            * ``meta_media_type``

        - from XML declaration
            * ``xml_encoding``

        - ``mismatch``: True if mismatch between XML declaration and HTTP
            header.
            Mismatch is True if any mismatches between HTTP header, XML
            declaration or textcontent (meta) are found. More detailed
            mismatch reports are written to the optional log or ``logtext``

            Mismatches are not necessarily errors as preferences are defined.
            For details see the specifications.

        - ``logtext``: if no log was given log reports are given here
    """

    def __init__(self):
        """Initialize all possible properties to ``None``, see class
        description
        """
        self.encoding = (
            self.mismatch
        ) = (
            self.logtext
        ) = (
            self.http_encoding
        ) = (
            self.http_media_type
        ) = self.meta_encoding = self.meta_media_type = self.xml_encoding = None

    def __str__(self):
        """Output the guessed encoding itself or the empty string."""
        if self.encoding:
            return self.encoding
        else:
            return ''

    def __repr__(self):
        return "<{}.{} object encoding={!r} mismatch={} at 0x{:x}>".format(
            self.__class__.__module__,
            self.__class__.__name__,
            self.encoding,
            self.mismatch,
            id(self),
        )


def buildlog(
    logname='encutils',
    level='INFO',
    stream=sys.stderr,
    filename=None,
    filemode="w",
    format='%(levelname)s\t%(message)s',
):
    """Helper to build a basic log

    - if `filename` is given returns a log logging to `filename` with
      mode `filemode`
    - else uses a log streaming to `stream` which defaults to
      `sys.stderr`
    - `level` defines the level of the log
    - `format` defines the formatter format of the log

    :returns:
        a log with the name `logname`
    """
    import logging

    log = logging.getLogger(logname)

    if filename:
        hdlr = logging.FileHandler(filename, filemode)
    else:
        hdlr = logging.StreamHandler(stream)

    formatter = logging.Formatter(format)
    hdlr.setFormatter(formatter)

    log.addHandler(hdlr)
    log.setLevel(logging.__dict__.get(level, logging.INFO))

    return log


def _getTextTypeByMediaType(media_type, log=None):
    """
    :returns:
        type as defined by constants in this class
    """
    if not media_type:
        return _OTHER_TYPE
    xml_application_types = [
        r'application/.*?\+xml',
        'application/xml',
        'application/xml-dtd',
        'application/xml-external-parsed-entity',
    ]
    xml_text_types = [r'text\/.*?\+xml', 'text/xml', 'text/xml-external-parsed-entity']

    media_type = media_type.strip().lower()

    if media_type in xml_application_types or re.match(
        xml_application_types[0], media_type, re.I | re.S | re.X
    ):
        return _XML_APPLICATION_TYPE
    elif media_type in xml_text_types or re.match(
        xml_text_types[0], media_type, re.I | re.S | re.X
    ):
        return _XML_TEXT_TYPE
    elif media_type == 'text/html':
        return _HTML_TEXT_TYPE
    elif media_type == 'text/css':
        return _TEXT_UTF8
    elif media_type.startswith('text/'):
        return _TEXT_TYPE
    else:
        return _OTHER_TYPE


def _getTextType(text, log=None):
    """Check if given text is XML (**naive test!**)
    used if no content-type given
    """
    if text[:30].find('<?xml version=') != -1:
        return _XML_APPLICATION_TYPE
    else:
        return _OTHER_TYPE


def encodingByMediaType(media_type, log=None):
    """
    :param media_type:
        a media type like "text/html"
    :returns:
        a default encoding for given `media_type`. For example
        ``"utf-8"`` for ``media_type="application/xml"``.

        If no default encoding is available returns ``None``.

        Refers to RFC 3023 and HTTP MIME specification.
    """
    defaultencodings = {
        _XML_APPLICATION_TYPE: 'utf-8',
        _XML_TEXT_TYPE: 'ascii',
        _HTML_TEXT_TYPE: 'iso-8859-1',  # should be None?
        _TEXT_TYPE: 'iso-8859-1',  # should be None?
        _TEXT_UTF8: 'utf-8',
        _OTHER_TYPE: None,
    }

    texttype = _getTextTypeByMediaType(media_type)
    encoding = defaultencodings.get(texttype, None)

    if log:
        if not encoding:
            log.debug('"%s" Media-Type has no default encoding', media_type)
        else:
            log.debug('Default encoding for Media Type "%s": %s', media_type, encoding)
    return encoding


def getHTTPInfo(response, log=None):
    """
    :param response:
        a HTTP response object
    :returns:
        ``(media_type, encoding)`` information from the `response`
        Content-Type HTTP header. (Case of headers is ignored.)
        May be ``(None, None)`` e.g. if no Content-Type header is
        available.
    """
    info = response.info()
    media_type, encoding = info.get_content_type(), info.get_content_charset()

    if encoding:
        encoding = encoding.lower()

    if log:
        log.info('HTTP media_type: %s', media_type)
        log.info('HTTP encoding: %s', encoding)

    return media_type, encoding


def getMetaInfo(text, log=None):
    """
    :param text:
        a byte string
    :returns:
        ``(media_type, encoding)`` information from (first)
        X/HTML Content-Type ``<meta>`` element if available in `text`.

        XHTML format::

            <meta http-equiv="Content-Type"
                  content="media_type;charset=encoding" />
    """
    p = _MetaHTMLParser()

    try:
        p.feed(text)
    except html.parser.HTMLParseError:
        pass

    if p.content_type:
        m = Message()
        m['content-type'] = p.content_type

        media_type = m.get_content_type()
        encoding = m.get_param('charset')  # defaults to None
        if encoding:
            encoding = encoding.lower()
        if log:
            log.info('HTML META media_type: %s', media_type)
            log.info('HTML META encoding: %s', encoding)
    else:
        media_type = encoding = None

    return media_type, encoding


def detectXMLEncoding(fp, log=None, includeDefault=True):  # noqa: C901
    """Attempt to detect the character encoding of the xml file
    given by a file object `fp`. `fp` must not be a codec wrapped file
    object! `fp` may be a string or unicode string though.

    Based on a recipe by Lars Tiede:
    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/363841
    which itself is based on Paul Prescotts recipe:
    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52257

    :returns:
        - if detection of the BOM succeeds, the codec name of the
          corresponding unicode charset is returned

        - if BOM detection fails, the xml declaration is searched for
          the encoding attribute and its value returned. the "<"
          character has to be the very first in the file then (it's xml
          standard after all).

        - if BOM and xml declaration fail, utf-8 is returned according
          to XML 1.0.
    """
    if isinstance(fp, str):
        fp = io.StringIO(fp)

    # detection using BOM

    # the BOMs we know, by their pattern
    bomDict = {  # bytepattern: name
        (0x00, 0x00, 0xFE, 0xFF): "utf_32_be",
        (0xFF, 0xFE, 0x00, 0x00): "utf_32_le",
        (0xFE, 0xFF, None, None): "utf_16_be",
        (0xFF, 0xFE, None, None): "utf_16_le",
        (0xEF, 0xBB, 0xBF, None): "utf-8",
    }

    # go to beginning of file and get the first 4 bytes
    oldFP = fp.tell()
    fp.seek(0)
    (byte1, byte2, byte3, byte4) = tuple(map(ord, fp.read(4)))

    # try bom detection using 4 bytes, 3 bytes, or 2 bytes
    bomDetection = bomDict.get((byte1, byte2, byte3, byte4))
    if not bomDetection:
        bomDetection = bomDict.get((byte1, byte2, byte3, None))
        if not bomDetection:
            bomDetection = bomDict.get((byte1, byte2, None, None))

    # if BOM detected, we're done :-)
    if bomDetection:
        if log:
            log.info('XML BOM encoding: %s' % bomDetection)
        fp.seek(oldFP)
        return bomDetection

    # still here? BOM detection failed.
    #  now that BOM detection has failed we assume one byte character
    #  encoding behaving ASCII

    # search xml declaration for encoding attribute

    # assume xml declaration fits into the first 2 KB (*cough*)
    fp.seek(0)
    buffer = fp.read(2048)

    # set up regular expression
    xmlDeclPattern = r"""
    ^<\?xml             # w/o BOM, xmldecl starts with <?xml at the first byte
    .+?                 # some chars (version info), matched minimal
    encoding=           # encoding attribute begins
    ["']                # attribute start delimiter
    (?P<encstr>         # what's matched in the brackets will be named encstr
     [^"']+              # every character not delimiter (not overly exact!)
    )                   # closes the brackets pair for the named group
    ["']                # attribute end delimiter
    .*?                 # some chars optionally (standalone decl or whitespace)
    \?>                 # xmldecl end
    """
    xmlDeclRE = re.compile(xmlDeclPattern, re.VERBOSE)

    # search and extract encoding string
    match = xmlDeclRE.search(buffer)
    fp.seek(oldFP)
    if match:
        enc = match.group("encstr").lower()
        if log:
            log.info('XML encoding="%s"' % enc)
        return enc
    else:
        if includeDefault:
            if log:
                log.info('XML encoding default utf-8')
            return 'utf-8'
        else:
            return None


def tryEncodings(text, log=None):  # noqa: C901
    """If installed uses chardet http://chardet.feedparser.org/ to detect
    encoding, else tries different encodings on `text` and returns the one
    that does not raise an exception which is not very advanced or may
    be totally wrong. The tried encoding are in order 'ascii', 'iso-8859-1',
    'windows-1252' (which probably will never happen as 'iso-8859-1' can decode
    these strings too) and lastly 'utf-8'.

    :param text:
        a byte string
    :returns:
        Working encoding or ``None`` if no encoding does work at all.

        The returned encoding might nevertheless be not the one intended by
        the author as it is only checked if the text might be encoded in
        that encoding. Some texts might be working in "iso-8859-1" *and*
        "windows-1252" *and* "ascii" *and* "utf-8" and ...
    """
    try:
        import chardet

        encoding = chardet.detect(text)["encoding"]

    except ImportError:
        msg = 'Using simplified encoding detection, you might want to install chardet.'
        if log:
            log.warn(msg)
        else:
            print(msg)

        encodings = (
            'ascii',
            'iso-8859-1',
            # 'windows-1252', # test later
            'utf-8',
        )
        encoding = None
        for e in encodings:
            try:
                text.decode(e)
            except UnicodeDecodeError:
                pass
            else:
                if 'iso-8859-1' == e:
                    try:
                        if '€' in text.decode('windows-1252'):
                            return 'windows-1252'
                    except UnicodeDecodeError:
                        pass

                return e

    return encoding


def getEncodingInfo(response=None, text='', log=None, url=None):  # noqa: C901
    """Find all encoding related information in given `text`.

    Information in headers of supplied HTTPResponse, possible XML
    declaration and X/HTML ``<meta>`` elements are used.

    :param response:
        HTTP response object, e.g. via ``urllib.urlopen('url')``
    :param text:
        a byte string to guess encoding for. XML prolog with
        encoding pseudo attribute or HTML meta element will be used to detect
        the encoding
    :param url:
        When given fetches document at `url` and all needed information.
        No `reponse` or `text` parameters are needed in this case.
    :param log:
        an optional logging logger to which messages may go, if
        no log given all log messages are available from resulting
        ``EncodingInfo``

    :returns:
        instance of :class:`EncodingInfo`.

    How the resulting encoding is retrieved:

    XML
        RFC 3023 states if media type given in the Content-Type HTTP header is
        application/xml, application/xml-dtd,
        application/xml-external-parsed-entity, or any one of the subtypes of
        application/xml such as application/atom+xml or application/rss+xml
        etc then the character encoding is determined in this order:

        1. the encoding given in the charset parameter of the Content-Type HTTP
        header, or
        2. the encoding given in the encoding attribute of the XML declaration
        within the document, or
        3. utf-8.

        Mismatch possibilities:
            - HTTP + XMLdecla
            - HTTP + HTMLmeta

            application/xhtml+xml ?
                XMLdecla + HTMLmeta


        If the media type given in the Content-Type HTTP header is text/xml,
        text/xml-external-parsed-entity, or a subtype like text/Anything+xml,
        the encoding attribute of the XML declaration is ignored completely
        and the character encoding is determined in the order:
        1. the encoding given in the charset parameter of the Content-Type HTTP
        header, or
        2. ascii.

        No mismatch possible.


        If no media type is given the XML encoding pseuso attribute is used
        if present.

        No mismatch possible.

    HTML
        For HTML served as text/html:
            http://www.w3.org/TR/REC-html40/charset.html#h-5.2.2

        1. An HTTP "charset" parameter in a "Content-Type" field.
           (maybe defaults to ISO-8859-1, but should not assume this)
        2. A META declaration with "http-equiv" set to "Content-Type" and a
           value set for "charset".
        3. The charset attribute set on an element that designates an external
           resource. (NOT IMPLEMENTED HERE YET)

        Mismatch possibilities:
            - HTTP + HTMLmeta

    TEXT
        For most text/* types the encoding will be reported as iso-8859-1.
        Exceptions are XML formats send as text/* mime type (see above) and
        text/css which has a default encoding of UTF-8.
    """
    if url:
        # may cause IOError which is raised
        response = urllib.request.urlopen(url)

    if text is None:
        # read text from response only if not explicitly given
        try:
            text = response.read()
        except OSError:
            pass

    if text is None:
        # text must be a string (not None)
        text = ''

    encinfo = EncodingInfo()

    logstream = io.StringIO()
    if not log:
        log = buildlog(stream=logstream, format='%(message)s')

    # HTTP
    if response:
        encinfo.http_media_type, encinfo.http_encoding = getHTTPInfo(response, log)
        texttype = _getTextTypeByMediaType(encinfo.http_media_type, log)
    else:
        # check if maybe XML or (TODO:) HTML
        texttype = _getTextType(text, log)

    # XML only served as application/xml ! #(also XHTML served as text/html)
    if texttype == _XML_APPLICATION_TYPE:  # or texttype == _XML_TEXT_TYPE:
        try:
            encinfo.xml_encoding = detectXMLEncoding(text, log)
        except (AttributeError, ValueError):
            encinfo.xml_encoding = None

    # XML (also XHTML served as text/html)
    if texttype == _HTML_TEXT_TYPE:
        try:
            encinfo.xml_encoding = detectXMLEncoding(text, log, includeDefault=False)
        except (AttributeError, ValueError):
            encinfo.xml_encoding = None

    # HTML
    if texttype == _HTML_TEXT_TYPE or texttype == _TEXT_TYPE:
        encinfo.meta_media_type, encinfo.meta_encoding = getMetaInfo(text, log)

    # guess
    # 1. HTTP charset?
    encinfo.encoding = encinfo.http_encoding
    encinfo.mismatch = False

    # 2. media_type?
    #   XML application/...
    if texttype == _XML_APPLICATION_TYPE:
        if not encinfo.encoding:
            encinfo.encoding = encinfo.xml_encoding
            # xml_encoding has default of utf-8

    #   text/html
    elif texttype == _HTML_TEXT_TYPE:
        if not encinfo.encoding:
            encinfo.encoding = encinfo.meta_encoding
        if not encinfo.encoding:
            encinfo.encoding = encodingByMediaType(encinfo.http_media_type)
        if not encinfo.encoding:
            encinfo.encoding = tryEncodings(text)

    #   text/... + xml or text/*
    elif texttype == _XML_TEXT_TYPE or texttype == _TEXT_TYPE:
        if not encinfo.encoding:
            encinfo.encoding = encodingByMediaType(encinfo.http_media_type)

    elif texttype == _TEXT_UTF8:
        if not encinfo.encoding:
            encinfo.encoding = encodingByMediaType(encinfo.http_media_type)

    # possible mismatches, checks if present at all and then if equal
    # HTTP + XML
    if (
        encinfo.http_encoding
        and encinfo.xml_encoding
        and encinfo.http_encoding != encinfo.xml_encoding
    ):
        encinfo.mismatch = True
        log.warn(
            '"%s" (HTTP) != "%s" (XML) encoding mismatch'
            % (encinfo.http_encoding, encinfo.xml_encoding)
        )
    # HTTP + Meta
    if (
        encinfo.http_encoding
        and encinfo.meta_encoding
        and encinfo.http_encoding != encinfo.meta_encoding
    ):
        encinfo.mismatch = True
        log.warning(
            '"%s" (HTTP) != "%s" (HTML <meta>) encoding mismatch'
            % (encinfo.http_encoding, encinfo.meta_encoding)
        )
    # XML + Meta
    if (
        encinfo.xml_encoding
        and encinfo.meta_encoding
        and encinfo.xml_encoding != encinfo.meta_encoding
    ):
        encinfo.mismatch = True
        log.warning(
            '"%s" (XML) != "%s" (HTML <meta>) encoding mismatch'
            % (encinfo.xml_encoding, encinfo.meta_encoding)
        )

    log.info(
        'Encoding (probably): %s (Mismatch: %s)', encinfo.encoding, encinfo.mismatch
    )

    encinfo.logtext = logstream.getvalue()
    return encinfo


if __name__ == '__main__':
    import pydoc

    pydoc.help(__name__)

env_pyrecodes/lib/python3.9/site-packages/encutils/__init__.py
NikolaBlagojevic/pyrecodes
"""A clean customisable Sphinx documentation theme."""

__version__ = "2023.08.19"

import hashlib
import logging
import os
from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional

import sphinx.application
from docutils import nodes
from pygments.formatters import HtmlFormatter
from pygments.style import Style
from pygments.token import Text
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.environment.adapters.toctree import TocTree
from sphinx.errors import ConfigError
from sphinx.highlighting import PygmentsBridge
from sphinx.transforms.post_transforms import SphinxPostTransform

from .navigation import get_navigation_tree

THEME_PATH = (Path(__file__).parent / "theme" / "furo").resolve()

logger = logging.getLogger(__name__)

# GLOBAL STATE
_KNOWN_STYLES_IN_USE: Dict[str, Optional[Style]] = {
    "light": None,
    "dark": None,
}


class WrapTableAndMathInAContainerTransform(SphinxPostTransform):
    """A Sphinx post-transform that wraps `table` and `div.math` in a container `div`.

    This makes it possible to handle these overflowing the content-width, which is
    necessary in a responsive theme.
    """

    formats = ("html",)
    default_priority = 500

    def run(self, **kwargs: Any) -> None:
        """Perform the post-transform on `self.document`."""
        get_nodes = (
            self.document.findall  # docutils 0.18+
            if hasattr(self.document, "findall")
            else self.document.traverse  # docutils <= 0.17.x
        )
        for node in list(get_nodes(nodes.table)):
            new_node = nodes.container(classes=["table-wrapper"])
            new_node.update_all_atts(node)
            node.parent.replace(node, new_node)
            new_node.append(node)

        for node in list(get_nodes(nodes.math_block)):
            new_node = nodes.container(classes=["math-wrapper"])
            new_node.update_all_atts(node)
            node.parent.replace(node, new_node)
            new_node.append(node)


def has_not_enough_items_to_show_toc(
    builder: StandaloneHTMLBuilder, docname: str
) -> bool:
    """Check if the toc has one or fewer items."""
    assert builder.env

    toctree = TocTree(builder.env).get_toc_for(docname, builder)
    try:
        self_toctree = toctree[0][1]
    except IndexError:
        val = True
    else:
        # There's only the page's own toctree in there.
        val = len(self_toctree) == 1 and self_toctree[0].tagname == "toctree"
    return val


def get_pygments_style_colors(
    style: Style, *, fallbacks: Dict[str, str]
) -> Dict[str, str]:
    """Get background/foreground colors for given pygments style."""
    background = style.background_color
    text_colors = style.style_for_token(Text)
    foreground = text_colors["color"]

    if not background:
        background = fallbacks["background"]

    if not foreground:
        foreground = fallbacks["foreground"]
    else:
        foreground = f"#{foreground}"

    return {"background": background, "foreground": foreground}


@lru_cache(maxsize=2)
def get_colors_for_codeblocks(
    highlighter: PygmentsBridge, *, fg: str, bg: str
) -> Dict[str, str]:
    """Get background/foreground colors for given pygments style."""
    return get_pygments_style_colors(
        highlighter.formatter_args["style"],
        fallbacks={
            "foreground": fg,
            "background": bg,
        },
    )


def _compute_navigation_tree(context: Dict[str, Any]) -> str:
    # The navigation tree, generated from the sphinx-provided ToC tree.
    if "toctree" in context:
        toctree = context["toctree"]
        toctree_html = toctree(
            collapse=False,
            titles_only=True,
            maxdepth=-1,
            includehidden=True,
        )
    else:
        toctree_html = ""

    return get_navigation_tree(toctree_html)


def _compute_hide_toc(
    context: Dict[str, Any],
    *,
    builder: StandaloneHTMLBuilder,
    docname: str,
) -> bool:
    # Should the table of contents be hidden?
    file_meta = context.get("meta", None) or {}
    if "hide-toc" in file_meta:
        return True
    elif "toc" not in context:
        return True
    elif not context["toc"]:
        return True

    return has_not_enough_items_to_show_toc(builder, docname)


@lru_cache(maxsize=None)
def _asset_hash(path: str) -> str:
    """Append a `?digest=` to an url based on the file content."""
    full_path = THEME_PATH / "static" / path
    digest = hashlib.sha1(full_path.read_bytes()).hexdigest()

    return f"_static/{path}?digest={digest}"


def _add_asset_hashes(static: List[str], add_digest_to: List[str]) -> None:
    if sphinx.version_info >= (7, 1):
        # https://github.com/sphinx-doc/sphinx/pull/11415 added the relevant
        # functionality to Sphinx, so we don't need to do anything.
        return

    for asset in add_digest_to:
        index = static.index("_static/" + asset)
        static[index].filename = _asset_hash(asset)  # type: ignore


def _html_page_context(
    app: sphinx.application.Sphinx,
    pagename: str,
    templatename: str,
    context: Dict[str, Any],
    doctree: Any,
) -> None:
    if "css_files" in context:
        if "_static/styles/furo.css" not in [c.filename for c in context["css_files"]]:
            raise ConfigError(
                "This documentation is not using `furo.css` as the stylesheet. "
                "If you have set `html_style` in your conf.py file, remove it."
            )

        _add_asset_hashes(
            context["css_files"],
            ["styles/furo.css", "styles/furo-extensions.css"],
        )
    if "scripts" in context:
        _add_asset_hashes(
            context["scripts"],
            ["scripts/furo.js"],
        )

    # Basic constants
    context["furo_version"] = __version__

    # Values computed from page-level context.
    context["furo_navigation_tree"] = _compute_navigation_tree(context)
    context["furo_hide_toc"] = _compute_hide_toc(
        context, builder=app.builder, docname=pagename
    )

    # Inject information about styles
    context["furo_pygments"] = {
        "light": get_pygments_style_colors(
            _KNOWN_STYLES_IN_USE["light"],
            fallbacks=dict(
                foreground="black",
                background="white",
            ),
        ),
        "dark": get_pygments_style_colors(
            _KNOWN_STYLES_IN_USE["dark"],
            fallbacks=dict(
                foreground="white",
                background="black",
            ),
        ),
    }


def _builder_inited(app: sphinx.application.Sphinx) -> None:
    if "furo" in app.config.extensions:
        raise ConfigError(
            "Did you list 'furo' in the `extensions` in conf.py? "
            "If so, please remove it. Furo does not work with non-HTML builders "
            "and specifying it as an `html_theme` is sufficient."
        )

    if not isinstance(app.builder, StandaloneHTMLBuilder) or app.builder.name not in {
        "html",
        "dirhtml",
    }:
        raise ConfigError(
            "Furo is being used as an extension in a non-HTML build. "
            "This should not happen."
        )

    # Our JS file needs to be loaded as soon as possible.
    app.add_js_file("scripts/furo.js", priority=200)

    # 500 is the default priority for extensions, we want this after this.
    app.add_css_file("styles/furo-extensions.css", priority=600)

    builder = app.builder
    assert (
        builder.highlighter is not None
    ), "there should be a default style known to Sphinx"
    assert (
        builder.dark_highlighter is None
    ), "this shouldn't be a dark style known to Sphinx"
    update_known_styles_state(app)

    def _update_default(key: str, *, new_default: Any) -> None:
        app.config.values[key] = (new_default, *app.config.values[key][1:])

    # Change the default permalinks icon
    _update_default("html_permalinks_icon", new_default="#")


def update_known_styles_state(app: sphinx.application.Sphinx) -> None:
    """Update a global store of known styles of this application."""
    global _KNOWN_STYLES_IN_USE

    _KNOWN_STYLES_IN_USE = {
        "light": _get_light_style(app),
        "dark": _get_dark_style(app),
    }


def _get_light_style(app: sphinx.application.Sphinx) -> Style:
    return app.builder.highlighter.formatter_args["style"]


def _get_dark_style(app: sphinx.application.Sphinx) -> Style:
    dark_style = app.config.pygments_dark_style
    return PygmentsBridge("html", dark_style).formatter_args["style"]


def _get_styles(formatter: HtmlFormatter, *, prefix: str) -> Iterator[str]:
    """Get styles out of a formatter, where everything has the correct prefix."""
    for line in formatter.get_linenos_style_defs():
        yield f"{prefix} {line}"
    yield from formatter.get_background_style_defs(prefix)
    yield from formatter.get_token_style_defs(prefix)


def get_pygments_stylesheet() -> str:
    """Generate the theme-specific pygments.css.

    There is no way to tell Sphinx how the theme handles dark mode; at this time.
    """
    light_formatter = PygmentsBridge.html_formatter(style=_KNOWN_STYLES_IN_USE["light"])
    dark_formatter = PygmentsBridge.html_formatter(style=_KNOWN_STYLES_IN_USE["dark"])

    lines: List[str] = []

    lines.extend(_get_styles(light_formatter, prefix=".highlight"))

    lines.append("@media not print {")

    dark_prefix = 'body[data-theme="dark"] .highlight'
    lines.extend(_get_styles(dark_formatter, prefix=dark_prefix))

    not_light_prefix = 'body:not([data-theme="light"]) .highlight'
    lines.append("@media (prefers-color-scheme: dark) {")
    lines.extend(_get_styles(dark_formatter, prefix=not_light_prefix))
    lines.append("}")

    lines.append("}")

    return "\n".join(lines)


# Yup, we overwrite the default pygments.css file, because it can't possibly respect
# the needs of this theme.
def _overwrite_pygments_css(
    app: sphinx.application.Sphinx,
    exception: Optional[Exception],
) -> None:
    if exception is not None:
        return

    assert app.builder
    with open(
        os.path.join(app.builder.outdir, "_static", "pygments.css"),
        "w",
        encoding="utf-8",
    ) as f:
        f.write(get_pygments_stylesheet())


def setup(app: sphinx.application.Sphinx) -> Dict[str, Any]:
    """Entry point for sphinx theming."""
    app.require_sphinx("6.0")

    app.add_config_value(
        "pygments_dark_style", default="native", rebuild="env", types=[str]
    )

    app.add_html_theme("furo", str(THEME_PATH))

    app.add_post_transform(WrapTableAndMathInAContainerTransform)

    app.connect("html-page-context", _html_page_context)
    app.connect("builder-inited", _builder_inited)
    app.connect("build-finished", _overwrite_pygments_css)

    return {
        "parallel_read_safe": True,
        "parallel_write_safe": True,
        "version": __version__,
    }

env_pyrecodes/lib/python3.9/site-packages/furo/__init__.py
NikolaBlagojevic/pyrecodes
"""geographiclib: geodesic routines from GeographicLib"""

__version_info__ = (2,
                    0,
                    0)
"""GeographicLib version as a tuple"""

__version__ = "2.0"
"""GeographicLib version as a string"""

__release_date__ = "2022-04-23"
"""GeographicLib release date"""

env_pyrecodes/lib/python3.9/site-packages/geographiclib/__init__.py
NikolaBlagojevic/pyrecodes
"""accumulator.py: transcription of GeographicLib::Accumulator class."""
# accumulator.py
#
# This is a rather literal translation of the GeographicLib::Accumulator class
# from to python.  See the documentation for the C++ class for more information
# at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# Copyright (c) Charles Karney (2011-2019) <charles@karney.com> and
# licensed under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

from geographiclib.geomath import Math

class Accumulator:
  """Like math.fsum, but allows a running sum"""

  def Set(self, y):
    """Set value from argument"""
    if isinstance(y, Accumulator):
      self._s, self._t = y._s, y._t
    else:
      self._s, self._t = float(y), 0.0

  def __init__(self, y = 0.0):
    """Constructor"""
    self._s = self._t = 0.0
    self.Set(y)

  def Add(self, y):
    """Add a value"""
    # Here's Shewchuk's solution...
    # hold exact sum as [s, t, u]
    y, u = Math.sum(y, self._t)             # Accumulate starting at
    self._s, self._t = Math.sum(y, self._s) # least significant end
    # Start is _s, _t decreasing and non-adjacent.  Sum is now (s + t + u)
    # exactly with s, t, u non-adjacent and in decreasing order (except
    # for possible zeros).  The following code tries to normalize the
    # result.  Ideally, we want _s = round(s+t+u) and _u = round(s+t+u -
    # _s).  The follow does an approximate job (and maintains the
    # decreasing non-adjacent property).  Here are two "failures" using
    # 3-bit floats:
    #
    # Case 1: _s is not equal to round(s+t+u) -- off by 1 ulp
    # [12, -1] - 8 -> [4, 0, -1] -> [4, -1] = 3 should be [3, 0] = 3
    #
    # Case 2: _s+_t is not as close to s+t+u as it shold be
    # [64, 5] + 4 -> [64, 8, 1] -> [64,  8] = 72 (off by 1)
    #                    should be [80, -7] = 73 (exact)
    #
    # "Fixing" these problems is probably not worth the expense.  The
    # representation inevitably leads to small errors in the accumulated
    # values.  The additional errors illustrated here amount to 1 ulp of
    # the less significant word during each addition to the Accumulator
    # and an additional possible error of 1 ulp in the reported sum.
    #
    # Incidentally, the "ideal" representation described above is not
    # canonical, because _s = round(_s + _t) may not be true.  For
    # example, with 3-bit floats:
    #
    # [128, 16] + 1 -> [160, -16] -- 160 = round(145).
    # But [160, 0] - 16 -> [128, 16] -- 128 = round(144).
    #
    if self._s == 0:            # This implies t == 0,
      self._s = u               # so result is u
    else:
      self._t += u              # otherwise just accumulate u to t.

  def Sum(self, y = 0.0):
    """Return sum + y"""
    if y == 0.0:
      return self._s
    b = Accumulator(self)
    b.Add(y)
    return b._s

  def Negate(self):
    """Negate sum"""
    self._s *= -1
    self._t *= -1

  def Remainder(self, y):
    """Remainder on division by y"""
    self._s = Math.remainder(self._s, y)
    self.Add(0.0)

env_pyrecodes/lib/python3.9/site-packages/geographiclib/accumulator.py
NikolaBlagojevic/pyrecodes
"""Define the WGS84 ellipsoid"""
# constants.py
#
# This is a translation of the GeographicLib::Constants class to python.  See
# the documentation for the C++ class for more information at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# Copyright (c) Charles Karney (2011-2016) <charles@karney.com> and
# licensed under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

class Constants:
  """
  Constants describing the WGS84 ellipsoid
  """

  WGS84_a = 6378137.0           # meters
  """the equatorial radius in meters of the WGS84 ellipsoid in meters"""
  WGS84_f = 1/298.257223563
  """the flattening of the WGS84 ellipsoid, 1/298.257223563"""

env_pyrecodes/lib/python3.9/site-packages/geographiclib/constants.py
NikolaBlagojevic/pyrecodes
"""Define the :class:`~geographiclib.geodesic.Geodesic` class

The ellipsoid parameters are defined by the constructor.  The direct and
inverse geodesic problems are solved by

  * :meth:`~geographiclib.geodesic.Geodesic.Inverse` Solve the inverse
    geodesic problem
  * :meth:`~geographiclib.geodesic.Geodesic.Direct` Solve the direct
    geodesic problem
  * :meth:`~geographiclib.geodesic.Geodesic.ArcDirect` Solve the direct
    geodesic problem in terms of spherical arc length

:class:`~geographiclib.geodesicline.GeodesicLine` objects can be created
with

  * :meth:`~geographiclib.geodesic.Geodesic.Line`
  * :meth:`~geographiclib.geodesic.Geodesic.DirectLine`
  * :meth:`~geographiclib.geodesic.Geodesic.ArcDirectLine`
  * :meth:`~geographiclib.geodesic.Geodesic.InverseLine`

:class:`~geographiclib.polygonarea.PolygonArea` objects can be created
with

  * :meth:`~geographiclib.geodesic.Geodesic.Polygon`

The public attributes for this class are

  * :attr:`~geographiclib.geodesic.Geodesic.a`
    :attr:`~geographiclib.geodesic.Geodesic.f`

*outmask* and *caps* bit masks are

  * :const:`~geographiclib.geodesic.Geodesic.EMPTY`
  * :const:`~geographiclib.geodesic.Geodesic.LATITUDE`
  * :const:`~geographiclib.geodesic.Geodesic.LONGITUDE`
  * :const:`~geographiclib.geodesic.Geodesic.AZIMUTH`
  * :const:`~geographiclib.geodesic.Geodesic.DISTANCE`
  * :const:`~geographiclib.geodesic.Geodesic.STANDARD`
  * :const:`~geographiclib.geodesic.Geodesic.DISTANCE_IN`
  * :const:`~geographiclib.geodesic.Geodesic.REDUCEDLENGTH`
  * :const:`~geographiclib.geodesic.Geodesic.GEODESICSCALE`
  * :const:`~geographiclib.geodesic.Geodesic.AREA`
  * :const:`~geographiclib.geodesic.Geodesic.ALL`
  * :const:`~geographiclib.geodesic.Geodesic.LONG_UNROLL`

:Example:

    >>> from geographiclib.geodesic import Geodesic
    >>> # The geodesic inverse problem
    ... Geodesic.WGS84.Inverse(-41.32, 174.81, 40.96, -5.50)
    {'lat1': -41.32,
     'a12': 179.6197069334283,
     's12': 19959679.26735382,
     'lat2': 40.96,
     'azi2': 18.825195123248392,
     'azi1': 161.06766998615882,
     'lon1': 174.81,
     'lon2': -5.5}

"""
# geodesic.py
#
# This is a rather literal translation of the GeographicLib::Geodesic class to
# python.  See the documentation for the C++ class for more information at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# The algorithms are derived in
#
#    Charles F. F. Karney,
#    Algorithms for geodesics, J. Geodesy 87, 43-55 (2013),
#    https://doi.org/10.1007/s00190-012-0578-z
#    Addenda: https://geographiclib.sourceforge.io/geod-addenda.html
#
# Copyright (c) Charles Karney (2011-2022) <charles@karney.com> and licensed
# under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

import math
import sys
from geographiclib.geomath import Math
from geographiclib.constants import Constants
from geographiclib.geodesiccapability import GeodesicCapability

class Geodesic:
  """Solve geodesic problems"""

  GEOGRAPHICLIB_GEODESIC_ORDER = 6
  nA1_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nC1_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nC1p_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nA2_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nC2_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nA3_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nA3x_ = nA3_
  nC3_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nC3x_ = (nC3_ * (nC3_ - 1)) // 2
  nC4_ = GEOGRAPHICLIB_GEODESIC_ORDER
  nC4x_ = (nC4_ * (nC4_ + 1)) // 2
  maxit1_ = 20
  maxit2_ = maxit1_ + sys.float_info.mant_dig + 10

  tiny_ = math.sqrt(sys.float_info.min)
  tol0_ = sys.float_info.epsilon
  tol1_ = 200 * tol0_
  tol2_ = math.sqrt(tol0_)
  tolb_ = tol0_ * tol2_
  xthresh_ = 1000 * tol2_

  CAP_NONE = GeodesicCapability.CAP_NONE
  CAP_C1   = GeodesicCapability.CAP_C1
  CAP_C1p  = GeodesicCapability.CAP_C1p
  CAP_C2   = GeodesicCapability.CAP_C2
  CAP_C3   = GeodesicCapability.CAP_C3
  CAP_C4   = GeodesicCapability.CAP_C4
  CAP_ALL  = GeodesicCapability.CAP_ALL
  CAP_MASK = GeodesicCapability.CAP_MASK
  OUT_ALL  = GeodesicCapability.OUT_ALL
  OUT_MASK = GeodesicCapability.OUT_MASK

  @staticmethod
  def _SinCosSeries(sinp, sinx, cosx, c):
    """Private: Evaluate a trig series using Clenshaw summation."""
    # Evaluate
    # y = sinp ? sum(c[i] * sin( 2*i    * x), i, 1, n) :
    #            sum(c[i] * cos((2*i+1) * x), i, 0, n-1)
    # using Clenshaw summation.  N.B. c[0] is unused for sin series
    # Approx operation count = (n + 5) mult and (2 * n + 2) add
    k = len(c)                  # Point to one beyond last element
    n = k - sinp
    ar = 2 * (cosx - sinx) * (cosx + sinx) # 2 * cos(2 * x)
    y1 = 0                                 # accumulators for sum
    if n & 1:
      k -= 1; y0 = c[k]
    else:
      y0 = 0
    # Now n is even
    n = n // 2
    while n:                    # while n--:
      n -= 1
      # Unroll loop x 2, so accumulators return to their original role
      k -= 1; y1 = ar * y0 - y1 + c[k]
      k -= 1; y0 = ar * y1 - y0 + c[k]
    return ( 2 * sinx * cosx * y0 if sinp # sin(2 * x) * y0
             else cosx * (y0 - y1) )      # cos(x) * (y0 - y1)

  @staticmethod
  def _Astroid(x, y):
    """Private: solve astroid equation."""
    # Solve k^4+2*k^3-(x^2+y^2-1)*k^2-2*y^2*k-y^2 = 0 for positive root k.
    # This solution is adapted from Geocentric::Reverse.
    p = Math.sq(x)
    q = Math.sq(y)
    r = (p + q - 1) / 6
    if not(q == 0 and r <= 0):
      # Avoid possible division by zero when r = 0 by multiplying equations
      # for s and t by r^3 and r, resp.
      S = p * q / 4            # S = r^3 * s
      r2 = Math.sq(r)
      r3 = r * r2
      # The discriminant of the quadratic equation for T3.  This is zero on
      # the evolute curve p^(1/3)+q^(1/3) = 1
      disc = S * (S + 2 * r3)
      u = r
      if disc >= 0:
        T3 = S + r3
        # Pick the sign on the sqrt to maximize abs(T3).  This minimizes loss
        # of precision due to cancellation.  The result is unchanged because
        # of the way the T is used in definition of u.
        T3 += -math.sqrt(disc) if T3 < 0 else math.sqrt(disc) # T3 = (r * t)^3
        # N.B. cbrt always returns the real root.  cbrt(-8) = -2.
        T = Math.cbrt(T3)       # T = r * t
        # T can be zero; but then r2 / T -> 0.
        u += T + (r2 / T if T != 0 else 0)
      else:
        # T is complex, but the way u is defined the result is real.
        ang = math.atan2(math.sqrt(-disc), -(S + r3))
        # There are three possible cube roots.  We choose the root which
        # avoids cancellation.  Note that disc < 0 implies that r < 0.
        u += 2 * r * math.cos(ang / 3)
      v = math.sqrt(Math.sq(u) + q) # guaranteed positive
      # Avoid loss of accuracy when u < 0.
      uv = q / (v - u) if u < 0 else u + v # u+v, guaranteed positive
      w = (uv - q) / (2 * v)               # positive?
      # Rearrange expression for k to avoid loss of accuracy due to
      # subtraction.  Division by 0 not possible because uv > 0, w >= 0.
      k = uv / (math.sqrt(uv + Math.sq(w)) + w) # guaranteed positive
    else:                                       # q == 0 && r <= 0
      # y = 0 with |x| <= 1.  Handle this case directly.
      # for y small, positive root is k = abs(y)/sqrt(1-x^2)
      k = 0
    return k

  @staticmethod
  def _A1m1f(eps):
    """Private: return A1-1."""
    coeff = [
      1, 4, 64, 0, 256,
    ]
    m = Geodesic.nA1_//2
    t = Math.polyval(m, coeff, 0, Math.sq(eps)) / coeff[m + 1]
    return (t + eps) / (1 - eps)

  @staticmethod
  def _C1f(eps, c):
    """Private: return C1."""
    coeff = [
      -1, 6, -16, 32,
      -9, 64, -128, 2048,
      9, -16, 768,
      3, -5, 512,
      -7, 1280,
      -7, 2048,
    ]
    eps2 = Math.sq(eps)
    d = eps
    o = 0
    for l in range(1, Geodesic.nC1_ + 1): # l is index of C1p[l]
      m = (Geodesic.nC1_ - l) // 2        # order of polynomial in eps^2
      c[l] = d * Math.polyval(m, coeff, o, eps2) / coeff[o + m + 1]
      o += m + 2
      d *= eps

  @staticmethod
  def _C1pf(eps, c):
    """Private: return C1'"""
    coeff = [
      205, -432, 768, 1536,
      4005, -4736, 3840, 12288,
      -225, 116, 384,
      -7173, 2695, 7680,
      3467, 7680,
      38081, 61440,
    ]
    eps2 = Math.sq(eps)
    d = eps
    o = 0
    for l in range(1, Geodesic.nC1p_ + 1): # l is index of C1p[l]
      m = (Geodesic.nC1p_ - l) // 2 # order of polynomial in eps^2
      c[l] = d * Math.polyval(m, coeff, o, eps2) / coeff[o + m + 1]
      o += m + 2
      d *= eps

  @staticmethod
  def _A2m1f(eps):
    """Private: return A2-1"""
    coeff = [
      -11, -28, -192, 0, 256,
    ]
    m = Geodesic.nA2_//2
    t = Math.polyval(m, coeff, 0, Math.sq(eps)) / coeff[m + 1]
    return (t - eps) / (1 + eps)

  @staticmethod
  def _C2f(eps, c):
    """Private: return C2"""
    coeff = [
      1, 2, 16, 32,
      35, 64, 384, 2048,
      15, 80, 768,
      7, 35, 512,
      63, 1280,
      77, 2048,
    ]
    eps2 = Math.sq(eps)
    d = eps
    o = 0
    for l in range(1, Geodesic.nC2_ + 1): # l is index of C2[l]
      m = (Geodesic.nC2_ - l) // 2        # order of polynomial in eps^2
      c[l] = d * Math.polyval(m, coeff, o, eps2) / coeff[o + m + 1]
      o += m + 2
      d *= eps

  def __init__(self, a, f):
    """Construct a Geodesic object

    :param a: the equatorial radius of the ellipsoid in meters
    :param f: the flattening of the ellipsoid

    An exception is thrown if *a* or the polar semi-axis *b* = *a* (1 -
    *f*) is not a finite positive quantity.

    """

    self.a = float(a)
    """The equatorial radius in meters (readonly)"""
    self.f = float(f)
    """The flattening (readonly)"""
    self._f1 = 1 - self.f
    self._e2 = self.f * (2 - self.f)
    self._ep2 = self._e2 / Math.sq(self._f1) # e2 / (1 - e2)
    self._n = self.f / ( 2 - self.f)
    self._b = self.a * self._f1
    # authalic radius squared
    self._c2 = (Math.sq(self.a) + Math.sq(self._b) *
                (1 if self._e2 == 0 else
                 (math.atanh(math.sqrt(self._e2)) if self._e2 > 0 else
                  math.atan(math.sqrt(-self._e2))) /
                 math.sqrt(abs(self._e2))))/2
    # The sig12 threshold for "really short".  Using the auxiliary sphere
    # solution with dnm computed at (bet1 + bet2) / 2, the relative error in
    # the azimuth consistency check is sig12^2 * abs(f) * min(1, 1-f/2) / 2.
    # (Error measured for 1/100 < b/a < 100 and abs(f) >= 1/1000.  For a given
    # f and sig12, the max error occurs for lines near the pole.  If the old
    # rule for computing dnm = (dn1 + dn2)/2 is used, then the error increases
    # by a factor of 2.)  Setting this equal to epsilon gives sig12 = etol2.
    # Here 0.1 is a safety factor (error decreased by 100) and max(0.001,
    # abs(f)) stops etol2 getting too large in the nearly spherical case.
    self._etol2 = 0.1 * Geodesic.tol2_ / math.sqrt( max(0.001, abs(self.f)) *
                                                    min(1.0, 1-self.f/2) / 2 )
    if not(math.isfinite(self.a) and self.a > 0):
      raise ValueError("Equatorial radius is not positive")
    if not(math.isfinite(self._b) and self._b > 0):
      raise ValueError("Polar semi-axis is not positive")
    self._A3x = list(range(Geodesic.nA3x_))
    self._C3x = list(range(Geodesic.nC3x_))
    self._C4x = list(range(Geodesic.nC4x_))
    self._A3coeff()
    self._C3coeff()
    self._C4coeff()

  def _A3coeff(self):
    """Private: return coefficients for A3"""
    coeff = [
      -3, 128,
      -2, -3, 64,
      -1, -3, -1, 16,
      3, -1, -2, 8,
      1, -1, 2,
      1, 1,
    ]
    o = 0; k = 0
    for j in range(Geodesic.nA3_ - 1, -1, -1): # coeff of eps^j
      m = min(Geodesic.nA3_ - j - 1, j) # order of polynomial in n
      self._A3x[k] = Math.polyval(m, coeff, o, self._n) / coeff[o + m + 1]
      k += 1
      o += m + 2

  def _C3coeff(self):
    """Private: return coefficients for C3"""
    coeff = [
      3, 128,
      2, 5, 128,
      -1, 3, 3, 64,
      -1, 0, 1, 8,
      -1, 1, 4,
      5, 256,
      1, 3, 128,
      -3, -2, 3, 64,
      1, -3, 2, 32,
      7, 512,
      -10, 9, 384,
      5, -9, 5, 192,
      7, 512,
      -14, 7, 512,
      21, 2560,
    ]
    o = 0; k = 0
    for l in range(1, Geodesic.nC3_): # l is index of C3[l]
      for j in range(Geodesic.nC3_ - 1, l - 1, -1): # coeff of eps^j
        m = min(Geodesic.nC3_ - j - 1, j) # order of polynomial in n
        self._C3x[k] = Math.polyval(m, coeff, o, self._n) / coeff[o + m + 1]
        k += 1
        o += m + 2

  def _C4coeff(self):
    """Private: return coefficients for C4"""
    coeff = [
      97, 15015,
      1088, 156, 45045,
      -224, -4784, 1573, 45045,
      -10656, 14144, -4576, -858, 45045,
      64, 624, -4576, 6864, -3003, 15015,
      100, 208, 572, 3432, -12012, 30030, 45045,
      1, 9009,
      -2944, 468, 135135,
      5792, 1040, -1287, 135135,
      5952, -11648, 9152, -2574, 135135,
      -64, -624, 4576, -6864, 3003, 135135,
      8, 10725,
      1856, -936, 225225,
      -8448, 4992, -1144, 225225,
      -1440, 4160, -4576, 1716, 225225,
      -136, 63063,
      1024, -208, 105105,
      3584, -3328, 1144, 315315,
      -128, 135135,
      -2560, 832, 405405,
      128, 99099,
    ]
    o = 0; k = 0
    for l in range(Geodesic.nC4_): # l is index of C4[l]
      for j in range(Geodesic.nC4_ - 1, l - 1, -1): # coeff of eps^j
        m = Geodesic.nC4_ - j - 1 # order of polynomial in n
        self._C4x[k] = Math.polyval(m, coeff, o, self._n) / coeff[o + m + 1]
        k += 1
        o += m + 2

  def _A3f(self, eps):
    """Private: return A3"""
    # Evaluate A3
    return Math.polyval(Geodesic.nA3_ - 1, self._A3x, 0, eps)

  def _C3f(self, eps, c):
    """Private: return C3"""
    # Evaluate C3
    # Elements c[1] thru c[nC3_ - 1] are set
    mult = 1
    o = 0
    for l in range(1, Geodesic.nC3_): # l is index of C3[l]
      m = Geodesic.nC3_ - l - 1       # order of polynomial in eps
      mult *= eps
      c[l] = mult * Math.polyval(m, self._C3x, o, eps)
      o += m + 1

  def _C4f(self, eps, c):
    """Private: return C4"""
    # Evaluate C4 coeffs by Horner's method
    # Elements c[0] thru c[nC4_ - 1] are set
    mult = 1
    o = 0
    for l in range(Geodesic.nC4_): # l is index of C4[l]
      m = Geodesic.nC4_ - l - 1    # order of polynomial in eps
      c[l] = mult * Math.polyval(m, self._C4x, o, eps)
      o += m + 1
      mult *= eps

  # return s12b, m12b, m0, M12, M21
  def _Lengths(self, eps, sig12,
               ssig1, csig1, dn1, ssig2, csig2, dn2, cbet1, cbet2, outmask,
               # Scratch areas of the right size
               C1a, C2a):
    """Private: return a bunch of lengths"""
    # Return s12b, m12b, m0, M12, M21, where
    # m12b = (reduced length)/_b; s12b = distance/_b,
    # m0 = coefficient of secular term in expression for reduced length.
    outmask &= Geodesic.OUT_MASK
    # outmask & DISTANCE: set s12b
    # outmask & REDUCEDLENGTH: set m12b & m0
    # outmask & GEODESICSCALE: set M12 & M21

    s12b = m12b = m0 = M12 = M21 = math.nan
    if outmask & (Geodesic.DISTANCE | Geodesic.REDUCEDLENGTH |
                  Geodesic.GEODESICSCALE):
      A1 = Geodesic._A1m1f(eps)
      Geodesic._C1f(eps, C1a)
      if outmask & (Geodesic.REDUCEDLENGTH | Geodesic.GEODESICSCALE):
        A2 = Geodesic._A2m1f(eps)
        Geodesic._C2f(eps, C2a)
        m0x = A1 - A2
        A2 = 1 + A2
      A1 = 1 + A1
    if outmask & Geodesic.DISTANCE:
      B1 = (Geodesic._SinCosSeries(True, ssig2, csig2, C1a) -
            Geodesic._SinCosSeries(True, ssig1, csig1, C1a))
      # Missing a factor of _b
      s12b = A1 * (sig12 + B1)
      if outmask & (Geodesic.REDUCEDLENGTH | Geodesic.GEODESICSCALE):
        B2 = (Geodesic._SinCosSeries(True, ssig2, csig2, C2a) -
              Geodesic._SinCosSeries(True, ssig1, csig1, C2a))
        J12 = m0x * sig12 + (A1 * B1 - A2 * B2)
    elif outmask & (Geodesic.REDUCEDLENGTH | Geodesic.GEODESICSCALE):
      # Assume here that nC1_ >= nC2_
      for l in range(1, Geodesic.nC2_):
        C2a[l] = A1 * C1a[l] - A2 * C2a[l]
      J12 = m0x * sig12 + (Geodesic._SinCosSeries(True, ssig2, csig2, C2a) -
                           Geodesic._SinCosSeries(True, ssig1, csig1, C2a))
    if outmask & Geodesic.REDUCEDLENGTH:
      m0 = m0x
      # Missing a factor of _b.
      # Add parens around (csig1 * ssig2) and (ssig1 * csig2) to ensure
      # accurate cancellation in the case of coincident points.
      m12b = (dn2 * (csig1 * ssig2) - dn1 * (ssig1 * csig2) -
              csig1 * csig2 * J12)
    if outmask & Geodesic.GEODESICSCALE:
      csig12 = csig1 * csig2 + ssig1 * ssig2
      t = self._ep2 * (cbet1 - cbet2) * (cbet1 + cbet2) / (dn1 + dn2)
      M12 = csig12 + (t * ssig2 - csig2 * J12) * ssig1 / dn1
      M21 = csig12 - (t * ssig1 - csig1 * J12) * ssig2 / dn2
    return s12b, m12b, m0, M12, M21

  # return sig12, salp1, calp1, salp2, calp2, dnm
  def _InverseStart(self, sbet1, cbet1, dn1, sbet2, cbet2, dn2,
                    lam12, slam12, clam12,
                    # Scratch areas of the right size
                    C1a, C2a):
    """Private: Find a starting value for Newton's method."""
    # Return a starting point for Newton's method in salp1 and calp1 (function
    # value is -1).  If Newton's method doesn't need to be used, return also
    # salp2 and calp2 and function value is sig12.
    sig12 = -1; salp2 = calp2 = dnm = math.nan # Return values
    # bet12 = bet2 - bet1 in [0, pi); bet12a = bet2 + bet1 in (-pi, 0]
    sbet12 = sbet2 * cbet1 - cbet2 * sbet1
    cbet12 = cbet2 * cbet1 + sbet2 * sbet1
    # Volatile declaration needed to fix inverse cases
    # 88.202499451857 0 -88.202499451857 179.981022032992859592
    # 89.262080389218 0 -89.262080389218 179.992207982775375662
    # 89.333123580033 0 -89.333123580032997687 179.99295812360148422
    # which otherwise fail with g++ 4.4.4 x86 -O3
    sbet12a = sbet2 * cbet1
    sbet12a += cbet2 * sbet1

    shortline = cbet12 >= 0 and sbet12 < 0.5 and cbet2 * lam12 < 0.5
    if shortline:
      sbetm2 = Math.sq(sbet1 + sbet2)
      # sin((bet1+bet2)/2)^2
      # =  (sbet1 + sbet2)^2 / ((sbet1 + sbet2)^2 + (cbet1 + cbet2)^2)
      sbetm2 /= sbetm2 + Math.sq(cbet1 + cbet2)
      dnm = math.sqrt(1 + self._ep2 * sbetm2)
      omg12 = lam12 / (self._f1 * dnm)
      somg12 = math.sin(omg12); comg12 = math.cos(omg12)
    else:
      somg12 = slam12; comg12 = clam12

    salp1 = cbet2 * somg12
    calp1 = (
      sbet12 + cbet2 * sbet1 * Math.sq(somg12) / (1 + comg12) if comg12 >= 0
      else sbet12a - cbet2 * sbet1 * Math.sq(somg12) / (1 - comg12))

    ssig12 = math.hypot(salp1, calp1)
    csig12 = sbet1 * sbet2 + cbet1 * cbet2 * comg12

    if shortline and ssig12 < self._etol2:
      # really short lines
      salp2 = cbet1 * somg12
      calp2 = sbet12 - cbet1 * sbet2 * (Math.sq(somg12) / (1 + comg12)
                                        if comg12 >= 0 else 1 - comg12)
      salp2, calp2 = Math.norm(salp2, calp2)
      # Set return value
      sig12 = math.atan2(ssig12, csig12)
    elif (abs(self._n) >= 0.1 or # Skip astroid calc if too eccentric
          csig12 >= 0 or
          ssig12 >= 6 * abs(self._n) * math.pi * Math.sq(cbet1)):
      # Nothing to do, zeroth order spherical approximation is OK
      pass
    else:
      # Scale lam12 and bet2 to x, y coordinate system where antipodal point
      # is at origin and singular point is at y = 0, x = -1.
      # real y, lamscale, betscale
      lam12x = math.atan2(-slam12, -clam12)
      if self.f >= 0:            # In fact f == 0 does not get here
        # x = dlong, y = dlat
        k2 = Math.sq(sbet1) * self._ep2
        eps = k2 / (2 * (1 + math.sqrt(1 + k2)) + k2)
        lamscale = self.f * cbet1 * self._A3f(eps) * math.pi
        betscale = lamscale * cbet1
        x = lam12x / lamscale
        y = sbet12a / betscale
      else:                     # _f < 0
        # x = dlat, y = dlong
        cbet12a = cbet2 * cbet1 - sbet2 * sbet1
        bet12a = math.atan2(sbet12a, cbet12a)
        # real m12b, m0, dummy
        # In the case of lon12 = 180, this repeats a calculation made in
        # Inverse.
        dummy, m12b, m0, dummy, dummy = self._Lengths(
          self._n, math.pi + bet12a, sbet1, -cbet1, dn1, sbet2, cbet2, dn2,
          cbet1, cbet2, Geodesic.REDUCEDLENGTH, C1a, C2a)
        x = -1 + m12b / (cbet1 * cbet2 * m0 * math.pi)
        betscale = (sbet12a / x if x < -0.01
                    else -self.f * Math.sq(cbet1) * math.pi)
        lamscale = betscale / cbet1
        y = lam12x / lamscale

      if y > -Geodesic.tol1_ and x > -1 - Geodesic.xthresh_:
        # strip near cut
        if self.f >= 0:
          salp1 = min(1.0, -x); calp1 = - math.sqrt(1 - Math.sq(salp1))
        else:
          calp1 = max((0.0 if x > -Geodesic.tol1_ else -1.0), x)
          salp1 = math.sqrt(1 - Math.sq(calp1))
      else:
        # Estimate alp1, by solving the astroid problem.
        #
        # Could estimate alpha1 = theta + pi/2, directly, i.e.,
        #   calp1 = y/k; salp1 = -x/(1+k);  for _f >= 0
        #   calp1 = x/(1+k); salp1 = -y/k;  for _f < 0 (need to check)
        #
        # However, it's better to estimate omg12 from astroid and use
        # spherical formula to compute alp1.  This reduces the mean number of
        # Newton iterations for astroid cases from 2.24 (min 0, max 6) to 2.12
        # (min 0 max 5).  The changes in the number of iterations are as
        # follows:
        #
        # change percent
        #    1       5
        #    0      78
        #   -1      16
        #   -2       0.6
        #   -3       0.04
        #   -4       0.002
        #
        # The histogram of iterations is (m = number of iterations estimating
        # alp1 directly, n = number of iterations estimating via omg12, total
        # number of trials = 148605):
        #
        #  iter    m      n
        #    0   148    186
        #    1 13046  13845
        #    2 93315 102225
        #    3 36189  32341
        #    4  5396      7
        #    5   455      1
        #    6    56      0
        #
        # Because omg12 is near pi, estimate work with omg12a = pi - omg12
        k = Geodesic._Astroid(x, y)
        omg12a = lamscale * ( -x * k/(1 + k) if self.f >= 0
                              else -y * (1 + k)/k )
        somg12 = math.sin(omg12a); comg12 = -math.cos(omg12a)
        # Update spherical estimate of alp1 using omg12 instead of lam12
        salp1 = cbet2 * somg12
        calp1 = sbet12a - cbet2 * sbet1 * Math.sq(somg12) / (1 - comg12)
    # Sanity check on starting guess.  Backwards check allows NaN through.
    if not (salp1 <= 0):
      salp1, calp1 = Math.norm(salp1, calp1)
    else:
      salp1 = 1; calp1 = 0
    return sig12, salp1, calp1, salp2, calp2, dnm

  # return lam12, salp2, calp2, sig12, ssig1, csig1, ssig2, csig2, eps,
  # domg12, dlam12
  def _Lambda12(self, sbet1, cbet1, dn1, sbet2, cbet2, dn2, salp1, calp1,
                slam120, clam120, diffp,
                # Scratch areas of the right size
                C1a, C2a, C3a):
    """Private: Solve hybrid problem"""
    if sbet1 == 0 and calp1 == 0:
      # Break degeneracy of equatorial line.  This case has already been
      # handled.
      calp1 = -Geodesic.tiny_

    # sin(alp1) * cos(bet1) = sin(alp0)
    salp0 = salp1 * cbet1
    calp0 = math.hypot(calp1, salp1 * sbet1) # calp0 > 0

    # real somg1, comg1, somg2, comg2, lam12
    # tan(bet1) = tan(sig1) * cos(alp1)
    # tan(omg1) = sin(alp0) * tan(sig1) = tan(omg1)=tan(alp1)*sin(bet1)
    ssig1 = sbet1; somg1 = salp0 * sbet1
    csig1 = comg1 = calp1 * cbet1
    ssig1, csig1 = Math.norm(ssig1, csig1)
    # Math.norm(somg1, comg1); -- don't need to normalize!

    # Enforce symmetries in the case abs(bet2) = -bet1.  Need to be careful
    # about this case, since this can yield singularities in the Newton
    # iteration.
    # sin(alp2) * cos(bet2) = sin(alp0)
    salp2 = salp0 / cbet2 if cbet2 != cbet1 else salp1
    # calp2 = sqrt(1 - sq(salp2))
    #       = sqrt(sq(calp0) - sq(sbet2)) / cbet2
    # and subst for calp0 and rearrange to give (choose positive sqrt
    # to give alp2 in [0, pi/2]).
    calp2 = (math.sqrt(Math.sq(calp1 * cbet1) +
                       ((cbet2 - cbet1) * (cbet1 + cbet2) if cbet1 < -sbet1
                        else (sbet1 - sbet2) * (sbet1 + sbet2))) / cbet2
             if cbet2 != cbet1 or abs(sbet2) != -sbet1 else abs(calp1))
    # tan(bet2) = tan(sig2) * cos(alp2)
    # tan(omg2) = sin(alp0) * tan(sig2).
    ssig2 = sbet2; somg2 = salp0 * sbet2
    csig2 = comg2 = calp2 * cbet2
    ssig2, csig2 = Math.norm(ssig2, csig2)
    # Math.norm(somg2, comg2); -- don't need to normalize!

    # sig12 = sig2 - sig1, limit to [0, pi]
    sig12 = math.atan2(max(0.0, csig1 * ssig2 - ssig1 * csig2) + 0.0,
                                csig1 * csig2 + ssig1 * ssig2)

    # omg12 = omg2 - omg1, limit to [0, pi]
    somg12 = max(0.0, comg1 * somg2 - somg1 * comg2) + 0.0
    comg12 =          comg1 * comg2 + somg1 * somg2
    # eta = omg12 - lam120
    eta = math.atan2(somg12 * clam120 - comg12 * slam120,
                     comg12 * clam120 + somg12 * slam120)

    # real B312
    k2 = Math.sq(calp0) * self._ep2
    eps = k2 / (2 * (1 + math.sqrt(1 + k2)) + k2)
    self._C3f(eps, C3a)
    B312 = (Geodesic._SinCosSeries(True, ssig2, csig2, C3a) -
            Geodesic._SinCosSeries(True, ssig1, csig1, C3a))
    domg12 =  -self.f * self._A3f(eps) * salp0 * (sig12 + B312)
    lam12 = eta + domg12

    if diffp:
      if calp2 == 0:
        dlam12 = - 2 * self._f1 * dn1 / sbet1
      else:
        dummy, dlam12, dummy, dummy, dummy = self._Lengths(
          eps, sig12, ssig1, csig1, dn1, ssig2, csig2, dn2, cbet1, cbet2,
          Geodesic.REDUCEDLENGTH, C1a, C2a)
        dlam12 *= self._f1 / (calp2 * cbet2)
    else:
      dlam12 = math.nan

    return (lam12, salp2, calp2, sig12, ssig1, csig1, ssig2, csig2, eps,
            domg12, dlam12)

  # return a12, s12, salp1, calp1, salp2, calp2, m12, M12, M21, S12
  def _GenInverse(self, lat1, lon1, lat2, lon2, outmask):
    """Private: General version of the inverse problem"""
    a12 = s12 = m12 = M12 = M21 = S12 = math.nan # return vals

    outmask &= Geodesic.OUT_MASK
    # Compute longitude difference (AngDiff does this carefully).  Result is
    # in [-180, 180] but -180 is only for west-going geodesics.  180 is for
    # east-going and meridional geodesics.
    lon12, lon12s = Math.AngDiff(lon1, lon2)
    # Make longitude difference positive.
    lonsign = math.copysign(1, lon12)
    lon12 = lonsign * lon12; lon12s = lonsign * lon12s
    lam12 = math.radians(lon12)
    # Calculate sincos of lon12 + error (this applies AngRound internally).
    slam12, clam12 = Math.sincosde(lon12, lon12s)
    lon12s = (180 - lon12) - lon12s # the supplementary longitude difference

    # If really close to the equator, treat as on equator.
    lat1 = Math.AngRound(Math.LatFix(lat1))
    lat2 = Math.AngRound(Math.LatFix(lat2))
    # Swap points so that point with higher (abs) latitude is point 1
    # If one latitude is a nan, then it becomes lat1.
    swapp = -1 if abs(lat1) < abs(lat2) or math.isnan(lat2) else 1
    if swapp < 0:
      lonsign *= -1
      lat2, lat1 = lat1, lat2
    # Make lat1 <= 0
    latsign = math.copysign(1, -lat1)
    lat1 *= latsign
    lat2 *= latsign
    # Now we have
    #
    #     0 <= lon12 <= 180
    #     -90 <= lat1 <= 0
    #     lat1 <= lat2 <= -lat1
    #
    # longsign, swapp, latsign register the transformation to bring the
    # coordinates to this canonical form.  In all cases, 1 means no change was
    # made.  We make these transformations so that there are few cases to
    # check, e.g., on verifying quadrants in atan2.  In addition, this
    # enforces some symmetries in the results returned.

    # real phi, sbet1, cbet1, sbet2, cbet2, s12x, m12x

    sbet1, cbet1 = Math.sincosd(lat1); sbet1 *= self._f1
    # Ensure cbet1 = +epsilon at poles
    sbet1, cbet1 = Math.norm(sbet1, cbet1); cbet1 = max(Geodesic.tiny_, cbet1)

    sbet2, cbet2 = Math.sincosd(lat2); sbet2 *= self._f1
    # Ensure cbet2 = +epsilon at poles
    sbet2, cbet2 = Math.norm(sbet2, cbet2); cbet2 = max(Geodesic.tiny_, cbet2)

    # If cbet1 < -sbet1, then cbet2 - cbet1 is a sensitive measure of the
    # |bet1| - |bet2|.  Alternatively (cbet1 >= -sbet1), abs(sbet2) + sbet1 is
    # a better measure.  This logic is used in assigning calp2 in Lambda12.
    # Sometimes these quantities vanish and in that case we force bet2 = +/-
    # bet1 exactly.  An example where is is necessary is the inverse problem
    # 48.522876735459 0 -48.52287673545898293 179.599720456223079643
    # which failed with Visual Studio 10 (Release and Debug)

    if cbet1 < -sbet1:
      if cbet2 == cbet1:
        sbet2 = math.copysign(sbet1, sbet2)
    else:
      if abs(sbet2) == -sbet1:
        cbet2 = cbet1

    dn1 = math.sqrt(1 + self._ep2 * Math.sq(sbet1))
    dn2 = math.sqrt(1 + self._ep2 * Math.sq(sbet2))

    # real a12, sig12, calp1, salp1, calp2, salp2
    # index zero elements of these arrays are unused
    C1a = list(range(Geodesic.nC1_ + 1))
    C2a = list(range(Geodesic.nC2_ + 1))
    C3a = list(range(Geodesic.nC3_))

    meridian = lat1 == -90 or slam12 == 0

    if meridian:

      # Endpoints are on a single full meridian, so the geodesic might lie on
      # a meridian.

      calp1 = clam12; salp1 = slam12 # Head to the target longitude
      calp2 = 1.0; salp2 = 0.0       # At the target we're heading north

      # tan(bet) = tan(sig) * cos(alp)
      ssig1 = sbet1; csig1 = calp1 * cbet1
      ssig2 = sbet2; csig2 = calp2 * cbet2

      # sig12 = sig2 - sig1
      sig12 = math.atan2(max(0.0, csig1 * ssig2 - ssig1 * csig2) + 0.0,
                                  csig1 * csig2 + ssig1 * ssig2)

      s12x, m12x, dummy, M12, M21 = self._Lengths(
        self._n, sig12, ssig1, csig1, dn1, ssig2, csig2, dn2, cbet1, cbet2,
        outmask | Geodesic.DISTANCE | Geodesic.REDUCEDLENGTH, C1a, C2a)

      # Add the check for sig12 since zero length geodesics might yield m12 <
      # 0.  Test case was
      #
      #    echo 20.001 0 20.001 0 | GeodSolve -i
      #
      # In fact, we will have sig12 > pi/2 for meridional geodesic which is
      # not a shortest path.
      if sig12 < 1 or m12x >= 0:
        if (sig12 < 3 * Geodesic.tiny_ or
            # Prevent negative s12 or m12 for short lines
            (sig12 < Geodesic.tol0_ and (s12x < 0 or m12x < 0))):
          sig12 = m12x = s12x = 0.0
        m12x *= self._b
        s12x *= self._b
        a12 = math.degrees(sig12)
      else:
        # m12 < 0, i.e., prolate and too close to anti-podal
        meridian = False
    # end if meridian:

    # somg12 == 2 marks that it needs to be calculated
    somg12 = 2.0; comg12 = 0.0; omg12 = 0.0
    if (not meridian and
        sbet1 == 0 and   # and sbet2 == 0
        # Mimic the way Lambda12 works with calp1 = 0
        (self.f <= 0 or lon12s >= self.f * 180)):

      # Geodesic runs along equator
      calp1 = calp2 = 0.0; salp1 = salp2 = 1.0
      s12x = self.a * lam12
      sig12 = omg12 = lam12 / self._f1
      m12x = self._b * math.sin(sig12)
      if outmask & Geodesic.GEODESICSCALE:
        M12 = M21 = math.cos(sig12)
      a12 = lon12 / self._f1

    elif not meridian:

      # Now point1 and point2 belong within a hemisphere bounded by a
      # meridian and geodesic is neither meridional or equatorial.

      # Figure a starting point for Newton's method
      sig12, salp1, calp1, salp2, calp2, dnm = self._InverseStart(
        sbet1, cbet1, dn1, sbet2, cbet2, dn2, lam12, slam12, clam12, C1a, C2a)

      if sig12 >= 0:
        # Short lines (InverseStart sets salp2, calp2, dnm)
        s12x = sig12 * self._b * dnm
        m12x = (Math.sq(dnm) * self._b * math.sin(sig12 / dnm))
        if outmask & Geodesic.GEODESICSCALE:
          M12 = M21 = math.cos(sig12 / dnm)
        a12 = math.degrees(sig12)
        omg12 = lam12 / (self._f1 * dnm)
      else:

        # Newton's method.  This is a straightforward solution of f(alp1) =
        # lambda12(alp1) - lam12 = 0 with one wrinkle.  f(alp) has exactly one
        # root in the interval (0, pi) and its derivative is positive at the
        # root.  Thus f(alp) is positive for alp > alp1 and negative for alp <
        # alp1.  During the course of the iteration, a range (alp1a, alp1b) is
        # maintained which brackets the root and with each evaluation of f(alp)
        # the range is shrunk if possible.  Newton's method is restarted
        # whenever the derivative of f is negative (because the new value of
        # alp1 is then further from the solution) or if the new estimate of
        # alp1 lies outside (0,pi); in this case, the new starting guess is
        # taken to be (alp1a + alp1b) / 2.
        # real ssig1, csig1, ssig2, csig2, eps
        numit = 0
        tripn = tripb = False
        # Bracketing range
        salp1a = Geodesic.tiny_; calp1a = 1.0
        salp1b = Geodesic.tiny_; calp1b = -1.0

        while numit < Geodesic.maxit2_:
          # the WGS84 test set: mean = 1.47, sd = 1.25, max = 16
          # WGS84 and random input: mean = 2.85, sd = 0.60
          (v, salp2, calp2, sig12, ssig1, csig1, ssig2, csig2,
           eps, domg12, dv) = self._Lambda12(
             sbet1, cbet1, dn1, sbet2, cbet2, dn2,
             salp1, calp1, slam12, clam12, numit < Geodesic.maxit1_,
             C1a, C2a, C3a)
          # Reversed test to allow escape with NaNs
          if tripb or not (abs(v) >= (8 if tripn else 1) * Geodesic.tol0_):
            break
          # Update bracketing values
          if v > 0 and (numit > Geodesic.maxit1_ or
                        calp1/salp1 > calp1b/salp1b):
            salp1b = salp1; calp1b = calp1
          elif v < 0 and (numit > Geodesic.maxit1_ or
                          calp1/salp1 < calp1a/salp1a):
            salp1a = salp1; calp1a = calp1

          numit += 1
          if numit < Geodesic.maxit1_ and dv > 0:
            dalp1 = -v/dv
            sdalp1 = math.sin(dalp1); cdalp1 = math.cos(dalp1)
            nsalp1 = salp1 * cdalp1 + calp1 * sdalp1
            if nsalp1 > 0 and abs(dalp1) < math.pi:
              calp1 = calp1 * cdalp1 - salp1 * sdalp1
              salp1 = nsalp1
              salp1, calp1 = Math.norm(salp1, calp1)
              # In some regimes we don't get quadratic convergence because
              # slope -> 0.  So use convergence conditions based on epsilon
              # instead of sqrt(epsilon).
              tripn = abs(v) <= 16 * Geodesic.tol0_
              continue
          # Either dv was not positive or updated value was outside
          # legal range.  Use the midpoint of the bracket as the next
          # estimate.  This mechanism is not needed for the WGS84
          # ellipsoid, but it does catch problems with more eccentric
          # ellipsoids.  Its efficacy is such for
          # the WGS84 test set with the starting guess set to alp1 = 90deg:
          # the WGS84 test set: mean = 5.21, sd = 3.93, max = 24
          # WGS84 and random input: mean = 4.74, sd = 0.99
          salp1 = (salp1a + salp1b)/2
          calp1 = (calp1a + calp1b)/2
          salp1, calp1 = Math.norm(salp1, calp1)
          tripn = False
          tripb = (abs(salp1a - salp1) + (calp1a - calp1) < Geodesic.tolb_ or
                   abs(salp1 - salp1b) + (calp1 - calp1b) < Geodesic.tolb_)

        lengthmask = (outmask |
                      (Geodesic.DISTANCE
                       if (outmask & (Geodesic.REDUCEDLENGTH |
                                      Geodesic.GEODESICSCALE))
                       else Geodesic.EMPTY))
        s12x, m12x, dummy, M12, M21 = self._Lengths(
          eps, sig12, ssig1, csig1, dn1, ssig2, csig2, dn2, cbet1, cbet2,
          lengthmask, C1a, C2a)

        m12x *= self._b
        s12x *= self._b
        a12 = math.degrees(sig12)
        if outmask & Geodesic.AREA:
          # omg12 = lam12 - domg12
          sdomg12 = math.sin(domg12); cdomg12 = math.cos(domg12)
          somg12 = slam12 * cdomg12 - clam12 * sdomg12
          comg12 = clam12 * cdomg12 + slam12 * sdomg12

    # end elif not meridian

    if outmask & Geodesic.DISTANCE:
      s12 = 0.0 + s12x          # Convert -0 to 0

    if outmask & Geodesic.REDUCEDLENGTH:
      m12 = 0.0 + m12x          # Convert -0 to 0

    if outmask & Geodesic.AREA:
      # From Lambda12: sin(alp1) * cos(bet1) = sin(alp0)
      salp0 = salp1 * cbet1
      calp0 = math.hypot(calp1, salp1 * sbet1) # calp0 > 0
      # real alp12
      if calp0 != 0 and salp0 != 0:
        # From Lambda12: tan(bet) = tan(sig) * cos(alp)
        ssig1 = sbet1; csig1 = calp1 * cbet1
        ssig2 = sbet2; csig2 = calp2 * cbet2
        k2 = Math.sq(calp0) * self._ep2
        eps = k2 / (2 * (1 + math.sqrt(1 + k2)) + k2)
        # Multiplier = a^2 * e^2 * cos(alpha0) * sin(alpha0).
        A4 = Math.sq(self.a) * calp0 * salp0 * self._e2
        ssig1, csig1 = Math.norm(ssig1, csig1)
        ssig2, csig2 = Math.norm(ssig2, csig2)
        C4a = list(range(Geodesic.nC4_))
        self._C4f(eps, C4a)
        B41 = Geodesic._SinCosSeries(False, ssig1, csig1, C4a)
        B42 = Geodesic._SinCosSeries(False, ssig2, csig2, C4a)
        S12 = A4 * (B42 - B41)
      else:
        # Avoid problems with indeterminate sig1, sig2 on equator
        S12 = 0.0

      if not meridian and somg12 == 2.0:
        somg12 = math.sin(omg12); comg12 = math.cos(omg12)

      if (not meridian and
          # omg12 < 3/4 * pi
          comg12 > -0.7071 and   # Long difference not too big
          sbet2 - sbet1 < 1.75): # Lat difference not too big
        # Use tan(Gamma/2) = tan(omg12/2)
        # * (tan(bet1/2)+tan(bet2/2))/(1+tan(bet1/2)*tan(bet2/2))
        # with tan(x/2) = sin(x)/(1+cos(x))
        domg12 = 1 + comg12; dbet1 = 1 + cbet1; dbet2 = 1 + cbet2
        alp12 = 2 * math.atan2( somg12 * ( sbet1 * dbet2 + sbet2 * dbet1 ),
                                domg12 * ( sbet1 * sbet2 + dbet1 * dbet2 ) )
      else:
        # alp12 = alp2 - alp1, used in atan2 so no need to normalize
        salp12 = salp2 * calp1 - calp2 * salp1
        calp12 = calp2 * calp1 + salp2 * salp1
        # The right thing appears to happen if alp1 = +/-180 and alp2 = 0, viz
        # salp12 = -0 and alp12 = -180.  However this depends on the sign
        # being attached to 0 correctly.  The following ensures the correct
        # behavior.
        if salp12 == 0 and calp12 < 0:
          salp12 = Geodesic.tiny_ * calp1
          calp12 = -1.0
        alp12 = math.atan2(salp12, calp12)
      S12 += self._c2 * alp12
      S12 *= swapp * lonsign * latsign
      # Convert -0 to 0
      S12 += 0.0

    # Convert calp, salp to azimuth accounting for lonsign, swapp, latsign.
    if swapp < 0:
      salp2, salp1 = salp1, salp2
      calp2, calp1 = calp1, calp2
      if outmask & Geodesic.GEODESICSCALE:
        M21, M12 = M12, M21

    salp1 *= swapp * lonsign; calp1 *= swapp * latsign
    salp2 *= swapp * lonsign; calp2 *= swapp * latsign

    return a12, s12, salp1, calp1, salp2, calp2, m12, M12, M21, S12

  def Inverse(self, lat1, lon1, lat2, lon2,
              outmask = GeodesicCapability.STANDARD):
    """Solve the inverse geodesic problem

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param lat2: latitude of the second point in degrees
    :param lon2: longitude of the second point in degrees
    :param outmask: the :ref:`output mask <outmask>`
    :return: a :ref:`dict`

    Compute geodesic between (*lat1*, *lon1*) and (*lat2*, *lon2*).
    The default value of *outmask* is STANDARD, i.e., the *lat1*,
    *lon1*, *azi1*, *lat2*, *lon2*, *azi2*, *s12*, *a12* entries are
    returned.

    """

    a12, s12, salp1,calp1, salp2,calp2, m12, M12, M21, S12 = self._GenInverse(
      lat1, lon1, lat2, lon2, outmask)
    outmask &= Geodesic.OUT_MASK
    if outmask & Geodesic.LONG_UNROLL:
      lon12, e = Math.AngDiff(lon1, lon2)
      lon2 = (lon1 + lon12) + e
    else:
      lon2 = Math.AngNormalize(lon2)
    result = {'lat1': Math.LatFix(lat1),
              'lon1': lon1 if outmask & Geodesic.LONG_UNROLL else
              Math.AngNormalize(lon1),
              'lat2': Math.LatFix(lat2),
              'lon2': lon2}
    result['a12'] = a12
    if outmask & Geodesic.DISTANCE: result['s12'] = s12
    if outmask & Geodesic.AZIMUTH:
      result['azi1'] = Math.atan2d(salp1, calp1)
      result['azi2'] = Math.atan2d(salp2, calp2)
    if outmask & Geodesic.REDUCEDLENGTH: result['m12'] = m12
    if outmask & Geodesic.GEODESICSCALE:
      result['M12'] = M12; result['M21'] = M21
    if outmask & Geodesic.AREA: result['S12'] = S12
    return result

  # return a12, lat2, lon2, azi2, s12, m12, M12, M21, S12
  def _GenDirect(self, lat1, lon1, azi1, arcmode, s12_a12, outmask):
    """Private: General version of direct problem"""
    from geographiclib.geodesicline import GeodesicLine
    # Automatically supply DISTANCE_IN if necessary
    if not arcmode: outmask |= Geodesic.DISTANCE_IN
    line = GeodesicLine(self, lat1, lon1, azi1, outmask)
    return line._GenPosition(arcmode, s12_a12, outmask)

  def Direct(self, lat1, lon1, azi1, s12,
             outmask = GeodesicCapability.STANDARD):
    """Solve the direct geodesic problem

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param azi1: azimuth at the first point in degrees
    :param s12: the distance from the first point to the second in
      meters
    :param outmask: the :ref:`output mask <outmask>`
    :return: a :ref:`dict`

    Compute geodesic starting at (*lat1*, *lon1*) with azimuth *azi1*
    and length *s12*.  The default value of *outmask* is STANDARD, i.e.,
    the *lat1*, *lon1*, *azi1*, *lat2*, *lon2*, *azi2*, *s12*, *a12*
    entries are returned.

    """

    a12, lat2, lon2, azi2, s12, m12, M12, M21, S12 = self._GenDirect(
      lat1, lon1, azi1, False, s12, outmask)
    outmask &= Geodesic.OUT_MASK
    result = {'lat1': Math.LatFix(lat1),
              'lon1': lon1 if outmask & Geodesic.LONG_UNROLL else
              Math.AngNormalize(lon1),
              'azi1': Math.AngNormalize(azi1),
              's12': s12}
    result['a12'] = a12
    if outmask & Geodesic.LATITUDE: result['lat2'] = lat2
    if outmask & Geodesic.LONGITUDE: result['lon2'] = lon2
    if outmask & Geodesic.AZIMUTH: result['azi2'] = azi2
    if outmask & Geodesic.REDUCEDLENGTH: result['m12'] = m12
    if outmask & Geodesic.GEODESICSCALE:
      result['M12'] = M12; result['M21'] = M21
    if outmask & Geodesic.AREA: result['S12'] = S12
    return result

  def ArcDirect(self, lat1, lon1, azi1, a12,
                outmask = GeodesicCapability.STANDARD):
    """Solve the direct geodesic problem in terms of spherical arc length

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param azi1: azimuth at the first point in degrees
    :param a12: spherical arc length from the first point to the second
      in degrees
    :param outmask: the :ref:`output mask <outmask>`
    :return: a :ref:`dict`

    Compute geodesic starting at (*lat1*, *lon1*) with azimuth *azi1*
    and arc length *a12*.  The default value of *outmask* is STANDARD,
    i.e., the *lat1*, *lon1*, *azi1*, *lat2*, *lon2*, *azi2*, *s12*,
    *a12* entries are returned.

    """

    a12, lat2, lon2, azi2, s12, m12, M12, M21, S12 = self._GenDirect(
      lat1, lon1, azi1, True, a12, outmask)
    outmask &= Geodesic.OUT_MASK
    result = {'lat1': Math.LatFix(lat1),
              'lon1': lon1 if outmask & Geodesic.LONG_UNROLL else
              Math.AngNormalize(lon1),
              'azi1': Math.AngNormalize(azi1),
              'a12': a12}
    if outmask & Geodesic.DISTANCE: result['s12'] = s12
    if outmask & Geodesic.LATITUDE: result['lat2'] = lat2
    if outmask & Geodesic.LONGITUDE: result['lon2'] = lon2
    if outmask & Geodesic.AZIMUTH: result['azi2'] = azi2
    if outmask & Geodesic.REDUCEDLENGTH: result['m12'] = m12
    if outmask & Geodesic.GEODESICSCALE:
      result['M12'] = M12; result['M21'] = M21
    if outmask & Geodesic.AREA: result['S12'] = S12
    return result

  def Line(self, lat1, lon1, azi1,
           caps = GeodesicCapability.STANDARD |
           GeodesicCapability.DISTANCE_IN):
    """Return a GeodesicLine object

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param azi1: azimuth at the first point in degrees
    :param caps: the :ref:`capabilities <outmask>`
    :return: a :class:`~geographiclib.geodesicline.GeodesicLine`

    This allows points along a geodesic starting at (*lat1*, *lon1*),
    with azimuth *azi1* to be found.  The default value of *caps* is
    STANDARD | DISTANCE_IN, allowing direct geodesic problem to be
    solved.

    """

    from geographiclib.geodesicline import GeodesicLine
    return GeodesicLine(self, lat1, lon1, azi1, caps)

  def _GenDirectLine(self, lat1, lon1, azi1, arcmode, s12_a12,
                     caps = GeodesicCapability.STANDARD |
                     GeodesicCapability.DISTANCE_IN):
    """Private: general form of DirectLine"""
    from geographiclib.geodesicline import GeodesicLine
    # Automatically supply DISTANCE_IN if necessary
    if not arcmode: caps |= Geodesic.DISTANCE_IN
    line = GeodesicLine(self, lat1, lon1, azi1, caps)
    if arcmode:
      line.SetArc(s12_a12)
    else:
      line.SetDistance(s12_a12)
    return line

  def DirectLine(self, lat1, lon1, azi1, s12,
                 caps = GeodesicCapability.STANDARD |
                 GeodesicCapability.DISTANCE_IN):
    """Define a GeodesicLine object in terms of the direct geodesic
    problem specified in terms of spherical arc length

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param azi1: azimuth at the first point in degrees
    :param s12: the distance from the first point to the second in
      meters
    :param caps: the :ref:`capabilities <outmask>`
    :return: a :class:`~geographiclib.geodesicline.GeodesicLine`

    This function sets point 3 of the GeodesicLine to correspond to
    point 2 of the direct geodesic problem.  The default value of *caps*
    is STANDARD | DISTANCE_IN, allowing direct geodesic problem to be
    solved.

    """

    return self._GenDirectLine(lat1, lon1, azi1, False, s12, caps)

  def ArcDirectLine(self, lat1, lon1, azi1, a12,
                 caps = GeodesicCapability.STANDARD |
                 GeodesicCapability.DISTANCE_IN):
    """Define a GeodesicLine object in terms of the direct geodesic
    problem specified in terms of spherical arc length

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param azi1: azimuth at the first point in degrees
    :param a12: spherical arc length from the first point to the second
      in degrees
    :param caps: the :ref:`capabilities <outmask>`
    :return: a :class:`~geographiclib.geodesicline.GeodesicLine`

    This function sets point 3 of the GeodesicLine to correspond to
    point 2 of the direct geodesic problem.  The default value of *caps*
    is STANDARD | DISTANCE_IN, allowing direct geodesic problem to be
    solved.

    """

    return self._GenDirectLine(lat1, lon1, azi1, True, a12, caps)

  def InverseLine(self, lat1, lon1, lat2, lon2,
                  caps = GeodesicCapability.STANDARD |
                  GeodesicCapability.DISTANCE_IN):
    """Define a GeodesicLine object in terms of the invese geodesic problem

    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param lat2: latitude of the second point in degrees
    :param lon2: longitude of the second point in degrees
    :param caps: the :ref:`capabilities <outmask>`
    :return: a :class:`~geographiclib.geodesicline.GeodesicLine`

    This function sets point 3 of the GeodesicLine to correspond to
    point 2 of the inverse geodesic problem.  The default value of *caps*
    is STANDARD | DISTANCE_IN, allowing direct geodesic problem to be
    solved.

    """

    from geographiclib.geodesicline import GeodesicLine
    a12, _, salp1, calp1, _, _, _, _, _, _ = self._GenInverse(
      lat1, lon1, lat2, lon2, 0)
    azi1 = Math.atan2d(salp1, calp1)
    if caps & (Geodesic.OUT_MASK & Geodesic.DISTANCE_IN):
      caps |= Geodesic.DISTANCE
    line = GeodesicLine(self, lat1, lon1, azi1, caps, salp1, calp1)
    line.SetArc(a12)
    return line

  def Polygon(self, polyline = False):
    """Return a PolygonArea object

    :param polyline: if True then the object describes a polyline
      instead of a polygon
    :return: a :class:`~geographiclib.polygonarea.PolygonArea`

    """

    from geographiclib.polygonarea import PolygonArea
    return PolygonArea(self, polyline)

  EMPTY         = GeodesicCapability.EMPTY
  """No capabilities, no output."""
  LATITUDE      = GeodesicCapability.LATITUDE
  """Calculate latitude *lat2*."""
  LONGITUDE     = GeodesicCapability.LONGITUDE
  """Calculate longitude *lon2*."""
  AZIMUTH       = GeodesicCapability.AZIMUTH
  """Calculate azimuths *azi1* and *azi2*."""
  DISTANCE      = GeodesicCapability.DISTANCE
  """Calculate distance *s12*."""
  STANDARD      = GeodesicCapability.STANDARD
  """All of the above."""
  DISTANCE_IN   = GeodesicCapability.DISTANCE_IN
  """Allow distance *s12* to be used as input in the direct geodesic
  problem."""
  REDUCEDLENGTH = GeodesicCapability.REDUCEDLENGTH
  """Calculate reduced length *m12*."""
  GEODESICSCALE = GeodesicCapability.GEODESICSCALE
  """Calculate geodesic scales *M12* and *M21*."""
  AREA          = GeodesicCapability.AREA
  """Calculate area *S12*."""
  ALL           = GeodesicCapability.ALL
  """All of the above."""
  LONG_UNROLL   = GeodesicCapability.LONG_UNROLL
  """Unroll longitudes, rather than reducing them to the range
  [-180d,180d].

  """

Geodesic.WGS84 = Geodesic(Constants.WGS84_a, Constants.WGS84_f)
"""Instantiation for the WGS84 ellipsoid"""

env_pyrecodes/lib/python3.9/site-packages/geographiclib/geodesic.py
NikolaBlagojevic/pyrecodes
"""geodesiccapability.py: capability constants for geodesic{,line}.py"""
# geodesiccapability.py
#
# This gathers the capability constants need by geodesic.py and
# geodesicline.py.  See the documentation for the GeographicLib::Geodesic class
# for more information at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# Copyright (c) Charles Karney (2011-2014) <charles@karney.com> and licensed
# under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

class GeodesicCapability:
  """
  Capability constants shared between Geodesic and GeodesicLine.
  """

  CAP_NONE = 0
  CAP_C1   = 1 << 0
  CAP_C1p  = 1 << 1
  CAP_C2   = 1 << 2
  CAP_C3   = 1 << 3
  CAP_C4   = 1 << 4
  CAP_ALL  = 0x1F
  CAP_MASK = CAP_ALL
  OUT_ALL  = 0x7F80
  OUT_MASK = 0xFF80             # Includes LONG_UNROLL
  EMPTY         = 0
  LATITUDE      = 1 << 7  | CAP_NONE
  LONGITUDE     = 1 << 8  | CAP_C3
  AZIMUTH       = 1 << 9  | CAP_NONE
  DISTANCE      = 1 << 10 | CAP_C1
  STANDARD      = LATITUDE | LONGITUDE | AZIMUTH | DISTANCE
  DISTANCE_IN   = 1 << 11 | CAP_C1 | CAP_C1p
  REDUCEDLENGTH = 1 << 12 | CAP_C1 | CAP_C2
  GEODESICSCALE = 1 << 13 | CAP_C1 | CAP_C2
  AREA          = 1 << 14 | CAP_C4
  LONG_UNROLL   = 1 << 15
  ALL           = OUT_ALL | CAP_ALL # Does not include LONG_UNROLL

env_pyrecodes/lib/python3.9/site-packages/geographiclib/geodesiccapability.py
NikolaBlagojevic/pyrecodes
"""Define the :class:`~geographiclib.geodesicline.GeodesicLine` class

The constructor defines the starting point of the line.  Points on the
line are given by

  * :meth:`~geographiclib.geodesicline.GeodesicLine.Position` position
    given in terms of distance
  * :meth:`~geographiclib.geodesicline.GeodesicLine.ArcPosition` position
    given in terms of spherical arc length

A reference point 3 can be defined with

  * :meth:`~geographiclib.geodesicline.GeodesicLine.SetDistance` set
    position of 3 in terms of the distance from the starting point
  * :meth:`~geographiclib.geodesicline.GeodesicLine.SetArc` set
    position of 3 in terms of the spherical arc length from the starting point

The object can also be constructed by

  * :meth:`Geodesic.Line <geographiclib.geodesic.Geodesic.Line>`
  * :meth:`Geodesic.DirectLine <geographiclib.geodesic.Geodesic.DirectLine>`
  * :meth:`Geodesic.ArcDirectLine
    <geographiclib.geodesic.Geodesic.ArcDirectLine>`
  * :meth:`Geodesic.InverseLine <geographiclib.geodesic.Geodesic.InverseLine>`

The public attributes for this class are

  * :attr:`~geographiclib.geodesicline.GeodesicLine.a`
    :attr:`~geographiclib.geodesicline.GeodesicLine.f`
    :attr:`~geographiclib.geodesicline.GeodesicLine.caps`
    :attr:`~geographiclib.geodesicline.GeodesicLine.lat1`
    :attr:`~geographiclib.geodesicline.GeodesicLine.lon1`
    :attr:`~geographiclib.geodesicline.GeodesicLine.azi1`
    :attr:`~geographiclib.geodesicline.GeodesicLine.salp1`
    :attr:`~geographiclib.geodesicline.GeodesicLine.calp1`
    :attr:`~geographiclib.geodesicline.GeodesicLine.s13`
    :attr:`~geographiclib.geodesicline.GeodesicLine.a13`

"""
# geodesicline.py
#
# This is a rather literal translation of the GeographicLib::GeodesicLine class
# to python.  See the documentation for the C++ class for more information at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# The algorithms are derived in
#
#    Charles F. F. Karney,
#    Algorithms for geodesics, J. Geodesy 87, 43-55 (2013),
#    https://doi.org/10.1007/s00190-012-0578-z
#    Addenda: https://geographiclib.sourceforge.io/geod-addenda.html
#
# Copyright (c) Charles Karney (2011-2022) <charles@karney.com> and licensed
# under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

import math
from geographiclib.geomath import Math
from geographiclib.geodesiccapability import GeodesicCapability

class GeodesicLine:
  """Points on a geodesic path"""

  def __init__(self, geod, lat1, lon1, azi1,
               caps = GeodesicCapability.STANDARD |
               GeodesicCapability.DISTANCE_IN,
               salp1 = math.nan, calp1 = math.nan):
    """Construct a GeodesicLine object

    :param geod: a :class:`~geographiclib.geodesic.Geodesic` object
    :param lat1: latitude of the first point in degrees
    :param lon1: longitude of the first point in degrees
    :param azi1: azimuth at the first point in degrees
    :param caps: the :ref:`capabilities <outmask>`

    This creates an object allowing points along a geodesic starting at
    (*lat1*, *lon1*), with azimuth *azi1* to be found.  The default
    value of *caps* is STANDARD | DISTANCE_IN.  The optional parameters
    *salp1* and *calp1* should not be supplied; they are part of the
    private interface.

    """

    from geographiclib.geodesic import Geodesic
    self.a = geod.a
    """The equatorial radius in meters (readonly)"""
    self.f = geod.f
    """The flattening (readonly)"""
    self._b = geod._b
    self._c2 = geod._c2
    self._f1 = geod._f1
    self.caps = (caps | Geodesic.LATITUDE | Geodesic.AZIMUTH |
                  Geodesic.LONG_UNROLL)
    """the capabilities (readonly)"""

    # Guard against underflow in salp0
    self.lat1 = Math.LatFix(lat1)
    """the latitude of the first point in degrees (readonly)"""
    self.lon1 = lon1
    """the longitude of the first point in degrees (readonly)"""
    if math.isnan(salp1) or math.isnan(calp1):
      self.azi1 = Math.AngNormalize(azi1)
      self.salp1, self.calp1 = Math.sincosd(Math.AngRound(azi1))
    else:
      self.azi1 = azi1
      """the azimuth at the first point in degrees (readonly)"""
      self.salp1 = salp1
      """the sine of the azimuth at the first point (readonly)"""
      self.calp1 = calp1
      """the cosine of the azimuth at the first point (readonly)"""

    # real cbet1, sbet1
    sbet1, cbet1 = Math.sincosd(Math.AngRound(self.lat1)); sbet1 *= self._f1
    # Ensure cbet1 = +epsilon at poles
    sbet1, cbet1 = Math.norm(sbet1, cbet1); cbet1 = max(Geodesic.tiny_, cbet1)
    self._dn1 = math.sqrt(1 + geod._ep2 * Math.sq(sbet1))

    # Evaluate alp0 from sin(alp1) * cos(bet1) = sin(alp0),
    self._salp0 = self.salp1 * cbet1 # alp0 in [0, pi/2 - |bet1|]
    # Alt: calp0 = hypot(sbet1, calp1 * cbet1).  The following
    # is slightly better (consider the case salp1 = 0).
    self._calp0 = math.hypot(self.calp1, self.salp1 * sbet1)
    # Evaluate sig with tan(bet1) = tan(sig1) * cos(alp1).
    # sig = 0 is nearest northward crossing of equator.
    # With bet1 = 0, alp1 = pi/2, we have sig1 = 0 (equatorial line).
    # With bet1 =  pi/2, alp1 = -pi, sig1 =  pi/2
    # With bet1 = -pi/2, alp1 =  0 , sig1 = -pi/2
    # Evaluate omg1 with tan(omg1) = sin(alp0) * tan(sig1).
    # With alp0 in (0, pi/2], quadrants for sig and omg coincide.
    # No atan2(0,0) ambiguity at poles since cbet1 = +epsilon.
    # With alp0 = 0, omg1 = 0 for alp1 = 0, omg1 = pi for alp1 = pi.
    self._ssig1 = sbet1; self._somg1 = self._salp0 * sbet1
    self._csig1 = self._comg1 = (cbet1 * self.calp1
                                 if sbet1 != 0 or self.calp1 != 0 else 1)
    # sig1 in (-pi, pi]
    self._ssig1, self._csig1 = Math.norm(self._ssig1, self._csig1)
    # No need to normalize
    # self._somg1, self._comg1 = Math.norm(self._somg1, self._comg1)

    self._k2 = Math.sq(self._calp0) * geod._ep2
    eps = self._k2 / (2 * (1 + math.sqrt(1 + self._k2)) + self._k2)

    if self.caps & Geodesic.CAP_C1:
      self._A1m1 = Geodesic._A1m1f(eps)
      self._C1a = list(range(Geodesic.nC1_ + 1))
      Geodesic._C1f(eps, self._C1a)
      self._B11 = Geodesic._SinCosSeries(
        True, self._ssig1, self._csig1, self._C1a)
      s = math.sin(self._B11); c = math.cos(self._B11)
      # tau1 = sig1 + B11
      self._stau1 = self._ssig1 * c + self._csig1 * s
      self._ctau1 = self._csig1 * c - self._ssig1 * s
      # Not necessary because C1pa reverts C1a
      #    _B11 = -_SinCosSeries(true, _stau1, _ctau1, _C1pa)

    if self.caps & Geodesic.CAP_C1p:
      self._C1pa = list(range(Geodesic.nC1p_ + 1))
      Geodesic._C1pf(eps, self._C1pa)

    if self.caps & Geodesic.CAP_C2:
      self._A2m1 = Geodesic._A2m1f(eps)
      self._C2a = list(range(Geodesic.nC2_ + 1))
      Geodesic._C2f(eps, self._C2a)
      self._B21 = Geodesic._SinCosSeries(
        True, self._ssig1, self._csig1, self._C2a)

    if self.caps & Geodesic.CAP_C3:
      self._C3a = list(range(Geodesic.nC3_))
      geod._C3f(eps, self._C3a)
      self._A3c = -self.f * self._salp0 * geod._A3f(eps)
      self._B31 = Geodesic._SinCosSeries(
        True, self._ssig1, self._csig1, self._C3a)

    if self.caps & Geodesic.CAP_C4:
      self._C4a = list(range(Geodesic.nC4_))
      geod._C4f(eps, self._C4a)
      # Multiplier = a^2 * e^2 * cos(alpha0) * sin(alpha0)
      self._A4 = Math.sq(self.a) * self._calp0 * self._salp0 * geod._e2
      self._B41 = Geodesic._SinCosSeries(
        False, self._ssig1, self._csig1, self._C4a)
    self.s13 = math.nan
    """the distance between point 1 and point 3 in meters (readonly)"""
    self.a13 = math.nan
    """the arc length between point 1 and point 3 in degrees (readonly)"""

  # return a12, lat2, lon2, azi2, s12, m12, M12, M21, S12
  def _GenPosition(self, arcmode, s12_a12, outmask):
    """Private: General solution of position along geodesic"""
    from geographiclib.geodesic import Geodesic
    a12 = lat2 = lon2 = azi2 = s12 = m12 = M12 = M21 = S12 = math.nan
    outmask &= self.caps & Geodesic.OUT_MASK
    if not (arcmode or
            (self.caps & (Geodesic.OUT_MASK & Geodesic.DISTANCE_IN))):
      # Uninitialized or impossible distance calculation requested
      return a12, lat2, lon2, azi2, s12, m12, M12, M21, S12

    # Avoid warning about uninitialized B12.
    B12 = 0.0; AB1 = 0.0
    if arcmode:
      # Interpret s12_a12 as spherical arc length
      sig12 = math.radians(s12_a12)
      ssig12, csig12 = Math.sincosd(s12_a12)
    else:
      # Interpret s12_a12 as distance
      tau12 = s12_a12 / (self._b * (1 + self._A1m1))
      tau12 = tau12 if math.isfinite(tau12) else math.nan
      s = math.sin(tau12); c = math.cos(tau12)
      # tau2 = tau1 + tau12
      B12 = - Geodesic._SinCosSeries(True,
                                    self._stau1 * c + self._ctau1 * s,
                                    self._ctau1 * c - self._stau1 * s,
                                    self._C1pa)
      sig12 = tau12 - (B12 - self._B11)
      ssig12 = math.sin(sig12); csig12 = math.cos(sig12)
      if abs(self.f) > 0.01:
        # Reverted distance series is inaccurate for |f| > 1/100, so correct
        # sig12 with 1 Newton iteration.  The following table shows the
        # approximate maximum error for a = WGS_a() and various f relative to
        # GeodesicExact.
        #     erri = the error in the inverse solution (nm)
        #     errd = the error in the direct solution (series only) (nm)
        #     errda = the error in the direct solution (series + 1 Newton) (nm)
        #
        #       f     erri  errd errda
        #     -1/5    12e6 1.2e9  69e6
        #     -1/10  123e3  12e6 765e3
        #     -1/20   1110 108e3  7155
        #     -1/50  18.63 200.9 27.12
        #     -1/100 18.63 23.78 23.37
        #     -1/150 18.63 21.05 20.26
        #      1/150 22.35 24.73 25.83
        #      1/100 22.35 25.03 25.31
        #      1/50  29.80 231.9 30.44
        #      1/20   5376 146e3  10e3
        #      1/10  829e3  22e6 1.5e6
        #      1/5   157e6 3.8e9 280e6
        ssig2 = self._ssig1 * csig12 + self._csig1 * ssig12
        csig2 = self._csig1 * csig12 - self._ssig1 * ssig12
        B12 = Geodesic._SinCosSeries(True, ssig2, csig2, self._C1a)
        serr = ((1 + self._A1m1) * (sig12 + (B12 - self._B11)) -
                s12_a12 / self._b)
        sig12 = sig12 - serr / math.sqrt(1 + self._k2 * Math.sq(ssig2))
        ssig12 = math.sin(sig12); csig12 = math.cos(sig12)
        # Update B12 below

    # real omg12, lam12, lon12
    # real ssig2, csig2, sbet2, cbet2, somg2, comg2, salp2, calp2
    # sig2 = sig1 + sig12
    ssig2 = self._ssig1 * csig12 + self._csig1 * ssig12
    csig2 = self._csig1 * csig12 - self._ssig1 * ssig12
    dn2 = math.sqrt(1 + self._k2 * Math.sq(ssig2))
    if outmask & (
      Geodesic.DISTANCE | Geodesic.REDUCEDLENGTH | Geodesic.GEODESICSCALE):
      if arcmode or abs(self.f) > 0.01:
        B12 = Geodesic._SinCosSeries(True, ssig2, csig2, self._C1a)
      AB1 = (1 + self._A1m1) * (B12 - self._B11)
    # sin(bet2) = cos(alp0) * sin(sig2)
    sbet2 = self._calp0 * ssig2
    # Alt: cbet2 = hypot(csig2, salp0 * ssig2)
    cbet2 = math.hypot(self._salp0, self._calp0 * csig2)
    if cbet2 == 0:
      # I.e., salp0 = 0, csig2 = 0.  Break the degeneracy in this case
      cbet2 = csig2 = Geodesic.tiny_
    # tan(alp0) = cos(sig2)*tan(alp2)
    salp2 = self._salp0; calp2 = self._calp0 * csig2 # No need to normalize

    if outmask & Geodesic.DISTANCE:
      s12 = self._b * ((1 + self._A1m1) * sig12 + AB1) if arcmode else s12_a12

    if outmask & Geodesic.LONGITUDE:
      # tan(omg2) = sin(alp0) * tan(sig2)
      somg2 = self._salp0 * ssig2; comg2 = csig2 # No need to normalize
      E = math.copysign(1, self._salp0)          # East or west going?
      # omg12 = omg2 - omg1
      omg12 = (E * (sig12
                    - (math.atan2(          ssig2,       csig2) -
                       math.atan2(    self._ssig1, self._csig1))
                    + (math.atan2(E *       somg2,       comg2) -
                       math.atan2(E * self._somg1, self._comg1)))
               if outmask & Geodesic.LONG_UNROLL
               else math.atan2(somg2 * self._comg1 - comg2 * self._somg1,
                               comg2 * self._comg1 + somg2 * self._somg1))
      lam12 = omg12 + self._A3c * (
        sig12 + (Geodesic._SinCosSeries(True, ssig2, csig2, self._C3a)
                 - self._B31))
      lon12 = math.degrees(lam12)
      lon2 = (self.lon1 + lon12 if outmask & Geodesic.LONG_UNROLL else
              Math.AngNormalize(Math.AngNormalize(self.lon1) +
                                Math.AngNormalize(lon12)))

    if outmask & Geodesic.LATITUDE:
      lat2 = Math.atan2d(sbet2, self._f1 * cbet2)

    if outmask & Geodesic.AZIMUTH:
      azi2 = Math.atan2d(salp2, calp2)

    if outmask & (Geodesic.REDUCEDLENGTH | Geodesic.GEODESICSCALE):
      B22 = Geodesic._SinCosSeries(True, ssig2, csig2, self._C2a)
      AB2 = (1 + self._A2m1) * (B22 - self._B21)
      J12 = (self._A1m1 - self._A2m1) * sig12 + (AB1 - AB2)
      if outmask & Geodesic.REDUCEDLENGTH:
        # Add parens around (_csig1 * ssig2) and (_ssig1 * csig2) to ensure
        # accurate cancellation in the case of coincident points.
        m12 = self._b * ((      dn2 * (self._csig1 * ssig2) -
                          self._dn1 * (self._ssig1 * csig2))
                         - self._csig1 * csig2 * J12)
      if outmask & Geodesic.GEODESICSCALE:
        t = (self._k2 * (ssig2 - self._ssig1) *
             (ssig2 + self._ssig1) / (self._dn1 + dn2))
        M12 = csig12 + (t * ssig2 - csig2 * J12) * self._ssig1 / self._dn1
        M21 = csig12 - (t * self._ssig1 - self._csig1 * J12) * ssig2 / dn2

    if outmask & Geodesic.AREA:
      B42 = Geodesic._SinCosSeries(False, ssig2, csig2, self._C4a)
      # real salp12, calp12
      if self._calp0 == 0 or self._salp0 == 0:
        # alp12 = alp2 - alp1, used in atan2 so no need to normalize
        salp12 = salp2 * self.calp1 - calp2 * self.salp1
        calp12 = calp2 * self.calp1 + salp2 * self.salp1
      else:
        # tan(alp) = tan(alp0) * sec(sig)
        # tan(alp2-alp1) = (tan(alp2) -tan(alp1)) / (tan(alp2)*tan(alp1)+1)
        # = calp0 * salp0 * (csig1-csig2) / (salp0^2 + calp0^2 * csig1*csig2)
        # If csig12 > 0, write
        #   csig1 - csig2 = ssig12 * (csig1 * ssig12 / (1 + csig12) + ssig1)
        # else
        #   csig1 - csig2 = csig1 * (1 - csig12) + ssig12 * ssig1
        # No need to normalize
        salp12 = self._calp0 * self._salp0 * (
          self._csig1 * (1 - csig12) + ssig12 * self._ssig1 if csig12 <= 0
          else ssig12 * (self._csig1 * ssig12 / (1 + csig12) + self._ssig1))
        calp12 = (Math.sq(self._salp0) +
                  Math.sq(self._calp0) * self._csig1 * csig2)
      S12 = (self._c2 * math.atan2(salp12, calp12) +
             self._A4 * (B42 - self._B41))

    a12 = s12_a12 if arcmode else math.degrees(sig12)
    return a12, lat2, lon2, azi2, s12, m12, M12, M21, S12

  def Position(self, s12, outmask = GeodesicCapability.STANDARD):
    """Find the position on the line given *s12*

    :param s12: the distance from the first point to the second in
      meters
    :param outmask: the :ref:`output mask <outmask>`
    :return: a :ref:`dict`

    The default value of *outmask* is STANDARD, i.e., the *lat1*,
    *lon1*, *azi1*, *lat2*, *lon2*, *azi2*, *s12*, *a12* entries are
    returned.  The :class:`~geographiclib.geodesicline.GeodesicLine`
    object must have been constructed with the DISTANCE_IN capability.

    """

    from geographiclib.geodesic import Geodesic
    result = {'lat1': self.lat1,
              'lon1': self.lon1 if outmask & Geodesic.LONG_UNROLL else
              Math.AngNormalize(self.lon1),
              'azi1': self.azi1, 's12': s12}
    a12, lat2, lon2, azi2, s12, m12, M12, M21, S12 = self._GenPosition(
      False, s12, outmask)
    outmask &= Geodesic.OUT_MASK
    result['a12'] = a12
    if outmask & Geodesic.LATITUDE: result['lat2'] = lat2
    if outmask & Geodesic.LONGITUDE: result['lon2'] = lon2
    if outmask & Geodesic.AZIMUTH: result['azi2'] = azi2
    if outmask & Geodesic.REDUCEDLENGTH: result['m12'] = m12
    if outmask & Geodesic.GEODESICSCALE:
      result['M12'] = M12; result['M21'] = M21
    if outmask & Geodesic.AREA: result['S12'] = S12
    return result

  def ArcPosition(self, a12, outmask = GeodesicCapability.STANDARD):
    """Find the position on the line given *a12*

    :param a12: spherical arc length from the first point to the second
      in degrees
    :param outmask: the :ref:`output mask <outmask>`
    :return: a :ref:`dict`

    The default value of *outmask* is STANDARD, i.e., the *lat1*,
    *lon1*, *azi1*, *lat2*, *lon2*, *azi2*, *s12*, *a12* entries are
    returned.

    """

    from geographiclib.geodesic import Geodesic
    result = {'lat1': self.lat1,
              'lon1': self.lon1 if outmask & Geodesic.LONG_UNROLL else
              Math.AngNormalize(self.lon1),
              'azi1': self.azi1, 'a12': a12}
    a12, lat2, lon2, azi2, s12, m12, M12, M21, S12 = self._GenPosition(
      True, a12, outmask)
    outmask &= Geodesic.OUT_MASK
    if outmask & Geodesic.DISTANCE: result['s12'] = s12
    if outmask & Geodesic.LATITUDE: result['lat2'] = lat2
    if outmask & Geodesic.LONGITUDE: result['lon2'] = lon2
    if outmask & Geodesic.AZIMUTH: result['azi2'] = azi2
    if outmask & Geodesic.REDUCEDLENGTH: result['m12'] = m12
    if outmask & Geodesic.GEODESICSCALE:
      result['M12'] = M12; result['M21'] = M21
    if outmask & Geodesic.AREA: result['S12'] = S12
    return result

  def SetDistance(self, s13):
    """Specify the position of point 3 in terms of distance

    :param s13: distance from point 1 to point 3 in meters

    """

    self.s13 = s13
    self.a13, _, _, _, _, _, _, _, _ = self._GenPosition(False, self.s13, 0)

  def SetArc(self, a13):
    """Specify the position of point 3 in terms of arc length

    :param a13: spherical arc length from point 1 to point 3 in degrees

    """

    from geographiclib.geodesic import Geodesic
    self.a13 = a13
    _, _, _, _, self.s13, _, _, _, _ = self._GenPosition(True, self.a13,
                                                         Geodesic.DISTANCE)

env_pyrecodes/lib/python3.9/site-packages/geographiclib/geodesicline.py
NikolaBlagojevic/pyrecodes
"""geomath.py: transcription of GeographicLib::Math class."""
# geomath.py
#
# This is a rather literal translation of the GeographicLib::Math class to
# python.  See the documentation for the C++ class for more information at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# Copyright (c) Charles Karney (2011-2021) <charles@karney.com> and
# licensed under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

import sys
import math

class Math:
  """
  Additional math routines for GeographicLib.
  """

  @staticmethod
  def sq(x):
    """Square a number"""

    return x * x

  @staticmethod
  def cbrt(x):
    """Real cube root of a number"""

    return math.copysign(math.pow(abs(x), 1/3.0), x)

  @staticmethod
  def norm(x, y):
    """Private: Normalize a two-vector."""

    r = (math.sqrt(Math.sq(x) + Math.sq(y))
         # hypot is inaccurate for 3.[89].  Problem reported by agdhruv
         # https://github.com/geopy/geopy/issues/466 ; see
         # https://bugs.python.org/issue43088
         # Visual Studio 2015 32-bit has a similar problem.
         if (3, 8) <= sys.version_info < (3, 10)
         else math.hypot(x, y))
    return x/r, y/r

  @staticmethod
  def sum(u, v):
    """Error free transformation of a sum."""

    # Error free transformation of a sum.  Note that t can be the same as one
    # of the first two arguments.
    s = u + v
    up = s - v
    vpp = s - up
    up -= u
    vpp -= v
    t = s if s == 0 else 0.0 - (up + vpp)
    # u + v =       s      + t
    #       = round(u + v) + t
    return s, t

  @staticmethod
  def polyval(N, p, s, x):
    """Evaluate a polynomial."""

    y = float(0 if N < 0 else p[s]) # make sure the returned value is a float
    while N > 0:
      N -= 1; s += 1
      y = y * x + p[s]
    return y

  @staticmethod
  def AngRound(x):
    """Private: Round an angle so that small values underflow to zero."""

    # The makes the smallest gap in x = 1/16 - nextafter(1/16, 0) = 1/2^57
    # for reals = 0.7 pm on the earth if x is an angle in degrees.  (This
    # is about 1000 times more resolution than we get with angles around 90
    # degrees.)  We use this to avoid having to deal with near singular
    # cases when x is non-zero but tiny (e.g., 1.0e-200).
    z = 1/16.0
    y = abs(x)
    # The compiler mustn't "simplify" z - (z - y) to y
    if y < z: y = z - (z - y)
    return math.copysign(y, x)

  @staticmethod
  def remainder(x, y):
    """remainder of x/y in the range [-y/2, y/2]."""

    return math.remainder(x, y) if math.isfinite(x) else math.nan

  @staticmethod
  def AngNormalize(x):
    """reduce angle to [-180,180]"""

    y = Math.remainder(x, 360)
    return math.copysign(180.0, x) if abs(y) == 180 else y

  @staticmethod
  def LatFix(x):
    """replace angles outside [-90,90] by NaN"""

    return math.nan if abs(x) > 90 else x

  @staticmethod
  def AngDiff(x, y):
    """compute y - x and reduce to [-180,180] accurately"""

    d, t = Math.sum(Math.remainder(-x, 360), Math.remainder(y, 360))
    d, t = Math.sum(Math.remainder(d, 360), t)
    if d == 0 or abs(d) == 180:
      d = math.copysign(d, y - x if t == 0 else -t)
    return d, t

  @staticmethod
  def sincosd(x):
    """Compute sine and cosine of x in degrees."""

    r = math.fmod(x, 360) if math.isfinite(x) else math.nan
    q = 0 if math.isnan(r) else int(round(r / 90))
    r -= 90 * q; r = math.radians(r)
    s = math.sin(r); c = math.cos(r)
    q = q % 4
    if   q == 1: s, c =  c, -s
    elif q == 2: s, c = -s, -c
    elif q == 3: s, c = -c,  s
    c = c + 0.0
    if s == 0: s = math.copysign(s, x)
    return s, c

  @staticmethod
  def sincosde(x, t):
    """Compute sine and cosine of (x + t) in degrees with x in [-180, 180]"""

    q = int(round(x / 90)) if math.isfinite(x) else 0
    r = x - 90 * q; r = math.radians(Math.AngRound(r + t))
    s = math.sin(r); c = math.cos(r)
    q = q % 4
    if   q == 1: s, c =  c, -s
    elif q == 2: s, c = -s, -c
    elif q == 3: s, c = -c,  s
    c = c + 0.0
    if s == 0: s = math.copysign(s, x)
    return s, c

  @staticmethod
  def atan2d(y, x):
    """compute atan2(y, x) with the result in degrees"""

    if abs(y) > abs(x):
      q = 2; x, y = y, x
    else:
      q = 0
    if x < 0:
      q += 1; x = -x
    ang = math.degrees(math.atan2(y, x))
    if   q == 1: ang = math.copysign(180, y) - ang
    elif q == 2: ang =                90      - ang
    elif q == 3: ang =               -90     + ang
    return ang

env_pyrecodes/lib/python3.9/site-packages/geographiclib/geomath.py
NikolaBlagojevic/pyrecodes
"""Define the :class:`~geographiclib.polygonarea.PolygonArea` class

The constructor initializes a empty polygon.  The available methods are

  * :meth:`~geographiclib.polygonarea.PolygonArea.Clear` reset the
    polygon
  * :meth:`~geographiclib.polygonarea.PolygonArea.AddPoint` add a vertex
    to the polygon
  * :meth:`~geographiclib.polygonarea.PolygonArea.AddEdge` add an edge
    to the polygon
  * :meth:`~geographiclib.polygonarea.PolygonArea.Compute` compute the
    properties of the polygon
  * :meth:`~geographiclib.polygonarea.PolygonArea.TestPoint` compute the
    properties of the polygon with a tentative additional vertex
  * :meth:`~geographiclib.polygonarea.PolygonArea.TestEdge` compute the
    properties of the polygon with a tentative additional edge

The public attributes for this class are

  * :attr:`~geographiclib.polygonarea.PolygonArea.earth`
    :attr:`~geographiclib.polygonarea.PolygonArea.polyline`
    :attr:`~geographiclib.polygonarea.PolygonArea.area0`
    :attr:`~geographiclib.polygonarea.PolygonArea.num`
    :attr:`~geographiclib.polygonarea.PolygonArea.lat1`
    :attr:`~geographiclib.polygonarea.PolygonArea.lon1`

"""
# polygonarea.py
#
# This is a rather literal translation of the GeographicLib::PolygonArea class
# to python.  See the documentation for the C++ class for more information at
#
#    https://geographiclib.sourceforge.io/html/annotated.html
#
# The algorithms are derived in
#
#    Charles F. F. Karney,
#    Algorithms for geodesics, J. Geodesy 87, 43-55 (2013),
#    https://doi.org/10.1007/s00190-012-0578-z
#    Addenda: https://geographiclib.sourceforge.io/geod-addenda.html
#
# Copyright (c) Charles Karney (2011-2022) <charles@karney.com> and licensed
# under the MIT/X11 License.  For more information, see
# https://geographiclib.sourceforge.io/
######################################################################

import math
from geographiclib.geomath import Math
from geographiclib.accumulator import Accumulator
from geographiclib.geodesic import Geodesic

class PolygonArea:
  """Area of a geodesic polygon"""

  @staticmethod
  def _transit(lon1, lon2):
    """Count crossings of prime meridian for AddPoint."""
    # Return 1 or -1 if crossing prime meridian in east or west direction.
    # Otherwise return zero.
    # Compute lon12 the same way as Geodesic::Inverse.
    lon12, _ = Math.AngDiff(lon1, lon2)
    lon1 = Math.AngNormalize(lon1)
    lon2 = Math.AngNormalize(lon2)
    return (1 if lon12 > 0 and ( lon1 < 0 <= lon2 or
                                (lon1 > 0 and lon2 == 0))
            else (-1 if lon12 < 0 and lon2 < 0 <= lon1 else 0))

  @staticmethod
  def _transitdirect(lon1, lon2):
    """Count crossings of prime meridian for AddEdge."""
    # We want to compute exactly
    #   int(floor(lon2 / 360)) - int(floor(lon1 / 360))
    lon1 = Math.remainder(lon1, 720.0); lon2 = Math.remainder(lon2, 720.0)
    return ( (0 if 0 <= lon2 < 360 else 1) -
             (0 if 0 <= lon1 < 360 else 1) )

  @staticmethod
  def _areareduceA(area, area0, crossings, reverse, sign):
    """Reduce accumulator area to allowed range."""
    area.Remainder(area0)
    if crossings & 1:
      area.Add( (1 if area.Sum() < 0 else -1) * area0/2 )
    # area is with the clockwise sense.  If !reverse convert to
    # counter-clockwise convention.
    if not reverse: area.Negate()
    # If sign put area in (-area0/2, area0/2], else put area in [0, area0)
    if sign:
      if area.Sum() > area0/2:
        area.Add( -area0 )
      elif area.Sum() <= -area0/2:
        area.Add(  area0 )
    else:
      if area.Sum() >= area0:
        area.Add( -area0 )
      elif area.Sum() < 0:
        area.Add(  area0 )

    return 0.0 + area.Sum()

  @staticmethod
  def _areareduceB(area, area0, crossings, reverse, sign):
    """Reduce double area to allowed range."""
    area = Math.remainder(area, area0)
    if crossings & 1:
      area += (1 if area < 0 else -1) * area0/2
    # area is with the clockwise sense.  If !reverse convert to
    # counter-clockwise convention.
    if not reverse: area *= -1
    # If sign put area in (-area0/2, area0/2], else put area in [0, area0)
    if sign:
      if area > area0/2:
        area -= area0
      elif area <= -area0/2:
        area += area0
    else:
      if area >= area0:
        area -= area0
      elif area < 0:
        area += area0

    return 0.0 + area

  def __init__(self, earth, polyline = False):
    """Construct a PolygonArea object

    :param earth: a :class:`~geographiclib.geodesic.Geodesic` object
    :param polyline: if true, treat object as a polyline instead of a polygon

    Initially the polygon has no vertices.
    """

    self.earth = earth
    """The geodesic object (readonly)"""
    self.polyline = polyline
    """Is this a polyline? (readonly)"""
    self.area0 = 4 * math.pi * earth._c2
    """The total area of the ellipsoid in meter^2 (readonly)"""
    self._mask = (Geodesic.LATITUDE | Geodesic.LONGITUDE |
                  Geodesic.DISTANCE |
                  (Geodesic.EMPTY if self.polyline else
                   Geodesic.AREA | Geodesic.LONG_UNROLL))
    if not self.polyline: self._areasum = Accumulator()
    self._perimetersum = Accumulator()
    self.num = 0
    """The current number of points in the polygon (readonly)"""
    self.lat1 = math.nan
    """The current latitude in degrees (readonly)"""
    self.lon1 = math.nan
    """The current longitude in degrees (readonly)"""
    self._crossings = 0
    self._lat0 = self._lon0 = math.nan

  def Clear(self):
    """Reset to empty polygon."""
    self.num = 0
    self._crossings = 0
    if not self.polyline: self._areasum.Set(0)
    self._perimetersum.Set(0)
    self._lat0 = self._lon0 = self.lat1 = self.lon1 = math.nan

  def AddPoint(self, lat, lon):
    """Add the next vertex to the polygon

    :param lat: the latitude of the point in degrees
    :param lon: the longitude of the point in degrees

    This adds an edge from the current vertex to the new vertex.
    """

    if self.num == 0:
      self._lat0 = self.lat1 = lat
      self._lon0 = self.lon1 = lon
    else:
      _, s12, _, _, _, _, _, _, _, S12 = self.earth._GenInverse(
        self.lat1, self.lon1, lat, lon, self._mask)
      self._perimetersum.Add(s12)
      if not self.polyline:
        self._areasum.Add(S12)
        self._crossings += PolygonArea._transit(self.lon1, lon)
      self.lat1 = lat
      self.lon1 = lon
    self.num += 1

  def AddEdge(self, azi, s):
    """Add the next edge to the polygon

    :param azi: the azimuth at the current the point in degrees
    :param s: the length of the edge in meters

    This specifies the new vertex in terms of the edge from the current
    vertex.

    """

    if self.num != 0:
      _, lat, lon, _, _, _, _, _, S12 = self.earth._GenDirect(
        self.lat1, self.lon1, azi, False, s, self._mask)
      self._perimetersum.Add(s)
      if not self.polyline:
        self._areasum.Add(S12)
        self._crossings += PolygonArea._transitdirect(self.lon1, lon)
      self.lat1 = lat
      self.lon1 = lon
      self.num += 1

  # return number, perimeter, area
  def Compute(self, reverse = False, sign = True):
    """Compute the properties of the polygon

    :param reverse: if true then clockwise (instead of
      counter-clockwise) traversal counts as a positive area
    :param sign: if true then return a signed result for the area if the
      polygon is traversed in the "wrong" direction instead of returning
      the area for the rest of the earth
    :return: a tuple of number, perimeter (meters), area (meters^2)

    Arbitrarily complex polygons are allowed.  In the case of
    self-intersecting polygons the area is accumulated "algebraically",
    e.g., the areas of the 2 loops in a figure-8 polygon will partially
    cancel.

    If the object is a polygon (and not a polyline), the perimeter
    includes the length of a final edge connecting the current point to
    the initial point.  If the object is a polyline, then area is nan.

    More points can be added to the polygon after this call.

    """
    if self.polyline: area = math.nan
    if self.num < 2:
      perimeter = 0.0
      if not self.polyline: area = 0.0
      return self.num, perimeter, area

    if self.polyline:
      perimeter = self._perimetersum.Sum()
      return self.num, perimeter, area

    _, s12, _, _, _, _, _, _, _, S12 = self.earth._GenInverse(
      self.lat1, self.lon1, self._lat0, self._lon0, self._mask)
    perimeter = self._perimetersum.Sum(s12)
    tempsum = Accumulator(self._areasum)
    tempsum.Add(S12)
    crossings = self._crossings + PolygonArea._transit(self.lon1, self._lon0)
    area = PolygonArea._areareduceA(tempsum, self.area0, crossings,
                                    reverse, sign)
    return self.num, perimeter, area

  # return number, perimeter, area
  def TestPoint(self, lat, lon, reverse = False, sign = True):
    """Compute the properties for a tentative additional vertex

    :param lat: the latitude of the point in degrees
    :param lon: the longitude of the point in degrees
    :param reverse: if true then clockwise (instead of
      counter-clockwise) traversal counts as a positive area
    :param sign: if true then return a signed result for the area if the
      polygon is traversed in the "wrong" direction instead of returning
      the area for the rest of the earth
    :return: a tuple of number, perimeter (meters), area (meters^2)

    """
    if self.polyline: area = math.nan
    if self.num == 0:
      perimeter = 0.0
      if not self.polyline: area = 0.0
      return 1, perimeter, area

    perimeter = self._perimetersum.Sum()
    tempsum = 0.0 if self.polyline else self._areasum.Sum()
    crossings = self._crossings; num = self.num + 1
    for i in ([0] if self.polyline else [0, 1]):
      _, s12, _, _, _, _, _, _, _, S12 = self.earth._GenInverse(
        self.lat1 if i == 0 else lat, self.lon1 if i == 0 else lon,
        self._lat0 if i != 0 else lat, self._lon0 if i != 0 else lon,
        self._mask)
      perimeter += s12
      if not self.polyline:
        tempsum += S12
        crossings += PolygonArea._transit(self.lon1 if i == 0 else lon,
                                         self._lon0 if i != 0 else lon)

    if self.polyline:
      return num, perimeter, area

    area = PolygonArea._areareduceB(tempsum, self.area0, crossings,
                                    reverse, sign)
    return num, perimeter, area

  # return num, perimeter, area
  def TestEdge(self, azi, s, reverse = False, sign = True):
    """Compute the properties for a tentative additional edge

    :param azi: the azimuth at the current the point in degrees
    :param s: the length of the edge in meters
    :param reverse: if true then clockwise (instead of
      counter-clockwise) traversal counts as a positive area
    :param sign: if true then return a signed result for the area if the
      polygon is traversed in the "wrong" direction instead of returning
      the area for the rest of the earth
    :return: a tuple of number, perimeter (meters), area (meters^2)

    """

    if self.num == 0:           # we don't have a starting point!
      return 0, math.nan, math.nan
    num = self.num + 1
    perimeter = self._perimetersum.Sum() + s
    if self.polyline:
      return num, perimeter, math.nan

    tempsum =  self._areasum.Sum()
    crossings = self._crossings
    _, lat, lon, _, _, _, _, _, S12 = self.earth._GenDirect(
      self.lat1, self.lon1, azi, False, s, self._mask)
    tempsum += S12
    crossings += PolygonArea._transitdirect(self.lon1, lon)
    _, s12, _, _, _, _, _, _, _, S12 = self.earth._GenInverse(
      lat, lon, self._lat0, self._lon0, self._mask)
    perimeter += s12
    tempsum += S12
    crossings += PolygonArea._transit(lon, self._lon0)

    area = PolygonArea._areareduceB(tempsum, self.area0, crossings,
                                    reverse, sign)
    return num, perimeter, area

env_pyrecodes/lib/python3.9/site-packages/geographiclib/polygonarea.py
NikolaBlagojevic/pyrecodes
"""Geodesic tests"""

import unittest
import math

from geographiclib.geodesic import Geodesic

class GeodesicTest(unittest.TestCase):
  """Geodesic test suite"""

  testcases = [
    [35.60777, -139.44815, 111.098748429560326,
     -11.17491, -69.95921, 129.289270889708762,
     8935244.5604818305, 80.50729714281974, 6273170.2055303837,
     0.16606318447386067, 0.16479116945612937, 12841384694976.432],
    [55.52454, 106.05087, 22.020059880982801,
     77.03196, 197.18234, 109.112041110671519,
     4105086.1713924406, 36.892740690445894, 3828869.3344387607,
     0.80076349608092607, 0.80101006984201008, 61674961290615.615],
    [-21.97856, 142.59065, -32.44456876433189,
     41.84138, 98.56635, -41.84359951440466,
     8394328.894657671, 75.62930491011522, 6161154.5773110616,
     0.24816339233950381, 0.24930251203627892, -6637997720646.717],
    [-66.99028, 112.2363, 173.73491240878403,
     -12.70631, 285.90344, 2.512956620913668,
     11150344.2312080241, 100.278634181155759, 6289939.5670446687,
     -0.17199490274700385, -0.17722569526345708, -121287239862139.744],
    [-17.42761, 173.34268, -159.033557661192928,
     -15.84784, 5.93557, -20.787484651536988,
     16076603.1631180673, 144.640108810286253, 3732902.1583877189,
     -0.81273638700070476, -0.81299800519154474, 97825992354058.708],
    [32.84994, 48.28919, 150.492927788121982,
     -56.28556, 202.29132, 48.113449399816759,
     16727068.9438164461, 150.565799985466607, 3147838.1910180939,
     -0.87334918086923126, -0.86505036767110637, -72445258525585.010],
    [6.96833, 52.74123, 92.581585386317712,
     -7.39675, 206.17291, 90.721692165923907,
     17102477.2496958388, 154.147366239113561, 2772035.6169917581,
     -0.89991282520302447, -0.89986892177110739, -1311796973197.995],
    [-50.56724, -16.30485, -105.439679907590164,
     -33.56571, -94.97412, -47.348547835650331,
     6455670.5118668696, 58.083719495371259, 5409150.7979815838,
     0.53053508035997263, 0.52988722644436602, 41071447902810.047],
    [-58.93002, -8.90775, 140.965397902500679,
     -8.91104, 133.13503, 19.255429433416599,
     11756066.0219864627, 105.755691241406877, 6151101.2270708536,
     -0.26548622269867183, -0.27068483874510741, -86143460552774.735],
    [-68.82867, -74.28391, 93.774347763114881,
     -50.63005, -8.36685, 34.65564085411343,
     3956936.926063544, 35.572254987389284, 3708890.9544062657,
     0.81443963736383502, 0.81420859815358342, -41845309450093.787],
    [-10.62672, -32.0898, -86.426713286747751,
     5.883, -134.31681, -80.473780971034875,
     11470869.3864563009, 103.387395634504061, 6184411.6622659713,
     -0.23138683500430237, -0.23155097622286792, 4198803992123.548],
    [-21.76221, 166.90563, 29.319421206936428,
     48.72884, 213.97627, 43.508671946410168,
     9098627.3986554915, 81.963476716121964, 6299240.9166992283,
     0.13965943368590333, 0.14152969707656796, 10024709850277.476],
    [-19.79938, -174.47484, 71.167275780171533,
     -11.99349, -154.35109, 65.589099775199228,
     2319004.8601169389, 20.896611684802389, 2267960.8703918325,
     0.93427001867125849, 0.93424887135032789, -3935477535005.785],
    [-11.95887, -116.94513, 92.712619830452549,
     4.57352, 7.16501, 78.64960934409585,
     13834722.5801401374, 124.688684161089762, 5228093.177931598,
     -0.56879356755666463, -0.56918731952397221, -9919582785894.853],
    [-87.85331, 85.66836, -65.120313040242748,
     66.48646, 16.09921, -4.888658719272296,
     17286615.3147144645, 155.58592449699137, 2635887.4729110181,
     -0.90697975771398578, -0.91095608883042767, 42667211366919.534],
    [1.74708, 128.32011, -101.584843631173858,
     -11.16617, 11.87109, -86.325793296437476,
     12942901.1241347408, 116.650512484301857, 5682744.8413270572,
     -0.44857868222697644, -0.44824490340007729, 10763055294345.653],
    [-25.72959, -144.90758, -153.647468693117198,
     -57.70581, -269.17879, -48.343983158876487,
     9413446.7452453107, 84.664533838404295, 6356176.6898881281,
     0.09492245755254703, 0.09737058264766572, 74515122850712.444],
    [-41.22777, 122.32875, 14.285113402275739,
     -7.57291, 130.37946, 10.805303085187369,
     3812686.035106021, 34.34330804743883, 3588703.8812128856,
     0.82605222593217889, 0.82572158200920196, -2456961531057.857],
    [11.01307, 138.25278, 79.43682622782374,
     6.62726, 247.05981, 103.708090215522657,
     11911190.819018408, 107.341669954114577, 6070904.722786735,
     -0.29767608923657404, -0.29785143390252321, 17121631423099.696],
    [-29.47124, 95.14681, -163.779130441688382,
     -27.46601, -69.15955, -15.909335945554969,
     13487015.8381145492, 121.294026715742277, 5481428.9945736388,
     -0.51527225545373252, -0.51556587964721788, 104679964020340.318]]

  def test_inverse(self):
    """Helper function for testing inverse calculation"""
    for l in GeodesicTest.testcases:
      (lat1, lon1, azi1, lat2, lon2, azi2,
       s12, a12, m12, M12, M21, S12) = l
      inv = Geodesic.WGS84.Inverse(lat1, lon1, lat2, lon2,
                     Geodesic.ALL | Geodesic.LONG_UNROLL)
      self.assertAlmostEqual(lon2, inv["lon2"], delta = 1e-13)
      self.assertAlmostEqual(azi1, inv["azi1"], delta = 1e-13)
      self.assertAlmostEqual(azi2, inv["azi2"], delta = 1e-13)
      self.assertAlmostEqual(s12, inv["s12"], delta = 1e-8)
      self.assertAlmostEqual(a12, inv["a12"], delta = 1e-13)
      self.assertAlmostEqual(m12, inv["m12"], delta = 1e-8)
      self.assertAlmostEqual(M12, inv["M12"], delta = 1e-15)
      self.assertAlmostEqual(M21, inv["M21"], delta = 1e-15)
      self.assertAlmostEqual(S12, inv["S12"], delta = 0.1)

  def test_direct(self):
    """Helper function for testing direct calculation"""
    for l in GeodesicTest.testcases:
      (lat1, lon1, azi1, lat2, lon2, azi2,
       s12, a12, m12, M12, M21, S12) = l
      direct = Geodesic.WGS84.Direct(lat1, lon1, azi1, s12,
                                     Geodesic.ALL | Geodesic.LONG_UNROLL)
      self.assertAlmostEqual(lat2, direct["lat2"], delta = 1e-13)
      self.assertAlmostEqual(lon2, direct["lon2"], delta = 1e-13)
      self.assertAlmostEqual(azi2, direct["azi2"], delta = 1e-13)
      self.assertAlmostEqual(a12, direct["a12"], delta = 1e-13)
      self.assertAlmostEqual(m12, direct["m12"], delta = 1e-8)
      self.assertAlmostEqual(M12, direct["M12"], delta = 1e-15)
      self.assertAlmostEqual(M21, direct["M21"], delta = 1e-15)
      self.assertAlmostEqual(S12, direct["S12"], delta = 0.1)

  def test_arcdirect(self):
    """Helper function for testing direct calculation with arc length"""
    for l in GeodesicTest.testcases:
      (lat1, lon1, azi1, lat2, lon2, azi2,
       s12, a12, m12, M12, M21, S12) = l
      direct = Geodesic.WGS84.ArcDirect(lat1, lon1, azi1, a12,
                                        Geodesic.ALL | Geodesic.LONG_UNROLL)
      self.assertAlmostEqual(lat2, direct["lat2"], delta = 1e-13)
      self.assertAlmostEqual(lon2, direct["lon2"], delta = 1e-13)
      self.assertAlmostEqual(azi2, direct["azi2"], delta = 1e-13)
      self.assertAlmostEqual(s12, direct["s12"], delta = 1e-8)
      self.assertAlmostEqual(m12, direct["m12"], delta = 1e-8)
      self.assertAlmostEqual(M12, direct["M12"], delta = 1e-15)
      self.assertAlmostEqual(M21, direct["M21"], delta = 1e-15)
      self.assertAlmostEqual(S12, direct["S12"], delta = 0.1)

class GeodSolveTest(unittest.TestCase):
  """GeodSolve tests"""

  def test_GeodSolve0(self):
    """GeodSolve0"""
    inv = Geodesic.WGS84.Inverse(40.6, -73.8, 49.01666667, 2.55)
    self.assertAlmostEqual(inv["azi1"], 53.47022, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 111.59367, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 5853226, delta = 0.5)

  def test_GeodSolve1(self):
    """GeodSolve1"""
    direct = Geodesic.WGS84.Direct(40.63972222, -73.77888889, 53.5, 5850e3)
    self.assertAlmostEqual(direct["lat2"], 49.01467, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], 2.56106, delta = 0.5e-5)
    self.assertAlmostEqual(direct["azi2"], 111.62947, delta = 0.5e-5)

  def test_GeodSolve2(self):
    """Check fix for antipodal prolate bug found 2010-09-04"""
    geod = Geodesic(6.4e6, -1/150.0)
    inv = geod.Inverse(0.07476, 0, -0.07476, 180)
    self.assertAlmostEqual(inv["azi1"], 90.00078, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 90.00078, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20106193, delta = 0.5)
    inv = geod.Inverse(0.1, 0, -0.1, 180)
    self.assertAlmostEqual(inv["azi1"], 90.00105, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 90.00105, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20106193, delta = 0.5)

  def test_GeodSolve4(self):
    """Check fix for short line bug found 2010-05-21"""
    inv = Geodesic.WGS84.Inverse(36.493349428792, 0,
                                 36.49334942879201, 0.0000008)
    self.assertAlmostEqual(inv["s12"], 0.072, delta = 0.5e-3)

  def test_GeodSolve5(self):
    """Check fix for point2=pole bug found 2010-05-03"""
    direct = Geodesic.WGS84.Direct(0.01777745589997, 30, 0, 10e6)
    self.assertAlmostEqual(direct["lat2"], 90, delta = 0.5e-5)
    if direct["lon2"] < 0:
      self.assertAlmostEqual(direct["lon2"], -150, delta = 0.5e-5)
      self.assertAlmostEqual(abs(direct["azi2"]), 180, delta = 0.5e-5)
    else:
      self.assertAlmostEqual(direct["lon2"], 30, delta = 0.5e-5)
      self.assertAlmostEqual(direct["azi2"], 0, delta = 0.5e-5)

  def test_GeodSolve6(self):
    """Check fix for volatile sbet12a bug found 2011-06-25 (gcc 4.4.4
    x86 -O3).  Found again on 2012-03-27 with tdm-mingw32 (g++ 4.6.1)."""
    inv = Geodesic.WGS84.Inverse(88.202499451857, 0,
                                 -88.202499451857, 179.981022032992859592)
    self.assertAlmostEqual(inv["s12"], 20003898.214, delta = 0.5e-3)
    inv = Geodesic.WGS84.Inverse(89.262080389218, 0,
                                 -89.262080389218, 179.992207982775375662)
    self.assertAlmostEqual(inv["s12"], 20003925.854, delta = 0.5e-3)
    inv = Geodesic.WGS84.Inverse(89.333123580033, 0,
                                 -89.333123580032997687, 179.99295812360148422)
    self.assertAlmostEqual(inv["s12"], 20003926.881, delta = 0.5e-3)

  def test_GeodSolve9(self):
    """Check fix for volatile x bug found 2011-06-25 (gcc 4.4.4 x86 -O3)"""
    inv = Geodesic.WGS84.Inverse(56.320923501171, 0,
                                 -56.320923501171, 179.664747671772880215)
    self.assertAlmostEqual(inv["s12"], 19993558.287, delta = 0.5e-3)

  def test_GeodSolve10(self):
    """Check fix for adjust tol1_ bug found 2011-06-25 (Visual Studio
    10 rel + debug)"""
    inv = Geodesic.WGS84.Inverse(52.784459512564, 0,
                                 -52.784459512563990912, 179.634407464943777557)
    self.assertAlmostEqual(inv["s12"], 19991596.095, delta = 0.5e-3)

  def test_GeodSolve11(self):
    """Check fix for bet2 = -bet1 bug found 2011-06-25 (Visual Studio
    10 rel + debug)"""
    inv = Geodesic.WGS84.Inverse(48.522876735459, 0,
                                 -48.52287673545898293, 179.599720456223079643)
    self.assertAlmostEqual(inv["s12"], 19989144.774, delta = 0.5e-3)

  def test_GeodSolve12(self):
    """Check fix for inverse geodesics on extreme prolate/oblate
    ellipsoids Reported 2012-08-29 Stefan Guenther
    <stefan.gunther@embl.de>; fixed 2012-10-07"""
    geod = Geodesic(89.8, -1.83)
    inv = geod.Inverse(0, 0, -10, 160)
    self.assertAlmostEqual(inv["azi1"], 120.27, delta = 1e-2)
    self.assertAlmostEqual(inv["azi2"], 105.15, delta = 1e-2)
    self.assertAlmostEqual(inv["s12"], 266.7, delta = 1e-1)

  def test_GeodSolve14(self):
    """Check fix for inverse ignoring lon12 = nan"""
    inv = Geodesic.WGS84.Inverse(0, 0, 1, math.nan)
    self.assertTrue(math.isnan(inv["azi1"]))
    self.assertTrue(math.isnan(inv["azi2"]))
    self.assertTrue(math.isnan(inv["s12"]))

  def test_GeodSolve15(self):
    """Initial implementation of Math::eatanhe was wrong for e^2 < 0.  This
    checks that this is fixed."""
    geod = Geodesic(6.4e6, -1/150.0)
    direct = geod.Direct(1, 2, 3, 4, Geodesic.AREA)
    self.assertAlmostEqual(direct["S12"], 23700, delta = 0.5)

  def test_GeodSolve17(self):
    """Check fix for LONG_UNROLL bug found on 2015-05-07"""
    direct = Geodesic.WGS84.Direct(40, -75, -10, 2e7,
                                   Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], -39, delta = 1)
    self.assertAlmostEqual(direct["lon2"], -254, delta = 1)
    self.assertAlmostEqual(direct["azi2"], -170, delta = 1)
    line = Geodesic.WGS84.Line(40, -75, -10)
    direct = line.Position(2e7, Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], -39, delta = 1)
    self.assertAlmostEqual(direct["lon2"], -254, delta = 1)
    self.assertAlmostEqual(direct["azi2"], -170, delta = 1)
    direct = Geodesic.WGS84.Direct(40, -75, -10, 2e7)
    self.assertAlmostEqual(direct["lat2"], -39, delta = 1)
    self.assertAlmostEqual(direct["lon2"], 105, delta = 1)
    self.assertAlmostEqual(direct["azi2"], -170, delta = 1)
    direct = line.Position(2e7)
    self.assertAlmostEqual(direct["lat2"], -39, delta = 1)
    self.assertAlmostEqual(direct["lon2"], 105, delta = 1)
    self.assertAlmostEqual(direct["azi2"], -170, delta = 1)

  def test_GeodSolve26(self):
    """Check 0/0 problem with area calculation on sphere 2015-09-08"""
    geod = Geodesic(6.4e6, 0)
    inv = geod.Inverse(1, 2, 3, 4, Geodesic.AREA)
    self.assertAlmostEqual(inv["S12"], 49911046115.0, delta = 0.5)

  def test_GeodSolve28(self):
    """Check for bad placement of assignment of r.a12 with |f| > 0.01 (bug in
    Java implementation fixed on 2015-05-19)."""
    geod = Geodesic(6.4e6, 0.1)
    direct = geod.Direct(1, 2, 10, 5e6)
    self.assertAlmostEqual(direct["a12"], 48.55570690, delta = 0.5e-8)

  def test_GeodSolve29(self):
    """Check longitude unrolling with inverse calculation 2015-09-16"""
    direct = Geodesic.WGS84.Inverse(0, 539, 0, 181)
    self.assertAlmostEqual(direct["lon1"], 179, delta = 1e-10)
    self.assertAlmostEqual(direct["lon2"], -179, delta = 1e-10)
    self.assertAlmostEqual(direct["s12"], 222639, delta = 0.5)
    direct = Geodesic.WGS84.Inverse(0, 539, 0, 181,
                                    Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lon1"], 539, delta = 1e-10)
    self.assertAlmostEqual(direct["lon2"], 541, delta = 1e-10)
    self.assertAlmostEqual(direct["s12"], 222639, delta = 0.5)

  def test_GeodSolve33(self):
    """Check max(-0.0,+0.0) issues 2015-08-22 (triggered by bugs in
    Octave -- sind(-0.0) = +0.0 -- and in some version of Visual
    Studio -- fmod(-0.0, 360.0) = +0.0."""
    inv = Geodesic.WGS84.Inverse(0, 0, 0, 179)
    self.assertAlmostEqual(inv["azi1"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 19926189, delta = 0.5)
    inv = Geodesic.WGS84.Inverse(0, 0, 0, 179.5)
    self.assertAlmostEqual(inv["azi1"], 55.96650, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 124.03350, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 19980862, delta = 0.5)
    inv = Geodesic.WGS84.Inverse(0, 0, 0, 180)
    self.assertAlmostEqual(inv["azi1"], 0.00000, delta = 0.5e-5)
    self.assertAlmostEqual(abs(inv["azi2"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20003931, delta = 0.5)
    inv = Geodesic.WGS84.Inverse(0, 0, 1, 180)
    self.assertAlmostEqual(inv["azi1"], 0.00000, delta = 0.5e-5)
    self.assertAlmostEqual(abs(inv["azi2"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 19893357, delta = 0.5)
    geod = Geodesic(6.4e6, 0)
    inv = geod.Inverse(0, 0, 0, 179)
    self.assertAlmostEqual(inv["azi1"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 19994492, delta = 0.5)
    inv = geod.Inverse(0, 0, 0, 180)
    self.assertAlmostEqual(inv["azi1"], 0.00000, delta = 0.5e-5)
    self.assertAlmostEqual(abs(inv["azi2"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20106193, delta = 0.5)
    inv = geod.Inverse(0, 0, 1, 180)
    self.assertAlmostEqual(inv["azi1"], 0.00000, delta = 0.5e-5)
    self.assertAlmostEqual(abs(inv["azi2"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 19994492, delta = 0.5)
    geod = Geodesic(6.4e6, -1/300.0)
    inv = geod.Inverse(0, 0, 0, 179)
    self.assertAlmostEqual(inv["azi1"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 19994492, delta = 0.5)
    inv = geod.Inverse(0, 0, 0, 180)
    self.assertAlmostEqual(inv["azi1"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 90.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20106193, delta = 0.5)
    inv = geod.Inverse(0, 0, 0.5, 180)
    self.assertAlmostEqual(inv["azi1"], 33.02493, delta = 0.5e-5)
    self.assertAlmostEqual(inv["azi2"], 146.97364, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20082617, delta = 0.5)
    inv = geod.Inverse(0, 0, 1, 180)
    self.assertAlmostEqual(inv["azi1"], 0.00000, delta = 0.5e-5)
    self.assertAlmostEqual(abs(inv["azi2"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(inv["s12"], 20027270, delta = 0.5)

  def test_GeodSolve55(self):
    """Check fix for nan + point on equator or pole not returning all nans in
    Geodesic::Inverse, found 2015-09-23."""
    inv = Geodesic.WGS84.Inverse(math.nan, 0, 0, 90)
    self.assertTrue(math.isnan(inv["azi1"]))
    self.assertTrue(math.isnan(inv["azi2"]))
    self.assertTrue(math.isnan(inv["s12"]))
    inv = Geodesic.WGS84.Inverse(math.nan, 0, 90, 9)
    self.assertTrue(math.isnan(inv["azi1"]))
    self.assertTrue(math.isnan(inv["azi2"]))
    self.assertTrue(math.isnan(inv["s12"]))

  def test_GeodSolve59(self):
    """Check for points close with longitudes close to 180 deg apart."""
    inv = Geodesic.WGS84.Inverse(5, 0.00000000000001, 10, 180)
    self.assertAlmostEqual(inv["azi1"], 0.000000000000035, delta = 1.5e-14)
    self.assertAlmostEqual(inv["azi2"], 179.99999999999996, delta = 1.5e-14)
    self.assertAlmostEqual(inv["s12"], 18345191.174332713, delta = 5e-9)

  def test_GeodSolve61(self):
    """Make sure small negative azimuths are west-going"""
    direct = Geodesic.WGS84.Direct(45, 0, -0.000000000000000003, 1e7,
                                   Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], 45.30632, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -180, delta = 0.5e-5)
    self.assertAlmostEqual(abs(direct["azi2"]), 180, delta = 0.5e-5)
    line = Geodesic.WGS84.InverseLine(45, 0, 80, -0.000000000000000003)
    direct = line.Position(1e7, Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], 45.30632, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -180, delta = 0.5e-5)
    self.assertAlmostEqual(abs(direct["azi2"]), 180, delta = 0.5e-5)

  def test_GeodSolve65(self):
    """Check for bug in east-going check in GeodesicLine (needed to check for
    sign of 0) and sign error in area calculation due to a bogus override
    of the code for alp12.  Found/fixed on 2015-12-19."""
    line = Geodesic.WGS84.InverseLine(30, -0.000000000000000001, -31, 180,
                                      Geodesic.ALL)
    direct = line.Position(1e7, Geodesic.ALL | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat1"], 30.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon1"], -0.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(abs(direct["azi1"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lat2"], -60.23169 , delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -0.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(abs(direct["azi2"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(direct["s12"] , 10000000  , delta = 0.5)
    self.assertAlmostEqual(direct["a12"] , 90.06544  , delta = 0.5e-5)
    self.assertAlmostEqual(direct["m12"] , 6363636   , delta = 0.5)
    self.assertAlmostEqual(direct["M12"] , -0.0012834, delta = 0.5e7)
    self.assertAlmostEqual(direct["M21"] , 0.0013749 , delta = 0.5e-7)
    self.assertAlmostEqual(direct["S12"] , 0         , delta = 0.5)
    direct = line.Position(2e7, Geodesic.ALL | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat1"], 30.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon1"], -0.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(abs(direct["azi1"]), 180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lat2"], -30.03547 , delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(direct["azi2"], -0.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(direct["s12"] , 20000000  , delta = 0.5)
    self.assertAlmostEqual(direct["a12"] , 179.96459 , delta = 0.5e-5)
    self.assertAlmostEqual(direct["m12"] , 54342     , delta = 0.5)
    self.assertAlmostEqual(direct["M12"] , -1.0045592, delta = 0.5e7)
    self.assertAlmostEqual(direct["M21"] , -0.9954339, delta = 0.5e-7)
    self.assertAlmostEqual(direct["S12"] , 127516405431022.0, delta = 0.5)

  def test_GeodSolve66(self):
    """Check for InverseLine if line is slightly west of S and that s13 is
    correctly set."""
    line = Geodesic.WGS84.InverseLine(-5, -0.000000000000002, -10, 180)
    direct = line.Position(2e7, Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], 4.96445   , delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -180.00000, delta = 0.5e-5)
    self.assertAlmostEqual(direct["azi2"], -0.00000  , delta = 0.5e-5)
    direct = line.Position(0.5 * line.s13,
                           Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], -87.52461 , delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -0.00000  , delta = 0.5e-5)
    self.assertAlmostEqual(direct["azi2"], -180.00000, delta = 0.5e-5)

  def test_GeodSolve71(self):
    """Check that DirectLine sets s13."""
    line = Geodesic.WGS84.DirectLine(1, 2, 45, 1e7)
    direct = line.Position(0.5 * line.s13,
                           Geodesic.STANDARD | Geodesic.LONG_UNROLL)
    self.assertAlmostEqual(direct["lat2"], 30.92625, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], 37.54640, delta = 0.5e-5)
    self.assertAlmostEqual(direct["azi2"], 55.43104, delta = 0.5e-5)

  def test_GeodSolve73(self):
    """Check for backwards from the pole bug reported by Anon on 2016-02-13.
    This only affected the Java implementation.  It was introduced in Java
    version 1.44 and fixed in 1.46-SNAPSHOT on 2016-01-17.
    Also the + sign on azi2 is a check on the normalizing of azimuths
    (converting -0.0 to +0.0)."""
    direct = Geodesic.WGS84.Direct(90, 10, 180, -1e6)
    self.assertAlmostEqual(direct["lat2"], 81.04623, delta = 0.5e-5)
    self.assertAlmostEqual(direct["lon2"], -170, delta = 0.5e-5)
    self.assertAlmostEqual(direct["azi2"], 0, delta = 0.5e-5)
    self.assertTrue(math.copysign(1, direct["azi2"]) > 0)

  def test_GeodSolve74(self):
    """Check fix for inaccurate areas, bug introduced in v1.46, fixed
    2015-10-16."""
    inv = Geodesic.WGS84.Inverse(54.1589, 15.3872, 54.1591, 15.3877,
                                 Geodesic.ALL)
    self.assertAlmostEqual(inv["azi1"], 55.723110355, delta = 5e-9)
    self.assertAlmostEqual(inv["azi2"], 55.723515675, delta = 5e-9)
    self.assertAlmostEqual(inv["s12"],  39.527686385, delta = 5e-9)
    self.assertAlmostEqual(inv["a12"],   0.000355495, delta = 5e-9)
    self.assertAlmostEqual(inv["m12"],  39.527686385, delta = 5e-9)
    self.assertAlmostEqual(inv["M12"],   0.999999995, delta = 5e-9)
    self.assertAlmostEqual(inv["M21"],   0.999999995, delta = 5e-9)
    self.assertAlmostEqual(inv["S12"], 286698586.30197, delta = 5e-4)

  def test_GeodSolve76(self):
    """The distance from Wellington and Salamanca (a classic failure of
    Vincenty)"""
    inv = Geodesic.WGS84.Inverse(-(41+19/60.0), 174+49/60.0,
                                 40+58/60.0, -(5+30/60.0))
    self.assertAlmostEqual(inv["azi1"], 160.39137649664, delta = 0.5e-11)
    self.assertAlmostEqual(inv["azi2"],  19.50042925176, delta = 0.5e-11)
    self.assertAlmostEqual(inv["s12"],  19960543.857179, delta = 0.5e-6)

  def test_GeodSolve78(self):
    """An example where the NGS calculator fails to converge"""
    inv = Geodesic.WGS84.Inverse(27.2, 0.0, -27.1, 179.5)
    self.assertAlmostEqual(inv["azi1"],  45.82468716758, delta = 0.5e-11)
    self.assertAlmostEqual(inv["azi2"], 134.22776532670, delta = 0.5e-11)
    self.assertAlmostEqual(inv["s12"],  19974354.765767, delta = 0.5e-6)

  def test_GeodSolve80(self):
    """Some tests to add code coverage: computing scale in special cases + zero
    length geodesic (includes GeodSolve80 - GeodSolve83) + using an incapable
    line."""
    inv = Geodesic.WGS84.Inverse(0, 0, 0, 90, Geodesic.GEODESICSCALE)
    self.assertAlmostEqual(inv["M12"], -0.00528427534, delta = 0.5e-10)
    self.assertAlmostEqual(inv["M21"], -0.00528427534, delta = 0.5e-10)

    inv = Geodesic.WGS84.Inverse(0, 0, 1e-6, 1e-6, Geodesic.GEODESICSCALE)
    self.assertAlmostEqual(inv["M12"], 1, delta = 0.5e-10)
    self.assertAlmostEqual(inv["M21"], 1, delta = 0.5e-10)

    inv = Geodesic.WGS84.Inverse(20.001, 0, 20.001, 0, Geodesic.ALL)
    self.assertAlmostEqual(inv["a12"], 0, delta = 1e-13)
    self.assertAlmostEqual(inv["s12"], 0, delta = 1e-8)
    self.assertAlmostEqual(inv["azi1"], 180, delta = 1e-13)
    self.assertAlmostEqual(inv["azi2"], 180, delta = 1e-13)
    self.assertAlmostEqual(inv["m12"], 0, delta =  1e-8)
    self.assertAlmostEqual(inv["M12"], 1, delta = 1e-15)
    self.assertAlmostEqual(inv["M21"], 1, delta = 1e-15)
    self.assertAlmostEqual(inv["S12"], 0, delta = 1e-10)
    self.assertTrue(math.copysign(1, inv["a12"]) > 0)
    self.assertTrue(math.copysign(1, inv["s12"]) > 0)
    self.assertTrue(math.copysign(1, inv["m12"]) > 0)

    inv = Geodesic.WGS84.Inverse(90, 0, 90, 180, Geodesic.ALL)
    self.assertAlmostEqual(inv["a12"], 0, delta = 1e-13)
    self.assertAlmostEqual(inv["s12"], 0, delta = 1e-8)
    self.assertAlmostEqual(inv["azi1"], 0, delta = 1e-13)
    self.assertAlmostEqual(inv["azi2"], 180, delta = 1e-13)
    self.assertAlmostEqual(inv["m12"], 0, delta = 1e-8)
    self.assertAlmostEqual(inv["M12"], 1, delta = 1e-15)
    self.assertAlmostEqual(inv["M21"], 1, delta = 1e-15)
    self.assertAlmostEqual(inv["S12"], 127516405431022.0, delta = 0.5)

    # An incapable line which can't take distance as input
    line = Geodesic.WGS84.Line(1, 2, 90, Geodesic.LATITUDE)
    direct = line.Position(1000, Geodesic.EMPTY)
    self.assertTrue(math.isnan(direct["a12"]))

  def test_GeodSolve84(self):
    """Tests for python implementation to check fix for range errors with
    {fmod,sin,cos}(inf) (includes GeodSolve84 - GeodSolve91)."""
    direct = Geodesic.WGS84.Direct(0, 0, 90, math.inf)
    self.assertTrue(math.isnan(direct["lat2"]))
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(math.isnan(direct["azi2"]))
    direct = Geodesic.WGS84.Direct(0, 0, 90, math.nan)
    self.assertTrue(math.isnan(direct["lat2"]))
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(math.isnan(direct["azi2"]))
    direct = Geodesic.WGS84.Direct(0, 0, math.inf, 1000)
    self.assertTrue(math.isnan(direct["lat2"]))
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(math.isnan(direct["azi2"]))
    direct = Geodesic.WGS84.Direct(0, 0, math.nan, 1000)
    self.assertTrue(math.isnan(direct["lat2"]))
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(math.isnan(direct["azi2"]))
    direct = Geodesic.WGS84.Direct(0, math.inf, 90, 1000)
    self.assertTrue(direct["lat1"] == 0)
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(direct["azi2"] == 90)
    direct = Geodesic.WGS84.Direct(0, math.nan, 90, 1000)
    self.assertTrue(direct["lat1"] == 0)
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(direct["azi2"] == 90)
    direct = Geodesic.WGS84.Direct(math.inf, 0, 90, 1000)
    self.assertTrue(math.isnan(direct["lat2"]))
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(math.isnan(direct["azi2"]))
    direct = Geodesic.WGS84.Direct(math.nan, 0, 90, 1000)
    self.assertTrue(math.isnan(direct["lat2"]))
    self.assertTrue(math.isnan(direct["lon2"]))
    self.assertTrue(math.isnan(direct["azi2"]))

  def test_GeodSolve92(self):
    """Check fix for inaccurate hypot with python 3.[89].  Problem reported
    by agdhruv https://github.com/geopy/geopy/issues/466 ; see
    https://bugs.python.org/issue43088"""
    inv = Geodesic.WGS84.Inverse(37.757540000000006, -122.47018,
                                 37.75754,           -122.470177)
    self.assertAlmostEqual(inv["azi1"], 89.99999923, delta = 1e-7  )
    self.assertAlmostEqual(inv["azi2"], 90.00000106, delta = 1e-7  )
    self.assertAlmostEqual(inv["s12"],   0.264,      delta = 0.5e-3)

  def test_GeodSolve94(self):
    """Check fix for lat2 = nan being treated as lat2 = 0 (bug found
       2021-07-26)"""
    inv = Geodesic.WGS84.Inverse(0, 0, math.nan, 90)
    self.assertTrue(math.isnan(inv["azi1"]))
    self.assertTrue(math.isnan(inv["azi2"]))
    self.assertTrue(math.isnan(inv["s12"]))

  def test_GeodSolve96(self):
    """Failure with long doubles found with test case from Nowak + Nowak Da
       Costa (2022).  Problem was using somg12 > 1 as a test that it needed
       to be set when roundoff could result in somg12 slightly bigger that 1.
       Found + fixed 2022-03-30."""
    geod = Geodesic(6378137, 1/298.257222101)
    inv = geod.Inverse(0, 0, 60.0832522871723, 89.8492185074635, Geodesic.AREA)
    self.assertAlmostEqual(inv["S12"], 42426932221845, delta = 0.5)

class PlanimeterTest(unittest.TestCase):
  """Planimeter tests"""

  polygon = Geodesic.WGS84.Polygon(False)
  polyline = Geodesic.WGS84.Polygon(True)

  @staticmethod
  def Planimeter(points):
    """Helper function for polygons"""
    PlanimeterTest.polygon.Clear()
    for p in points:
      PlanimeterTest.polygon.AddPoint(p[0], p[1])
    return PlanimeterTest.polygon.Compute(False, True)

  @staticmethod
  def PolyLength(points):
    """Helper function for polylines"""
    PlanimeterTest.polyline.Clear()
    for p in points:
      PlanimeterTest.polyline.AddPoint(p[0], p[1])
    return PlanimeterTest.polyline.Compute(False, True)

  def test_Planimeter0(self):
    """Check fix for pole-encircling bug found 2011-03-16"""
    points = [[89, 0], [89, 90], [89, 180], [89, 270]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 631819.8745, delta = 1e-4)
    self.assertAlmostEqual(area, 24952305678.0, delta = 1)
    points = [[-89, 0], [-89, 90], [-89, 180], [-89, 270]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 631819.8745, delta = 1e-4)
    self.assertAlmostEqual(area, -24952305678.0, delta = 1)

    points = [[0, -1], [-1, 0], [0, 1], [1, 0]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 627598.2731, delta = 1e-4)
    self.assertAlmostEqual(area, 24619419146.0, delta = 1)

    points = [[90, 0], [0, 0], [0, 90]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 30022685, delta = 1)
    self.assertAlmostEqual(area, 63758202715511.0, delta = 1)
    _, perimeter, area = PlanimeterTest.PolyLength(points)
    self.assertAlmostEqual(perimeter, 20020719, delta = 1)
    self.assertTrue(math.isnan(area))

  def test_Planimeter5(self):
    """Check fix for Planimeter pole crossing bug found 2011-06-24"""
    points = [[89, 0.1], [89, 90.1], [89, -179.9]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 539297, delta = 1)
    self.assertAlmostEqual(area, 12476152838.5, delta = 1)

  def test_Planimeter6(self):
    """Check fix for Planimeter lon12 rounding bug found 2012-12-03"""
    points = [[9, -0.00000000000001], [9, 180], [9, 0]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 36026861, delta = 1)
    self.assertAlmostEqual(area, 0, delta = 1)
    points = [[9, 0.00000000000001], [9, 0], [9, 180]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 36026861, delta = 1)
    self.assertAlmostEqual(area, 0, delta = 1)
    points = [[9, 0.00000000000001], [9, 180], [9, 0]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 36026861, delta = 1)
    self.assertAlmostEqual(area, 0, delta = 1)
    points = [[9, -0.00000000000001], [9, 0], [9, 180]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 36026861, delta = 1)
    self.assertAlmostEqual(area, 0, delta = 1)

  def test_Planimeter12(self):
    """Area of arctic circle (not really -- adjunct to rhumb-area test)"""
    points = [[66.562222222, 0], [66.562222222, 180], [66.562222222, 360]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 10465729, delta = 1)
    self.assertAlmostEqual(area, 0, delta = 1)

  def test_Planimeter12r(self):
    """Reverse area of arctic circle"""
    points = [[66.562222222, -0], [66.562222222, -180], [66.562222222, -360]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 10465729, delta = 1)
    self.assertAlmostEqual(area, 0, delta = 1)

  def test_Planimeter13(self):
    """Check encircling pole twice"""
    points = [[89,-360], [89,-240], [89,-120], [89,0], [89,120], [89,240]]
    _, perimeter, area = PlanimeterTest.Planimeter(points)
    self.assertAlmostEqual(perimeter, 1160741, delta = 1)
    self.assertAlmostEqual(area, 32415230256.0, delta = 1)

  def test_Planimeter15(self):
    """Coverage tests, includes Planimeter15 - Planimeter18 (combinations of
    reverse and sign) + calls to testpoint, testedge."""
    lat = [2, 1, 3]
    lon = [1, 2, 3]
    r = 18454562325.45119
    a0 = 510065621724088.5093   # ellipsoid area
    PlanimeterTest.polygon.Clear()
    PlanimeterTest.polygon.AddPoint(lat[0], lon[0])
    PlanimeterTest.polygon.AddPoint(lat[1], lon[1])
    _, _, area = PlanimeterTest.polygon.TestPoint(lat[2], lon[2], False, True)
    self.assertAlmostEqual(area, r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.TestPoint(lat[2], lon[2], False, False)
    self.assertAlmostEqual(area, r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.TestPoint(lat[2], lon[2], True, True)
    self.assertAlmostEqual(area, -r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.TestPoint(lat[2], lon[2], True, False)
    self.assertAlmostEqual(area, a0-r, delta = 0.5)
    inv = Geodesic.WGS84.Inverse(lat[1], lon[1], lat[2], lon[2])
    azi1 = inv["azi1"]
    s12 = inv["s12"]
    _, _, area = PlanimeterTest.polygon.TestEdge(azi1, s12, False, True)
    self.assertAlmostEqual(area, r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.TestEdge(azi1, s12, False, False)
    self.assertAlmostEqual(area, r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.TestEdge(azi1, s12, True, True)
    self.assertAlmostEqual(area, -r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.TestEdge(azi1, s12, True, False)
    self.assertAlmostEqual(area, a0-r, delta = 0.5)
    PlanimeterTest.polygon.AddPoint(lat[2], lon[2])
    _, _, area = PlanimeterTest.polygon.Compute(False, True)
    self.assertAlmostEqual(area, r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.Compute(False, False)
    self.assertAlmostEqual(area, r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.Compute(True, True)
    self.assertAlmostEqual(area, -r, delta = 0.5)
    _, _, area = PlanimeterTest.polygon.Compute(True, False)
    self.assertAlmostEqual(area, a0-r, delta = 0.5)

  def test_Planimeter19(self):
    """Coverage tests, includes Planimeter19 - Planimeter20 (degenerate
    polygons) + extra cases."""
    PlanimeterTest.polygon.Clear()
    _, perimeter, area = PlanimeterTest.polygon.Compute(False, True)
    self.assertTrue(area == 0)
    self.assertTrue(perimeter == 0)
    _, perimeter, area = PlanimeterTest.polygon.TestPoint(1, 1, False, True)
    self.assertTrue(area == 0)
    self.assertTrue(perimeter == 0)
    _, perimeter, area = PlanimeterTest.polygon.TestEdge(90, 1000, False, True)
    self.assertTrue(math.isnan(area))
    self.assertTrue(math.isnan(perimeter))
    PlanimeterTest.polygon.AddPoint(1, 1)
    _, perimeter, area = PlanimeterTest.polygon.Compute(False, True)
    self.assertTrue(area == 0)
    self.assertTrue(perimeter == 0)
    PlanimeterTest.polyline.Clear()
    _, perimeter, area = PlanimeterTest.polyline.Compute(False, True)
    self.assertTrue(perimeter == 0)
    _, perimeter, area = PlanimeterTest.polyline.TestPoint(1, 1, False, True)
    self.assertTrue(perimeter == 0)
    _, perimeter, area = PlanimeterTest.polyline.TestEdge(90, 1000, False, True)
    self.assertTrue(math.isnan(perimeter))
    PlanimeterTest.polyline.AddPoint(1, 1)
    _, perimeter, area = PlanimeterTest.polyline.Compute(False, True)
    self.assertTrue(perimeter == 0)
    PlanimeterTest.polygon.AddPoint(1, 1)
    _, perimeter, area = PlanimeterTest.polyline.TestEdge(90, 1000, False, True)
    self.assertAlmostEqual(perimeter, 1000, delta = 1e-10)
    _, perimeter, area = PlanimeterTest.polyline.TestPoint(2, 2, False, True)
    self.assertAlmostEqual(perimeter, 156876.149, delta = 0.5e-3)

  def test_Planimeter21(self):
    """Some tests to add code coverage: multiple circlings of pole (includes
    Planimeter21 - Planimeter28) + invocations via testpoint and testedge."""
    lat = 45
    azi = 39.2144607176828184218
    s = 8420705.40957178156285
    r = 39433884866571.4277     # Area for one circuit
    a0 = 510065621724088.5093   # Ellipsoid area
    PlanimeterTest.polygon.Clear()
    PlanimeterTest.polygon.AddPoint(lat,  60)
    PlanimeterTest.polygon.AddPoint(lat, 180)
    PlanimeterTest.polygon.AddPoint(lat, -60)
    PlanimeterTest.polygon.AddPoint(lat,  60)
    PlanimeterTest.polygon.AddPoint(lat, 180)
    PlanimeterTest.polygon.AddPoint(lat, -60)
    for i in [3, 4]:
      PlanimeterTest.polygon.AddPoint(lat,  60)
      PlanimeterTest.polygon.AddPoint(lat, 180)
      _, _, area = PlanimeterTest.polygon.TestPoint(lat, -60, False, True)
      self.assertAlmostEqual(area,  i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestPoint(lat, -60, False, False)
      self.assertAlmostEqual(area,  i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestPoint(lat, -60, True, True)
      self.assertAlmostEqual(area, -i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestPoint(lat, -60, True, False)
      self.assertAlmostEqual(area, -i*r + a0, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestEdge(azi, s, False, True)
      self.assertAlmostEqual(area,  i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestEdge(azi, s, False, False)
      self.assertAlmostEqual(area,  i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestEdge(azi, s, True, True)
      self.assertAlmostEqual(area, -i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.TestEdge(azi, s, True, False)
      self.assertAlmostEqual(area, -i*r + a0, delta = 0.5)
      PlanimeterTest.polygon.AddPoint(lat, -60)
      _, _, area = PlanimeterTest.polygon.Compute(False, True)
      self.assertAlmostEqual(area,  i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.Compute(False, False)
      self.assertAlmostEqual(area,  i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.Compute(True, True)
      self.assertAlmostEqual(area, -i*r, delta = 0.5)
      _, _, area = PlanimeterTest.polygon.Compute(True, False)
      self.assertAlmostEqual(area, -i*r + a0, delta = 0.5)

  def test_Planimeter29(self):
    """Check fix to transitdirect vs transit zero handling inconsistency"""
    PlanimeterTest.polygon.Clear()
    PlanimeterTest.polygon.AddPoint(0, 0)
    PlanimeterTest.polygon.AddEdge( 90, 1000)
    PlanimeterTest.polygon.AddEdge(  0, 1000)
    PlanimeterTest.polygon.AddEdge(-90, 1000)
    _, _, area = PlanimeterTest.polygon.Compute(False, True)
    # The area should be 1e6.  Prior to the fix it was 1e6 - A/2, where
    # A = ellipsoid area.
    self.assertAlmostEqual(area, 1000000.0, delta = 0.01)

env_pyrecodes/lib/python3.9/site-packages/geographiclib/test/test_geodesic.py
NikolaBlagojevic/pyrecodes
"""Geodesic tests"""

import unittest
import math
import sys

from geographiclib.geomath import Math
from geographiclib.geodesic import Geodesic

class SignTest(unittest.TestCase):
  """Sign test suite"""

  @staticmethod
  def equiv(x, y):
    """Test for equivalence"""

    return ( (math.isnan(x) and math.isnan(y)) or
             (x == y and math.copysign(1.0, x) == math.copysign(1.0, y)) )

  def test_AngRound(self):
    """Test special cases for AngRound"""
    eps = sys.float_info.epsilon
    self.assertTrue(SignTest.equiv(Math.AngRound(-eps/32), -eps/32))
    self.assertTrue(SignTest.equiv(Math.AngRound(-eps/64), -0.0   ))
    self.assertTrue(SignTest.equiv(Math.AngRound(-  0.0 ), -0.0   ))
    self.assertTrue(SignTest.equiv(Math.AngRound(   0.0 ), +0.0   ))
    self.assertTrue(SignTest.equiv(Math.AngRound( eps/64), +0.0   ))
    self.assertTrue(SignTest.equiv(Math.AngRound( eps/32), +eps/32))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-2*eps)/64), (1-2*eps)/64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps  )/64),  1.0     /64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/2)/64),  1.0     /64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/4)/64),  1.0     /64))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1.0     /64),  1.0     /64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps/2)/64),  1.0     /64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps  )/64),  1.0     /64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+2*eps)/64), (1+2*eps)/64))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps  )/32), (1-eps  )/32))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/2)/32),  1.0     /32))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/4)/32),  1.0     /32))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1.0     /32),  1.0     /32))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps/2)/32),  1.0     /32))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps  )/32), (1+eps  )/32))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps  )/16), (1-eps  )/16))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/2)/16), (1-eps/2)/16))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/4)/16),  1.0     /16))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1.0     /16),  1.0     /16))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps/4)/16),  1.0     /16))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps/2)/16),  1.0     /16))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps  )/16), (1+eps  )/16))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps  )/ 8), (1-eps  )/ 8))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/2)/ 8), (1-eps/2)/ 8))
    self.assertTrue(SignTest.equiv(Math.AngRound((1-eps/4)/ 8),  1.0     / 8))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps/2)/ 8),  1.0     / 8))
    self.assertTrue(SignTest.equiv(Math.AngRound((1+eps  )/ 8), (1+eps  )/ 8))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1-eps      ),  1-eps      ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1-eps/2    ),  1-eps/2    ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1-eps/4    ),  1          ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1.0        ),  1          ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1+eps/4    ),  1          ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1+eps/2    ),  1          ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 1+eps      ),  1+  eps    ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 90.0-64*eps),  90-64*eps  ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 90.0-32*eps),  90         ))
    self.assertTrue(SignTest.equiv(Math.AngRound( 90.0       ),  90         ))

  def test_sincosd(self):
    """Test special cases for sincosd"""
    inf = math.inf
    nan = math.nan
    s, c = Math.sincosd(-  inf)
    self.assertTrue(SignTest.equiv(s,  nan) and SignTest.equiv(c,  nan))
    s, c = Math.sincosd(-810.0)
    self.assertTrue(SignTest.equiv(s, -1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(-720.0)
    self.assertTrue(SignTest.equiv(s, -0.0) and SignTest.equiv(c, +1.0))
    s, c = Math.sincosd(-630.0)
    self.assertTrue(SignTest.equiv(s, +1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(-540.0)
    self.assertTrue(SignTest.equiv(s, -0.0) and SignTest.equiv(c, -1.0))
    s, c = Math.sincosd(-450.0)
    self.assertTrue(SignTest.equiv(s, -1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(-360.0)
    self.assertTrue(SignTest.equiv(s, -0.0) and SignTest.equiv(c, +1.0))
    s, c = Math.sincosd(-270.0)
    self.assertTrue(SignTest.equiv(s, +1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(-180.0)
    self.assertTrue(SignTest.equiv(s, -0.0) and SignTest.equiv(c, -1.0))
    s, c = Math.sincosd(- 90.0)
    self.assertTrue(SignTest.equiv(s, -1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(-  0.0)
    self.assertTrue(SignTest.equiv(s, -0.0) and SignTest.equiv(c, +1.0))
    s, c = Math.sincosd(+  0.0)
    self.assertTrue(SignTest.equiv(s, +0.0) and SignTest.equiv(c, +1.0))
    s, c = Math.sincosd(+ 90.0)
    self.assertTrue(SignTest.equiv(s, +1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(+180.0)
    self.assertTrue(SignTest.equiv(s, +0.0) and SignTest.equiv(c, -1.0))
    s, c = Math.sincosd(+270.0)
    self.assertTrue(SignTest.equiv(s, -1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(+360.0)
    self.assertTrue(SignTest.equiv(s, +0.0) and SignTest.equiv(c, +1.0))
    s, c = Math.sincosd(+450.0)
    self.assertTrue(SignTest.equiv(s, +1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(+540.0)
    self.assertTrue(SignTest.equiv(s, +0.0) and SignTest.equiv(c, -1.0))
    s, c = Math.sincosd(+630.0)
    self.assertTrue(SignTest.equiv(s, -1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(+720.0)
    self.assertTrue(SignTest.equiv(s, +0.0) and SignTest.equiv(c, +1.0))
    s, c = Math.sincosd(+810.0)
    self.assertTrue(SignTest.equiv(s, +1.0) and SignTest.equiv(c, +0.0))
    s, c = Math.sincosd(+  inf)
    self.assertTrue(SignTest.equiv(s,  nan) and SignTest.equiv(c,  nan))
    s, c = Math.sincosd(  nan)
    self.assertTrue(SignTest.equiv(s,  nan) and SignTest.equiv(c,  nan))

  def test_sincosd2(self):
    """Test accuracy of sincosd"""
    s1, c1 = Math.sincosd(         9.0)
    s2, c2 = Math.sincosd(        81.0)
    s3, c3 = Math.sincosd(-123456789.0)
    self.assertTrue(SignTest.equiv(s1, c2))
    self.assertTrue(SignTest.equiv(s1, s3))
    self.assertTrue(SignTest.equiv(c1, s2))
    self.assertTrue(SignTest.equiv(c1,-c3))

  def test_atan2d(self):
    """Test special cases for atan2d"""
    inf = math.inf
    nan = math.nan
    self.assertTrue(SignTest.equiv(Math.atan2d(+0.0 , -0.0 ), +180))
    self.assertTrue(SignTest.equiv(Math.atan2d(-0.0 , -0.0 ), -180))
    self.assertTrue(SignTest.equiv(Math.atan2d(+0.0 , +0.0 ), +0.0))
    self.assertTrue(SignTest.equiv(Math.atan2d(-0.0 , +0.0 ), -0.0))
    self.assertTrue(SignTest.equiv(Math.atan2d(+0.0 , -1.0 ), +180))
    self.assertTrue(SignTest.equiv(Math.atan2d(-0.0 , -1.0 ), -180))
    self.assertTrue(SignTest.equiv(Math.atan2d(+0.0 , +1.0 ), +0.0))
    self.assertTrue(SignTest.equiv(Math.atan2d(-0.0 , +1.0 ), -0.0))
    self.assertTrue(SignTest.equiv(Math.atan2d(-1.0 , +0.0 ),  -90))
    self.assertTrue(SignTest.equiv(Math.atan2d(-1.0 , -0.0 ),  -90))
    self.assertTrue(SignTest.equiv(Math.atan2d(+1.0 , +0.0 ),  +90))
    self.assertTrue(SignTest.equiv(Math.atan2d(+1.0 , -0.0 ),  +90))
    self.assertTrue(SignTest.equiv(Math.atan2d(+1.0 ,  -inf), +180))
    self.assertTrue(SignTest.equiv(Math.atan2d(-1.0 ,  -inf), -180))
    self.assertTrue(SignTest.equiv(Math.atan2d(+1.0 ,  +inf), +0.0))
    self.assertTrue(SignTest.equiv(Math.atan2d(-1.0 ,  +inf), -0.0))
    self.assertTrue(SignTest.equiv(Math.atan2d( +inf, +1.0 ),  +90))
    self.assertTrue(SignTest.equiv(Math.atan2d( +inf, -1.0 ),  +90))
    self.assertTrue(SignTest.equiv(Math.atan2d( -inf, +1.0 ),  -90))
    self.assertTrue(SignTest.equiv(Math.atan2d( -inf, -1.0 ),  -90))
    self.assertTrue(SignTest.equiv(Math.atan2d( +inf,  -inf), +135))
    self.assertTrue(SignTest.equiv(Math.atan2d( -inf,  -inf), -135))
    self.assertTrue(SignTest.equiv(Math.atan2d( +inf,  +inf),  +45))
    self.assertTrue(SignTest.equiv(Math.atan2d( -inf,  +inf),  -45))
    self.assertTrue(SignTest.equiv(Math.atan2d(  nan, +1.0 ),  nan))
    self.assertTrue(SignTest.equiv(Math.atan2d(+1.0 ,   nan),  nan))

  def test_atan2d2(self):
    """Test accuracy of atan2d"""
    s = 7e-16
    self.assertEqual(Math.atan2d(s, -1.0), 180 - Math.atan2d(s, 1.0))

  def test_sum(self):
    """Test special cases of sum"""
    s,_ = Math.sum(+9.0, -9.0); self.assertTrue(SignTest.equiv(s, +0.0))
    s,_ = Math.sum(-9.0, +9.0); self.assertTrue(SignTest.equiv(s, +0.0))
    s,_ = Math.sum(-0.0, +0.0); self.assertTrue(SignTest.equiv(s, +0.0))
    s,_ = Math.sum(+0.0, -0.0); self.assertTrue(SignTest.equiv(s, +0.0))
    s,_ = Math.sum(-0.0, -0.0); self.assertTrue(SignTest.equiv(s, -0.0))
    s,_ = Math.sum(+0.0, +0.0); self.assertTrue(SignTest.equiv(s, +0.0))

  def test_AngNormalize(self):
    """Test special cases of AngNormalize"""
    self.assertTrue(SignTest.equiv(Math.AngNormalize(-900.0), -180))
    self.assertTrue(SignTest.equiv(Math.AngNormalize(-720.0), -0.0))
    self.assertTrue(SignTest.equiv(Math.AngNormalize(-540.0), -180))
    self.assertTrue(SignTest.equiv(Math.AngNormalize(-360.0), -0.0))
    self.assertTrue(SignTest.equiv(Math.AngNormalize(-180.0), -180))
    self.assertTrue(SignTest.equiv(Math.AngNormalize(  -0.0), -0.0))
    self.assertTrue(SignTest.equiv(Math.AngNormalize(  +0.0), +0.0))
    self.assertTrue(SignTest.equiv(Math.AngNormalize( 180.0), +180))
    self.assertTrue(SignTest.equiv(Math.AngNormalize( 360.0), +0.0))
    self.assertTrue(SignTest.equiv(Math.AngNormalize( 540.0), +180))
    self.assertTrue(SignTest.equiv(Math.AngNormalize( 720.0), +0.0))
    self.assertTrue(SignTest.equiv(Math.AngNormalize( 900.0), +180))

  def test_AngDiff(self):
    """Test special cases of AngDiff"""
    eps = sys.float_info.epsilon
    s,_ = Math.AngDiff(+  0.0,+  0.0); self.assertTrue(SignTest.equiv(s,+0.0 ))
    s,_ = Math.AngDiff(+  0.0,-  0.0); self.assertTrue(SignTest.equiv(s,-0.0 ))
    s,_ = Math.AngDiff(-  0.0,+  0.0); self.assertTrue(SignTest.equiv(s,+0.0 ))
    s,_ = Math.AngDiff(-  0.0,-  0.0); self.assertTrue(SignTest.equiv(s,+0.0 ))
    s,_ = Math.AngDiff(+  5.0,+365.0); self.assertTrue(SignTest.equiv(s,+0.0 ))
    s,_ = Math.AngDiff(+365.0,+  5.0); self.assertTrue(SignTest.equiv(s,-0.0 ))
    s,_ = Math.AngDiff(+  5.0,+185.0); self.assertTrue(SignTest.equiv(s,+180.0))
    s,_ = Math.AngDiff(+185.0,+  5.0); self.assertTrue(SignTest.equiv(s,-180.0))
    s,_ = Math.AngDiff( +eps ,+180.0); self.assertTrue(SignTest.equiv(s,+180.0))
    s,_ = Math.AngDiff( -eps ,+180.0); self.assertTrue(SignTest.equiv(s,-180.0))
    s,_ = Math.AngDiff( +eps ,-180.0); self.assertTrue(SignTest.equiv(s,+180.0))
    s,_ = Math.AngDiff( -eps ,-180.0); self.assertTrue(SignTest.equiv(s,-180.0))

  def test_AngDiff2(self):
    """Test accuracy of AngDiff"""
    eps = sys.float_info.epsilon
    x = 138 + 128 * eps; y = -164; s,_ = Math.AngDiff(x, y)
    self.assertEqual(s, 58 - 128 * eps)

  def test_equatorial_coincident(self):
    """
    azimuth with coincident point on equator
    """
    # lat1 lat2 azi1/2
    C = [
      [ +0.0, -0.0, 180 ],
      [ -0.0, +0.0,   0 ]
    ]
    for l in C:
      (lat1, lat2, azi) = l
      inv = Geodesic.WGS84.Inverse(lat1, 0.0, lat2, 0.0)
      self.assertTrue(SignTest.equiv(inv["azi1"], azi))
      self.assertTrue(SignTest.equiv(inv["azi2"], azi))

  def test_equatorial_NS(self):
    """Does the nearly antipodal equatorial solution go north or south?"""
    # lat1 lat2 azi1 azi2
    C = [
      [ +0.0, +0.0,  56, 124],
      [ -0.0, -0.0, 124,  56]
    ]
    for l in C:
      (lat1, lat2, azi1, azi2) = l
      inv = Geodesic.WGS84.Inverse(lat1, 0.0, lat2, 179.5)
      self.assertAlmostEqual(inv["azi1"], azi1, delta = 1)
      self.assertAlmostEqual(inv["azi2"], azi2, delta = 1)

  def test_antipodal(self):
    """How does the exact antipodal equatorial path go N/S + E/W"""
    # lat1 lat2 lon2 azi1 azi2
    C = [
      [ +0.0, +0.0, +180,   +0.0, +180],
      [ -0.0, -0.0, +180, +180,   +0.0],
      [ +0.0, +0.0, -180,   -0.0, -180],
      [ -0.0, -0.0, -180, -180,   -0.0]
    ]
    for l in C:
      (lat1, lat2, lon2, azi1, azi2) = l
      inv = Geodesic.WGS84.Inverse(lat1, 0.0, lat2, lon2)
      self.assertTrue(SignTest.equiv(inv["azi1"], azi1))
      self.assertTrue(SignTest.equiv(inv["azi2"], azi2))

  def test_antipodal_prolate(self):
    """Antipodal points on the equator with prolate ellipsoid"""
    # lon2 azi1/2
    C = [
      [ +180, +90 ],
      [ -180, -90 ]
    ]
    geod = Geodesic(6.4e6, -1/300.0)
    for l in C:
      (lon2, azi) = l
      inv = geod.Inverse(0.0, 0.0, 0.0, lon2)
      self.assertTrue(SignTest.equiv(inv["azi1"], azi))
      self.assertTrue(SignTest.equiv(inv["azi2"], azi))

  def test_azimuth_0_180(self):
    """azimuths = +/-0 and +/-180 for the direct problem"""
    # azi1, lon2, azi2
    C = [
      [ +0.0, +180, +180 ],
      [ -0.0, -180, -180 ],
      [ +180, +180, +0.0 ],
      [ -180, -180, -0.0 ]
    ]
    for l in C:
      (azi1, lon2, azi2) = l
      direct = Geodesic.WGS84.Direct(0.0, 0.0, azi1, 15e6,
                                     Geodesic.STANDARD | Geodesic.LONG_UNROLL)
      self.assertTrue(SignTest.equiv(direct["lon2"], lon2))
      self.assertTrue(SignTest.equiv(direct["azi2"], azi2))

env_pyrecodes/lib/python3.9/site-packages/geographiclib/test/test_sign.py
NikolaBlagojevic/pyrecodes
"""
Compatibility shim for the vectorized geometry operations.

Uses PyGEOS if available/set, otherwise loops through Shapely geometries.

"""
import warnings

import numpy as np
import pandas as pd

import shapely
import shapely.geometry
import shapely.geos
import shapely.ops
import shapely.wkb
import shapely.wkt
import shapely.validation

from shapely.geometry.base import BaseGeometry

from . import _compat as compat

try:
    import pygeos
except ImportError:
    geos = None


_names = {
    "MISSING": None,
    "NAG": None,
    "POINT": "Point",
    "LINESTRING": "LineString",
    "LINEARRING": "LinearRing",
    "POLYGON": "Polygon",
    "MULTIPOINT": "MultiPoint",
    "MULTILINESTRING": "MultiLineString",
    "MULTIPOLYGON": "MultiPolygon",
    "GEOMETRYCOLLECTION": "GeometryCollection",
}

if compat.USE_SHAPELY_20 or compat.USE_PYGEOS:
    if compat.USE_SHAPELY_20:
        type_mapping = {p.value: _names[p.name] for p in shapely.GeometryType}
    else:
        type_mapping = {p.value: _names[p.name] for p in pygeos.GeometryType}
    geometry_type_ids = list(type_mapping.keys())
    geometry_type_values = np.array(list(type_mapping.values()), dtype=object)
else:
    type_mapping, geometry_type_ids, geometry_type_values = None, None, None


def isna(value):
    """
    Check if scalar value is NA-like (None, np.nan or pd.NA).

    Custom version that only works for scalars (returning True or False),
    as `pd.isna` also works for array-like input returning a boolean array.
    """
    if value is None:
        return True
    elif isinstance(value, float) and np.isnan(value):
        return True
    elif value is pd.NA:
        return True
    else:
        return False


def _pygeos_to_shapely(geom):
    if geom is None:
        return None

    if compat.PYGEOS_SHAPELY_COMPAT:
        # we can only use this compatible fast path for shapely < 2, because
        # shapely 2+ doesn't expose clone
        if not compat.SHAPELY_GE_20:
            geom = shapely.geos.lgeos.GEOSGeom_clone(geom._ptr)
            return shapely.geometry.base.geom_factory(geom)

    # fallback going through WKB
    if pygeos.is_empty(geom) and pygeos.get_type_id(geom) == 0:
        # empty point does not roundtrip through WKB
        return shapely.wkt.loads("POINT EMPTY")
    else:
        return shapely.wkb.loads(pygeos.to_wkb(geom))


def _shapely_to_pygeos(geom):
    if geom is None:
        return None

    if compat.PYGEOS_SHAPELY_COMPAT:
        return pygeos.from_shapely(geom)

    # fallback going through WKB
    if geom.is_empty and geom.geom_type == "Point":
        # empty point does not roundtrip through WKB
        return pygeos.from_wkt("POINT EMPTY")
    else:
        return pygeos.from_wkb(geom.wkb)


def from_shapely(data):
    """
    Convert a list or array of shapely objects to an object-dtype numpy
    array of validated geometry elements.

    """
    # First try a fast path for pygeos if possible, but do this in a try-except
    # block because pygeos.from_shapely only handles Shapely objects, while
    # the rest of this function is more forgiving (also __geo_interface__).
    if compat.USE_PYGEOS and compat.PYGEOS_SHAPELY_COMPAT:
        if not isinstance(data, np.ndarray):
            arr = np.empty(len(data), dtype=object)
            with compat.ignore_shapely2_warnings():
                arr[:] = data
        else:
            arr = data
        try:
            return pygeos.from_shapely(arr)
        except TypeError:
            pass

    out = []

    for geom in data:
        if compat.USE_PYGEOS and isinstance(geom, pygeos.Geometry):
            out.append(geom)
        elif isinstance(geom, BaseGeometry):
            if compat.USE_PYGEOS:
                out.append(_shapely_to_pygeos(geom))
            else:
                out.append(geom)
        elif hasattr(geom, "__geo_interface__"):
            geom = shapely.geometry.shape(geom)
            if compat.USE_PYGEOS:
                out.append(_shapely_to_pygeos(geom))
            else:
                out.append(geom)
        elif isna(geom):
            out.append(None)
        else:
            raise TypeError("Input must be valid geometry objects: {0}".format(geom))

    if compat.USE_PYGEOS:
        return np.array(out, dtype=object)
    else:
        # numpy can expand geometry collections into 2D arrays, use this
        # two-step construction to avoid this
        aout = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            aout[:] = out
        return aout


def to_shapely(data):
    if compat.USE_PYGEOS:
        out = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            out[:] = [_pygeos_to_shapely(geom) for geom in data]
        return out
    else:
        return data


def from_wkb(data):
    """
    Convert a list or array of WKB objects to a np.ndarray[geoms].
    """
    if compat.USE_SHAPELY_20:
        return shapely.from_wkb(data)
    if compat.USE_PYGEOS:
        return pygeos.from_wkb(data)

    out = []

    for geom in data:
        if not isna(geom) and len(geom):
            geom = shapely.wkb.loads(geom, hex=isinstance(geom, str))
        else:
            geom = None
        out.append(geom)

    aout = np.empty(len(data), dtype=object)
    with compat.ignore_shapely2_warnings():
        aout[:] = out
    return aout


def to_wkb(data, hex=False, **kwargs):
    if compat.USE_SHAPELY_20:
        return shapely.to_wkb(data, hex=hex, **kwargs)
    elif compat.USE_PYGEOS:
        return pygeos.to_wkb(data, hex=hex, **kwargs)
    else:
        if hex:
            out = [geom.wkb_hex if geom is not None else None for geom in data]
        else:
            out = [geom.wkb if geom is not None else None for geom in data]
        return np.array(out, dtype=object)


def from_wkt(data):
    """
    Convert a list or array of WKT objects to a np.ndarray[geoms].
    """
    if compat.USE_SHAPELY_20:
        return shapely.from_wkt(data)
    if compat.USE_PYGEOS:
        return pygeos.from_wkt(data)

    out = []

    for geom in data:
        if not isna(geom) and len(geom):
            if isinstance(geom, bytes):
                geom = geom.decode("utf-8")
            geom = shapely.wkt.loads(geom)
        else:
            geom = None
        out.append(geom)

    aout = np.empty(len(data), dtype=object)
    with compat.ignore_shapely2_warnings():
        aout[:] = out
    return aout


def to_wkt(data, **kwargs):
    if compat.USE_SHAPELY_20:
        return shapely.to_wkt(data, **kwargs)
    elif compat.USE_PYGEOS:
        return pygeos.to_wkt(data, **kwargs)
    else:
        out = [geom.wkt if geom is not None else None for geom in data]
        return np.array(out, dtype=object)


def _points_from_xy(x, y, z=None):
    # helper method for shapely-based function
    if not len(x) == len(y):
        raise ValueError("x and y arrays must be equal length.")
    if z is not None:
        if not len(z) == len(x):
            raise ValueError("z array must be same length as x and y.")
        geom = [shapely.geometry.Point(i, j, k) for i, j, k in zip(x, y, z)]
    else:
        geom = [shapely.geometry.Point(i, j) for i, j in zip(x, y)]
    return geom


def points_from_xy(x, y, z=None):

    x = np.asarray(x, dtype="float64")
    y = np.asarray(y, dtype="float64")
    if z is not None:
        z = np.asarray(z, dtype="float64")

    if compat.USE_SHAPELY_20:
        return shapely.points(x, y, z)
    elif compat.USE_PYGEOS:
        return pygeos.points(x, y, z)
    else:
        out = _points_from_xy(x, y, z)
        aout = np.empty(len(x), dtype=object)
        with compat.ignore_shapely2_warnings():
            aout[:] = out
        return aout


# -----------------------------------------------------------------------------
# Helper methods for the vectorized operations
# -----------------------------------------------------------------------------


def _binary_method(op, left, right, **kwargs):
    # type: (str, np.array[geoms], [np.array[geoms]/BaseGeometry]) -> array-like
    if isinstance(right, BaseGeometry):
        right = from_shapely([right])[0]
    return getattr(pygeos, op)(left, right, **kwargs)


def _binary_geo(op, left, right):
    # type: (str, np.array[geoms], [np.array[geoms]/BaseGeometry]) -> np.array[geoms]
    """Apply geometry-valued operation

    Supports:

    -   difference
    -   symmetric_difference
    -   intersection
    -   union

    Parameters
    ----------
    op: string
    right: np.array[geoms] or single shapely BaseGeoemtry
    """
    if isinstance(right, BaseGeometry):
        # intersection can return empty GeometryCollections, and if the
        # result are only those, numpy will coerce it to empty 2D array
        data = np.empty(len(left), dtype=object)
        with compat.ignore_shapely2_warnings():
            data[:] = [
                getattr(s, op)(right) if s is not None and right is not None else None
                for s in left
            ]
        return data
    elif isinstance(right, np.ndarray):
        if len(left) != len(right):
            msg = "Lengths of inputs do not match. Left: {0}, Right: {1}".format(
                len(left), len(right)
            )
            raise ValueError(msg)
        data = np.empty(len(left), dtype=object)
        with compat.ignore_shapely2_warnings():
            data[:] = [
                getattr(this_elem, op)(other_elem)
                if this_elem is not None and other_elem is not None
                else None
                for this_elem, other_elem in zip(left, right)
            ]
        return data
    else:
        raise TypeError("Type not known: {0} vs {1}".format(type(left), type(right)))


def _binary_predicate(op, left, right, *args, **kwargs):
    # type: (str, np.array[geoms], np.array[geoms]/BaseGeometry, args/kwargs)
    #        -> array[bool]
    """Binary operation on np.array[geoms] that returns a boolean ndarray

    Supports:

    -  contains
    -  disjoint
    -  intersects
    -  touches
    -  crosses
    -  within
    -  overlaps
    -  covers
    -  covered_by
    -  equals

    Parameters
    ----------
    op: string
    right: np.array[geoms] or single shapely BaseGeoemtry
    """
    # empty geometries are handled by shapely (all give False except disjoint)
    if isinstance(right, BaseGeometry):
        data = [
            getattr(s, op)(right, *args, **kwargs) if s is not None else False
            for s in left
        ]
        return np.array(data, dtype=bool)
    elif isinstance(right, np.ndarray):
        data = [
            getattr(this_elem, op)(other_elem, *args, **kwargs)
            if not (this_elem is None or other_elem is None)
            else False
            for this_elem, other_elem in zip(left, right)
        ]
        return np.array(data, dtype=bool)
    else:
        raise TypeError("Type not known: {0} vs {1}".format(type(left), type(right)))


def _binary_op_float(op, left, right, *args, **kwargs):
    # type: (str, np.array[geoms], np.array[geoms]/BaseGeometry, args/kwargs)
    #        -> array
    """Binary operation on np.array[geoms] that returns a ndarray"""
    # used for distance -> check for empty as we want to return np.nan instead 0.0
    # as shapely does currently (https://github.com/Toblerity/Shapely/issues/498)
    if isinstance(right, BaseGeometry):
        data = [
            getattr(s, op)(right, *args, **kwargs)
            if not (s is None or s.is_empty or right.is_empty)
            else np.nan
            for s in left
        ]
        return np.array(data, dtype=float)
    elif isinstance(right, np.ndarray):
        if len(left) != len(right):
            msg = "Lengths of inputs do not match. Left: {0}, Right: {1}".format(
                len(left), len(right)
            )
            raise ValueError(msg)
        data = [
            getattr(this_elem, op)(other_elem, *args, **kwargs)
            if not (this_elem is None or this_elem.is_empty)
            | (other_elem is None or other_elem.is_empty)
            else np.nan
            for this_elem, other_elem in zip(left, right)
        ]
        return np.array(data, dtype=float)
    else:
        raise TypeError("Type not known: {0} vs {1}".format(type(left), type(right)))


def _binary_op(op, left, right, *args, **kwargs):
    # type: (str, np.array[geoms], np.array[geoms]/BaseGeometry, args/kwargs)
    #        -> array
    """Binary operation on np.array[geoms] that returns a ndarray"""
    # pass empty to shapely (relate handles this correctly, project only
    # for linestrings and points)
    if op == "project":
        null_value = np.nan
        dtype = float
    elif op == "relate":
        null_value = None
        dtype = object
    else:
        raise AssertionError("wrong op")

    if isinstance(right, BaseGeometry):
        data = [
            getattr(s, op)(right, *args, **kwargs) if s is not None else null_value
            for s in left
        ]
        return np.array(data, dtype=dtype)
    elif isinstance(right, np.ndarray):
        if len(left) != len(right):
            msg = "Lengths of inputs do not match. Left: {0}, Right: {1}".format(
                len(left), len(right)
            )
            raise ValueError(msg)
        data = [
            getattr(this_elem, op)(other_elem, *args, **kwargs)
            if not (this_elem is None or other_elem is None)
            else null_value
            for this_elem, other_elem in zip(left, right)
        ]
        return np.array(data, dtype=dtype)
    else:
        raise TypeError("Type not known: {0} vs {1}".format(type(left), type(right)))


def _affinity_method(op, left, *args, **kwargs):
    # type: (str, np.array[geoms], ...) -> np.array[geoms]

    # not all shapely.affinity methods can handle empty geometries:
    # affine_transform itself works (as well as translate), but rotate, scale
    # and skew fail (they try to unpack the bounds).
    # Here: consistently returning empty geom for input empty geom
    left = to_shapely(left)
    out = []
    for geom in left:
        if geom is None or geom.is_empty:
            res = geom
        else:
            res = getattr(shapely.affinity, op)(geom, *args, **kwargs)
        out.append(res)
    data = np.empty(len(left), dtype=object)
    with compat.ignore_shapely2_warnings():
        data[:] = out
    return from_shapely(data)


# -----------------------------------------------------------------------------
# Vectorized operations
# -----------------------------------------------------------------------------


#
# Unary operations that return non-geometry (bool or float)
#


def _unary_op(op, left, null_value=False):
    # type: (str, np.array[geoms], Any) -> np.array
    """Unary operation that returns a Series"""
    data = [getattr(geom, op, null_value) for geom in left]
    return np.array(data, dtype=np.dtype(type(null_value)))


def is_valid(data):
    if compat.USE_SHAPELY_20:
        return shapely.is_valid(data)
    elif compat.USE_PYGEOS:
        return pygeos.is_valid(data)
    else:
        return _unary_op("is_valid", data, null_value=False)


def is_empty(data):
    if compat.USE_SHAPELY_20:
        return shapely.is_empty(data)
    elif compat.USE_PYGEOS:
        return pygeos.is_empty(data)
    else:
        return _unary_op("is_empty", data, null_value=False)


def is_simple(data):
    if compat.USE_SHAPELY_20:
        return shapely.is_simple(data)
    elif compat.USE_PYGEOS:
        return pygeos.is_simple(data)
    else:
        return _unary_op("is_simple", data, null_value=False)


def is_ring(data):
    if "Polygon" in geom_type(data):
        warnings.warn(
            "is_ring currently returns True for Polygons, which is not correct. "
            "This will be corrected to False in a future release.",
            FutureWarning,
            stacklevel=3,
        )
    if compat.USE_PYGEOS:
        return pygeos.is_ring(data) | pygeos.is_ring(pygeos.get_exterior_ring(data))
    else:
        # for polygons operates on the exterior, so can't use _unary_op()
        results = []
        for geom in data:
            if geom is None:
                results.append(False)
            elif geom.geom_type == "Polygon":
                results.append(geom.exterior.is_ring)
            elif geom.geom_type in ["LineString", "LinearRing"]:
                results.append(geom.is_ring)
            else:
                results.append(False)
        return np.array(results, dtype=bool)


def is_closed(data):
    if compat.USE_SHAPELY_20:
        return shapely.is_closed(data)
    elif compat.USE_PYGEOS:
        return pygeos.is_closed(data)
    else:
        return _unary_op("is_closed", data, null_value=False)


def has_z(data):
    if compat.USE_SHAPELY_20:
        return shapely.has_z(data)
    elif compat.USE_PYGEOS:
        return pygeos.has_z(data)
    else:
        return _unary_op("has_z", data, null_value=False)


def geom_type(data):
    if compat.USE_SHAPELY_20:
        res = shapely.get_type_id(data)
        return geometry_type_values[np.searchsorted(geometry_type_ids, res)]
    elif compat.USE_PYGEOS:
        res = pygeos.get_type_id(data)
        return geometry_type_values[np.searchsorted(geometry_type_ids, res)]
    else:
        return _unary_op("geom_type", data, null_value=None)


def area(data):
    if compat.USE_SHAPELY_20:
        return shapely.area(data)
    elif compat.USE_PYGEOS:
        return pygeos.area(data)
    else:
        return _unary_op("area", data, null_value=np.nan)


def length(data):
    if compat.USE_SHAPELY_20:
        return shapely.length(data)
    elif compat.USE_PYGEOS:
        return pygeos.length(data)
    else:
        return _unary_op("length", data, null_value=np.nan)


#
# Unary operations that return new geometries
#


def _unary_geo(op, left, *args, **kwargs):
    # type: (str, np.array[geoms]) -> np.array[geoms]
    """Unary operation that returns new geometries"""
    # ensure 1D output, see note above
    data = np.empty(len(left), dtype=object)
    with compat.ignore_shapely2_warnings():
        data[:] = [getattr(geom, op, None) for geom in left]
    return data


def boundary(data):
    if compat.USE_SHAPELY_20:
        return shapely.boundary(data)
    elif compat.USE_PYGEOS:
        return pygeos.boundary(data)
    else:
        return _unary_geo("boundary", data)


def centroid(data):
    if compat.USE_SHAPELY_20:
        return shapely.centroid(data)
    elif compat.USE_PYGEOS:
        return pygeos.centroid(data)
    else:
        return _unary_geo("centroid", data)


def convex_hull(data):
    if compat.USE_SHAPELY_20:
        return shapely.convex_hull(data)
    elif compat.USE_PYGEOS:
        return pygeos.convex_hull(data)
    else:
        return _unary_geo("convex_hull", data)


def envelope(data):
    if compat.USE_SHAPELY_20:
        return shapely.envelope(data)
    elif compat.USE_PYGEOS:
        return pygeos.envelope(data)
    else:
        return _unary_geo("envelope", data)


def exterior(data):
    if compat.USE_SHAPELY_20:
        return shapely.get_exterior_ring(data)
    elif compat.USE_PYGEOS:
        return pygeos.get_exterior_ring(data)
    else:
        return _unary_geo("exterior", data)


def interiors(data):
    data = to_shapely(data)
    has_non_poly = False
    inner_rings = []
    for geom in data:
        interior_ring_seq = getattr(geom, "interiors", None)
        # polygon case
        if interior_ring_seq is not None:
            inner_rings.append(list(interior_ring_seq))
        # non-polygon case
        else:
            has_non_poly = True
            inner_rings.append(None)
    if has_non_poly:
        warnings.warn(
            "Only Polygon objects have interior rings. For other "
            "geometry types, None is returned."
        )
    data = np.empty(len(data), dtype=object)
    with compat.ignore_shapely2_warnings():
        data[:] = inner_rings
    return data


def representative_point(data):
    if compat.USE_PYGEOS:
        return pygeos.point_on_surface(data)
    else:
        # method and not a property -> can't use _unary_geo
        out = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            out[:] = [
                geom.representative_point() if geom is not None else None
                for geom in data
            ]
        return out


#
# Binary predicates
#


def covers(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.covers(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("covers", data, other)
    else:
        return _binary_predicate("covers", data, other)


def covered_by(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.covered_by(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("covered_by", data, other)
    else:
        raise NotImplementedError(
            "covered_by is only implemented for pygeos, not shapely"
        )


def contains(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.contains(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("contains", data, other)
    else:
        return _binary_predicate("contains", data, other)


def crosses(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.crosses(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("crosses", data, other)
    else:
        return _binary_predicate("crosses", data, other)


def disjoint(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.disjoint(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("disjoint", data, other)
    else:
        return _binary_predicate("disjoint", data, other)


def equals(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.equals(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("equals", data, other)
    else:
        return _binary_predicate("equals", data, other)


def intersects(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.intersects(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("intersects", data, other)
    else:
        return _binary_predicate("intersects", data, other)


def overlaps(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.overlaps(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("overlaps", data, other)
    else:
        return _binary_predicate("overlaps", data, other)


def touches(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.touches(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("touches", data, other)
    else:
        return _binary_predicate("touches", data, other)


def within(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.within(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("within", data, other)
    else:
        return _binary_predicate("within", data, other)


def equals_exact(data, other, tolerance):
    if compat.USE_SHAPELY_20:
        return shapely.equals_exact(data, other, tolerance=tolerance)
    elif compat.USE_PYGEOS:
        return _binary_method("equals_exact", data, other, tolerance=tolerance)
    else:
        return _binary_predicate("equals_exact", data, other, tolerance=tolerance)


def almost_equals(self, other, decimal):
    if compat.USE_PYGEOS or compat.USE_SHAPELY_20:
        return self.equals_exact(other, 0.5 * 10 ** (-decimal))
    else:
        return _binary_predicate("almost_equals", self, other, decimal=decimal)


#
# Binary operations that return new geometries
#


def clip_by_rect(data, xmin, ymin, xmax, ymax):
    if compat.USE_PYGEOS:
        return pygeos.clip_by_rect(data, xmin, ymin, xmax, ymax)
    else:
        clipped_geometries = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            clipped_geometries[:] = [
                shapely.ops.clip_by_rect(s, xmin, ymin, xmax, ymax)
                if s is not None
                else None
                for s in data
            ]
        return clipped_geometries


def difference(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.difference(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("difference", data, other)
    else:
        return _binary_geo("difference", data, other)


def intersection(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.intersection(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("intersection", data, other)
    else:
        return _binary_geo("intersection", data, other)


def symmetric_difference(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.symmetric_difference(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("symmetric_difference", data, other)
    else:
        return _binary_geo("symmetric_difference", data, other)


def union(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.union(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("union", data, other)
    else:
        return _binary_geo("union", data, other)


#
# Other operations
#


def distance(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.distance(data, other)
    elif compat.USE_PYGEOS:
        return _binary_method("distance", data, other)
    else:
        return _binary_op_float("distance", data, other)


def buffer(data, distance, resolution=16, **kwargs):
    if compat.USE_SHAPELY_20:
        if compat.SHAPELY_G_20a1:
            return shapely.buffer(data, distance, quad_segs=resolution, **kwargs)
        else:
            # TODO: temporary keep this (so geopandas works with latest released
            # shapely, currently alpha1) until shapely beta1 is out
            return shapely.buffer(data, distance, quadsegs=resolution, **kwargs)
    elif compat.USE_PYGEOS:
        return pygeos.buffer(data, distance, quadsegs=resolution, **kwargs)
    else:
        out = np.empty(len(data), dtype=object)
        if isinstance(distance, np.ndarray):
            if len(distance) != len(data):
                raise ValueError(
                    "Length of distance sequence does not match "
                    "length of the GeoSeries"
                )

            with compat.ignore_shapely2_warnings():
                out[:] = [
                    geom.buffer(dist, resolution, **kwargs)
                    if geom is not None
                    else None
                    for geom, dist in zip(data, distance)
                ]
            return out

        with compat.ignore_shapely2_warnings():
            out[:] = [
                geom.buffer(distance, resolution, **kwargs)
                if geom is not None
                else None
                for geom in data
            ]
        return out


def interpolate(data, distance, normalized=False):
    if compat.USE_SHAPELY_20:
        return shapely.line_interpolate_point(data, distance, normalized=normalized)
    elif compat.USE_PYGEOS:
        try:
            return pygeos.line_interpolate_point(data, distance, normalized=normalized)
        except TypeError:  # support for pygeos<0.9
            return pygeos.line_interpolate_point(data, distance, normalize=normalized)
    else:
        out = np.empty(len(data), dtype=object)
        if isinstance(distance, np.ndarray):
            if len(distance) != len(data):
                raise ValueError(
                    "Length of distance sequence does not match "
                    "length of the GeoSeries"
                )
            with compat.ignore_shapely2_warnings():
                out[:] = [
                    geom.interpolate(dist, normalized=normalized)
                    for geom, dist in zip(data, distance)
                ]
            return out

        with compat.ignore_shapely2_warnings():
            out[:] = [
                geom.interpolate(distance, normalized=normalized) for geom in data
            ]
        return out


def simplify(data, tolerance, preserve_topology=True):
    if compat.USE_SHAPELY_20:
        return shapely.simplify(data, tolerance, preserve_topology=preserve_topology)
    elif compat.USE_PYGEOS:
        # preserve_topology has different default as pygeos!
        return pygeos.simplify(data, tolerance, preserve_topology=preserve_topology)
    else:
        # method and not a property -> can't use _unary_geo
        out = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            out[:] = [
                geom.simplify(tolerance, preserve_topology=preserve_topology)
                for geom in data
            ]
        return out


def _shapely_normalize(geom):
    """
    Small helper function for now because it is not yet available in Shapely.
    """
    from shapely.geos import lgeos
    from shapely.geometry.base import geom_factory
    from ctypes import c_void_p, c_int

    lgeos._lgeos.GEOSNormalize_r.restype = c_int
    lgeos._lgeos.GEOSNormalize_r.argtypes = [c_void_p, c_void_p]

    geom_cloned = lgeos.GEOSGeom_clone(geom._geom)
    lgeos._lgeos.GEOSNormalize_r(lgeos.geos_handle, geom_cloned)
    return geom_factory(geom_cloned)


def normalize(data):
    if compat.USE_SHAPELY_20:
        return shapely.normalize(data)
    elif compat.USE_PYGEOS:
        return pygeos.normalize(data)
    elif compat.SHAPELY_GE_18:
        out = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            out[:] = [geom.normalize() if geom is not None else None for geom in data]
    else:
        out = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            out[:] = [
                _shapely_normalize(geom) if geom is not None else None for geom in data
            ]
    return out


def make_valid(data):
    if compat.USE_SHAPELY_20:
        return shapely.make_valid(data)
    elif compat.USE_PYGEOS:
        return pygeos.make_valid(data)
    elif not compat.SHAPELY_GE_18:
        raise NotImplementedError(
            f"shapely >= 1.8 or PyGEOS is required, "
            f"version {shapely.__version__} is installed"
        )
    else:
        out = np.empty(len(data), dtype=object)
        with compat.ignore_shapely2_warnings():
            out[:] = [
                shapely.validation.make_valid(geom) if geom is not None else None
                for geom in data
            ]
    return out


def project(data, other, normalized=False):
    if compat.USE_SHAPELY_20:
        return shapely.line_locate_point(data, other, normalized=normalized)
    elif compat.USE_PYGEOS:
        try:
            return pygeos.line_locate_point(data, other, normalized=normalized)
        except TypeError:  # support for pygeos<0.9
            return pygeos.line_locate_point(data, other, normalize=normalized)
    else:
        return _binary_op("project", data, other, normalized=normalized)


def relate(data, other):
    if compat.USE_SHAPELY_20:
        return shapely.relate(data, other)
    data = to_shapely(data)
    if isinstance(other, np.ndarray):
        other = to_shapely(other)
    return _binary_op("relate", data, other)


def unary_union(data):
    if compat.USE_SHAPELY_20:
        data = shapely.union_all(data)
        if data is None:  # shapely 2.0a1
            return None
        if data.is_empty:  # shapely 2.0
            return None
        else:
            return data
    elif compat.USE_PYGEOS:
        return _pygeos_to_shapely(pygeos.union_all(data))
    else:
        data = [g for g in data if g is not None]
        if data:
            return shapely.ops.unary_union(data)
        else:
            return None


#
# Coordinate related properties
#


def get_x(data):
    if compat.USE_SHAPELY_20:
        return shapely.get_x(data)
    elif compat.USE_PYGEOS:
        return pygeos.get_x(data)
    else:
        return _unary_op("x", data, null_value=np.nan)


def get_y(data):
    if compat.USE_SHAPELY_20:
        return shapely.get_y(data)
    elif compat.USE_PYGEOS:
        return pygeos.get_y(data)
    else:
        return _unary_op("y", data, null_value=np.nan)


def get_z(data):
    if compat.USE_SHAPELY_20:
        return shapely.get_z(data)
    elif compat.USE_PYGEOS:
        return pygeos.get_z(data)
    else:
        data = [geom.z if geom.has_z else np.nan for geom in data]
        return np.array(data, dtype=np.dtype(float))


def bounds(data):
    if compat.USE_SHAPELY_20:
        return shapely.bounds(data)
    elif compat.USE_PYGEOS:
        return pygeos.bounds(data)
    # ensure that for empty arrays, the result has the correct shape
    if len(data) == 0:
        return np.empty((0, 4), dtype="float64")
    # need to explicitly check for empty (in addition to missing) geometries,
    # as those return an empty tuple, not resulting in a 2D array
    bounds = np.array(
        [
            geom.bounds
            if not (geom is None or geom.is_empty)
            else (np.nan, np.nan, np.nan, np.nan)
            for geom in data
        ]
    )
    return bounds


#
# Coordinate transformation
#


def transform(data, func):
    if compat.USE_SHAPELY_20 or compat.USE_PYGEOS:
        if compat.USE_SHAPELY_20:
            has_z = shapely.has_z(data)
            from shapely import get_coordinates, set_coordinates
        else:
            has_z = pygeos.has_z(data)
            from pygeos import get_coordinates, set_coordinates

        result = np.empty_like(data)

        coords = get_coordinates(data[~has_z], include_z=False)
        new_coords_z = func(coords[:, 0], coords[:, 1])
        result[~has_z] = set_coordinates(data[~has_z].copy(), np.array(new_coords_z).T)

        coords_z = get_coordinates(data[has_z], include_z=True)
        new_coords_z = func(coords_z[:, 0], coords_z[:, 1], coords_z[:, 2])
        result[has_z] = set_coordinates(data[has_z].copy(), np.array(new_coords_z).T)

        return result
    else:
        from shapely.ops import transform

        n = len(data)
        result = np.empty(n, dtype=object)
        for i in range(n):
            geom = data[i]
            if isna(geom):
                result[i] = geom
            else:
                result[i] = transform(func, geom)

        return result

env_pyrecodes/lib/python3.9/site-packages/geopandas/_vectorized.py
NikolaBlagojevic/pyrecodes

# This file was generated by 'versioneer.py' (0.21) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.

import json

version_json = '''
{
 "date": "2022-12-10T19:37:18+0100",
 "dirty": false,
 "error": null,
 "full-revisionid": "efcb3675d94935ee19b06c75467f9ccc24eb8843",
 "version": "0.12.2"
}
'''  # END VERSION_JSON


def get_versions():
    return json.loads(version_json)

env_pyrecodes/lib/python3.9/site-packages/geopandas/_version.py
NikolaBlagojevic/pyrecodes
from packaging.version import Version

import random

import numpy as np
import pandas as pd

from shapely.geometry import Point, Polygon, LineString
import pyproj

from geopandas import GeoSeries, GeoDataFrame, points_from_xy, datasets, read_file
from geopandas.array import from_shapely, from_wkb, from_wkt, GeometryArray

from geopandas.testing import assert_geodataframe_equal
import pytest


# pyproj 2.3.1 fixed a segfault for the case working in an environment with
# 'init' dicts (https://github.com/pyproj4/pyproj/issues/415)
PYPROJ_LT_231 = Version(pyproj.__version__) < Version("2.3.1")
PYPROJ_GE_3 = Version(pyproj.__version__) >= Version("3.0.0")


def _create_df(x, y=None, crs=None):
    y = y or x
    x = np.asarray(x)
    y = np.asarray(y)

    return GeoDataFrame(
        {"geometry": points_from_xy(x, y), "value1": x + y, "value2": x * y}, crs=crs
    )


def df_epsg26918():
    # EPSG:26918
    # Center coordinates
    # -1683723.64 6689139.23
    return _create_df(
        x=range(-1683723, -1683723 + 10, 1),
        y=range(6689139, 6689139 + 10, 1),
        crs="epsg:26918",
    )


def test_to_crs_transform():
    df = df_epsg26918()
    lonlat = df.to_crs(epsg=4326)
    utm = lonlat.to_crs(epsg=26918)
    assert_geodataframe_equal(df, utm, check_less_precise=True)


def test_to_crs_transform__missing_data():
    # https://github.com/geopandas/geopandas/issues/1573
    df = df_epsg26918()
    df.loc[3, "geometry"] = None
    lonlat = df.to_crs(epsg=4326)
    utm = lonlat.to_crs(epsg=26918)
    assert_geodataframe_equal(df, utm, check_less_precise=True)


def test_to_crs_transform__empty_data():
    df = df_epsg26918().iloc[:0]
    lonlat = df.to_crs(epsg=4326)
    utm = lonlat.to_crs(epsg=26918)
    assert_geodataframe_equal(df, utm, check_less_precise=True)


def test_to_crs_inplace():
    df = df_epsg26918()
    lonlat = df.to_crs(epsg=4326)
    df.to_crs(epsg=4326, inplace=True)
    assert_geodataframe_equal(df, lonlat, check_less_precise=True)


def test_to_crs_geo_column_name():
    # Test to_crs() with different geometry column name (GH#339)
    df = df_epsg26918()
    df = df.rename(columns={"geometry": "geom"})
    df.set_geometry("geom", inplace=True)
    lonlat = df.to_crs(epsg=4326)
    utm = lonlat.to_crs(epsg=26918)
    assert lonlat.geometry.name == "geom"
    assert utm.geometry.name == "geom"
    assert_geodataframe_equal(df, utm, check_less_precise=True)


def test_to_crs_dimension_z():
    # preserve z dimension
    arr = points_from_xy([1, 2], [2, 3], [3, 4], crs=4326)
    assert arr.has_z.all()
    result = arr.to_crs(epsg=3857)
    assert result.has_z.all()


def test_to_crs_dimension_mixed():
    s = GeoSeries([Point(1, 2), LineString([(1, 2, 3), (4, 5, 6)])], crs=2056)
    result = s.to_crs(epsg=4326)
    assert not result[0].is_empty
    assert result.has_z.tolist() == [False, True]
    roundtrip = result.to_crs(epsg=2056)
    # TODO replace with assert_geoseries_equal once we expose tolerance keyword
    # assert_geoseries_equal(roundtrip, s, check_less_precise=True)
    for a, b in zip(roundtrip, s):
        np.testing.assert_allclose(a.coords[:], b.coords[:], atol=0.01)


# -----------------------------------------------------------------------------
# Test different supported formats for CRS specification


@pytest.fixture(
    params=[
        4326,
        "epsg:4326",
        pytest.param(
            {"init": "epsg:4326"},
            marks=pytest.mark.skipif(PYPROJ_LT_231, reason="segfault"),
        ),
        "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs",
        {"proj": "latlong", "ellps": "WGS84", "datum": "WGS84", "no_defs": True},
    ],
    ids=["epsg_number", "epsg_string", "epsg_dict", "proj4_string", "proj4_dict"],
)
def epsg4326(request):
    if isinstance(request.param, int):
        return dict(epsg=request.param)
    return dict(crs=request.param)


@pytest.fixture(
    params=[
        26918,
        "epsg:26918",
        pytest.param(
            {"init": "epsg:26918", "no_defs": True},
            marks=pytest.mark.skipif(PYPROJ_LT_231, reason="segfault"),
        ),
        "+proj=utm +zone=18 +ellps=GRS80 +datum=NAD83 +units=m +no_defs ",
        {"proj": "utm", "zone": 18, "datum": "NAD83", "units": "m", "no_defs": True},
    ],
    ids=["epsg_number", "epsg_string", "epsg_dict", "proj4_string", "proj4_dict"],
)
def epsg26918(request):
    if isinstance(request.param, int):
        return dict(epsg=request.param)
    return dict(crs=request.param)


@pytest.mark.filterwarnings("ignore:'\\+init:DeprecationWarning")
@pytest.mark.filterwarnings("ignore:'\\+init:FutureWarning")
def test_transform2(epsg4326, epsg26918):
    # with PROJ >= 7, the transformation using EPSG code vs proj4 string is
    # slightly different due to use of grid files or not -> turn off network
    # to not use grid files at all for this test
    if PYPROJ_GE_3:
        pyproj.network.set_network_enabled(False)

    df = df_epsg26918()
    lonlat = df.to_crs(**epsg4326)
    utm = lonlat.to_crs(**epsg26918)
    # can't check for CRS equality, as the formats differ although representing
    # the same CRS
    assert_geodataframe_equal(df, utm, check_less_precise=True, check_crs=False)


def test_crs_axis_order__always_xy():
    df = GeoDataFrame(geometry=[Point(-1683723, 6689139)], crs="epsg:26918")
    lonlat = df.to_crs("epsg:4326")
    test_lonlat = GeoDataFrame(
        geometry=[Point(-110.1399901, 55.1350011)], crs="epsg:4326"
    )
    assert_geodataframe_equal(lonlat, test_lonlat, check_less_precise=True)


def test_skip_exact_same():
    df = df_epsg26918()
    utm = df.to_crs(df.crs)
    assert_geodataframe_equal(df, utm, check_less_precise=True)


# Test CRS on GeometryArray level
class TestGeometryArrayCRS:
    def setup_method(self):
        self.osgb = pyproj.CRS(27700)
        self.wgs = pyproj.CRS(4326)

        self.geoms = [Point(0, 0), Point(1, 1)]
        self.polys = [
            Polygon([(random.random(), random.random()) for i in range(3)])
            for _ in range(10)
        ]
        self.arr = from_shapely(self.polys, crs=27700)

    def test_array(self):
        arr = from_shapely(self.geoms)
        arr.crs = 27700
        assert arr.crs == self.osgb

        arr = from_shapely(self.geoms, crs=27700)
        assert arr.crs == self.osgb

        arr = GeometryArray(arr)
        assert arr.crs == self.osgb

        arr = GeometryArray(arr, crs=4326)
        assert arr.crs == self.wgs

    def test_series(self):
        s = GeoSeries(crs=27700)
        assert s.crs == self.osgb
        assert s.values.crs == self.osgb

        arr = from_shapely(self.geoms)
        s = GeoSeries(arr, crs=27700)
        assert s.crs == self.osgb
        assert s.values.crs == self.osgb

        # manually change CRS
        s.crs = 4326
        assert s.crs == self.wgs
        assert s.values.crs == self.wgs

        s = GeoSeries(self.geoms, crs=27700)
        assert s.crs == self.osgb
        assert s.values.crs == self.osgb

        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(arr)
        assert s.crs == self.osgb
        assert s.values.crs == self.osgb

        with pytest.raises(
            ValueError,
            match="CRS mismatch between CRS of the passed geometries and 'crs'",
        ):
            s = GeoSeries(arr, crs=4326)
        assert s.crs == self.osgb

    def test_dataframe(self):
        arr = from_shapely(self.geoms, crs=27700)
        df = GeoDataFrame(geometry=arr)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        arr = from_shapely(self.geoms)
        s = GeoSeries(arr, crs=27700)
        df = GeoDataFrame(geometry=s)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        # different passed CRS than array CRS is now an error
        match_str = "CRS mismatch between CRS of the passed geometries and 'crs'"
        with pytest.raises(ValueError, match=match_str):
            df = GeoDataFrame(geometry=s, crs=4326)
        with pytest.raises(ValueError, match=match_str):
            GeoDataFrame(geometry=s, crs=4326)
        with pytest.raises(ValueError, match=match_str):
            GeoDataFrame({"data": [1, 2], "geometry": s}, crs=4326)
        with pytest.raises(ValueError, match=match_str):
            GeoDataFrame(df, crs=4326).crs

        # manually change CRS
        arr = from_shapely(self.geoms)
        s = GeoSeries(arr, crs=27700)
        df = GeoDataFrame(geometry=s)
        df.crs = 4326
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df.geometry.values.crs == self.wgs

        with pytest.raises(ValueError, match="Assigning CRS to a GeoDataFrame without"):
            GeoDataFrame(self.geoms, columns=["geom"], crs=27700)
        with pytest.raises(ValueError, match="Assigning CRS to a GeoDataFrame without"):
            GeoDataFrame(crs=27700)

        df = GeoDataFrame(self.geoms, columns=["geom"])
        df = df.set_geometry("geom", crs=27700)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb
        assert df.geom.crs == self.osgb
        assert df.geom.values.crs == self.osgb

        df = GeoDataFrame(geometry=self.geoms, crs=27700)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        # new geometry with set CRS has priority over GDF CRS
        df = GeoDataFrame(geometry=self.geoms, crs=27700)
        df = df.set_geometry(self.geoms, crs=4326)
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df.geometry.values.crs == self.wgs

        arr = from_shapely(self.geoms)
        s = GeoSeries(arr, crs=27700)
        df = GeoDataFrame()
        df = df.set_geometry(s)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        arr = from_shapely(self.geoms, crs=27700)
        df = GeoDataFrame()
        df = df.set_geometry(arr)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        arr = from_shapely(self.geoms)
        df = GeoDataFrame({"col1": [1, 2], "geometry": arr}, crs=4326)
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df.geometry.values.crs == self.wgs

        arr = from_shapely(self.geoms, crs=4326)
        df = GeoDataFrame({"col1": [1, 2], "geometry": arr})
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df.geometry.values.crs == self.wgs

        # geometry column without geometry
        df = GeoDataFrame({"geometry": [0, 1]})
        with pytest.raises(
            ValueError,
            match="Assigning CRS to a GeoDataFrame without an active geometry",
        ):
            df.crs = 27700
        with pytest.raises(
            AttributeError,
            match="The CRS attribute of a GeoDataFrame without an active",
        ):
            assert df.crs == self.osgb

    def test_dataframe_getitem_without_geometry_column(self):
        df = GeoDataFrame({"col": range(10)}, geometry=self.arr)
        df["geom2"] = df.geometry.centroid
        subset = df[["col", "geom2"]]
        with pytest.raises(
            AttributeError,
            match="The CRS attribute of a GeoDataFrame without an active",
        ):
            assert subset.crs == self.osgb

    def test_dataframe_setitem(self):
        # new geometry CRS has priority over GDF CRS
        arr = from_shapely(self.geoms)
        s = GeoSeries(arr, crs=27700)
        df = GeoDataFrame()
        df["geometry"] = s
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        arr = from_shapely(self.geoms, crs=27700)
        df = GeoDataFrame()
        df["geometry"] = arr
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        # test to_crs case (GH1960)
        arr = from_shapely(self.geoms)
        df = GeoDataFrame({"col1": [1, 2], "geometry": arr}, crs=4326)
        df["geometry"] = df["geometry"].to_crs(27700)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        # test changing geometry crs not in the geometry column doesn't change the crs
        arr = from_shapely(self.geoms)
        df = GeoDataFrame(
            {"col1": [1, 2], "geometry": arr, "other_geom": arr}, crs=4326
        )
        df["other_geom"] = from_shapely(self.geoms, crs=27700)
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df["geometry"].crs == self.wgs
        assert df["other_geom"].crs == self.osgb

    def test_dataframe_setitem_without_geometry_column(self):
        arr = from_shapely(self.geoms)
        df = GeoDataFrame({"col1": [1, 2], "geometry": arr}, crs=4326)

        # override geometry with non geometry
        with pytest.warns(UserWarning):
            df["geometry"] = 1

        # assigning a list of geometry object doesn't have cached access to 4326
        df["geometry"] = self.geoms
        assert df.crs is None

    @pytest.mark.parametrize(
        "scalar", [None, Point(0, 0), LineString([(0, 0), (1, 1)])]
    )
    def test_scalar(self, scalar):
        df = GeoDataFrame()
        df["geometry"] = scalar
        df.crs = 4326
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df.geometry.values.crs == self.wgs

    @pytest.mark.filterwarnings("ignore:Accessing CRS")
    def test_crs_with_no_geom_fails(self):
        with pytest.raises(ValueError, match="Assigning CRS to a GeoDataFrame without"):
            df = GeoDataFrame()
            df.crs = 4326

    def test_read_file(self):
        nybb_filename = datasets.get_path("nybb")
        df = read_file(nybb_filename)
        assert df.crs == pyproj.CRS(2263)
        assert df.geometry.crs == pyproj.CRS(2263)
        assert df.geometry.values.crs == pyproj.CRS(2263)

    def test_multiple_geoms(self):
        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(self.geoms, crs=4326)
        df = GeoDataFrame(s, geometry=arr, columns=["col1"])
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb
        assert df.col1.crs == self.wgs
        assert df.col1.values.crs == self.wgs

    def test_multiple_geoms_set_geom(self):
        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(self.geoms, crs=4326)
        df = GeoDataFrame(s, geometry=arr, columns=["col1"])
        df = df.set_geometry("col1")
        assert df.crs == self.wgs
        assert df.geometry.crs == self.wgs
        assert df.geometry.values.crs == self.wgs
        assert df["geometry"].crs == self.osgb
        assert df["geometry"].values.crs == self.osgb

    def test_assign_cols(self):
        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(self.geoms, crs=4326)
        df = GeoDataFrame(s, geometry=arr, columns=["col1"])
        df["geom2"] = s
        df["geom3"] = s.values
        df["geom4"] = from_shapely(self.geoms)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb
        assert df.geom2.crs == self.wgs
        assert df.geom2.values.crs == self.wgs
        assert df.geom3.crs == self.wgs
        assert df.geom3.values.crs == self.wgs
        assert df.geom4.crs is None
        assert df.geom4.values.crs is None

    def test_copy(self):
        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(self.geoms, crs=4326)
        df = GeoDataFrame(s, geometry=arr, columns=["col1"])

        arr_copy = arr.copy()
        assert arr_copy.crs == arr.crs

        s_copy = s.copy()
        assert s_copy.crs == s.crs
        assert s_copy.values.crs == s.values.crs

        df_copy = df.copy()
        assert df_copy.crs == df.crs
        assert df_copy.geometry.crs == df.geometry.crs
        assert df_copy.geometry.values.crs == df.geometry.values.crs
        assert df_copy.col1.crs == df.col1.crs
        assert df_copy.col1.values.crs == df.col1.values.crs

    def test_rename(self):
        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(self.geoms, crs=4326)
        df = GeoDataFrame(s, geometry=arr, columns=["col1"])
        df = df.rename(columns={"geometry": "geom"}).set_geometry("geom")
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        df = df.rename_geometry("geom2")
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        df = df.rename(columns={"col1": "column1"})
        assert df.column1.crs == self.wgs
        assert df.column1.values.crs == self.wgs

    def test_geoseries_to_crs(self):
        s = GeoSeries(self.geoms, crs=27700)
        s = s.to_crs(4326)
        assert s.crs == self.wgs
        assert s.values.crs == self.wgs

        df = GeoDataFrame(geometry=s)
        assert df.crs == self.wgs
        df = df.to_crs(27700)
        assert df.crs == self.osgb
        assert df.geometry.crs == self.osgb
        assert df.geometry.values.crs == self.osgb

        # make sure that only active geometry is transformed
        arr = from_shapely(self.geoms, crs=4326)
        df["col1"] = arr
        df = df.to_crs(3857)
        assert df.col1.crs == self.wgs
        assert df.col1.values.crs == self.wgs

    def test_array_to_crs(self):
        arr = from_shapely(self.geoms, crs=27700)
        arr = arr.to_crs(4326)
        assert arr.crs == self.wgs

    def test_from_shapely(self):
        arr = from_shapely(self.geoms, crs=27700)
        assert arr.crs == self.osgb

    def test_from_wkb(self):
        L_wkb = [p.wkb for p in self.geoms]
        arr = from_wkb(L_wkb, crs=27700)
        assert arr.crs == self.osgb

    def test_from_wkt(self):
        L_wkt = [p.wkt for p in self.geoms]
        arr = from_wkt(L_wkt, crs=27700)
        assert arr.crs == self.osgb

    def test_points_from_xy(self):
        df = pd.DataFrame([{"x": x, "y": x, "z": x} for x in range(10)])
        arr = points_from_xy(df["x"], df["y"], crs=27700)
        assert arr.crs == self.osgb

    # setting CRS in GeoSeries should not set it in passed array without CRS
    def test_original(self):
        arr = from_shapely(self.geoms)
        s = GeoSeries(arr, crs=27700)
        assert arr.crs is None
        assert s.crs == self.osgb

    def test_ops(self):
        arr = self.arr
        bound = arr.boundary
        assert bound.crs == self.osgb

        cent = arr.centroid
        assert cent.crs == self.osgb

        hull = arr.convex_hull
        assert hull.crs == self.osgb

        envelope = arr.envelope
        assert envelope.crs == self.osgb

        exterior = arr.exterior
        assert exterior.crs == self.osgb

        representative_point = arr.representative_point()
        assert representative_point.crs == self.osgb

    def test_binary_ops(self):
        arr = self.arr
        quads = []
        while len(quads) < 10:
            geom = Polygon([(random.random(), random.random()) for i in range(4)])
            if geom.is_valid:
                quads.append(geom)

        arr2 = from_shapely(quads, crs=27700)

        difference = arr.difference(arr2)
        assert difference.crs == self.osgb

        intersection = arr.intersection(arr2)
        assert intersection.crs == self.osgb

        symmetric_difference = arr.symmetric_difference(arr2)
        assert symmetric_difference.crs == self.osgb

        union = arr.union(arr2)
        assert union.crs == self.osgb

    def test_other(self):
        arr = self.arr

        buffer = arr.buffer(5)
        assert buffer.crs == self.osgb

        interpolate = arr.exterior.interpolate(0.1)
        assert interpolate.crs == self.osgb

        simplify = arr.simplify(5)
        assert simplify.crs == self.osgb

    @pytest.mark.parametrize(
        "attr, arg",
        [
            ("affine_transform", ([0, 1, 1, 0, 0, 0],)),
            ("translate", ()),
            ("rotate", (10,)),
            ("scale", ()),
            ("skew", ()),
        ],
    )
    def test_affinity_methods(self, attr, arg):
        result = getattr(self.arr, attr)(*arg)

        assert result.crs == self.osgb

    def test_slice(self):
        s = GeoSeries(self.arr, crs=27700)
        assert s.iloc[1:].values.crs == self.osgb

        df = GeoDataFrame({"col1": self.arr}, geometry=s)
        assert df.iloc[1:].geometry.values.crs == self.osgb
        assert df.iloc[1:].col1.values.crs == self.osgb

    def test_concat(self):
        s = GeoSeries(self.arr, crs=27700)
        assert pd.concat([s, s]).values.crs == self.osgb

        df = GeoDataFrame({"col1": from_shapely(self.geoms, crs=4326)}, geometry=s)
        assert pd.concat([df, df]).geometry.values.crs == self.osgb
        assert pd.concat([df, df]).col1.values.crs == self.wgs

    def test_merge(self):
        arr = from_shapely(self.geoms, crs=27700)
        s = GeoSeries(self.geoms, crs=4326)
        df = GeoDataFrame({"col1": s}, geometry=arr)
        df2 = GeoDataFrame({"col2": s}, geometry=arr).rename_geometry("geom")
        merged = df.merge(df2, left_index=True, right_index=True)
        assert merged.col1.values.crs == self.wgs
        assert merged.geometry.values.crs == self.osgb
        assert merged.col2.values.crs == self.wgs
        assert merged.geom.values.crs == self.osgb
        assert merged.crs == self.osgb

    # make sure that geometry column from list has CRS (__setitem__)
    def test_setitem_geometry(self):
        arr = from_shapely(self.geoms, crs=27700)
        df = GeoDataFrame({"col1": [0, 1]}, geometry=arr)

        df["geometry"] = [g for g in df.geometry]
        assert df.geometry.values.crs == self.osgb

        df2 = GeoDataFrame({"col1": [0, 1]}, geometry=arr)
        df2["geometry"] = from_shapely(self.geoms, crs=4326)
        assert df2.geometry.values.crs == self.wgs

    def test_astype(self):
        arr = from_shapely(self.geoms, crs=27700)
        df = GeoDataFrame({"col1": [0, 1]}, geometry=arr)
        df2 = df.astype({"col1": str})
        assert df2.crs == self.osgb

    def test_apply(self):
        s = GeoSeries(self.arr)
        assert s.crs == 27700

        # apply preserves the CRS if the result is a GeoSeries
        result = s.apply(lambda x: x.centroid)
        assert result.crs == 27700

    def test_apply_geodataframe(self):
        df = GeoDataFrame({"col1": [0, 1]}, geometry=self.geoms, crs=27700)
        assert df.crs == 27700

        # apply preserves the CRS if the result is a GeoDataFrame
        result = df.apply(lambda col: col, axis=0)
        assert result.crs == 27700
        result = df.apply(lambda row: row, axis=1)
        assert result.crs == 27700


class TestSetCRS:
    @pytest.mark.parametrize(
        "constructor",
        [
            lambda geoms, crs: GeoSeries(geoms, crs=crs),
            lambda geoms, crs: GeoDataFrame(geometry=geoms, crs=crs),
        ],
        ids=["geoseries", "geodataframe"],
    )
    def test_set_crs(self, constructor):
        naive = constructor([Point(0, 0), Point(1, 1)], crs=None)
        assert naive.crs is None

        # by default returns a copy
        result = naive.set_crs(crs="EPSG:4326")
        assert result.crs == "EPSG:4326"
        assert naive.crs is None

        result = naive.set_crs(epsg=4326)
        assert result.crs == "EPSG:4326"
        assert naive.crs is None

        # with inplace=True
        result = naive.set_crs(crs="EPSG:4326", inplace=True)
        assert result is naive
        assert result.crs == naive.crs == "EPSG:4326"

        # raise for non-naive when crs would be overridden
        non_naive = constructor([Point(0, 0), Point(1, 1)], crs="EPSG:4326")
        assert non_naive.crs == "EPSG:4326"
        with pytest.raises(ValueError, match="already has a CRS"):
            non_naive.set_crs("EPSG:3857")

        # allow for equal crs
        result = non_naive.set_crs("EPSG:4326")
        assert result.crs == "EPSG:4326"

        # replace with allow_override=True
        result = non_naive.set_crs("EPSG:3857", allow_override=True)
        assert non_naive.crs == "EPSG:4326"
        assert result.crs == "EPSG:3857"

        result = non_naive.set_crs("EPSG:3857", allow_override=True, inplace=True)
        assert non_naive.crs == "EPSG:3857"
        assert result.crs == "EPSG:3857"

        # raise error when no crs is passed
        with pytest.raises(ValueError):
            naive.set_crs(crs=None, epsg=None)

env_pyrecodes/lib/python3.9/site-packages/geopandas/tests/test_crs.py
NikolaBlagojevic/pyrecodes
"""
Geopy can calculate geodesic distance between two points using the
`geodesic distance
<https://en.wikipedia.org/wiki/Geodesics_on_an_ellipsoid>`_ or the
`great-circle distance
<https://en.wikipedia.org/wiki/Great-circle_distance>`_,
with a default of the geodesic distance available as the function
``geopy.distance.distance``.

Great-circle distance (:class:`.great_circle`) uses a spherical model of
the earth, using the mean earth radius as defined by the International
Union of Geodesy and Geophysics, (2\\ *a* + *b*)/3 = 6371.0087714150598
kilometers approx 6371.009 km (for WGS-84), resulting in an error of up
to about 0.5%. The radius value is stored in
:const:`distance.EARTH_RADIUS`, so it can be customized (it should
always be in kilometers, however).

The geodesic distance is the shortest distance on the surface of an
ellipsoidal model of the earth.  The default algorithm uses the method
is given by `Karney (2013)
<https://doi.org/10.1007%2Fs00190-012-0578-z>`_ (:class:`.geodesic`);
this is accurate to round-off and always converges.

``geopy.distance.distance`` currently uses :class:`.geodesic`.

There are multiple popular ellipsoidal models,
and which one will be the most accurate depends on where your points are
located on the earth.  The default is the WGS-84 ellipsoid, which is the
most globally accurate.  geopy includes a few other models in the
:const:`distance.ELLIPSOIDS` dictionary::

                  model             major (km)   minor (km)     flattening
    ELLIPSOIDS = {'WGS-84':        (6378.137,    6356.7523142,  1 / 298.257223563),
                  'GRS-80':        (6378.137,    6356.7523141,  1 / 298.257222101),
                  'Airy (1830)':   (6377.563396, 6356.256909,   1 / 299.3249646),
                  'Intl 1924':     (6378.388,    6356.911946,   1 / 297.0),
                  'Clarke (1880)': (6378.249145, 6356.51486955, 1 / 293.465),
                  'GRS-67':        (6378.1600,   6356.774719,   1 / 298.25),
                  }

Here are examples of ``distance.distance`` usage, taking pair
of :code:`(lat, lon)` tuples::

    >>> from geopy import distance
    >>> newport_ri = (41.49008, -71.312796)
    >>> cleveland_oh = (41.499498, -81.695391)
    >>> print(distance.distance(newport_ri, cleveland_oh).miles)
    538.39044536

    >>> wellington = (-41.32, 174.81)
    >>> salamanca = (40.96, -5.50)
    >>> print(distance.distance(wellington, salamanca).km)
    19959.6792674

Using :class:`.great_circle` distance::

    >>> print(distance.great_circle(newport_ri, cleveland_oh).miles)
    536.997990696

You can change the ellipsoid model used by the geodesic formulas like so::

    >>> ne, cl = newport_ri, cleveland_oh
    >>> print(distance.geodesic(ne, cl, ellipsoid='GRS-80').miles)

The above model name will automatically be retrieved from the
:const:`distance.ELLIPSOIDS` dictionary. Alternatively, you can specify
the model values directly::

    >>> distance.geodesic(ne, cl, ellipsoid=(6377., 6356., 1 / 297.)).miles

Distances support simple arithmetic, making it easy to do things like
calculate the length of a path::

    >>> from geopy import Nominatim
    >>> d = distance.distance
    >>> g = Nominatim(user_agent="specify_your_app_name_here")
    >>> _, wa = g.geocode('Washington, DC')
    >>> _, pa = g.geocode('Palo Alto, CA')
    >>> print((d(ne, cl) + d(cl, wa) + d(wa, pa)).miles)
    3277.30439191


.. _distance_altitudes:

Currently all algorithms assume that altitudes of the points are either
zero (as in the examples above) or equal, and are relatively small.
Thus altitudes never affect the resulting distances::

    >>> from geopy import distance
    >>> newport_ri = (41.49008, -71.312796)
    >>> cleveland_oh = (41.499498, -81.695391)
    >>> print(distance.distance(newport_ri, cleveland_oh).km)
    866.4554329098687
    >>> newport_ri = (41.49008, -71.312796, 100)
    >>> cleveland_oh = (41.499498, -81.695391, 100)
    >>> print(distance.distance(newport_ri, cleveland_oh).km)
    866.4554329098687

If you need to calculate distances with elevation, then for short
distances the `Euclidean distance
<https://en.wikipedia.org/wiki/Euclidean_distance>`_ formula might give
a suitable approximation::

    >>> import math
    >>> from geopy import distance
    >>> p1 = (43.668613, 40.258916, 0.976)
    >>> p2 = (43.658852, 40.250839, 1.475)
    >>> flat_distance = distance.distance(p1[:2], p2[:2]).km
    >>> print(flat_distance)
    1.265133525952866
    >>> euclidian_distance = math.sqrt(flat_distance**2 + (p2[2] - p1[2])**2)
    >>> print(euclidian_distance)
    1.359986705262199

An attempt to calculate distances between points with different altitudes
would result in a :class:`ValueError` exception.

"""
from math import asin, atan2, cos, sin, sqrt

from geographiclib.geodesic import Geodesic

from geopy import units, util
from geopy.point import Point
from geopy.units import radians

# IUGG mean earth radius in kilometers, from
# https://en.wikipedia.org/wiki/Earth_radius#Mean_radius.  Using a
# sphere with this radius results in an error of up to about 0.5%.
EARTH_RADIUS = 6371.009

# From http://www.movable-type.co.uk/scripts/LatLongVincenty.html:
#   The most accurate and widely used globally-applicable model for the earth
#   ellipsoid is WGS-84, used in this script. Other ellipsoids offering a
#   better fit to the local geoid include Airy (1830) in the UK, International
#   1924 in much of Europe, Clarke (1880) in Africa, and GRS-67 in South
#   America. America (NAD83) and Australia (GDA) use GRS-80, functionally
#   equivalent to the WGS-84 ellipsoid.
ELLIPSOIDS = {
    # model           major (km)   minor (km)     flattening
    'WGS-84':        (6378.137, 6356.7523142, 1 / 298.257223563),
    'GRS-80':        (6378.137, 6356.7523141, 1 / 298.257222101),
    'Airy (1830)':   (6377.563396, 6356.256909, 1 / 299.3249646),
    'Intl 1924':     (6378.388, 6356.911946, 1 / 297.0),
    'Clarke (1880)': (6378.249145, 6356.51486955, 1 / 293.465),
    'GRS-67':        (6378.1600, 6356.774719, 1 / 298.25)
}


def cmp(a, b):
    return (a > b) - (a < b)


def lonlat(x, y, z=0):
    """
    ``geopy.distance.distance`` accepts coordinates in ``(y, x)``/``(lat, lon)``
    order, while some other libraries and systems might use
    ``(x, y)``/``(lon, lat)``.

    This function provides a convenient way to convert coordinates of the
    ``(x, y)``/``(lon, lat)`` format to a :class:`geopy.point.Point` instance.

    Example::

        >>> from geopy.distance import lonlat, distance
        >>> newport_ri_xy = (-71.312796, 41.49008)
        >>> cleveland_oh_xy = (-81.695391, 41.499498)
        >>> print(distance(lonlat(*newport_ri_xy), lonlat(*cleveland_oh_xy)).miles)
        538.3904453677203

    :param x: longitude
    :param y: latitude
    :param z: (optional) altitude
    :return: Point(latitude, longitude, altitude)
    """
    return Point(y, x, z)


def _ensure_same_altitude(a, b):
    if abs(a.altitude - b.altitude) > 1e-6:
        raise ValueError(
            'Calculating distance between points with different altitudes '
            'is not supported'
        )
    # Note: non-zero equal altitudes are fine: assuming that
    # the elevation is many times smaller than the Earth radius
    # it won't give much error.


class Distance:
    """
    Base class for other distance algorithms. Represents a distance.

    Can be used for units conversion::

        >>> from geopy.distance import Distance
        >>> Distance(miles=10).km
        16.09344

    Distance instances have all *distance* properties from :mod:`geopy.units`,
    e.g.: ``km``, ``m``, ``meters``, ``miles`` and so on.

    Distance instances are immutable.

    They support comparison::

        >>> from geopy.distance import Distance
        >>> Distance(kilometers=2) == Distance(meters=2000)
        True
        >>> Distance(kilometers=2) > Distance(miles=1)
        True

    String representation::

        >>> from geopy.distance import Distance
        >>> repr(Distance(kilometers=2))
        'Distance(2.0)'
        >>> str(Distance(kilometers=2))
        '2.0 km'
        >>> repr(Distance(miles=2))
        'Distance(3.218688)'
        >>> str(Distance(miles=2))
        '3.218688 km'

    Arithmetics::

        >>> from geopy.distance import Distance
        >>> -Distance(miles=2)
        Distance(-3.218688)
        >>> Distance(miles=2) + Distance(kilometers=1)
        Distance(4.218688)
        >>> Distance(miles=2) - Distance(kilometers=1)
        Distance(2.218688)
        >>> Distance(kilometers=6) * 5
        Distance(30.0)
        >>> Distance(kilometers=6) / 5
        Distance(1.2)
    """

    def __init__(self, *args, **kwargs):
        """
        There are 3 ways to create a distance:

        - From kilometers::

            >>> from geopy.distance import Distance
            >>> Distance(1.42)
            Distance(1.42)

        - From units::

            >>> from geopy.distance import Distance
            >>> Distance(kilometers=1.42)
            Distance(1.42)
            >>> Distance(miles=1)
            Distance(1.609344)

        - From points (for non-abstract distances only),
          calculated as a sum of distances between all points::

            >>> from geopy.distance import geodesic
            >>> geodesic((40, 160), (40.1, 160.1))
            Distance(14.003702498106215)
            >>> geodesic((40, 160), (40.1, 160.1), (40.2, 160.2))
            Distance(27.999954644813478)
        """

        kilometers = kwargs.pop('kilometers', 0)
        if len(args) == 1:
            # if we only get one argument we assume
            # it's a known distance instead of
            # calculating it first
            kilometers += args[0]
        elif len(args) > 1:
            for a, b in util.pairwise(args):
                kilometers += self.measure(a, b)

        kilometers += units.kilometers(**kwargs)
        self.__kilometers = kilometers

    def __add__(self, other):
        if isinstance(other, Distance):
            return self.__class__(self.kilometers + other.kilometers)
        else:
            raise TypeError(
                "Distance instance must be added with Distance instance."
            )

    def __neg__(self):
        return self.__class__(-self.kilometers)

    def __sub__(self, other):
        return self + -other

    def __mul__(self, other):
        if isinstance(other, Distance):
            raise TypeError(
                "Distance instance must be multiplicated with numbers."
            )
        else:
            return self.__class__(self.kilometers * other)

    def __rmul__(self, other):
        if isinstance(other, Distance):
            raise TypeError(
                "Distance instance must be multiplicated with numbers."
            )
        else:
            return self.__class__(other * self.kilometers)

    def __truediv__(self, other):
        if isinstance(other, Distance):
            return self.kilometers / other.kilometers
        else:
            return self.__class__(self.kilometers / other)

    def __floordiv__(self, other):
        if isinstance(other, Distance):
            return self.kilometers // other.kilometers
        else:
            return self.__class__(self.kilometers // other)

    def __abs__(self):
        return self.__class__(abs(self.kilometers))

    def __bool__(self):
        return bool(self.kilometers)

    def measure(self, a, b):
        # Intentionally not documented, because this method is not supposed
        # to be used directly.
        raise NotImplementedError("Distance is an abstract class")

    def destination(self, point, bearing, distance=None):
        """
        Calculate destination point using a starting point, bearing
        and a distance. This method works for non-abstract distances only.

        Example: a point 10 miles east from ``(34, 148)``::

            >>> import geopy.distance
            >>> geopy.distance.distance(miles=10).destination((34, 148), bearing=90)
            Point(33.99987666492774, 148.17419994321995, 0.0)

        :param point: Starting point.
        :type point: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param float bearing: Bearing in degrees: 0 -- North, 90 -- East,
            180 -- South, 270 or -90 -- West.

        :param distance: Distance, can be used to override
            this instance::

                >>> from geopy.distance import distance, Distance
                >>> distance(miles=10).destination((34, 148), bearing=90, \
distance=Distance(100))
                Point(33.995238229104764, 149.08238904409637, 0.0)

        :type distance: :class:`.Distance`

        :rtype: :class:`geopy.point.Point`
        """
        raise NotImplementedError("Distance is an abstract class")

    def __repr__(self):  # pragma: no cover
        return 'Distance(%s)' % self.kilometers

    def __str__(self):  # pragma: no cover
        return '%s km' % self.__kilometers

    def __cmp__(self, other):  # py2 only
        if isinstance(other, Distance):
            return cmp(self.kilometers, other.kilometers)
        else:
            return cmp(self.kilometers, other)

    def __hash__(self):
        return hash(self.kilometers)

    def __eq__(self, other):
        return self.__cmp__(other) == 0

    def __ne__(self, other):
        return self.__cmp__(other) != 0

    def __gt__(self, other):
        return self.__cmp__(other) > 0

    def __lt__(self, other):
        return self.__cmp__(other) < 0

    def __ge__(self, other):
        return self.__cmp__(other) >= 0

    def __le__(self, other):
        return self.__cmp__(other) <= 0

    @property
    def feet(self):
        return units.feet(kilometers=self.kilometers)

    @property
    def ft(self):
        return self.feet

    @property
    def kilometers(self):
        return self.__kilometers

    @property
    def km(self):
        return self.kilometers

    @property
    def m(self):
        return self.meters

    @property
    def meters(self):
        return units.meters(kilometers=self.kilometers)

    @property
    def mi(self):
        return self.miles

    @property
    def miles(self):
        return units.miles(kilometers=self.kilometers)

    @property
    def nautical(self):
        return units.nautical(kilometers=self.kilometers)

    @property
    def nm(self):
        return self.nautical


class great_circle(Distance):
    """
    Use spherical geometry to calculate the surface distance between
    points.

    Set which radius of the earth to use by specifying a ``radius`` keyword
    argument. It must be in kilometers. The default is to use the module
    constant `EARTH_RADIUS`, which uses the average great-circle radius.

    Example::

        >>> from geopy.distance import great_circle
        >>> newport_ri = (41.49008, -71.312796)
        >>> cleveland_oh = (41.499498, -81.695391)
        >>> print(great_circle(newport_ri, cleveland_oh).miles)
        536.997990696

    """

    def __init__(self, *args, **kwargs):
        self.RADIUS = kwargs.pop('radius', EARTH_RADIUS)
        super().__init__(*args, **kwargs)

    def measure(self, a, b):
        a, b = Point(a), Point(b)
        _ensure_same_altitude(a, b)

        lat1, lng1 = radians(degrees=a.latitude), radians(degrees=a.longitude)
        lat2, lng2 = radians(degrees=b.latitude), radians(degrees=b.longitude)

        sin_lat1, cos_lat1 = sin(lat1), cos(lat1)
        sin_lat2, cos_lat2 = sin(lat2), cos(lat2)

        delta_lng = lng2 - lng1
        cos_delta_lng, sin_delta_lng = cos(delta_lng), sin(delta_lng)

        d = atan2(sqrt((cos_lat2 * sin_delta_lng) ** 2 +
                       (cos_lat1 * sin_lat2 -
                        sin_lat1 * cos_lat2 * cos_delta_lng) ** 2),
                  sin_lat1 * sin_lat2 + cos_lat1 * cos_lat2 * cos_delta_lng)

        return self.RADIUS * d

    def destination(self, point, bearing, distance=None):
        point = Point(point)
        lat1 = units.radians(degrees=point.latitude)
        lng1 = units.radians(degrees=point.longitude)
        bearing = units.radians(degrees=bearing)

        if distance is None:
            distance = self
        if isinstance(distance, Distance):
            distance = distance.kilometers

        d_div_r = float(distance) / self.RADIUS

        lat2 = asin(
            sin(lat1) * cos(d_div_r) +
            cos(lat1) * sin(d_div_r) * cos(bearing)
        )

        lng2 = lng1 + atan2(
            sin(bearing) * sin(d_div_r) * cos(lat1),
            cos(d_div_r) - sin(lat1) * sin(lat2)
        )

        return Point(units.degrees(radians=lat2), units.degrees(radians=lng2))


GreatCircleDistance = great_circle


class geodesic(Distance):
    """
    Calculate the geodesic distance between points.

    Set which ellipsoidal model of the earth to use by specifying an
    ``ellipsoid`` keyword argument. The default is 'WGS-84', which is the
    most globally accurate model.  If ``ellipsoid`` is a string, it is
    looked up in the `ELLIPSOIDS` dictionary to obtain the major and minor
    semiaxes and the flattening. Otherwise, it should be a tuple with those
    values.  See the comments above the `ELLIPSOIDS` dictionary for
    more information.

    Example::

        >>> from geopy.distance import geodesic
        >>> newport_ri = (41.49008, -71.312796)
        >>> cleveland_oh = (41.499498, -81.695391)
        >>> print(geodesic(newport_ri, cleveland_oh).miles)
        538.390445368

    """

    def __init__(self, *args, **kwargs):
        self.ellipsoid_key = None
        self.ELLIPSOID = None
        self.geod = None
        self.set_ellipsoid(kwargs.pop('ellipsoid', 'WGS-84'))
        major, minor, f = self.ELLIPSOID
        super().__init__(*args, **kwargs)

    def set_ellipsoid(self, ellipsoid):
        if isinstance(ellipsoid, str):
            try:
                self.ELLIPSOID = ELLIPSOIDS[ellipsoid]
                self.ellipsoid_key = ellipsoid
            except KeyError:
                raise Exception(
                    "Invalid ellipsoid. See geopy.distance.ELLIPSOIDS"
                )
        else:
            self.ELLIPSOID = ellipsoid
            self.ellipsoid_key = None

    def measure(self, a, b):
        a, b = Point(a), Point(b)
        _ensure_same_altitude(a, b)
        lat1, lon1 = a.latitude, a.longitude
        lat2, lon2 = b.latitude, b.longitude

        if not (isinstance(self.geod, Geodesic) and
                self.geod.a == self.ELLIPSOID[0] and
                self.geod.f == self.ELLIPSOID[2]):
            self.geod = Geodesic(self.ELLIPSOID[0], self.ELLIPSOID[2])

        s12 = self.geod.Inverse(lat1, lon1, lat2, lon2,
                                Geodesic.DISTANCE)['s12']

        return s12

    def destination(self, point, bearing, distance=None):
        point = Point(point)
        lat1 = point.latitude
        lon1 = point.longitude
        azi1 = bearing

        if distance is None:
            distance = self
        if isinstance(distance, Distance):
            distance = distance.kilometers

        if not (isinstance(self.geod, Geodesic) and
                self.geod.a == self.ELLIPSOID[0] and
                self.geod.f == self.ELLIPSOID[2]):
            self.geod = Geodesic(self.ELLIPSOID[0], self.ELLIPSOID[2])

        r = self.geod.Direct(lat1, lon1, azi1, distance,
                             Geodesic.LATITUDE | Geodesic.LONGITUDE)

        return Point(r['lat2'], r['lon2'])


GeodesicDistance = geodesic

# Set the default distance formula
distance = GeodesicDistance

env_pyrecodes/lib/python3.9/site-packages/geopy/distance.py
NikolaBlagojevic/pyrecodes
from functools import partial
from urllib.parse import urlencode

from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.util import logger

__all__ = ("BANFrance", )


class BANFrance(Geocoder):
    """Geocoder using the Base Adresse Nationale France API.

    Documentation at:
        https://adresse.data.gouv.fr/api
    """

    geocode_path = '/search'
    reverse_path = '/reverse'

    def __init__(
            self,
            *,
            domain='api-adresse.data.gouv.fr',
            scheme=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None
    ):
        """

        :param str domain: Currently it is ``'api-adresse.data.gouv.fr'``, can
            be changed for testing purposes.

        :param str scheme:
            See :attr:`geopy.geocoders.options.default_scheme`.

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

        :param callable adapter_factory:
            See :attr:`geopy.geocoders.options.default_adapter_factory`.

            .. versionadded:: 2.0

        """
        super().__init__(
            scheme=scheme,
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
            adapter_factory=adapter_factory,
        )
        self.domain = domain.strip('/')

        self.geocode_api = (
            '%s://%s%s' % (self.scheme, self.domain, self.geocode_path)
        )
        self.reverse_api = (
            '%s://%s%s' % (self.scheme, self.domain, self.reverse_path)
        )

    def geocode(
            self,
            query,
            *,
            limit=None,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL
    ):
        """
        Return a location point by address.

        :param str query: The address or query you wish to geocode.

        :param int limit: Defines the maximum number of items in the
            response structure. If not provided and there are multiple
            results the BAN API will return 5 results by default.
            This will be reset to one if ``exactly_one`` is True.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """

        params = {
            'q': query,
        }

        if limit is not None:
            params['limit'] = limit

        url = "?".join((self.geocode_api, urlencode(params)))

        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def reverse(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL
    ):
        """
        Return an address by location point.

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """

        try:
            lat, lon = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")

        params = {
            'lat': lat,
            'lon': lon,
        }

        url = "?".join((self.reverse_api, urlencode(params)))
        logger.debug("%s.reverse: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def _parse_feature(self, feature):
        # Parse each resource.
        latitude = feature.get('geometry', {}).get('coordinates', [])[1]
        longitude = feature.get('geometry', {}).get('coordinates', [])[0]
        placename = feature.get('properties', {}).get('label')

        return Location(placename, (latitude, longitude), feature)

    def _parse_json(self, response, exactly_one):
        if response is None or 'features' not in response:
            return None
        features = response['features']
        if not len(features):
            return None
        if exactly_one:
            return self._parse_feature(features[0])
        else:
            return [self._parse_feature(feature) for feature in features]

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/banfrance.py
NikolaBlagojevic/pyrecodes
import asyncio
import functools
import inspect
import threading

from geopy import compat
from geopy.adapters import (
    AdapterHTTPError,
    BaseAsyncAdapter,
    BaseSyncAdapter,
    RequestsAdapter,
    URLLibAdapter,
    get_retry_after,
)
from geopy.exc import (
    ConfigurationError,
    GeocoderAuthenticationFailure,
    GeocoderInsufficientPrivileges,
    GeocoderQueryError,
    GeocoderQuotaExceeded,
    GeocoderRateLimited,
    GeocoderServiceError,
    GeocoderTimedOut,
)
from geopy.point import Point
from geopy.util import __version__, logger

__all__ = (
    "Geocoder",
    "options",
)

_DEFAULT_USER_AGENT = "geopy/%s" % __version__

_DEFAULT_ADAPTER_CLASS = next(
    adapter_cls
    for adapter_cls in (RequestsAdapter, URLLibAdapter,)
    if adapter_cls.is_available
)


class options:
    """The `options` object contains default configuration values for
    geocoders, e.g. `timeout` and `User-Agent`.
    Instead of passing a custom value to each geocoder individually, you can
    override a default value in this object.

    Please note that not all geocoders use all attributes of this object.
    For example, some geocoders don't respect the ``default_scheme``
    attribute. Refer to the specific geocoder's initializer doc for a list
    of parameters which that geocoder accepts.

    Example for overriding default ``timeout`` and ``user_agent``::

        >>> import geopy.geocoders
        >>> from geopy.geocoders import Nominatim
        >>> geopy.geocoders.options.default_user_agent = 'my_app/1'
        >>> geopy.geocoders.options.default_timeout = 7
        >>> geolocator = Nominatim()
        >>> print(geolocator.headers)
        {'User-Agent': 'my_app/1'}
        >>> print(geolocator.timeout)
        7

    Attributes:
        default_adapter_factory
            A callable which returns a :class:`geopy.adapters.BaseAdapter`
            instance. Adapters are different implementations of HTTP clients.
            See :mod:`geopy.adapters` for more info.

            This callable accepts two keyword args: ``proxies`` and ``ssl_context``.
            A class might be specified as this callable as well.

            Example::

                import geopy.geocoders
                geopy.geocoders.options.default_adapter_factory \
= geopy.adapters.URLLibAdapter

                geopy.geocoders.options.default_adapter_factory = (
                    lambda proxies, ssl_context: MyAdapter(
                        proxies=proxies, ssl_context=ssl_context, my_custom_arg=42
                    )
                )

            If `requests <https://requests.readthedocs.io>`_ package is
            installed, the default adapter is
            :class:`geopy.adapters.RequestsAdapter`. Otherwise it is
            :class:`geopy.adapters.URLLibAdapter`.

            .. versionadded:: 2.0

        default_proxies
            Tunnel requests through HTTP proxy.

            By default the system proxies are respected (e.g.
            `HTTP_PROXY` and `HTTPS_PROXY` env vars or platform-specific
            proxy settings, such as macOS or Windows native
            preferences -- see :func:`urllib.request.getproxies` for
            more details). The `proxies` value for using system proxies
            is ``None``.

            To disable system proxies and issue requests directly,
            explicitly pass an empty dict as a value for `proxies`: ``{}``.

            To use a custom HTTP proxy location, pass a string.
            Valid examples are:

            - ``"192.0.2.0:8080"``
            - ``"john:passw0rd@192.0.2.0:8080"``
            - ``"http://john:passw0rd@192.0.2.0:8080"``

            Please note:

            - Scheme part (``http://``) of the proxy is ignored.
            - Only `http` proxy is supported. Even if the proxy scheme
              is `https`, it will be ignored, and the connection between
              client and proxy would still be unencrypted.
              However, `https` requests via `http` proxy are still
              supported (via `HTTP CONNECT` method).


            Raw urllib-style `proxies` dict might be provided instead of
            a string:

            - ``{"https": "192.0.2.0:8080"}`` -- means that HTTP proxy
              would be used only for requests having `https` scheme.
              String `proxies` value is automatically used for both
              schemes, and is provided as a shorthand for the urllib-style
              `proxies` dict.

            For more information, see
            documentation on :func:`urllib.request.getproxies`.

        default_scheme
            Use ``'https'`` or ``'http'`` as the API URL's scheme.

        default_ssl_context
            An :class:`ssl.SSLContext` instance with custom TLS
            verification settings. Pass ``None`` to use the interpreter's
            defaults (that is to use the system's trusted CA certificates).

            To use the CA bundle used by `requests` library::

                import ssl
                import certifi
                import geopy.geocoders
                ctx = ssl.create_default_context(cafile=certifi.where())
                geopy.geocoders.options.default_ssl_context = ctx

            To disable TLS certificate verification completely::

                import ssl
                import geopy.geocoders
                ctx = ssl.create_default_context()
                ctx.check_hostname = False
                ctx.verify_mode = ssl.CERT_NONE
                geopy.geocoders.options.default_ssl_context = ctx

            See docs for the :class:`ssl.SSLContext` class for more examples.

        default_timeout
            Time, in seconds, to wait for the geocoding service to respond
            before raising a :class:`geopy.exc.GeocoderTimedOut` exception.
            Pass `None` to disable timeout.

        default_user_agent
            User-Agent header to send with the requests to geocoder API.
    """

    # Please keep the attributes sorted (Sphinx sorts them in the rendered
    # docs) and make sure that each attr has a corresponding section in
    # the docstring above.
    #
    # It's bad to have the attrs docs separated from the attrs
    # themselves. Although Sphinx supports docstrings for each attr [1],
    # this is not standardized and won't work with `help()` function and
    # in the ReadTheDocs (at least out of the box) [2].
    #
    # [1]: http://www.sphinx-doc.org/en/master/ext/autodoc.html#directive-autoattribute
    # [2]: https://github.com/rtfd/readthedocs.org/issues/855#issuecomment-261337038
    default_adapter_factory = _DEFAULT_ADAPTER_CLASS
    default_proxies = None
    default_scheme = 'https'
    default_ssl_context = None
    default_timeout = 1
    default_user_agent = _DEFAULT_USER_AGENT


# Create an object which `repr` returns 'DEFAULT_SENTINEL'. Sphinx (docs) uses
# this value when generating method's signature.
DEFAULT_SENTINEL = type('object', (object,),
                        {'__repr__': lambda self: 'DEFAULT_SENTINEL'})()

ERROR_CODE_MAP = {
    400: GeocoderQueryError,
    401: GeocoderAuthenticationFailure,
    402: GeocoderQuotaExceeded,
    403: GeocoderInsufficientPrivileges,
    407: GeocoderAuthenticationFailure,
    408: GeocoderTimedOut,
    412: GeocoderQueryError,
    413: GeocoderQueryError,
    414: GeocoderQueryError,
    429: GeocoderRateLimited,
    502: GeocoderServiceError,
    503: GeocoderTimedOut,
    504: GeocoderTimedOut
}

NONE_RESULT = object()  # special return value for `_geocoder_exception_handler`


class Geocoder:
    """
    Template object for geocoders.
    """

    def __init__(
            self,
            *,
            scheme=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None
    ):
        self.scheme = scheme or options.default_scheme
        if self.scheme not in ('http', 'https'):
            raise ConfigurationError(
                'Supported schemes are `http` and `https`.'
            )
        self.timeout = (timeout if timeout is not DEFAULT_SENTINEL
                        else options.default_timeout)
        self.proxies = (proxies if proxies is not DEFAULT_SENTINEL
                        else options.default_proxies)
        self.headers = {'User-Agent': user_agent or options.default_user_agent}
        self.ssl_context = (ssl_context if ssl_context is not DEFAULT_SENTINEL
                            else options.default_ssl_context)

        if isinstance(self.proxies, str):
            self.proxies = {'http': self.proxies, 'https': self.proxies}

        if adapter_factory is None:
            adapter_factory = options.default_adapter_factory
        self.adapter = adapter_factory(
            proxies=self.proxies,
            ssl_context=self.ssl_context,
        )
        if isinstance(self.adapter, BaseSyncAdapter):
            self.__run_async = False
        elif isinstance(self.adapter, BaseAsyncAdapter):
            self.__run_async = True
        else:
            raise ConfigurationError(
                "Adapter %r must extend either BaseSyncAdapter or BaseAsyncAdapter"
                % (type(self.adapter),)
            )

    def __enter__(self):
        """Context manager for synchronous adapters. At exit all
        open connections will be closed.

        In synchronous mode context manager usage is not required,
        and connections will be automatically closed by garbage collection.
        """
        if self.__run_async:
            raise TypeError("`async with` must be used with async adapters")
        res = self.adapter.__enter__()
        assert res is self.adapter, "adapter's __enter__ must return `self`"
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.adapter.__exit__(exc_type, exc_val, exc_tb)

    async def __aenter__(self):
        """Context manager for asynchronous adapters. At exit all
        open connections will be closed.

        In asynchronous mode context manager usage is not required,
        however, it is strongly advised to avoid warnings about
        resources leaks.
        """
        if not self.__run_async:
            raise TypeError("`async with` cannot be used with sync adapters")
        res = await self.adapter.__aenter__()
        assert res is self.adapter, "adapter's __enter__ must return `self`"
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self.adapter.__aexit__(exc_type, exc_val, exc_tb)

    def _coerce_point_to_string(self, point, output_format="%(lat)s,%(lon)s"):
        """
        Do the right thing on "point" input. For geocoders with reverse
        methods.
        """
        if not isinstance(point, Point):
            point = Point(point)

        # Altitude is silently dropped.
        #
        # Geocoding services (almost?) always consider only lat and lon
        # in queries, so altitude doesn't affect the request.
        # A non-zero altitude should not raise an exception
        # though, because PoIs are assumed to span the whole
        # altitude axis (i.e. not just the 0km plane).
        return output_format % dict(lat=_format_coordinate(point.latitude),
                                    lon=_format_coordinate(point.longitude))

    def _format_bounding_box(
        self, bbox, output_format="%(lat1)s,%(lon1)s,%(lat2)s,%(lon2)s"
    ):
        """
        Transform bounding box boundaries to a string matching
        `output_format` from the following formats:

            - [Point(lat1, lon1), Point(lat2, lon2)]
            - [[lat1, lon1], [lat2, lon2]]
            - ["lat1,lon1", "lat2,lon2"]

        It is guaranteed that lat1 <= lat2 and lon1 <= lon2.
        """
        if len(bbox) != 2:
            raise GeocoderQueryError("Unsupported format for a bounding box")
        p1, p2 = bbox
        p1, p2 = Point(p1), Point(p2)
        return output_format % dict(lat1=min(p1.latitude, p2.latitude),
                                    lon1=min(p1.longitude, p2.longitude),
                                    lat2=max(p1.latitude, p2.latitude),
                                    lon2=max(p1.longitude, p2.longitude))

    def _geocoder_exception_handler(self, error):
        """
        Geocoder-specific exceptions handler.
        Override if custom exceptions processing is needed.
        For example, raising an appropriate GeocoderQuotaExceeded on non-200
        response with a textual message in the body about the exceeded quota.

        Return `NONE_RESULT` to have the geocoding call return `None` (meaning
        empty result).
        """
        pass

    def _call_geocoder(
            self,
            url,
            callback,
            *,
            timeout=DEFAULT_SENTINEL,
            is_json=True,
            headers=None
    ):
        """
        For a generated query URL, get the results.
        """

        req_headers = self.headers.copy()
        if headers:
            req_headers.update(headers)

        timeout = (timeout if timeout is not DEFAULT_SENTINEL
                   else self.timeout)

        try:
            if is_json:
                result = self.adapter.get_json(url, timeout=timeout, headers=req_headers)
            else:
                result = self.adapter.get_text(url, timeout=timeout, headers=req_headers)
            if self.__run_async:
                async def fut():
                    try:
                        res = callback(await result)
                        if inspect.isawaitable(res):
                            res = await res
                        return res
                    except Exception as error:
                        res = self._adapter_error_handler(error)
                        if res is NONE_RESULT:
                            return None
                        raise

                return fut()
            else:
                return callback(result)
        except Exception as error:
            res = self._adapter_error_handler(error)
            if res is NONE_RESULT:
                return None
            raise

    def _adapter_error_handler(self, error):
        if isinstance(error, AdapterHTTPError):
            if error.text:
                logger.info(
                    'Received an HTTP error (%s): %s',
                    error.status_code,
                    error.text,
                    exc_info=False,
                )
            res = self._geocoder_exception_handler(error)
            if res is NONE_RESULT:
                return NONE_RESULT
            exc_cls = ERROR_CODE_MAP.get(error.status_code, GeocoderServiceError)
            if issubclass(exc_cls, GeocoderRateLimited):
                raise exc_cls(
                    str(error), retry_after=get_retry_after(error.headers)
                ) from error
            else:
                raise exc_cls(str(error)) from error
        else:
            res = self._geocoder_exception_handler(error)
            if res is NONE_RESULT:
                return NONE_RESULT

    # def geocode(self, query, *, exactly_one=True, timeout=DEFAULT_SENTINEL):
    #     raise NotImplementedError()

    # def reverse(self, query, *, exactly_one=True, timeout=DEFAULT_SENTINEL):
    #     raise NotImplementedError()


def _format_coordinate(coordinate):
    if abs(coordinate) >= 1:
        return coordinate  # use the default arbitrary precision scientific notation
    return f"{coordinate:.7f}"


def _synchronized(func):
    """A decorator for geocoder methods which makes the method always run
    under a lock. The lock is reentrant.

    This decorator transparently handles sync and async working modes.
    """

    sync_lock = threading.RLock()

    def locked_sync(self, *args, **kwargs):
        with sync_lock:
            return func(self, *args, **kwargs)

    # At the moment this decorator is evaluated we don't know if we
    # will work in sync or async mode.
    # But we shouldn't create the asyncio Lock in sync mode to avoid
    # unwanted implicit loop initialization.
    async_lock = None  # asyncio.Lock()
    async_lock_task = None  # support reentrance

    async def locked_async(self, *args, **kwargs):
        nonlocal async_lock
        nonlocal async_lock_task

        if async_lock is None:
            async_lock = asyncio.Lock()

        if async_lock.locked():
            assert async_lock_task is not None
            if compat.current_task() is async_lock_task:
                res = func(self, *args, **kwargs)
                if inspect.isawaitable(res):
                    res = await res
                return res

        async with async_lock:
            async_lock_task = compat.current_task()
            try:
                res = func(self, *args, **kwargs)
                if inspect.isawaitable(res):
                    res = await res
                return res
            finally:
                async_lock_task = None

    @functools.wraps(func)
    def f(self, *args, **kwargs):
        run_async = isinstance(self.adapter, BaseAsyncAdapter)
        if run_async:
            return locked_async(self, *args, **kwargs)
        else:
            return locked_sync(self, *args, **kwargs)

    return f

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/base.py
NikolaBlagojevic/pyrecodes
import base64
import collections.abc
import hashlib
import hmac
from calendar import timegm
from datetime import datetime
from functools import partial
from urllib.parse import urlencode

from geopy.exc import (
    ConfigurationError,
    GeocoderQueryError,
    GeocoderQuotaExceeded,
    GeocoderServiceError,
    GeocoderUnavailable,
)
from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.timezone import ensure_pytz_is_installed, from_timezone_name
from geopy.util import logger

__all__ = ("GoogleV3", )


class GoogleV3(Geocoder):
    """Geocoder using the Google Maps v3 API.

    Documentation at:
        https://developers.google.com/maps/documentation/geocoding/

    Pricing details:
        https://developers.google.com/maps/documentation/geocoding/usage-and-billing
    """

    api_path = '/maps/api/geocode/json'
    timezone_path = '/maps/api/timezone/json'

    def __init__(
            self,
            api_key=None,
            *,
            domain='maps.googleapis.com',
            scheme=None,
            client_id=None,
            secret_key=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None,
            channel=''
    ):
        """

        :param str api_key: The API key required by Google to perform
            geocoding requests, mandatory (unless premier is used,
            then both ``client_id`` and ``secret_key`` must be specified
            instead).
            API keys are managed through
            the Google APIs console (https://code.google.com/apis/console).
            Make sure to have both ``Geocoding API`` and ``Time Zone API``
            services enabled for this API key.

            .. versionchanged:: 2.1
               Previously a warning has been emitted when neither ``api_key``
               nor premier were specified. Now a :class:`geopy.exc.ConfigurationError`
               is raised.

        :param str domain: Should be the localized Google Maps domain to
            connect to. The default is 'maps.googleapis.com', but if you're
            geocoding address in the UK (for example), you may want to set it
            to 'maps.google.co.uk' to properly bias results.

        :param str scheme:
            See :attr:`geopy.geocoders.options.default_scheme`.

        :param str client_id: If using premier, the account client id.

        :param str secret_key: If using premier, the account secret key.

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

        :param callable adapter_factory:
            See :attr:`geopy.geocoders.options.default_adapter_factory`.

            .. versionadded:: 2.0

        :param str channel: If using premier, the channel identifier.
        """
        super().__init__(
            scheme=scheme,
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
            adapter_factory=adapter_factory,
        )
        if client_id and not secret_key:
            raise ConfigurationError('Must provide secret_key with client_id.')
        if secret_key and not client_id:
            raise ConfigurationError('Must provide client_id with secret_key.')

        self.premier = bool(client_id and secret_key)
        self.client_id = client_id
        self.secret_key = secret_key

        if not self.premier and not api_key:
            raise ConfigurationError(
                'Since July 2018 Google requires each request to have an API key. '
                'Pass a valid `api_key` to GoogleV3 geocoder to fix this error. '
                'See https://developers.google.com/maps/documentation/geocoding/usage-and-billing'  # noqa
            )

        self.api_key = api_key
        self.domain = domain.strip('/')

        self.channel = channel

        self.api = '%s://%s%s' % (self.scheme, self.domain, self.api_path)
        self.tz_api = '%s://%s%s' % (self.scheme, self.domain, self.timezone_path)

    def _get_signed_url(self, params):
        """
        Returns a Premier account signed url. Docs on signature:
            https://developers.google.com/maps/documentation/business/webservices/auth#digital_signatures
        """
        params['client'] = self.client_id

        if self.channel:
            params['channel'] = self.channel

        path = "?".join((self.api_path, urlencode(params)))
        signature = hmac.new(
            base64.urlsafe_b64decode(self.secret_key),
            path.encode('utf-8'),
            hashlib.sha1
        )
        signature = base64.urlsafe_b64encode(
            signature.digest()
        ).decode('utf-8')
        return '%s://%s%s&signature=%s' % (
            self.scheme, self.domain, path, signature
        )

    def _format_components_param(self, components):
        """
        Format the components dict to something Google understands.
        """
        component_items = []

        if isinstance(components, collections.abc.Mapping):
            component_items = components.items()
        elif (
            isinstance(components, collections.abc.Sequence)
            and not isinstance(components, (str, bytes))
        ):
            component_items = components
        else:
            raise ValueError(
                '`components` parameter must be of type `dict` or `list`')

        return "|".join(
            ":".join(item) for item in component_items
        )

    def geocode(
            self,
            query=None,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            bounds=None,
            region=None,
            components=None,
            place_id=None,
            language=None,
            sensor=False
    ):
        """
        Return a location point by address.

        :param str query: The address or query you wish to geocode. Optional,
            if ``components`` param is set::

                >>> g.geocode(components={"city": "Paris", "country": "FR"})
                Location(France, (46.227638, 2.213749, 0.0))

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :type bounds: list or tuple of 2 items of :class:`geopy.point.Point` or
            ``(latitude, longitude)`` or ``"%(latitude)s, %(longitude)s"``.
        :param bounds: The bounding box of the viewport within which
            to bias geocode results more prominently.
            Example: ``[Point(22, 180), Point(-22, -180)]``.

        :param str region: The region code, specified as a ccTLD
            ("top-level domain") two-character value.

        :type components: dict or list
        :param components: Restricts to an area. Can use any combination of:
            `route`, `locality`, `administrative_area`, `postal_code`,
            `country`.

            Pass a list of tuples if you want to specify multiple components of
            the same type, e.g.:

                >>> [('administrative_area', 'VA'), ('administrative_area', 'Arlington')]

        :param str place_id: Retrieve a Location using a Place ID.
            Cannot be not used with ``query`` or ``bounds`` parameters.

                >>> g.geocode(place_id='ChIJOcfP0Iq2j4ARDrXUa7ZWs34')

        :param str language: The language in which to return results.

        :param bool sensor: Whether the geocoding request comes from a
            device with a location sensor.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        params = {
            'sensor': str(sensor).lower()
        }
        if place_id and (bounds or query):
            raise ValueError(
                'Only one of the `query` or `place id` or `bounds` '
                ' parameters must be entered.')

        if place_id is not None:
            params['place_id'] = place_id

        if query is not None:
            params['address'] = query

        if query is None and place_id is None and not components:
            raise ValueError('Either `query` or `components` or `place_id` '
                             'must be set.')

        if self.api_key:
            params['key'] = self.api_key
        if bounds:
            params['bounds'] = self._format_bounding_box(
                bounds, "%(lat1)s,%(lon1)s|%(lat2)s,%(lon2)s")
        if region:
            params['region'] = region
        if components:
            params['components'] = self._format_components_param(components)
        if language:
            params['language'] = language

        if self.premier:
            url = self._get_signed_url(params)
        else:
            url = "?".join((self.api, urlencode(params)))

        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def reverse(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            language=None,
            sensor=False
    ):
        """
        Return an address by location point.

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param str language: The language in which to return results.

        :param bool sensor: Whether the geocoding request comes from a
            device with a location sensor.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """

        params = {
            'latlng': self._coerce_point_to_string(query),
            'sensor': str(sensor).lower()
        }
        if language:
            params['language'] = language
        if self.api_key:
            params['key'] = self.api_key

        if not self.premier:
            url = "?".join((self.api, urlencode(params)))
        else:
            url = self._get_signed_url(params)

        logger.debug("%s.reverse: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def reverse_timezone(self, query, *, at_time=None, timeout=DEFAULT_SENTINEL):
        """
        Find the timezone a point in `query` was in for a specified `at_time`.

        `None` will be returned for points without an assigned
        Olson timezone id (e.g. for Antarctica).

        :param query: The coordinates for which you want a timezone.
        :type query: :class:`geopy.point.Point`, list or tuple of (latitude,
            longitude), or string as "%(latitude)s, %(longitude)s"

        :param at_time: The time at which you want the timezone of this
            location. This is optional, and defaults to the time that the
            function is called in UTC. Timezone-aware datetimes are correctly
            handled and naive datetimes are silently treated as UTC.
        :type at_time: :class:`datetime.datetime` or None

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None`` or :class:`geopy.timezone.Timezone`.
        """
        ensure_pytz_is_installed()

        location = self._coerce_point_to_string(query)

        timestamp = self._normalize_timezone_at_time(at_time)

        params = {
            "location": location,
            "timestamp": timestamp,
        }
        if self.api_key:
            params['key'] = self.api_key
        url = "?".join((self.tz_api, urlencode(params)))

        logger.debug("%s.reverse_timezone: %s", self.__class__.__name__, url)
        return self._call_geocoder(url, self._parse_json_timezone, timeout=timeout)

    def _parse_json_timezone(self, response):
        self._check_status(response)

        timezone_id = response.get("timeZoneId")
        if timezone_id is None:
            # Google returns `status: ZERO_RESULTS` for uncovered
            # points (e.g. for Antarctica), so there's nothing
            # meaningful to be returned as the `raw` response,
            # hence we return `None`.
            return None
        return from_timezone_name(timezone_id, raw=response)

    def _normalize_timezone_at_time(self, at_time):
        if at_time is None:
            timestamp = timegm(datetime.utcnow().utctimetuple())
        elif isinstance(at_time, datetime):
            # Naive datetimes are silently treated as UTC.
            # Timezone-aware datetimes are handled correctly.
            timestamp = timegm(at_time.utctimetuple())
        else:
            raise GeocoderQueryError(
                "`at_time` must be an instance of `datetime.datetime`"
            )
        return timestamp

    def _parse_json(self, page, exactly_one=True):
        '''Returns location, (latitude, longitude) from json feed.'''

        places = page.get('results', [])
        self._check_status(page)
        if not places:
            return None

        def parse_place(place):
            '''Get the location, lat, lng from a single json place.'''
            location = place.get('formatted_address')
            latitude = place['geometry']['location']['lat']
            longitude = place['geometry']['location']['lng']
            return Location(location, (latitude, longitude), place)

        if exactly_one:
            return parse_place(places[0])
        else:
            return [parse_place(place) for place in places]

    def _check_status(self, response):
        # https://developers.google.com/maps/documentation/geocoding/requests-geocoding#StatusCodes
        status = response.get('status')
        if status == 'OK':
            return
        if status == 'ZERO_RESULTS':
            return

        error_message = response.get('error_message')
        # https://developers.google.com/maps/documentation/geocoding/requests-geocoding#ErrorMessages
        #   When the geocoder returns a status code other than OK, there *may*
        #   be an additional error_message field within the Geocoding response
        #   object.

        if status in ('OVER_QUERY_LIMIT', 'OVER_DAILY_LIMIT'):
            raise GeocoderQuotaExceeded(
                error_message or
                'The given key has gone over the requests limit in the 24'
                ' hour period or has submitted too many requests in too'
                ' short a period of time'
            )
        elif status == 'REQUEST_DENIED':
            raise GeocoderQueryError(error_message or 'Your request was denied')
        elif status == 'INVALID_REQUEST':
            raise GeocoderQueryError(
                error_message or 'Probably missing address or latlng'
            )
        elif status == 'UNKNOWN_ERROR':
            raise GeocoderUnavailable(error_message or 'Server error')
        else:
            # Unknown (undocumented) status.
            raise GeocoderServiceError(error_message or 'Unknown error')

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/google.py
NikolaBlagojevic/pyrecodes
import warnings
import xml.etree.ElementTree as ET
from functools import partial
from urllib.parse import urlencode

from geopy.exc import GeocoderQueryError
from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.util import logger

__all__ = ("IGNFrance", )


class IGNFrance(Geocoder):
    """Geocoder using the IGN France GeoCoder OpenLS API.

    Documentation at:
        https://geoservices.ign.fr/services-web-essentiels
    """

    xml_request = """<?xml version="1.0" encoding="UTF-8"?>
    <XLS version="1.2"
        xmlns="http://www.opengis.net/xls"
        xmlns:gml="http://www.opengis.net/gml"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://www.opengis.net/xls
        http://schemas.opengis.net/ols/1.2/olsAll.xsd">
        <RequestHeader srsName="epsg:4326"/>
        <Request methodName="{method_name}"
                 maximumResponses="{maximum_responses}"
                 requestID=""
                 version="1.2">
            {sub_request}
        </Request>
    </XLS>"""

    api_path = '/essentiels/geoportail/ols'

    def __init__(
            self,
            api_key=None,
            *,
            username=None,
            password=None,
            referer=None,
            domain='wxs.ign.fr',
            scheme=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None
    ):
        """

        :param str api_key: Not used.

            .. deprecated:: 2.3
                IGNFrance geocoding methods no longer accept or require
                authentication, see `<https://geoservices.ign.fr/actualites/2021-10-04-evolution-des-modalites-dacces-aux-services-web>`_.
                This parameter is scheduled for removal in geopy 3.0.

        :param str username: Not used.

            .. deprecated:: 2.3
                See the `api_key` deprecation note.

        :param str password: Not used.

            .. deprecated:: 2.3
                See the `api_key` deprecation note.

        :param str referer: Not used.

            .. deprecated:: 2.3
                See the `api_key` deprecation note.

        :param str domain: Currently it is ``'wxs.ign.fr'``, can
            be changed for testing purposes for developer API
            e.g ``'gpp3-wxs.ign.fr'`` at the moment.

        :param str scheme:
            See :attr:`geopy.geocoders.options.default_scheme`.

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

        :param callable adapter_factory:
            See :attr:`geopy.geocoders.options.default_adapter_factory`.

            .. versionadded:: 2.0
        """  # noqa
        super().__init__(
            scheme=scheme,
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
            adapter_factory=adapter_factory,
        )

        if api_key or username or password or referer:
            warnings.warn(
                "IGNFrance no longer accepts or requires authentication, "
                "so api_key, username, password and referer are not used "
                "anymore. These arguments should be removed. "
                "In geopy 3 these options will be removed, causing "
                "an error instead of this warning.",
                DeprecationWarning,
                stacklevel=2,
            )

        self.domain = domain.strip('/')
        api_path = self.api_path
        self.api = '%s://%s%s' % (self.scheme, self.domain, api_path)

    def geocode(
            self,
            query,
            *,
            query_type='StreetAddress',
            maximum_responses=25,
            is_freeform=False,
            filtering=None,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL
    ):
        """
        Return a location point by address.

        :param str query: The query string to be geocoded.

        :param str query_type: The type to provide for geocoding. It can be
            `PositionOfInterest`, `StreetAddress` or `CadastralParcel`.
            `StreetAddress` is the default choice if none provided.

        :param int maximum_responses: The maximum number of responses
            to ask to the API in the query body.

        :param str is_freeform: Set if return is structured with
            freeform structure or a more structured returned.
            By default, value is False.

        :param str filtering: Provide string that help setting geocoder
            filter. It contains an XML string. See examples in documentation
            and ignfrance.py file in directory tests.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """

        # Check if acceptable query type
        if query_type not in ['PositionOfInterest',
                              'StreetAddress',
                              'CadastralParcel']:
            raise GeocoderQueryError("""You did not provided a query_type the
            webservice can consume. It should be PositionOfInterest,
            'StreetAddress or CadastralParcel""")

        # Check query validity for CadastralParcel
        if query_type == 'CadastralParcel' and len(query.strip()) != 14:
            raise GeocoderQueryError("""You must send a string of fourteen
                characters long to match the cadastre required code""")

        sub_request = """
                <GeocodeRequest returnFreeForm="{is_freeform}">
                    <Address countryCode="{query_type}">
                        <freeFormAddress>{query}</freeFormAddress>
                        {filtering}
                    </Address>
                </GeocodeRequest>
        """

        xml_request = self.xml_request.format(
            method_name='LocationUtilityService',
            sub_request=sub_request,
            maximum_responses=maximum_responses
        )

        # Manage type change for xml case sensitive
        if is_freeform:
            is_freeform = 'true'
        else:
            is_freeform = 'false'

        # Manage filtering value
        if filtering is None:
            filtering = ''

        # Create query using parameters
        request_string = xml_request.format(
            is_freeform=is_freeform,
            query=query,
            query_type=query_type,
            filtering=filtering
        )

        params = {
            'xls': request_string
        }

        url = "?".join((self.api, urlencode(params)))

        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        callback = partial(
            self._parse_xml, is_freeform=is_freeform, exactly_one=exactly_one
        )
        return self._request_raw_content(url, callback, timeout=timeout)

    def reverse(
            self,
            query,
            *,
            reverse_geocode_preference=('StreetAddress', ),
            maximum_responses=25,
            filtering='',
            exactly_one=True,
            timeout=DEFAULT_SENTINEL
    ):
        """
        Return an address by location point.

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param list reverse_geocode_preference: Enable to set expected results
            type. It can be `StreetAddress` or `PositionOfInterest`.
            Default is set to `StreetAddress`.

        :param int maximum_responses: The maximum number of responses
            to ask to the API in the query body.

        :param str filtering: Provide string that help setting geocoder
            filter. It contains an XML string. See examples in documentation
            and ignfrance.py file in directory tests.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """

        sub_request = """
            <ReverseGeocodeRequest>
                {reverse_geocode_preference}
                <Position>
                  <gml:Point>
                    <gml:pos>{query}</gml:pos>
                  </gml:Point>
                  {filtering}
                </Position>
            </ReverseGeocodeRequest>
        """

        xml_request = self.xml_request.format(
            method_name='ReverseGeocodeRequest',
            sub_request=sub_request,
            maximum_responses=maximum_responses
        )

        for pref in reverse_geocode_preference:
            if pref not in ('StreetAddress', 'PositionOfInterest'):
                raise GeocoderQueryError(
                    '`reverse_geocode_preference` must contain '
                    'one or more of: StreetAddress, PositionOfInterest'
                )

        point = self._coerce_point_to_string(query, "%(lat)s %(lon)s")
        reverse_geocode_preference = '\n'.join(
            '<ReverseGeocodePreference>%s</ReverseGeocodePreference>' % pref
            for pref
            in reverse_geocode_preference
        )

        request_string = xml_request.format(
            maximum_responses=maximum_responses,
            query=point,
            reverse_geocode_preference=reverse_geocode_preference,
            filtering=filtering
        )

        url = "?".join((self.api, urlencode({'xls': request_string})))

        logger.debug("%s.reverse: %s", self.__class__.__name__, url)
        callback = partial(
            self._parse_xml,
            exactly_one=exactly_one,
            is_reverse=True,
            is_freeform='false'
        )
        return self._request_raw_content(url, callback, timeout=timeout)

    def _parse_xml(self,
                   page,
                   is_reverse=False,
                   is_freeform=False,
                   exactly_one=True):
        """
        Returns location, (latitude, longitude) from XML feed
        and transform to json
        """
        # Parse the page
        tree = ET.fromstring(page.encode('utf-8'))

        # Clean tree from namespace to facilitate XML manipulation
        def remove_namespace(doc, namespace):
            """Remove namespace in the document in place."""
            ns = '{%s}' % namespace
            nsl = len(ns)
            for elem in doc.iter():
                if elem.tag.startswith(ns):
                    elem.tag = elem.tag[nsl:]

        remove_namespace(tree, 'http://www.opengis.net/gml')
        remove_namespace(tree, 'http://www.opengis.net/xls')
        remove_namespace(tree, 'http://www.opengis.net/xlsext')

        # Return places as json instead of XML
        places = self._xml_to_json_places(tree, is_reverse=is_reverse)

        if not places:
            return None
        if exactly_one:
            return self._parse_place(places[0], is_freeform=is_freeform)
        else:
            return [
                self._parse_place(
                    place,
                    is_freeform=is_freeform
                ) for place in places
            ]

    def _xml_to_json_places(self, tree, is_reverse=False):
        """
        Transform the xml ElementTree due to XML webservice return to json
        """

        select_multi = (
            'GeocodedAddress'
            if not is_reverse
            else 'ReverseGeocodedLocation'
        )

        adresses = tree.findall('.//' + select_multi)
        places = []

        sel_pl = './/Address/Place[@type="{}"]'
        for adr in adresses:
            el = {}
            el['pos'] = adr.find('./Point/pos')
            el['street'] = adr.find('.//Address/StreetAddress/Street')
            el['freeformaddress'] = adr.find('.//Address/freeFormAddress')
            el['municipality'] = adr.find(sel_pl.format('Municipality'))
            el['numero'] = adr.find(sel_pl.format('Numero'))
            el['feuille'] = adr.find(sel_pl.format('Feuille'))
            el['section'] = adr.find(sel_pl.format('Section'))
            el['departement'] = adr.find(sel_pl.format('Departement'))
            el['commune_absorbee'] = adr.find(sel_pl.format('CommuneAbsorbee'))
            el['commune'] = adr.find(sel_pl.format('Commune'))
            el['insee'] = adr.find(sel_pl.format('INSEE'))
            el['qualite'] = adr.find(sel_pl.format('Qualite'))
            el['territoire'] = adr.find(sel_pl.format('Territoire'))
            el['id'] = adr.find(sel_pl.format('ID'))
            el['id_tr'] = adr.find(sel_pl.format('ID_TR'))
            el['bbox'] = adr.find(sel_pl.format('Bbox'))
            el['nature'] = adr.find(sel_pl.format('Nature'))
            el['postal_code'] = adr.find('.//Address/PostalCode')
            el['extended_geocode_match_code'] = adr.find(
                './/ExtendedGeocodeMatchCode'
            )

            place = {}

            def testContentAttrib(selector, key):
                """
                Helper to select by attribute and if not attribute,
                value set to empty string
                """
                return selector.attrib.get(
                    key,
                    None
                ) if selector is not None else None

            place['accuracy'] = testContentAttrib(
                adr.find('.//GeocodeMatchCode'), 'accuracy')

            place['match_type'] = testContentAttrib(
                adr.find('.//GeocodeMatchCode'), 'matchType')

            place['building'] = testContentAttrib(
                adr.find('.//Address/StreetAddress/Building'), 'number')

            place['search_centre_distance'] = testContentAttrib(
                adr.find('.//SearchCentreDistance'), 'value')

            for key, value in iter(el.items()):
                if value is not None:
                    place[key] = value.text
                else:
                    place[key] = None

            # We check if lat lng is not empty and unpack accordingly
            if place['pos']:
                lat, lng = place['pos'].split(' ')
                place['lat'] = lat.strip()
                place['lng'] = lng.strip()
            else:
                place['lat'] = place['lng'] = None

            # We removed the unused key
            place.pop("pos", None)
            places.append(place)

        return places

    def _request_raw_content(self, url, callback, *, timeout):
        """
        Send the request to get raw content.
        """
        return self._call_geocoder(
            url,
            callback,
            timeout=timeout,
            is_json=False,
        )

    def _parse_place(self, place, is_freeform=None):
        """
        Get the location, lat, lng and place from a single json place.
        """
        # When freeform already so full address
        if is_freeform == 'true':
            location = place.get('freeformaddress')
        else:
            # For parcelle
            if place.get('numero'):
                location = place.get('street')
            else:
                # When classic geocoding
                # or when reverse geocoding
                location = "%s %s" % (
                    place.get('postal_code', ''),
                    place.get('commune', ''),
                )
                if place.get('street'):
                    location = "%s, %s" % (
                        place.get('street', ''),
                        location,
                    )
                if place.get('building'):
                    location = "%s %s" % (
                        place.get('building', ''),
                        location,
                    )

        return Location(location, (place.get('lat'), place.get('lng')), place)

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/ignfrance.py
NikolaBlagojevic/pyrecodes
from functools import partial
from urllib.parse import quote, urlencode

from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.point import Point
from geopy.util import logger

__all__ = ("MapBox", )


class MapBox(Geocoder):
    """Geocoder using the Mapbox API.

    Documentation at:
        https://www.mapbox.com/api-documentation/
    """

    api_path = '/geocoding/v5/mapbox.places/%(query)s.json/'

    def __init__(
            self,
            api_key,
            *,
            scheme=None,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None,
            domain='api.mapbox.com',
            referer=None
    ):
        """
        :param str api_key: The API key required by Mapbox to perform
            geocoding requests. API keys are managed through Mapox's account
            page (https://www.mapbox.com/account/access-tokens).

        :param str scheme:
            See :attr:`geopy.geocoders.options.default_scheme`.

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

        :param callable adapter_factory:
            See :attr:`geopy.geocoders.options.default_adapter_factory`.

            .. versionadded:: 2.0

        :param str domain: base api domain for mapbox

        :param str referer: The URL used to satisfy the URL restriction of
            mapbox tokens.

            .. versionadded:: 2.3
        """
        super().__init__(
            scheme=scheme,
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
            adapter_factory=adapter_factory,
        )
        self.api_key = api_key
        self.domain = domain.strip('/')
        self.api = "%s://%s%s" % (self.scheme, self.domain, self.api_path)
        if referer:
            self.headers['Referer'] = referer

    def _parse_json(self, json, exactly_one=True):
        '''Returns location, (latitude, longitude) from json feed.'''
        features = json['features']
        if features == []:
            return None

        def parse_feature(feature):
            location = feature['place_name']
            longitude = feature['geometry']['coordinates'][0]
            latitude = feature['geometry']['coordinates'][1]
            return Location(location, (latitude, longitude), feature)
        if exactly_one:
            return parse_feature(features[0])
        else:
            return [parse_feature(feature) for feature in features]

    def geocode(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            proximity=None,
            country=None,
            language=None,
            bbox=None
    ):
        """
        Return a location point by address.

        :param str query: The address or query you wish to geocode.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param proximity: A coordinate to bias local results based on a provided
            location.
        :type proximity: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param country: Country to filter result in form of
            ISO 3166-1 alpha-2 country code (e.g. ``FR``).
            Might be a Python list of strings.

        :type country: str or list

        :param str language: This parameter controls the language of the text supplied in
            responses, and also affects result scoring, with results matching the user’s
            query in the requested language being preferred over results that match in
            another language. You can pass two letters country codes (ISO 639-1).

            .. versionadded:: 2.3

        :param bbox: The bounding box of the viewport within which
            to bias geocode results more prominently.
            Example: ``[Point(22, 180), Point(-22, -180)]``.
        :type bbox: list or tuple of 2 items of :class:`geopy.point.Point` or
            ``(latitude, longitude)`` or ``"%(latitude)s, %(longitude)s"``.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        params = {}

        params['access_token'] = self.api_key
        if bbox:
            params['bbox'] = self._format_bounding_box(
                bbox, "%(lon1)s,%(lat1)s,%(lon2)s,%(lat2)s")

        if not country:
            country = []
        if isinstance(country, str):
            country = [country]
        if country:
            params['country'] = ",".join(country)

        if proximity:
            p = Point(proximity)
            params['proximity'] = "%s,%s" % (p.longitude, p.latitude)

        if language:
            params['language'] = language

        quoted_query = quote(query.encode('utf-8'))
        url = "?".join((self.api % dict(query=quoted_query),
                        urlencode(params)))
        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def reverse(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL
    ):
        """
        Return an address by location point.

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        params = {}
        params['access_token'] = self.api_key

        point = self._coerce_point_to_string(query, "%(lon)s,%(lat)s")
        quoted_query = quote(point.encode('utf-8'))
        url = "?".join((self.api % dict(query=quoted_query),
                        urlencode(params)))
        logger.debug("%s.reverse: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/mapbox.py
NikolaBlagojevic/pyrecodes
import collections.abc
from functools import partial
from urllib.parse import urlencode

from geopy.exc import ConfigurationError, GeocoderQueryError
from geopy.geocoders.base import _DEFAULT_USER_AGENT, DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.util import logger

__all__ = ("Nominatim", )

_DEFAULT_NOMINATIM_DOMAIN = 'nominatim.openstreetmap.org'
_REJECTED_USER_AGENTS = (
    # Various sample user-agent strings mentioned in docs:
    "my-application",
    "my_app/1",
    "my_user_agent/1.0",
    "specify_your_app_name_here",
    _DEFAULT_USER_AGENT,
)


class Nominatim(Geocoder):
    """Nominatim geocoder for OpenStreetMap data.

    Documentation at:
        https://nominatim.org/release-docs/develop/api/Overview/

    .. attention::
       Using Nominatim with the default `user_agent` is strongly discouraged,
       as it violates Nominatim's Usage Policy
       https://operations.osmfoundation.org/policies/nominatim/
       and may possibly cause 403 and 429 HTTP errors. Please make sure
       to specify a custom `user_agent` with
       ``Nominatim(user_agent="my-application")`` or by
       overriding the default `user_agent`:
       ``geopy.geocoders.options.default_user_agent = "my-application"``.
       An exception will be thrown if a custom `user_agent` is not specified.
    """

    structured_query_params = {
        'street',
        'city',
        'county',
        'state',
        'country',
        'postalcode',
    }

    geocode_path = '/search'
    reverse_path = '/reverse'

    def __init__(
            self,
            *,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            domain=_DEFAULT_NOMINATIM_DOMAIN,
            scheme=None,
            user_agent=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None
            # Make sure to synchronize the changes of this signature in the
            # inheriting classes (e.g. PickPoint).
    ):
        """

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str domain: Domain where the target Nominatim service
            is hosted.

        :param str scheme:
            See :attr:`geopy.geocoders.options.default_scheme`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

        :param callable adapter_factory:
            See :attr:`geopy.geocoders.options.default_adapter_factory`.

            .. versionadded:: 2.0
        """
        super().__init__(
            scheme=scheme,
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
            adapter_factory=adapter_factory,
        )

        self.domain = domain.strip('/')

        if (self.domain == _DEFAULT_NOMINATIM_DOMAIN
                and self.headers['User-Agent'] in _REJECTED_USER_AGENTS):
            raise ConfigurationError(
                'Using Nominatim with default or sample `user_agent` "%s" is '
                'strongly discouraged, as it violates Nominatim\'s ToS '
                'https://operations.osmfoundation.org/policies/nominatim/ '
                'and may possibly cause 403 and 429 HTTP errors. '
                'Please specify a custom `user_agent` with '
                '`Nominatim(user_agent="my-application")` or by '
                'overriding the default `user_agent`: '
                '`geopy.geocoders.options.default_user_agent = "my-application"`.'
                % self.headers['User-Agent']
            )

        self.api = "%s://%s%s" % (self.scheme, self.domain, self.geocode_path)
        self.reverse_api = "%s://%s%s" % (self.scheme, self.domain, self.reverse_path)

    def _construct_url(self, base_api, params):
        """
        Construct geocoding request url.
        The method can be overridden in Nominatim-based geocoders in order
        to extend URL parameters.

        :param str base_api: Geocoding function base address - self.api
            or self.reverse_api.

        :param dict params: Geocoding params.

        :return: string URL.
        """
        return "?".join((base_api, urlencode(params)))

    def geocode(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            limit=None,
            addressdetails=False,
            language=False,
            geometry=None,
            extratags=False,
            country_codes=None,
            viewbox=None,
            bounded=False,
            featuretype=None,
            namedetails=False
    ):
        """
        Return a location point by address.

        :param query: The address, query or a structured query
            you wish to geocode.

            For a structured query, provide a dictionary whose keys
            are one of: `street`, `city`, `county`, `state`, `country`, or
            `postalcode`. For more information, see Nominatim's
            documentation for `structured requests`:

                https://nominatim.org/release-docs/develop/api/Search

        :type query: dict or str

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param int limit: Maximum amount of results to return from Nominatim.
            Unless exactly_one is set to False, limit will always be 1.

        :param bool addressdetails: If you want in *Location.raw* to include
            address details such as house_number, city_district, postcode, etc
            (in a structured form) set it to True

        :param str language: Preferred language in which to return results.
            Either uses standard
            `RFC2616 <http://www.ietf.org/rfc/rfc2616.txt>`_
            accept-language string or a simple comma-separated
            list of language codes.

        :param str geometry: If present, specifies whether the geocoding
            service should return the result's geometry in `wkt`, `svg`,
            `kml`, or `geojson` formats. This is available via the
            `raw` attribute on the returned :class:`geopy.location.Location`
            object.

        :param bool extratags: Include additional information in the result if available,
            e.g. wikipedia link, opening hours.

        :param country_codes: Limit search results
            to a specific country (or a list of countries).
            A country_code should be the ISO 3166-1alpha2 code,
            e.g. ``gb`` for the United Kingdom, ``de`` for Germany, etc.

        :type country_codes: str or list

        :type viewbox: list or tuple of 2 items of :class:`geopy.point.Point` or
            ``(latitude, longitude)`` or ``"%(latitude)s, %(longitude)s"``.

        :param viewbox: Prefer this area to find search results. By default this is
            treated as a hint, if you want to restrict results to this area,
            specify ``bounded=True`` as well.
            Example: ``[Point(22, 180), Point(-22, -180)]``.

        :param bool bounded: Restrict the results to only items contained
            within the bounding ``viewbox``.

        :param str featuretype: If present, restrict results to certain type of features.
            Allowed values: `country`, `state`, `city`, `settlement`.

        :param bool namedetails: If you want in *Location.raw* to include
            namedetails, set it to True. This will be a list of alternative names,
            including language variants, etc.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """

        if isinstance(query, collections.abc.Mapping):
            params = {
                key: val
                for key, val
                in query.items()
                if key in self.structured_query_params
            }
        else:
            params = {'q': query}

        params.update({
            'format': 'json'
        })

        if exactly_one:
            params['limit'] = 1
        elif limit is not None:
            limit = int(limit)
            if limit < 1:
                raise ValueError("Limit cannot be less than 1")
            params['limit'] = limit

        if viewbox:
            params['viewbox'] = self._format_bounding_box(
                viewbox, "%(lon1)s,%(lat1)s,%(lon2)s,%(lat2)s")

        if bounded:
            params['bounded'] = 1

        if not country_codes:
            country_codes = []
        if isinstance(country_codes, str):
            country_codes = [country_codes]
        if country_codes:
            params['countrycodes'] = ",".join(country_codes)

        if addressdetails:
            params['addressdetails'] = 1

        if namedetails:
            params['namedetails'] = 1

        if language:
            params['accept-language'] = language

        if extratags:
            params['extratags'] = True

        if geometry is not None:
            geometry = geometry.lower()
            if geometry == 'wkt':
                params['polygon_text'] = 1
            elif geometry == 'svg':
                params['polygon_svg'] = 1
            elif geometry == 'kml':
                params['polygon_kml'] = 1
            elif geometry == 'geojson':
                params['polygon_geojson'] = 1
            else:
                raise GeocoderQueryError(
                    "Invalid geometry format. Must be one of: "
                    "wkt, svg, kml, geojson."
                )

        if featuretype:
            params['featuretype'] = featuretype

        url = self._construct_url(self.api, params)
        logger.debug("%s.geocode: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def reverse(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            language=False,
            addressdetails=True,
            zoom=None,
            namedetails=False,
    ):
        """
        Return an address by location point.

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param str language: Preferred language in which to return results.
            Either uses standard
            `RFC2616 <http://www.ietf.org/rfc/rfc2616.txt>`_
            accept-language string or a simple comma-separated
            list of language codes.

        :param bool addressdetails: Whether or not to include address details,
            such as city, county, state, etc. in *Location.raw*

        :param int zoom: Level of detail required for the address,
            an integer in range from 0 (country level) to 18 (building level),
            default is 18.

        :param bool namedetails: If you want in *Location.raw* to include
            namedetails, set it to True. This will be a list of alternative names,
            including language variants, etc.

            .. versionadded:: 2.3

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.

        """
        try:
            lat, lon = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")
        params = {
            'lat': lat,
            'lon': lon,
            'format': 'json',
        }
        if language:
            params['accept-language'] = language

        params['addressdetails'] = 1 if addressdetails else 0

        if zoom is not None:
            params['zoom'] = zoom

        if namedetails:
            params['namedetails'] = 1

        url = self._construct_url(self.reverse_api, params)
        logger.debug("%s.reverse: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def _parse_code(self, place):
        # Parse each resource.
        latitude = place.get('lat', None)
        longitude = place.get('lon', None)
        placename = place.get('display_name', None)
        if latitude is not None and longitude is not None:
            latitude = float(latitude)
            longitude = float(longitude)
        return Location(placename, (latitude, longitude), place)

    def _parse_json(self, places, exactly_one):
        if not places:
            return None

        if isinstance(places, collections.abc.Mapping) and 'error' in places:
            if places['error'] == 'Unable to geocode':
                # no results in reverse
                return None
            else:
                raise GeocoderQueryError(places['error'])

        if not isinstance(places, collections.abc.Sequence):
            places = [places]
        if exactly_one:
            return self._parse_code(places[0])
        else:
            return [self._parse_code(place) for place in places]

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/nominatim.py
NikolaBlagojevic/pyrecodes
import warnings
from functools import partial
from urllib.parse import urlencode

from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.location import Location
from geopy.util import logger

__all__ = ("Pelias", )


class Pelias(Geocoder):
    """Pelias geocoder.

    Documentation at:
        https://github.com/pelias/documentation

    See also :class:`geopy.geocoders.GeocodeEarth` which is a Pelias-based
    service provided by the developers of Pelias itself.
    """

    geocode_path = '/v1/search'
    reverse_path = '/v1/reverse'

    def __init__(
            self,
            domain,
            api_key=None,
            *,
            timeout=DEFAULT_SENTINEL,
            proxies=DEFAULT_SENTINEL,
            user_agent=None,
            scheme=None,
            ssl_context=DEFAULT_SENTINEL,
            adapter_factory=None
            # Make sure to synchronize the changes of this signature in the
            # inheriting classes (e.g. GeocodeEarth).
    ):
        """
        :param str domain: Specify a domain for Pelias API.

        :param str api_key: Pelias API key, optional.

        :param int timeout:
            See :attr:`geopy.geocoders.options.default_timeout`.

        :param dict proxies:
            See :attr:`geopy.geocoders.options.default_proxies`.

        :param str user_agent:
            See :attr:`geopy.geocoders.options.default_user_agent`.

        :param str scheme:
            See :attr:`geopy.geocoders.options.default_scheme`.

        :type ssl_context: :class:`ssl.SSLContext`
        :param ssl_context:
            See :attr:`geopy.geocoders.options.default_ssl_context`.

        :param callable adapter_factory:
            See :attr:`geopy.geocoders.options.default_adapter_factory`.

            .. versionadded:: 2.0

        """
        super().__init__(
            scheme=scheme,
            timeout=timeout,
            proxies=proxies,
            user_agent=user_agent,
            ssl_context=ssl_context,
            adapter_factory=adapter_factory,
        )

        self.api_key = api_key
        self.domain = domain.strip('/')

        self.geocode_api = (
            '%s://%s%s' % (self.scheme, self.domain, self.geocode_path)
        )
        self.reverse_api = (
            '%s://%s%s' % (self.scheme, self.domain, self.reverse_path)
        )

    def geocode(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            boundary_rect=None,
            countries=None,
            country_bias=None,
            language=None
    ):
        """
        Return a location point by address.

        :param str query: The address or query you wish to geocode.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :type boundary_rect: list or tuple of 2 items of :class:`geopy.point.Point`
            or ``(latitude, longitude)`` or ``"%(latitude)s, %(longitude)s"``.
        :param boundary_rect: Coordinates to restrict search within.
            Example: ``[Point(22, 180), Point(-22, -180)]``.

        :param list countries: A list of country codes specified in
            `ISO 3166-1 alpha-2 or alpha-3
            <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3>`_
            format, e.g. ``['USA', 'CAN']``.
            This is a hard filter.

            .. versionadded:: 2.3

        :param str country_bias: Bias results to this country (ISO alpha-3).

            .. deprecated:: 2.3
                Use ``countries`` instead. This option behaves the same way,
                i.e. it's not a soft filter as the name suggests.
                This parameter is scheduled for removal in geopy 3.0.

        :param str language: Preferred language in which to return results.
            Either uses standard
            `RFC2616 <http://www.ietf.org/rfc/rfc2616.txt>`_
            accept-language string or a simple comma-separated
            list of language codes.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        params = {'text': query}

        if self.api_key:
            params.update({
                'api_key': self.api_key
            })

        if boundary_rect:
            lon1, lat1, lon2, lat2 = self._format_bounding_box(
                boundary_rect, "%(lon1)s,%(lat1)s,%(lon2)s,%(lat2)s").split(',')
            params['boundary.rect.min_lon'] = lon1
            params['boundary.rect.min_lat'] = lat1
            params['boundary.rect.max_lon'] = lon2
            params['boundary.rect.max_lat'] = lat2

        if country_bias:
            warnings.warn(
                "`country_bias` is deprecated, because it's not "
                "a soft filter as the name suggests. Pass a list to the "
                "`countries` option instead, which behaves the same "
                "way. In geopy 3 the `country_bias` option will be removed.",
                DeprecationWarning,
                stacklevel=2,
            )
            params['boundary.country'] = country_bias

        if countries:
            params['boundary.country'] = ",".join(countries)

        if language:
            params["lang"] = language

        url = "?".join((self.geocode_api, urlencode(params)))
        logger.debug("%s.geocode_api: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def reverse(
            self,
            query,
            *,
            exactly_one=True,
            timeout=DEFAULT_SENTINEL,
            language=None
    ):
        """
        Return an address by location point.

        :param query: The coordinates for which you wish to obtain the
            closest human-readable addresses.
        :type query: :class:`geopy.point.Point`, list or tuple of ``(latitude,
            longitude)``, or string as ``"%(latitude)s, %(longitude)s"``.

        :param bool exactly_one: Return one result or a list of results, if
            available.

        :param int timeout: Time, in seconds, to wait for the geocoding service
            to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
            exception. Set this only if you wish to override, on this call
            only, the value set during the geocoder's initialization.

        :param str language: Preferred language in which to return results.
            Either uses standard
            `RFC2616 <http://www.ietf.org/rfc/rfc2616.txt>`_
            accept-language string or a simple comma-separated
            list of language codes.

        :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
            ``exactly_one=False``.
        """
        try:
            lat, lon = self._coerce_point_to_string(query).split(',')
        except ValueError:
            raise ValueError("Must be a coordinate pair or Point")
        params = {
            'point.lat': lat,
            'point.lon': lon,
        }

        if language:
            params['lang'] = language

        if self.api_key:
            params.update({
                'api_key': self.api_key
            })

        url = "?".join((self.reverse_api, urlencode(params)))
        logger.debug("%s.reverse: %s", self.__class__.__name__, url)
        callback = partial(self._parse_json, exactly_one=exactly_one)
        return self._call_geocoder(url, callback, timeout=timeout)

    def _parse_code(self, feature):
        # Parse each resource.
        latitude = feature.get('geometry', {}).get('coordinates', [])[1]
        longitude = feature.get('geometry', {}).get('coordinates', [])[0]
        placename = feature.get('properties', {}).get('name')
        return Location(placename, (latitude, longitude), feature)

    def _parse_json(self, response, exactly_one):
        if response is None:
            return None
        features = response['features']
        if not len(features):
            return None
        if exactly_one:
            return self._parse_code(features[0])
        else:
            return [self._parse_code(feature) for feature in features]

env_pyrecodes/lib/python3.9/site-packages/geopy/geocoders/pelias.py
NikolaBlagojevic/pyrecodes
from pathlib import Path

import numpy as np

from ..config import known_extensions
from .request import InitializationError, IOMode
from .v3_plugin_api import ImageProperties, PluginV3


def _legacy_default_index(format):
    if format._name == "FFMPEG":
        index = Ellipsis
    elif format._name == "GIF-PIL":
        index = Ellipsis
    else:
        index = 0

    return index


class LegacyPlugin(PluginV3):
    """A plugin to  make old (v2.9) plugins compatible with v3.0

    .. depreciated:: 2.9
        `legacy_get_reader` will be removed in a future version of imageio.
        `legacy_get_writer` will be removed in a future version of imageio.

    This plugin is a wrapper around the old FormatManager class and exposes
    all the old plugins via the new API. On top of this it has
    ``legacy_get_reader`` and ``legacy_get_writer`` methods to allow using
    it with the v2.9 API.

    Methods
    -------
    read(index=None, **kwargs)
        Read the image at position ``index``.
    write(image, **kwargs)
        Write image to the URI.
    iter(**kwargs)
        Iteratively yield images from the given URI.
    get_meta(index=None)
        Return the metadata for the image at position ``index``.
    legacy_get_reader(**kwargs)
        Returns the v2.9 image reader. (depreciated)
    legacy_get_writer(**kwargs)
        Returns the v2.9 image writer. (depreciated)

    Examples
    --------

    >>> import imageio.v3 as iio
    >>> with iio.imopen("/path/to/image.tiff", "r", legacy_mode=True) as file:
    >>>     reader = file.legacy_get_reader()  # depreciated
    >>>     for im in file.iter():
    >>>         print(im.shape)

    """

    def __init__(self, request, legacy_plugin):
        """Instantiate a new Legacy Plugin

        Parameters
        ----------
        uri : {str, pathlib.Path, bytes, file}
            The resource to load the image from, e.g. a filename, pathlib.Path,
            http address or file object, see the docs for more info.
        legacy_plugin : Format
            The (legacy) format to use to interface with the URI.

        """
        self._request = request
        self._format = legacy_plugin

        source = (
            "<bytes>"
            if isinstance(self._request.raw_uri, bytes)
            else self._request.raw_uri
        )
        if self._request.mode.io_mode == IOMode.read:
            if not self._format.can_read(request):
                raise InitializationError(
                    f"`{self._format.name}`" f" can not read `{source}`."
                )
        else:
            if not self._format.can_write(request):
                raise InitializationError(
                    f"`{self._format.name}`" f" can not write to `{source}`."
                )

    def legacy_get_reader(self, **kwargs):
        """legacy_get_reader(**kwargs)

        a utility method to provide support vor the V2.9 API

        Parameters
        ----------
        kwargs : ...
            Further keyword arguments are passed to the reader. See :func:`.help`
            to see what arguments are available for a particular format.
        """

        # Note: this will break thread-safety
        self._request._kwargs = kwargs

        # safeguard for DICOM plugin reading from folders
        try:
            assert Path(self._request.filename).is_dir()
        except OSError:
            pass  # not a valid path on this OS
        except AssertionError:
            pass  # not a folder
        else:
            return self._format.get_reader(self._request)

        self._request.get_file().seek(0)
        return self._format.get_reader(self._request)

    def read(self, *, index=None, **kwargs):
        """
        Parses the given URI and creates a ndarray from it.

        Parameters
        ----------
        index : {integer, None}
            If the URI contains a list of ndimages return the index-th
            image. If None, stack all images into an ndimage along the
            0-th dimension (equivalent to np.stack(imgs, axis=0)).
        kwargs : ...
            Further keyword arguments are passed to the reader. See
            :func:`.help` to see what arguments are available for a particular
            format.

        Returns
        -------
        ndimage : np.ndarray
            A numpy array containing the decoded image data.

        """

        if index is None:
            index = _legacy_default_index(self._format)

        if index is Ellipsis:
            img = np.stack([im for im in self.iter(**kwargs)])
            return img

        reader = self.legacy_get_reader(**kwargs)
        return reader.get_data(index)

    def legacy_get_writer(self, **kwargs):
        """legacy_get_writer(**kwargs)

        Returns a :class:`.Writer` object which can be used to write data
        and meta data to the specified file.

        Parameters
        ----------
        kwargs : ...
            Further keyword arguments are passed to the writer. See :func:`.help`
            to see what arguments are available for a particular format.
        """

        # Note: this will break thread-safety
        self._request._kwargs = kwargs
        return self._format.get_writer(self._request)

    def write(self, ndimage, *, is_batch=None, metadata=None, **kwargs):
        """
        Write an ndimage to the URI specified in path.

        If the URI points to a file on the current host and the file does not
        yet exist it will be created. If the file exists already, it will be
        appended if possible; otherwise, it will be replaced.

        Parameters
        ----------
        ndimage : numpy.ndarray
            The ndimage or list of ndimages to write.
        is_batch : bool
            If True, treat the supplied ndimage as a batch of images. If False,
            treat the supplied ndimage as a single image. If None, try to
            determine ``is_batch`` from the ndimage's shape and ndim.
        metadata : dict
            The metadata passed to write alongside the image.
        kwargs : ...
            Further keyword arguments are passed to the writer. See
            :func:`.help` to see what arguments are available for a
            particular format.


        Returns
        -------
        buffer : bytes
            When writing to the special target "<bytes>", this function will
            return the encoded image data as a bytes string. Otherwise it
            returns None.

        Notes
        -----
        Automatically determining ``is_batch`` may fail for some images due to
        shape aliasing. For example, it may classify a channel-first color image
        as a batch of gray images. In most cases this automatic deduction works
        fine (it has for almost a decade), but if you do have one of those edge
        cases (or are worried that you might) consider explicitly setting
        ``is_batch``.

        """

        if is_batch or isinstance(ndimage, (list, tuple)):
            pass  # ndimage is list of images
        elif is_batch is False:
            ndimage = [ndimage]
        else:
            # Write the largest possible block by guessing the meaning of each
            # dimension from the shape/ndim and then checking if any batch
            # dimensions are left.
            ndimage = np.asanyarray(ndimage)
            batch_dims = ndimage.ndim

            # two spatial dimensions
            batch_dims = max(batch_dims - 2, 0)

            # packed (channel-last) image
            if ndimage.ndim >= 3 and ndimage.shape[-1] < 5:
                batch_dims = max(batch_dims - 1, 0)

            # format supports volumetric images
            ext_infos = known_extensions.get(self._request.extension, list())
            for ext_info in ext_infos:
                if self._format.name in ext_info.priority and ext_info.volume_support:
                    batch_dims = max(batch_dims - 1, 0)
                    break

            if batch_dims == 0:
                ndimage = [ndimage]

        with self.legacy_get_writer(**kwargs) as writer:
            for image in ndimage:
                image = np.asanyarray(image)

                if image.ndim < 2:
                    raise ValueError(
                        "The image must have at least two spatial dimensions."
                    )

                if not np.issubdtype(image.dtype, np.number) and not np.issubdtype(
                    image.dtype, bool
                ):
                    raise ValueError(
                        f"All images have to be numeric, and not `{image.dtype}`."
                    )

                writer.append_data(image, metadata)

        return writer.request.get_result()

    def iter(self, **kwargs):
        """Iterate over a list of ndimages given by the URI

        Parameters
        ----------
        kwargs : ...
            Further keyword arguments are passed to the reader. See
            :func:`.help` to see what arguments are available for a particular
            format.
        """

        reader = self.legacy_get_reader(**kwargs)
        for image in reader:
            yield image

    def properties(self, index=None):
        """Standardized ndimage metadata.

        Parameters
        ----------
        index : int
            The index of the ndimage for which to return properties. If the
            index is out of bounds a ``ValueError`` is raised. If ``None``,
            return the properties for the ndimage stack. If this is impossible,
            e.g., due to shape missmatch, an exception will be raised.

        Returns
        -------
        properties : ImageProperties
            A dataclass filled with standardized image metadata.

        """

        if index is None:
            index = _legacy_default_index(self._format)

        # for backwards compatibility ... actually reads pixel data :(
        if index is Ellipsis:
            image = self.read(index=0)
            n_images = self.legacy_get_reader().get_length()
            return ImageProperties(
                shape=(n_images, *image.shape),
                dtype=image.dtype,
                n_images=n_images,
                is_batch=True,
            )

        image = self.read(index=index)
        return ImageProperties(
            shape=image.shape,
            dtype=image.dtype,
            is_batch=False,
        )

    def get_meta(self, *, index=None):
        """Read ndimage metadata from the URI

        Parameters
        ----------
        index : {integer, None}
            If the URI contains a list of ndimages return the metadata
            corresponding to the index-th image. If None, behavior depends on
            the used api

            Legacy-style API: return metadata of the first element (index=0)
            New-style API: Behavior depends on the used Plugin.

        Returns
        -------
        metadata : dict
            A dictionary of metadata.

        """

        return self.metadata(index=index, exclude_applied=False)

    def metadata(self, index=None, exclude_applied: bool = True):
        """Format-Specific ndimage metadata.

        Parameters
        ----------
        index : int
            The index of the ndimage to read. If the index is out of bounds a
            ``ValueError`` is raised. If ``None``, global metadata is returned.
        exclude_applied : bool
            This parameter exists for compatibility and has no effect. Legacy
            plugins always report all metadata they find.

        Returns
        -------
        metadata : dict
            A dictionary filled with format-specific metadata fields and their
            values.

        """

        if index is None:
            index = _legacy_default_index(self._format)

        return self.legacy_get_reader().get_meta_data(index=index)

    def __del__(self) -> None:
        pass
        # turns out we can't close the file here for LegacyPlugin
        # because it would break backwards compatibility
        # with legacy_get_writer and legacy_get_reader
        # self._request.finish()

env_pyrecodes/lib/python3.9/site-packages/imageio/core/legacy_plugin_wrapper.py
NikolaBlagojevic/pyrecodes
from . import Request
from ..typing import ArrayLike
import numpy as np
from typing import Optional, Dict, Any, Tuple, Union, List, Iterator
from dataclasses import dataclass


@dataclass
class ImageProperties:
    """Standardized Metadata

    ImageProperties represent a set of standardized metadata that is available
    under the same name for every supported format. If the ImageResource (or
    format) does not specify the value, a sensible default value is chosen
    instead.

    Attributes
    ----------
    shape : Tuple[int, ...]
        The shape of the loaded ndimage.
    dtype : np.dtype
        The dtype of the loaded ndimage.
    n_images : int
        Number of images in the file if ``index=...``, `None` for single images.
    is_batch : bool
        If True, the first dimension of the ndimage represents a batch dimension
        along which several images are stacked.
    spacing : Tuple
        A tuple describing the spacing between pixels along each axis of the
        ndimage. If the spacing is uniform along an axis the value corresponding
        to that axis is a single float. If the spacing is non-uniform, the value
        corresponding to that axis is a tuple in which the i-th element
        indicates the spacing between the i-th and (i+1)-th pixel along that
        axis.

    """

    shape: Tuple[int, ...]
    dtype: np.dtype
    n_images: Optional[int] = None
    is_batch: bool = False
    spacing: Optional[tuple] = None


class PluginV3:
    """A ImageIO Plugin.

    This is an abstract plugin that documents the v3 plugin API interface. A
    plugin is an adapter/wrapper around a backend that converts a request from
    iio.core (e.g., read an image from file) into a sequence of instructions for
    the backend that fullfill the request.

    Plugin authors may choose to subclass this class when implementing a new
    plugin, but aren't obliged to do so. As long as the plugin class implements
    the interface (methods) described below the ImageIO core will treat it just
    like any other plugin.


    Parameters
    ----------
    request : iio.Request
        A request object that represents the users intent. It provides a
        standard interface to access the various ImageResources and serves them
        to the plugin as a file object (or file). Check the docs for details.
    **kwargs : Any
        Additional configuration arguments for the plugin or backend. Usually
        these match the configuration arguments available on the backend and
        are forwarded to it.


    Raises
    ------
    InitializationError
        During ``__init__`` the plugin tests if it can fulfill the request. If
        it can't, e.g., because the request points to a file in the wrong
        format, then it should raise an ``InitializationError`` and provide a
        reason for failure. This reason may be reported to the user.
    ImportError
        Plugins will be imported dynamically when listed in
        ``iio.config.known_plugins`` to fullfill requests. This way, users only
        have to load plugins/backends they actually use. If this plugin's backend
        is not installed, it should raise an ``ImportError`` either during
        module import or during class construction.

    Notes
    -----
    Upon successful construction the plugin takes ownership of the provided
    request. This means that it is the plugin's responsibility to call
    request.finish() to close the resource when it is no longer needed.

    Plugins _must_ implement a context manager that closes and cleans any
    resources held by the plugin upon exit.

    """

    def __init__(self, request: Request) -> None:
        """Initialize a new Plugin Instance.

        See Plugin's docstring for detailed documentation.

        Notes
        -----
        The implementation here stores the request as a local variable that is
        exposed using a @property below. If you inherit from PluginV3, remember
        to call ``super().__init__(request)``.

        """

        self._request = request

    def read(self, *, index: int = 0) -> np.ndarray:
        """Read a ndimage.

        The ``read`` method loads a (single) ndimage, located at ``index`` from
        the requested ImageResource.

        It is at the plugin's descretion to decide (and document) what
        constitutes a single ndimage. A sensible way to make this decision is to
        choose based on the ImageResource's format and on what users will expect
        from such a format. For example, a sensible choice for a TIFF file
        produced by an ImageJ hyperstack is to read it as a volumetric ndimage
        (1 color dimension followed by 3 spatial dimensions). On the other hand,
        a sensible choice for a MP4 file produced by Davinci Resolve is to treat
        each frame as a ndimage (2 spatial dimensions followed by 1 color
        dimension).

        The value ``index=None`` is special. It requests the plugin to load all
        ndimages in the file and stack them along a new first axis. For example,
        if a MP4 file is read with ``index=None`` and the plugin identifies
        single frames as ndimages, then the plugin should read all frames and
        stack them into a new ndimage which now contains a time axis as its
        first axis. If a PNG file (single image format) is read with
        ``index=None`` the plugin does a very similar thing: It loads all
        ndimages in the file (here it's just one) and stacks them along a new
        first axis, effectively prepending an axis with size 1 to the image. If
        a plugin does not wish to support ``index=None`` it should set a more
        sensible default and raise a ``ValueError`` when requested to read using
        ``index=None``.

        Parameters
        ----------
        index : int
            If the ImageResource contains multiple ndimages, and index is an
            integer, select the index-th ndimage from among them and return it.
            If index is an ellipsis (...), read all ndimages in the file and
            stack them along a new batch dimension. If index is None, let the
            plugin decide. If the index is out of bounds a ``ValueError`` is
            raised.
        **kwargs : Any
            The read method may accept any number of plugin-specific keyword
            arguments to further customize the read behavior. Usually these
            match the arguments available on the backend and are forwarded to
            it.

        Returns
        -------
        ndimage : np.ndarray
            A ndimage containing decoded pixel data (sometimes called bitmap).

        Notes
        -----
        The ImageResource from which the plugin should read is managed by the
        provided request object. Directly accessing the managed ImageResource is
        _not_ permitted. Instead, you can get FileLike access to the
        ImageResource via request.get_file().

        If the backend doesn't support reading from FileLike objects, you can
        request a temporary file to pass to the backend via
        ``request.get_local_filename()``. This is, however, not very performant
        (involves copying the Request's content into a temporary file), so you
        should avoid doing this whenever possible. Consider it a fallback method
        in case all else fails.

        """
        raise NotImplementedError()

    def write(self, ndimage: Union[ArrayLike, List[ArrayLike]]) -> Optional[bytes]:
        """Write a ndimage to a ImageResource.

        The ``write`` method encodes the given ndimage into the format handled
        by the backend and writes it to the ImageResource. It overwrites
        any content that may have been previously stored in the file.

        If the backend supports only a single format then it must check if
        the ImageResource matches that format and raise an exception if not.
        Typically, this should be done during initialization in the form of a
        ``InitializationError``.

        If the backend supports more than one format it must determine the
        requested/desired format. Usually this can be done by inspecting the
        ImageResource (e.g., by checking ``request.extension``), or by providing
        a mechanism to explicitly set the format (perhaps with a - sensible -
        default value). If the plugin can not determine the desired format, it
        **must not** write to the ImageResource, but raise an exception instead.

        If the backend supports at least one format that can hold multiple
        ndimages it should be capable of handling ndimage batches and lists of
        ndimages. If the ``ndimage`` input is a list of ndimages, the plugin
        should not assume that the ndimages are not stackable, i.e., ndimages
        may have different shapes. Otherwise, the ``ndimage`` may be a batch of
        multiple ndimages stacked along the first axis of the array. The plugin
        must be able to discover this, either automatically or via additional
        `kwargs`. If there is ambiguity in the process, the plugin must clearly
        document what happens in such cases and, if possible, describe how to
        resolve this ambiguity.

        Parameters
        ----------
        ndimage : ArrayLike
            The ndimage to encode and write to the current ImageResource.
        **kwargs : Any
            The write method may accept any number of plugin-specific keyword
            arguments to customize the writing behavior. Usually these match the
            arguments available on the backend and are forwarded to it.

        Returns
        -------
        encoded_image : bytes or None
            If the chosen ImageResource is the special target ``"<bytes>"`` then
            write should return a byte string containing the encoded image data.
            Otherwise, it returns None.

        Notes
        -----
        The ImageResource to which the plugin should write to is managed by the
        provided request object. Directly accessing the managed ImageResource is
        _not_ permitted. Instead, you can get FileLike access to the
        ImageResource via request.get_file().

        If the backend doesn't support writing to FileLike objects, you can
        request a temporary file to pass to the backend via
        ``request.get_local_filename()``. This is, however, not very performant
        (involves copying the Request's content from a temporary file), so you
        should avoid doing this whenever possible. Consider it a fallback method
        in case all else fails.

        """
        raise NotImplementedError()

    def iter(self) -> Iterator[np.ndarray]:
        """Iterate the ImageResource.

        This method returns a generator that yields ndimages in the order in which
        they appear in the file. This is roughly equivalent to::

            idx = 0
            while True:
                try:
                    yield self.read(index=idx)
                except ValueError:
                    break

        It works very similar to ``read``, and you can consult the documentation
        of that method for additional information on desired behavior.

        Parameters
        ----------
        **kwargs : Any
            The iter method may accept any number of plugin-specific keyword
            arguments to further customize the reading/iteration behavior.
            Usually these match the arguments available on the backend and are
            forwarded to it.

        Yields
        ------
        ndimage : np.ndarray
            A ndimage containing decoded pixel data (sometimes called bitmap).

        See Also
        --------
        PluginV3.read

        """
        raise NotImplementedError()

    def properties(self, index: int = 0) -> ImageProperties:
        """Standardized ndimage metadata.

        Parameters
        ----------
        index : int
            If the ImageResource contains multiple ndimages, and index is an
        integer, select the index-th ndimage from among them and return its
        properties. If index is an ellipsis (...), read all ndimages in the file
        and stack them along a new batch dimension and return their properties.
        If index is None, the plugin decides the default.

        Returns
        -------
        properties : ImageProperties
            A dataclass filled with standardized image metadata.

        """
        raise NotImplementedError()

    def metadata(self, index: int = 0, exclude_applied: bool = True) -> Dict[str, Any]:
        """Format-Specific ndimage metadata.

        The method reads metadata stored in the ImageResource and returns it as
        a python dict. The plugin is free to choose which name to give a piece
        of metadata; however, if possible, it should match the name given by the
        format. There is no requirement regarding the fields a plugin must
        expose; however, if a plugin does expose any,``exclude_applied`` applies
        to these fields.

        If the plugin does return metadata items, it must check the value of
        ``exclude_applied`` before returning them. If ``exclude applied`` is
        True, then any metadata item that would be applied to an ndimage
        returned by ``read`` (or ``iter``) must not be returned. This is done to
        avoid confusion; for example, if an ImageResource defines the ExIF
        rotation tag, and the plugin applies the rotation to the data before
        returning it, then ``exclude_applied`` prevents confusion on whether the
        tag was already applied or not.

        The `kwarg` ``index`` behaves similar to its counterpart in ``read``
        with one exception: If the ``index`` is None, then global metadata is
        returned instead of returning a combination of all metadata items. If
        there is no global metadata, the Plugin should return an empty dict or
        raise an exception.

        Parameters
        ----------
        index : int
            If the ImageResource contains multiple ndimages, and index is an
            integer, select the index-th ndimage from among them and return its
            metadata. If index is an ellipsis (...), return global metadata. If
            index is None, the plugin decides the default.
        exclude_applied : bool
            If True (default), do not report metadata fields that the plugin
            would apply/consume while reading the image.

        Returns
        -------
        metadata : dict
            A dictionary filled with format-specific metadata fields and their
            values.

        """
        raise NotImplementedError()

    def close(self) -> None:
        """Close the ImageResource.

        This method allows a plugin to behave similar to the python build-in ``open``::

            image_file = my_plugin(Request, "r")
            ...
            image_file.close()

        It is used by the context manager and deconstructor below to avoid leaking
        ImageResources. If the plugin has no other cleanup to do it doesn't have
        to overwrite this method itself and can rely on the implementation
        below.

        """

        self.request.finish()

    @property
    def request(self) -> Request:
        return self._request

    def __enter__(self) -> "PluginV3":
        return self

    def __exit__(self, type, value, traceback) -> None:
        self.close()

    def __del__(self) -> None:
        self.close()

env_pyrecodes/lib/python3.9/site-packages/imageio/core/v3_plugin_api.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
# imageio is distributed under the terms of the (new) BSD License.

"""Read/Write video using FFMPEG

.. note::
    We are in the process of (slowly) replacing this plugin with a new one that
    is based on `pyav <https://pyav.org/docs/stable/>`_. It is faster and more
    flexible than the plugin documented here. Check the :mod:`pyav
    plugin's documentation <imageio.plugins.pyav>` for more information about
    this plugin.

Backend Library: https://github.com/imageio/imageio-ffmpeg

.. note::
    To use this plugin you have to install its backend::

        pip install imageio[ffmpeg]


The ffmpeg format provides reading and writing for a wide range of movie formats
such as .avi, .mpeg, .mp4, etc. as well as the ability to read streams from
webcams and USB cameras. It is based on ffmpeg and is inspired by/based `moviepy
<https://github.com/Zulko/moviepy/>`_ by Zulko.

Parameters for reading
----------------------
fps : scalar
    The number of frames per second to read the data at. Default None (i.e.
    read at the file's own fps). One can use this for files with a
    variable fps, or in cases where imageio is unable to correctly detect
    the fps. In case of trouble opening camera streams, it may help to set an
    explicit fps value matching a framerate supported by the camera.
loop : bool
    If True, the video will rewind as soon as a frame is requested
    beyond the last frame. Otherwise, IndexError is raised. Default False.
    Setting this to True will internally call ``count_frames()``,
    and set the reader's length to that value instead of inf.
size : str | tuple
    The frame size (i.e. resolution) to read the images, e.g.
    (100, 100) or "640x480". For camera streams, this allows setting
    the capture resolution. For normal video data, ffmpeg will
    rescale the data.
dtype : str | type
    The dtype for the output arrays. Determines the bit-depth that
    is requested from ffmpeg. Supported dtypes: uint8, uint16.
    Default: uint8.
pixelformat : str
    The pixel format for the camera to use (e.g. "yuyv422" or
    "gray"). The camera needs to support the format in order for
    this to take effect. Note that the images produced by this
    reader are always RGB.
input_params : list
    List additional arguments to ffmpeg for input file options.
    (Can also be provided as ``ffmpeg_params`` for backwards compatibility)
    Example ffmpeg arguments to use aggressive error handling:
    ['-err_detect', 'aggressive']
output_params : list
    List additional arguments to ffmpeg for output file options (i.e. the
    stream being read by imageio).
print_info : bool
    Print information about the video file as reported by ffmpeg.

Parameters for writing
----------------------
fps : scalar
    The number of frames per second. Default 10.
codec : str
    the video codec to use. Default 'libx264', which represents the
    widely available mpeg4. Except when saving .wmv files, then the
    defaults is 'msmpeg4' which is more commonly supported for windows
quality : float | None
    Video output quality. Default is 5. Uses variable bit rate. Highest
    quality is 10, lowest is 0. Set to None to prevent variable bitrate
    flags to FFMPEG so you can manually specify them using output_params
    instead. Specifying a fixed bitrate using 'bitrate' disables this
    parameter.
bitrate : int | None
    Set a constant bitrate for the video encoding. Default is None causing
    'quality' parameter to be used instead.  Better quality videos with
    smaller file sizes will result from using the 'quality'  variable
    bitrate parameter rather than specifiying a fixed bitrate with this
    parameter.
pixelformat: str
    The output video pixel format. Default is 'yuv420p' which most widely
    supported by video players.
input_params : list
    List additional arguments to ffmpeg for input file options (i.e. the
    stream that imageio provides).
output_params : list
    List additional arguments to ffmpeg for output file options.
    (Can also be provided as ``ffmpeg_params`` for backwards compatibility)
    Example ffmpeg arguments to use only intra frames and set aspect ratio:
    ['-intra', '-aspect', '16:9']
ffmpeg_log_level: str
    Sets ffmpeg output log level.  Default is "warning".
    Values can be "quiet", "panic", "fatal", "error", "warning", "info"
    "verbose", or "debug". Also prints the FFMPEG command being used by
    imageio if "info", "verbose", or "debug".
macro_block_size: int
    Size constraint for video. Width and height, must be divisible by this
    number. If not divisible by this number imageio will tell ffmpeg to
    scale the image up to the next closest size
    divisible by this number. Most codecs are compatible with a macroblock
    size of 16 (default), some can go smaller (4, 8). To disable this
    automatic feature set it to None or 1, however be warned many players
    can't decode videos that are odd in size and some codecs will produce
    poor results or fail. See https://en.wikipedia.org/wiki/Macroblock.
audio_path : str | None
    Audio path of any audio that needs to be written. Defaults to nothing,
    so no audio will be written. Please note, when writing shorter video
    than the original, ffmpeg will not truncate the audio track; it
    will maintain its original length and be longer than the video.
audio_codec : str | None
    The audio codec to use. Defaults to nothing, but if an audio_path has
    been provided ffmpeg will attempt to set a default codec.

Notes
-----
If you are using anaconda and ``anaconda/ffmpeg`` you will not be able to
encode/decode H.264 (likely due to licensing concerns). If you need this
format on anaconda install ``conda-forge/ffmpeg`` instead.

You can use the ``IMAGEIO_FFMPEG_EXE`` environment variable to force using a
specific ffmpeg executable.

To get the number of frames before having read them all, you can use the
``reader.count_frames()`` method (the reader will then use
``imageio_ffmpeg.count_frames_and_secs()`` to get the exact number of frames,
note that this operation can take a few seconds on large files). Alternatively,
the number of frames can be estimated from the fps and duration in the meta data
(though these values themselves are not always present/reliable).

"""

import re
import sys
import time
import logging
import platform
import threading
import subprocess as sp
import imageio_ffmpeg

import numpy as np

from ..core import Format, image_as_uint

logger = logging.getLogger(__name__)

# Get camera format
if sys.platform.startswith("win"):
    CAM_FORMAT = "dshow"  # dshow or vfwcap
elif sys.platform.startswith("linux"):
    CAM_FORMAT = "video4linux2"
elif sys.platform.startswith("darwin"):
    CAM_FORMAT = "avfoundation"
else:  # pragma: no cover
    CAM_FORMAT = "unknown-cam-format"


def download(directory=None, force_download=False):  # pragma: no cover
    raise RuntimeError(
        "imageio.ffmpeg.download() has been deprecated. "
        "Use 'pip install imageio-ffmpeg' instead.'"
    )


# For backwards compatibility - we dont use this ourselves
def get_exe():  # pragma: no cover
    """Wrapper for imageio_ffmpeg.get_ffmpeg_exe()"""

    return imageio_ffmpeg.get_ffmpeg_exe()


class FfmpegFormat(Format):
    """Read/Write ImageResources using FFMPEG.

    See :mod:`imageio.plugins.ffmpeg`
    """

    def _can_read(self, request):
        # Read from video stream?
        # Note that we could write the _video flag here, but a user might
        # select this format explicitly (and this code is not run)
        if re.match(r"<video(\d+)>", request.filename):
            return True

        # Read from file that we know?
        if request.extension in self.extensions:
            return True

    def _can_write(self, request):
        if request.extension in self.extensions:
            return True

    # --

    class Reader(Format.Reader):
        _frame_catcher = None
        _read_gen = None

        def _get_cam_inputname(self, index):
            if sys.platform.startswith("linux"):
                return "/dev/" + self.request._video[1:-1]

            elif sys.platform.startswith("win"):
                # Ask ffmpeg for list of dshow device names
                ffmpeg_api = imageio_ffmpeg
                cmd = [
                    ffmpeg_api.get_ffmpeg_exe(),
                    "-list_devices",
                    "true",
                    "-f",
                    CAM_FORMAT,
                    "-i",
                    "dummy",
                ]
                # Set `shell=True` in sp.run to prevent popup of a command
                # line window in frozen applications. Note: this would be a
                # security vulnerability if user-input goes into the cmd.
                # Note that the ffmpeg process returns with exit code 1 when
                # using `-list_devices` (or `-list_options`), even if the
                # command is successful, so we set `check=False` explicitly.
                completed_process = sp.run(
                    cmd,
                    stdout=sp.PIPE,
                    stderr=sp.PIPE,
                    encoding="utf-8",
                    shell=True,
                    check=False,
                )

                # Return device name at index
                try:
                    name = parse_device_names(completed_process.stderr)[index]
                except IndexError:
                    raise IndexError("No ffdshow camera at index %i." % index)
                return "video=%s" % name

            elif sys.platform.startswith("darwin"):
                # Appears that newer ffmpeg builds don't support -list-devices
                # on OS X. But you can directly open the camera by index.
                name = str(index)
                return name

            else:  # pragma: no cover
                return "??"

        def _open(
            self,
            loop=False,
            size=None,
            dtype=None,
            pixelformat=None,
            print_info=False,
            ffmpeg_params=None,
            input_params=None,
            output_params=None,
            fps=None,
        ):
            # Get generator functions
            self._ffmpeg_api = imageio_ffmpeg
            # Process input args
            self._arg_loop = bool(loop)
            if size is None:
                self._arg_size = None
            elif isinstance(size, tuple):
                self._arg_size = "%ix%i" % size
            elif isinstance(size, str) and "x" in size:
                self._arg_size = size
            else:
                raise ValueError('FFMPEG size must be tuple of "NxM"')
            if pixelformat is None:
                pass
            elif not isinstance(pixelformat, str):
                raise ValueError("FFMPEG pixelformat must be str")
            if dtype is None:
                self._dtype = np.dtype("uint8")
            else:
                self._dtype = np.dtype(dtype)
                allowed_dtypes = ["uint8", "uint16"]
                if self._dtype.name not in allowed_dtypes:
                    raise ValueError(
                        "dtype must be one of: {}".format(", ".join(allowed_dtypes))
                    )
            self._arg_pixelformat = pixelformat
            self._arg_input_params = input_params or []
            self._arg_output_params = output_params or []
            self._arg_input_params += ffmpeg_params or []  # backward compat
            # Write "_video"_arg - indicating webcam support
            self.request._video = None
            regex_match = re.match(r"<video(\d+)>", self.request.filename)
            if regex_match:
                self.request._video = self.request.filename
            # Get local filename
            if self.request._video:
                index = int(regex_match.group(1))
                self._filename = self._get_cam_inputname(index)
            else:
                self._filename = self.request.get_local_filename()
                # When passed to ffmpeg on command line, carets need to be escaped.
                self._filename = self._filename.replace("^", "^^")
            # Determine pixel format and depth
            self._depth = 3
            if self._dtype.name == "uint8":
                self._pix_fmt = "rgb24"
                self._bytes_per_channel = 1
            else:
                self._pix_fmt = "rgb48le"
                self._bytes_per_channel = 2
            # Initialize parameters
            self._pos = -1
            self._meta = {"plugin": "ffmpeg"}
            self._lastread = None

            # Calculating this from fps and duration is not accurate,
            # and calculating it exactly with ffmpeg_api.count_frames_and_secs
            # takes too long to do for each video. But we need it for looping.
            self._nframes = float("inf")
            if self._arg_loop and not self.request._video:
                self._nframes = self.count_frames()
            self._meta["nframes"] = self._nframes

            # Specify input framerate? (only on macOS)
            # Ideally we'd get the supported framerate from the metadata, but we get the
            # metadata when we boot ffmpeg ... maybe we could refactor this so we can
            # get the metadata beforehand, but for now we'll just give it 2 tries on MacOS,
            # one with fps 30 and one with fps 15.
            need_input_fps = need_output_fps = False
            if self.request._video and platform.system().lower() == "darwin":
                if "-framerate" not in str(self._arg_input_params):
                    need_input_fps = True
                if not self.request.kwargs.get("fps", None):
                    need_output_fps = True
            if need_input_fps:
                self._arg_input_params.extend(["-framerate", str(float(30))])
            if need_output_fps:
                self._arg_output_params.extend(["-r", str(float(30))])

            # Start ffmpeg subprocess and get meta information
            try:
                self._initialize()
            except IndexError:
                # Specify input framerate again, this time different.
                if need_input_fps:
                    self._arg_input_params[-1] = str(float(15))
                    self._initialize()
                else:
                    raise

            # For cameras, create thread that keeps reading the images
            if self.request._video:
                self._frame_catcher = FrameCatcher(self._read_gen)

            # For reference - but disabled, because it is inaccurate
            # if self._meta["nframes"] == float("inf"):
            #     if self._meta.get("fps", 0) > 0:
            #         if self._meta.get("duration", 0) > 0:
            #             n = round(self._meta["duration"] * self._meta["fps"])
            #             self._meta["nframes"] = int(n)

        def _close(self):
            # First close the frame catcher, because we cannot close the gen
            # if the frame catcher thread is using it
            if self._frame_catcher is not None:
                self._frame_catcher.stop_me()
                self._frame_catcher = None
            if self._read_gen is not None:
                self._read_gen.close()
                self._read_gen = None

        def count_frames(self):
            """Count the number of frames. Note that this can take a few
            seconds for large files. Also note that it counts the number
            of frames in the original video and does not take a given fps
            into account.
            """
            # This would have been nice, but this does not work :(
            # oargs = []
            # if self.request.kwargs.get("fps", None):
            #     fps = float(self.request.kwargs["fps"])
            #     oargs += ["-r", "%.02f" % fps]
            cf = self._ffmpeg_api.count_frames_and_secs
            return cf(self._filename)[0]

        def _get_length(self):
            return self._nframes  # only not inf if loop is True

        def _get_data(self, index):
            """Reads a frame at index. Note for coders: getting an
            arbitrary frame in the video with ffmpeg can be painfully
            slow if some decoding has to be done. This function tries
            to avoid fectching arbitrary frames whenever possible, by
            moving between adjacent frames."""
            # Modulo index (for looping)
            if self._arg_loop and self._nframes < float("inf"):
                index %= self._nframes

            if index == self._pos:
                return self._lastread, dict(new=False)
            elif index < 0:
                raise IndexError("Frame index must be >= 0")
            elif index >= self._nframes:
                raise IndexError("Reached end of video")
            else:
                if (index < self._pos) or (index > self._pos + 100):
                    self._initialize(index)
                else:
                    self._skip_frames(index - self._pos - 1)
                result, is_new = self._read_frame()
                self._pos = index
                return result, dict(new=is_new)

        def _get_meta_data(self, index):
            return self._meta

        def _initialize(self, index=0):
            # Close the current generator, and thereby terminate its subprocess
            if self._read_gen is not None:
                self._read_gen.close()

            iargs = []
            oargs = []

            # Create input args
            iargs += self._arg_input_params
            if self.request._video:
                iargs += ["-f", CAM_FORMAT]
                if self._arg_pixelformat:
                    iargs += ["-pix_fmt", self._arg_pixelformat]
                if self._arg_size:
                    iargs += ["-s", self._arg_size]
            elif index > 0:  # re-initialize  / seek
                # Note: only works if we initialized earlier, and now have meta
                # Some info here: https://trac.ffmpeg.org/wiki/Seeking
                # There are two ways to seek, one before -i (input_params) and
                # after (output_params). The former is fast, because it uses
                # keyframes, the latter is slow but accurate. According to
                # the article above, the fast method should also be accurate
                # from ffmpeg version 2.1, however in version 4.1 our tests
                # start failing again. Not sure why, but we can solve this
                # by combining slow and fast. Seek the long stretch using
                # the fast method, and seek the last 10s the slow way.
                starttime = index / self._meta["fps"]
                seek_slow = min(10, starttime)
                seek_fast = starttime - seek_slow
                # We used to have this epsilon earlier, when we did not use
                # the slow seek. I don't think we need it anymore.
                # epsilon = -1 / self._meta["fps"] * 0.1
                iargs += ["-ss", "%.06f" % (seek_fast)]
                oargs += ["-ss", "%.06f" % (seek_slow)]

            # Output args, for writing to pipe
            if self._arg_size:
                oargs += ["-s", self._arg_size]
            if self.request.kwargs.get("fps", None):
                fps = float(self.request.kwargs["fps"])
                oargs += ["-r", "%.02f" % fps]
            oargs += self._arg_output_params

            # Get pixelformat and bytes per pixel
            pix_fmt = self._pix_fmt
            bpp = self._depth * self._bytes_per_channel

            # Create generator
            rf = self._ffmpeg_api.read_frames
            self._read_gen = rf(
                self._filename, pix_fmt, bpp, input_params=iargs, output_params=oargs
            )

            # Read meta data. This start the generator (and ffmpeg subprocess)
            if self.request._video:
                # With cameras, catch error and turn into IndexError
                try:
                    meta = self._read_gen.__next__()
                except IOError as err:
                    err_text = str(err)
                    if "darwin" in sys.platform:
                        if "Unknown input format: 'avfoundation'" in err_text:
                            err_text += (
                                "Try installing FFMPEG using "
                                "home brew to get a version with "
                                "support for cameras."
                            )
                    raise IndexError(
                        "No (working) camera at {}.\n\n{}".format(
                            self.request._video, err_text
                        )
                    )
                else:
                    self._meta.update(meta)
            elif index == 0:
                self._meta.update(self._read_gen.__next__())
            else:
                self._read_gen.__next__()  # we already have meta data

        def _skip_frames(self, n=1):
            """Reads and throws away n frames"""
            for i in range(n):
                self._read_gen.__next__()
            self._pos += n

        def _read_frame(self):
            # Read and convert to numpy array
            w, h = self._meta["size"]
            framesize = w * h * self._depth * self._bytes_per_channel
            # t0 = time.time()

            # Read frame
            if self._frame_catcher:  # pragma: no cover - camera thing
                s, is_new = self._frame_catcher.get_frame()
            else:
                s = self._read_gen.__next__()
                is_new = True

            # Check
            if len(s) != framesize:
                raise RuntimeError(
                    "Frame is %i bytes, but expected %i." % (len(s), framesize)
                )

            result = np.frombuffer(s, dtype=self._dtype).copy()
            result = result.reshape((h, w, self._depth))
            # t1 = time.time()
            # print('etime', t1-t0)

            # Store and return
            self._lastread = result
            return result, is_new

    # --

    class Writer(Format.Writer):
        _write_gen = None

        def _open(
            self,
            fps=10,
            codec="libx264",
            bitrate=None,
            pixelformat="yuv420p",
            ffmpeg_params=None,
            input_params=None,
            output_params=None,
            ffmpeg_log_level="quiet",
            quality=5,
            macro_block_size=16,
            audio_path=None,
            audio_codec=None,
        ):
            self._ffmpeg_api = imageio_ffmpeg
            self._filename = self.request.get_local_filename()
            self._pix_fmt = None
            self._depth = None
            self._size = None

        def _close(self):
            if self._write_gen is not None:
                self._write_gen.close()
                self._write_gen = None

        def _append_data(self, im, meta):
            # Get props of image
            h, w = im.shape[:2]
            size = w, h
            depth = 1 if im.ndim == 2 else im.shape[2]

            # Ensure that image is in uint8
            im = image_as_uint(im, bitdepth=8)
            # To be written efficiently, ie. without creating an immutable
            # buffer, by calling im.tobytes() the array must be contiguous.
            if not im.flags.c_contiguous:
                # checkign the flag is a micro optimization.
                # the image will be a numpy subclass. See discussion
                # https://github.com/numpy/numpy/issues/11804
                im = np.ascontiguousarray(im)

            # Set size and initialize if not initialized yet
            if self._size is None:
                map = {1: "gray", 2: "gray8a", 3: "rgb24", 4: "rgba"}
                self._pix_fmt = map.get(depth, None)
                if self._pix_fmt is None:
                    raise ValueError("Image must have 1, 2, 3 or 4 channels")
                self._size = size
                self._depth = depth
                self._initialize()

            # Check size of image
            if size != self._size:
                raise ValueError("All images in a movie should have same size")
            if depth != self._depth:
                raise ValueError(
                    "All images in a movie should have same " "number of channels"
                )

            assert self._write_gen is not None  # Check status

            # Write. Yes, we can send the data in as a numpy array
            self._write_gen.send(im)

        def set_meta_data(self, meta):
            raise RuntimeError(
                "The ffmpeg format does not support setting " "meta data."
            )

        def _initialize(self):
            # Close existing generator
            if self._write_gen is not None:
                self._write_gen.close()

            # Get parameters
            # Use None to let imageio-ffmpeg (or ffmpeg) select good results
            fps = self.request.kwargs.get("fps", 10)
            codec = self.request.kwargs.get("codec", None)
            bitrate = self.request.kwargs.get("bitrate", None)
            quality = self.request.kwargs.get("quality", None)
            input_params = self.request.kwargs.get("input_params") or []
            output_params = self.request.kwargs.get("output_params") or []
            output_params += self.request.kwargs.get("ffmpeg_params") or []
            pixelformat = self.request.kwargs.get("pixelformat", None)
            macro_block_size = self.request.kwargs.get("macro_block_size", 16)
            ffmpeg_log_level = self.request.kwargs.get("ffmpeg_log_level", None)
            audio_path = self.request.kwargs.get("audio_path", None)
            audio_codec = self.request.kwargs.get("audio_codec", None)

            macro_block_size = macro_block_size or 1  # None -> 1

            # Create generator
            self._write_gen = self._ffmpeg_api.write_frames(
                self._filename,
                self._size,
                pix_fmt_in=self._pix_fmt,
                pix_fmt_out=pixelformat,
                fps=fps,
                quality=quality,
                bitrate=bitrate,
                codec=codec,
                macro_block_size=macro_block_size,
                ffmpeg_log_level=ffmpeg_log_level,
                input_params=input_params,
                output_params=output_params,
                audio_path=audio_path,
                audio_codec=audio_codec,
            )

            # Seed the generator (this is where the ffmpeg subprocess starts)
            self._write_gen.send(None)


class FrameCatcher(threading.Thread):
    """Thread to keep reading the frame data from stdout. This is
    useful when streaming from a webcam. Otherwise, if the user code
    does not grab frames fast enough, the buffer will fill up, leading
    to lag, and ffmpeg can also stall (experienced on Linux). The
    get_frame() method always returns the last available image.
    """

    def __init__(self, gen):
        self._gen = gen
        self._frame = None
        self._frame_is_new = False
        self._lock = threading.RLock()
        threading.Thread.__init__(self)
        self.daemon = True  # do not let this thread hold up Python shutdown
        self._should_stop = False
        self.start()

    def stop_me(self):
        self._should_stop = True
        while self.is_alive():
            time.sleep(0.001)

    def get_frame(self):
        while self._frame is None:  # pragma: no cover - an init thing
            time.sleep(0.001)
        with self._lock:
            is_new = self._frame_is_new
            self._frame_is_new = False  # reset
            return self._frame, is_new

    def run(self):
        # This runs in the worker thread
        try:
            while not self._should_stop:
                time.sleep(0)  # give control to other threads
                frame = self._gen.__next__()
                with self._lock:
                    self._frame = frame
                    self._frame_is_new = True
        except (StopIteration, EOFError):
            pass


def parse_device_names(ffmpeg_output):
    """Parse the output of the ffmpeg -list-devices command"""
    # Collect device names - get [friendly_name, alt_name] of each
    device_names = []
    in_video_devices = False
    for line in ffmpeg_output.splitlines():
        if line.startswith("[dshow"):
            logger.debug(line)
            line = line.split("]", 1)[1].strip()
            if in_video_devices and line.startswith('"'):
                friendly_name = line[1:-1]
                device_names.append([friendly_name, ""])
            elif in_video_devices and line.lower().startswith("alternative name"):
                alt_name = line.split(" name ", 1)[1].strip()[1:-1]
                if sys.platform.startswith("win"):
                    alt_name = alt_name.replace("&", "^&")  # Tested to work
                else:
                    alt_name = alt_name.replace("&", "\\&")  # Does this work?
                device_names[-1][-1] = alt_name
            elif "video devices" in line:
                in_video_devices = True
            elif "devices" in line:
                # set False for subsequent "devices" sections
                in_video_devices = False
    # Post-process, see #441
    # prefer friendly names, use alt name if two cams have same friendly name
    device_names2 = []
    for friendly_name, alt_name in device_names:
        if friendly_name not in device_names2:
            device_names2.append(friendly_name)
        elif alt_name:
            device_names2.append(alt_name)
        else:
            device_names2.append(friendly_name)  # duplicate, but not much we can do
    return device_names2

env_pyrecodes/lib/python3.9/site-packages/imageio/plugins/ffmpeg.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
# imageio is distributed under the terms of the (new) BSD License.

""" Read/Write images using Pillow/PIL.

Backend Library: `Pillow <https://pillow.readthedocs.io/en/stable/>`_

Plugin that wraps the the Pillow library. Pillow is a friendly fork of PIL
(Python Image Library) and supports reading and writing of common formats (jpg,
png, gif, tiff, ...). For, the complete list of features and supported formats
please refer to pillows official docs (see the Backend Library link).

Parameters
----------
request : Request
    A request object representing the resource to be operated on.

Methods
-------

.. autosummary::
    :toctree: _plugins/pillow

    PillowPlugin.read
    PillowPlugin.write
    PillowPlugin.iter
    PillowPlugin.get_meta

"""

from io import BytesIO
from typing import Callable, Optional, Dict, Any, Tuple, cast, Iterator, Union, List
import numpy as np
from PIL import Image, UnidentifiedImageError, ImageSequence, ExifTags  # type: ignore
from ..core.request import Request, IOMode, InitializationError, URI_BYTES
from ..core.v3_plugin_api import PluginV3, ImageProperties
import warnings
from ..typing import ArrayLike


def _exif_orientation_transform(orientation: int, mode: str) -> Callable:
    # get transformation that transforms an image from a
    # given EXIF orientation into the standard orientation

    # -1 if the mode has color channel, 0 otherwise
    axis = -2 if Image.getmodebands(mode) > 1 else -1

    EXIF_ORIENTATION = {
        1: lambda x: x,
        2: lambda x: np.flip(x, axis=axis),
        3: lambda x: np.rot90(x, k=2),
        4: lambda x: np.flip(x, axis=axis - 1),
        5: lambda x: np.flip(np.rot90(x, k=3), axis=axis),
        6: lambda x: np.rot90(x, k=1),
        7: lambda x: np.flip(np.rot90(x, k=1), axis=axis),
        8: lambda x: np.rot90(x, k=3),
    }

    return EXIF_ORIENTATION[orientation]


class PillowPlugin(PluginV3):
    def __init__(self, request: Request) -> None:
        """Instantiate a new Pillow Plugin Object

        Parameters
        ----------
        request : {Request}
            A request object representing the resource to be operated on.

        """

        super().__init__(request)

        self._image: Image = None

        if request.mode.io_mode == IOMode.read:
            try:
                with Image.open(request.get_file()):
                    # Check if it is generally possible to read the image.
                    # This will not read any data and merely try to find a
                    # compatible pillow plugin (ref: the pillow docs).
                    pass
            except UnidentifiedImageError:
                if request._uri_type == URI_BYTES:
                    raise InitializationError(
                        "Pillow can not read the provided bytes."
                    ) from None
                else:
                    raise InitializationError(
                        f"Pillow can not read {request.raw_uri}."
                    ) from None

            self._image = Image.open(self._request.get_file())
        else:
            extension = self.request.extension or self.request.format_hint
            if extension is None:
                warnings.warn(
                    "Can't determine file format to write as. You _must_"
                    " set `format` during write or the call will fail. Use "
                    "`extension` to supress this warning. ",
                    UserWarning,
                )
                return

            tirage = [Image.preinit, Image.init]
            for format_loader in tirage:
                format_loader()
                if extension in Image.registered_extensions().keys():
                    return

            raise InitializationError(
                f"Pillow can not write `{extension}` files."
            ) from None

    def close(self) -> None:
        if self._image:
            self._image.close()

        self._request.finish()

    def read(
        self, *, index=None, mode=None, rotate=False, apply_gamma=False, as_gray=None
    ) -> np.ndarray:
        """
        Parses the given URI and creates a ndarray from it.

        Parameters
        ----------
        index : int
            If the ImageResource contains multiple ndimages, and index is an
            integer, select the index-th ndimage from among them and return it.
            If index is an ellipsis (...), read all ndimages in the file and
            stack them along a new batch dimension and return them. If index is
            None, this plugin reads the first image of the file (index=0) unless
            the image is a GIF or APNG, in which case all images are read
            (index=...).
        mode : str
            Convert the image to the given mode before returning it. If None,
            the mode will be left unchanged. Possible modes can be found at:
            https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes
        rotate : bool
            If set to ``True`` and the image contains an EXIF orientation tag,
            apply the orientation before returning the ndimage.
        apply_gamma : bool
            If ``True`` and the image contains metadata about gamma, apply gamma
            correction to the image.
        as_gray : bool
            Deprecated. Exists to raise a constructive error message.

        Returns
        -------
        ndimage : ndarray
            A numpy array containing the loaded image data

        Notes
        -----
        If you read a paletted image (e.g. GIF) then the plugin will apply the
        palette by default. Should you wish to read the palette indices of each
        pixel use ``mode="P"``. The coresponding color pallete can be found in
        the image's metadata using the ``palette`` key when metadata is
        extracted using the ``exclude_applied=False`` kwarg. The latter is
        needed, as palettes are applied by default and hence excluded by default
        to keep metadata and pixel data consistent.

        """

        if as_gray is not None:
            raise TypeError(
                "The keyword `as_gray` is no longer supported."
                "Use `mode='L'` instead."
            )

        if index is None:
            if self._image.format == "GIF":
                index = Ellipsis
            elif self._image.custom_mimetype == "image/apng":
                index = Ellipsis
            else:
                index = 0

        if isinstance(index, int):
            # will raise IO error if index >= number of frames in image
            self._image.seek(index)
            image = self._apply_transforms(self._image, mode, rotate, apply_gamma)
            return image
        else:
            iterator = self.iter(mode=mode, rotate=rotate, apply_gamma=apply_gamma)
            image = np.stack([im for im in iterator], axis=0)
            return image

    def iter(
        self, *, mode: str = None, rotate: bool = False, apply_gamma: bool = False
    ) -> Iterator[np.ndarray]:
        """
        Iterate over all ndimages/frames in the URI

        Parameters
        ----------
        mode : {str, None}
            Convert the image to the given mode before returning it. If None,
            the mode will be left unchanged. Possible modes can be found at:
            https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes
        rotate : {bool}
            If set to ``True`` and the image contains an EXIF orientation tag,
            apply the orientation before returning the ndimage.
        apply_gamma : {bool}
            If ``True`` and the image contains metadata about gamma, apply gamma
            correction to the image.
        """

        for im in ImageSequence.Iterator(self._image):
            yield self._apply_transforms(im, mode, rotate, apply_gamma)

    def _apply_transforms(self, image, mode, rotate, apply_gamma) -> np.ndarray:
        if mode is not None:
            image = image.convert(mode)
        elif image.mode == "P":
            # adjust for pillow9 changes
            # see: https://github.com/python-pillow/Pillow/issues/5929
            image = image.convert(image.palette.mode)
        image = np.asarray(image)

        meta = self.metadata(index=self._image.tell(), exclude_applied=False)
        if rotate and "Orientation" in meta:
            transformation = _exif_orientation_transform(
                meta["Orientation"], self._image.mode
            )
            image = transformation(image)

        if apply_gamma and "gamma" in meta:
            gamma = float(meta["gamma"])
            scale = float(65536 if image.dtype == np.uint16 else 255)
            gain = 1.0
            image = ((image / scale) ** gamma) * scale * gain + 0.4999
            image = np.round(image).astype(np.uint8)

        return image

    def write(
        self,
        ndimage: Union[ArrayLike, List[ArrayLike]],
        *,
        mode: str = None,
        format: str = None,
        is_batch: bool = None,
        **kwargs,
    ) -> Optional[bytes]:
        """
        Write an ndimage to the URI specified in path.

        If the URI points to a file on the current host and the file does not
        yet exist it will be created. If the file exists already, it will be
        appended if possible; otherwise, it will be replaced.

        If necessary, the image is broken down along the leading dimension to
        fit into individual frames of the chosen format. If the format doesn't
        support multiple frames, and IOError is raised.

        Parameters
        ----------
        image : ndarray or list
            The ndimage to write. If a list is given each element is expected to
            be an ndimage.
        mode : str
            Specify the image's color format. If None (default), the mode is
            inferred from the array's shape and dtype. Possible modes can be
            found at:
            https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes
        format : str
            Optional format override. If omitted, the format to use is
            determined from the filename extension. If a file object was used
            instead of a filename, this parameter must always be used.
        is_batch : bool
            Explicitly tell the writer that ``image`` is a batch of images
            (True) or not (False). If None, the writer will guess this from the
            provided ``mode`` or ``image.shape``. While the latter often works,
            it may cause problems for small images due to aliasing of spatial
            and color-channel axes.
        kwargs : ...
            Extra arguments to pass to pillow. If a writer doesn't recognise an
            option, it is silently ignored. The available options are described
            in pillow's `image format documentation
            <https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html>`_
            for each writer.

        Notes
        -----
        When writing batches of very narrow (2-4 pixels wide) gray images set
        the ``mode`` explicitly to avoid the batch being identified as a colored
        image.

        """
        if "fps" in kwargs:
            raise TypeError(
                "The keyword `fps` is no longer supported. Use `duration`"
                "(in ms) instead, e.g. `fps=50` == `duration=20` (1000 * 1/50)."
            )

        extension = self.request.extension or self.request.format_hint

        save_args = {
            "format": format or Image.registered_extensions()[extension],
        }

        if isinstance(ndimage, list):
            ndimage = np.stack(ndimage, axis=0)
            is_batch = True
        else:
            ndimage = np.asarray(ndimage)

        # check if ndimage is a batch of frames/pages (e.g. for writing GIF)
        # if mode is given, use it; otherwise fall back to image.ndim only
        if is_batch is not None:
            pass
        elif mode is not None:
            is_batch = (
                ndimage.ndim > 3 if Image.getmodebands(mode) > 1 else ndimage.ndim > 2
            )
        elif ndimage.ndim == 2:
            is_batch = False
        elif ndimage.ndim == 3 and ndimage.shape[-1] == 1:
            raise ValueError("Can't write images with one color channel.")
        elif ndimage.ndim == 3 and ndimage.shape[-1] in [2, 3, 4]:
            # Note: this makes a channel-last assumption
            is_batch = False
        else:
            is_batch = True

        if not is_batch:
            ndimage = ndimage[None, ...]

        pil_frames = list()
        for frame in ndimage:
            pil_frame = Image.fromarray(frame, mode=mode)
            if "bits" in kwargs:
                pil_frame = pil_frame.quantize(colors=2 ** kwargs["bits"])
            pil_frames.append(pil_frame)
        primary_image, other_images = pil_frames[0], pil_frames[1:]

        if is_batch:
            save_args["save_all"] = True
            save_args["append_images"] = other_images

        save_args.update(kwargs)
        primary_image.save(self._request.get_file(), **save_args)

        if self._request._uri_type == URI_BYTES:
            file = cast(BytesIO, self._request.get_file())
            return file.getvalue()

        return None

    def get_meta(self, *, index=0) -> Dict[str, Any]:
        return self.metadata(index=index, exclude_applied=False)

    def metadata(
        self, index: int = None, exclude_applied: bool = True
    ) -> Dict[str, Any]:
        """Read ndimage metadata.

        Parameters
        ----------
        index : {integer, None}
            If the ImageResource contains multiple ndimages, and index is an
            integer, select the index-th ndimage from among them and return its
            metadata. If index is an ellipsis (...), read and return global
            metadata. If index is None, this plugin reads metadata from the
            first image of the file (index=0) unless the image is a GIF or APNG,
            in which case global metadata is read (index=...).
        exclude_applied : bool
            If True, exclude metadata fields that are applied to the image while
            reading. For example, if the binary data contains a rotation flag,
            the image is rotated by default and the rotation flag is excluded
            from the metadata to avoid confusion.

        Returns
        -------
        metadata : dict
            A dictionary of format-specific metadata.

        """

        if index is None:
            if self._image.format == "GIF":
                index = Ellipsis
            elif self._image.custom_mimetype == "image/apng":
                index = Ellipsis
            else:
                index = 0

        if isinstance(index, int) and self._image.tell() != index:
            self._image.seek(index)

        metadata = self._image.info.copy()
        metadata["mode"] = self._image.mode
        metadata["shape"] = self._image.size

        if self._image.mode == "P" and not exclude_applied:
            metadata["palette"] = np.asarray(tuple(self._image.palette.colors.keys()))

        if self._image.getexif():
            exif_data = {
                ExifTags.TAGS.get(key, "unknown"): value
                for key, value in dict(self._image.getexif()).items()
            }
            exif_data.pop("unknown", None)
            metadata.update(exif_data)

        if exclude_applied:
            metadata.pop("Orientation", None)

        return metadata

    def properties(self, index: int = None) -> ImageProperties:
        """Standardized ndimage metadata
        Parameters
        ----------
        index : int
            If the ImageResource contains multiple ndimages, and index is an
            integer, select the index-th ndimage from among them and return its
            properties. If index is an ellipsis (...), read and return the
            properties of all ndimages in the file stacked along a new batch
            dimension. If index is None, this plugin reads and returns the
            properties of the first image (index=0) unless the image is a GIF or
            APNG, in which case it reads and returns the properties all images
            (index=...).

        Returns
        -------
        properties : ImageProperties
            A dataclass filled with standardized image metadata.

        Notes
        -----
        This does not decode pixel data and is fast for large images.

        """

        if index is None:
            if self._image.format == "GIF":
                index = Ellipsis
            elif self._image.custom_mimetype == "image/apng":
                index = Ellipsis
            else:
                index = 0

        if index is Ellipsis:
            self._image.seek(0)
        else:
            self._image.seek(index)

        if self._image.mode == "P":
            # mode of palette images is determined by their palette
            mode = self._image.palette.mode
        else:
            mode = self._image.mode

        width: int = self._image.width
        height: int = self._image.height
        shape: Tuple[int, ...] = (height, width)

        n_frames: int = self._image.n_frames
        if index is ...:
            shape = (n_frames, *shape)

        dummy = np.asarray(Image.new(mode, (1, 1)))
        pil_shape: Tuple[int, ...] = dummy.shape
        if len(pil_shape) > 2:
            shape = (*shape, *pil_shape[2:])

        return ImageProperties(
            shape=shape,
            dtype=dummy.dtype,
            n_images=n_frames if index is Ellipsis else None,
            is_batch=index is Ellipsis,
        )

env_pyrecodes/lib/python3.9/site-packages/imageio/plugins/pillow.py
NikolaBlagojevic/pyrecodes
"""Read/Write TIFF files using tifffile.

.. note::
    To use this plugin you need to have `tifffile
    <https://github.com/cgohlke/tifffile>`_ installed::

        pip install tifffile

This plugin wraps tifffile, a powerfull library to manipulate TIFF files. It
superseeds our previous tifffile plugin and aims to expose all the features of
tifffile.

The plugin treats individual TIFF series as ndimages. A series is a sequence of
TIFF pages that, when combined describe a meaningful unit, e.g., a volumetric
image (where each slice is stored on an individual page) or a multi-color
staining picture (where each stain is stored on an individual page). Different
TIFF flavors/variants use series in different ways and, as such, the resulting
reading behavior may vary depending on the program used while creating a
particular TIFF file.

Methods
-------
.. note::
    Check the respective function for a list of supported kwargs and detailed
    documentation.

.. autosummary::
    :toctree:

    TifffilePlugin.read
    TifffilePlugin.iter
    TifffilePlugin.write
    TifffilePlugin.properties
    TifffilePlugin.metadata

Additional methods available inside the :func:`imopen <imageio.v3.imopen>`
context:

.. autosummary::
    :toctree:

    TifffilePlugin.iter_pages

"""

from io import BytesIO
from typing import Any, Dict, Optional, cast
import warnings
import datetime

import numpy as np
import tifffile

from ..core.request import URI_BYTES, InitializationError, Request
from ..core.v3_plugin_api import ImageProperties, PluginV3
from ..typing import ArrayLike


def _get_resolution(page):
    """Get the resolution in a py3.7 compatible way"""

    metadata = {
        # uncomment once py 3.7 reached EoL - in fact, refactor this
        # function :)
        # "resolution_unit": page.resolutionunit,
        # "resolution": page.resolution,
    }

    if 296 in page.tags:
        metadata["resolution_unit"] = page.tags[296].value.value

    if 282 in page.tags and 283 in page.tags and 296 in page.tags:
        resolution_x = page.tags[282].value
        resolution_y = page.tags[283].value
        if resolution_x[1] == 0 or resolution_y[1] == 0:
            warnings.warn(
                "Ignoring resulution metadata, "
                "because at least one direction has a 0 denominator.",
                RuntimeWarning,
            )
        else:
            metadata["resolution"] = (
                resolution_x[0] / resolution_x[1],
                resolution_y[0] / resolution_y[1],
            )

    return metadata


def _get_datatime(page):
    """Get the datetime in a python 3.7 compatible way"""

    metadata = {
        # uncomment once python 3.7 is EoL
        # "datetime": page.datetime,
    }

    try:
        metadata["datetime"] = datetime.datetime.strptime(
            page.tags[306].value, "%Y:%m:%d %H:%M:%S"
        )
    except KeyError:
        pass

    return metadata


class TifffilePlugin(PluginV3):
    """Support for tifffile as backend.

    Parameters
    ----------
    request : iio.Request
        A request object that represents the users intent. It provides a
        standard interface for a plugin to access the various ImageResources.
        Check the docs for details.
    kwargs : Any
        Additional kwargs are forwarded to tifffile's constructor, i.e.
        to ``TiffFile`` for reading or ``TiffWriter`` for writing.

    """

    def __init__(self, request: Request, **kwargs) -> None:
        super().__init__(request)
        self._fh = None

        if request.mode.io_mode == "r":
            try:
                self._fh = tifffile.TiffFile(request.get_file(), **kwargs)
            except tifffile.tifffile.TiffFileError:
                raise InitializationError("Tifffile can not read this file.")
        else:
            self._fh = tifffile.TiffWriter(request.get_file(), **kwargs)

    # ---------------------
    # Standard V3 Interface
    # ---------------------

    def read(self, *, index: int = None, page: int = None, **kwargs) -> np.ndarray:
        """Read a ndimage or page.

        The ndimage returned depends on the value of both ``index`` and
        ``page``. ``index`` selects the series to read and ``page`` allows
        selecting a single page from the selected series. If ``index=None``,
        ``page`` is understood as a flat index, i.e., the selection ignores
        individual series inside the file. If both ``index`` and ``page`` are
        ``None``, then all the series are read and returned as a batch.

        Parameters
        ----------
        index : int
            If ``int``, select the ndimage (series) located at that index inside
            the file and return ``page`` from it. If ``None`` and ``page`` is
            ``int`` read the page located at that (flat) index inside the file.
            If ``None`` and ``page=None``, read all ndimages from the file and
            return them as a batch.
        page : int
            If ``None`` return the full selected ndimage. If ``int``, read the
            page at the selected index and return it.
        kwargs : Any
            Additional kwargs are forwarded to TiffFile's ``as_array`` method.

        Returns
        -------
        ndarray : np.ndarray
            The decoded ndimage or page.
        """

        if "key" not in kwargs:
            kwargs["key"] = page
        elif page is not None:
            raise ValueError("Can't use `page` and `key` at the same time.")

        # set plugin default for ``index``
        if index is not None and "series" in kwargs:
            raise ValueError("Can't use `series` and `index` at the same time.")
        elif "series" in kwargs:
            index = kwargs.pop("series")
        elif index is not None:
            pass
        else:
            index = 0

        if index is Ellipsis and page is None:
            # read all series in the file and return them as a batch
            ndimage = np.stack([x for x in self.iter(**kwargs)])
        else:
            index = None if index is Ellipsis else index
            ndimage = self._fh.asarray(series=index, **kwargs)

        return ndimage

    def iter(self, **kwargs) -> np.ndarray:
        """Yield ndimages from the TIFF.

        Parameters
        ----------
        kwargs : Any
            Additional kwargs are forwarded to the TiffPageSeries' ``as_array``
            method.

        Yields
        ------
        ndimage : np.ndarray
            A decoded ndimage.
        """

        for sequence in self._fh.series:
            yield sequence.asarray(**kwargs)

    def write(
        self, ndimage: ArrayLike, *, is_batch: bool = False, **kwargs
    ) -> Optional[bytes]:
        """Save a ndimage as TIFF.

        Parameters
        ----------
        ndimage : ArrayLike
            The ndimage to encode and write to the ImageResource.
        is_batch : bool
            If True, the first dimension of the given ndimage is treated as a
            batch dimension and each element will create a new series.
        kwargs : Any
            Additional kwargs are forwarded to TiffWriter's ``write`` method.

        Returns
        -------
        encoded_image : bytes
            If the ImageResource is ``"<bytes>"``, return the encoded bytes.
            Otherwise write returns None.

        Notes
        -----
        Incremental writing is supported. Subsequent calls to ``write`` will
        create new series unless ``contiguous=True`` is used, in which case the
        call to write will append to the current series.

        """

        if not is_batch:
            ndimage = np.asarray(ndimage)[None, :]

        for image in ndimage:
            self._fh.write(image, **kwargs)

        if self._request._uri_type == URI_BYTES:
            self._fh.close()
            file = cast(BytesIO, self._request.get_file())
            return file.getvalue()

    def metadata(
        self, *, index: int = Ellipsis, page: int = None, exclude_applied: bool = True
    ) -> Dict[str, Any]:
        """Format-Specific TIFF metadata.

        The metadata returned depends on the value of both ``index`` and
        ``page``. ``index`` selects a series and ``page`` allows selecting a
        single page from the selected series. If ``index=Ellipsis``, ``page`` is
        understood as a flat index, i.e., the selection ignores individual
        series inside the file. If ``index=Ellipsis`` and ``page=None`` then
        global (file-level) metadata is returned.

        Parameters
        ----------
        index : int
            Select the series of which to extract metadata from. If Ellipsis, treat
            page as a flat index into the file's pages.
        page : int
            If not None, select the page of which to extract metadata from. If
            None, read series-level metadata or, if ``index=...`` global,
            file-level metadata.
        exclude_applied : bool
            For API compatibility. Currently ignored.

        Returns
        -------
        metadata : dict
            A dictionary with information regarding the tiff flavor (file-level)
            or tiff tags (page-level).
        """

        if index is not Ellipsis and page is not None:
            target = self._fh.series[index].pages[page]
        elif index is not Ellipsis and page is None:
            # This is based on my understanding that series-level metadata is
            # stored in the first TIFF page.
            target = self._fh.series[index].pages[0]
        elif index is Ellipsis and page is not None:
            target = self._fh.pages[page]
        else:
            target = None

        metadata = {}
        if target is None:
            # return file-level metadata
            metadata["byteorder"] = self._fh.byteorder

            for flag in tifffile.TIFF.FILE_FLAGS:
                flag_value = getattr(self._fh, "is_" + flag)
                metadata["is_" + flag] = flag_value

                if flag_value and hasattr(self._fh, flag + "_metadata"):
                    flavor_metadata = getattr(self._fh, flag + "_metadata")
                    if isinstance(flavor_metadata, tuple):
                        metadata.update(flavor_metadata[0])
                    else:
                        metadata.update(flavor_metadata)
        else:
            # tifffile may return a TiffFrame instead of a page
            target = target.keyframe

            metadata.update({tag.name: tag.value for tag in target.tags})
            metadata.update(
                {
                    "planar_configuration": target.planarconfig,
                    "compression": target.compression,
                    "predictor": target.predictor,
                    "orientation": None,  # TODO
                    "description1": target.description1,
                    "description": target.description,
                    "software": target.software,
                    # update once python 3.7 reached EoL
                    **_get_resolution(target),
                    **_get_datatime(target),
                }
            )

        return metadata

    def properties(self, *, index: int = None, page: int = None) -> ImageProperties:
        """Standardized metadata.

        The properties returned depend on the value of both ``index`` and
        ``page``. ``index`` selects a series and ``page`` allows selecting a
        single page from the selected series. If ``index=Ellipsis``, ``page`` is
        understood as a flat index, i.e., the selection ignores individual
        series inside the file. If ``index=Ellipsis`` and ``page=None`` then
        global (file-level) properties are returned. If ``index=Ellipsis``
        and ``page=...``, file-level properties for the flattened index are
        returned.

        Parameters
        ----------
        index : int
            If ``int``, select the ndimage (series) located at that index inside
            the file. If ``Ellipsis`` and ``page`` is ``int`` extract the
            properties of the page located at that (flat) index inside the file.
            If ``Ellipsis`` and ``page=None``, return the properties for the
            batch of all ndimages in the file.
        page : int
            If ``None`` return the properties of the full ndimage. If ``...``
            return the properties of the flattened index. If ``int``,
            return the properties of the page at the selected index only.

        Returns
        -------
        image_properties : ImageProperties
            The standardized metadata (properties) of the selected ndimage or series.

        """
        index = index or 0
        page_idx = 0 if page in (None, Ellipsis) else page

        if index is Ellipsis:
            target_page = self._fh.pages[page_idx]
        else:
            target_page = self._fh.series[index].pages[page_idx]

        if index is Ellipsis and page is None:
            n_series = len(self._fh.series)
            props = ImageProperties(
                shape=(n_series, *target_page.shape),
                dtype=target_page.dtype,
                n_images=n_series,
                is_batch=True,
                spacing=_get_resolution(target_page)["resolution"],
            )
        elif index is Ellipsis and page is Ellipsis:
            n_pages = len(self._fh.pages)
            props = ImageProperties(
                shape=(n_pages, *target_page.shape),
                dtype=target_page.dtype,
                n_images=n_pages,
                is_batch=True,
                spacing=_get_resolution(target_page)["resolution"],
            )
        else:
            props = ImageProperties(
                shape=target_page.shape,
                dtype=target_page.dtype,
                is_batch=False,
                spacing=_get_resolution(target_page)["resolution"],
            )

        return props

    def close(self) -> None:
        if self._fh is not None:
            self._fh.close()

        super().close()

    # ------------------------------
    # Add-on Interface inside imopen
    # ------------------------------

    def iter_pages(self, index=..., **kwargs):
        """Yield pages from a TIFF file.

        This generator walks over the flat index of the pages inside an
        ImageResource and yields them in order.

        Parameters
        ----------
        index : int
            The index of the series to yield pages from. If Ellipsis, walk over
            the file's flat index (and ignore individual series).
        kwargs : Any
            Additional kwargs are passed to TiffPage's ``as_array`` method.

        Yields
        ------
        page : np.ndarray
            A page stored inside the TIFF file.

        """

        if index is Ellipsis:
            pages = self._fh.pages
        else:
            pages = self._fh.series[index]

        for page in pages:
            yield page.asarray(**kwargs)

env_pyrecodes/lib/python3.9/site-packages/imageio/plugins/tifffile_v3.py
NikolaBlagojevic/pyrecodes
import io
import os
import re
import struct
from xml.etree import ElementTree


__all__ = ["get", "getDPI", "__version__"]


__version__ = "1.4.1"


_UNIT_KM = -3
_UNIT_100M = -2
_UNIT_10M = -1
_UNIT_1M = 0
_UNIT_10CM = 1
_UNIT_CM = 2
_UNIT_MM = 3
_UNIT_0_1MM = 4
_UNIT_0_01MM = 5
_UNIT_UM = 6
_UNIT_INCH = 6

_TIFF_TYPE_SIZES = {
  1: 1,
  2: 1,
  3: 2,
  4: 4,
  5: 8,
  6: 1,
  7: 1,
  8: 2,
  9: 4,
  10: 8,
  11: 4,
  12: 8,
}


def _convertToDPI(density, unit):
    if unit == _UNIT_KM:
        return int(density * 0.0000254 + 0.5)
    elif unit == _UNIT_100M:
        return int(density * 0.000254 + 0.5)
    elif unit == _UNIT_10M:
        return int(density * 0.00254 + 0.5)
    elif unit == _UNIT_1M:
        return int(density * 0.0254 + 0.5)
    elif unit == _UNIT_10CM:
        return int(density * 0.254 + 0.5)
    elif unit == _UNIT_CM:
        return int(density * 2.54 + 0.5)
    elif unit == _UNIT_MM:
        return int(density * 25.4 + 0.5)
    elif unit == _UNIT_0_1MM:
        return density * 254
    elif unit == _UNIT_0_01MM:
        return density * 2540
    elif unit == _UNIT_UM:
        return density * 25400
    return density


def _convertToPx(value):
    matched = re.match(r"(\d+(?:\.\d+)?)?([a-z]*)$", value)
    if not matched:
        raise ValueError("unknown length value: %s" % value)

    length, unit = matched.groups()
    if unit == "":
        return float(length)
    elif unit == "cm":
        return float(length) * 96 / 2.54
    elif unit == "mm":
        return float(length) * 96 / 2.54 / 10
    elif unit == "in":
        return float(length) * 96
    elif unit == "pc":
        return float(length) * 96 / 6
    elif unit == "pt":
        return float(length) * 96 / 6
    elif unit == "px":
        return float(length)

    raise ValueError("unknown unit type: %s" % unit)


def get(filepath):
    """
    Return (width, height) for a given img file content
    no requirements
    :type filepath: Union[bytes, str, pathlib.Path]
    :rtype Tuple[int, int]
    """
    height = -1
    width = -1

    if isinstance(filepath, io.BytesIO):  # file-like object
        fhandle = filepath
    else:
        fhandle = open(filepath, 'rb')

    try:
        head = fhandle.read(31)
        size = len(head)
        # handle GIFs
        if size >= 10 and head[:6] in (b'GIF87a', b'GIF89a'):
            # Check to see if content_type is correct
            try:
                width, height = struct.unpack("<hh", head[6:10])
            except struct.error:
                raise ValueError("Invalid GIF file")
        # see png edition spec bytes are below chunk length then and finally the
        elif size >= 24 and head.startswith(b'\211PNG\r\n\032\n') and head[12:16] == b'IHDR':
            try:
                width, height = struct.unpack(">LL", head[16:24])
            except struct.error:
                raise ValueError("Invalid PNG file")
        # Maybe this is for an older PNG version.
        elif size >= 16 and head.startswith(b'\211PNG\r\n\032\n'):
            # Check to see if we have the right content type
            try:
                width, height = struct.unpack(">LL", head[8:16])
            except struct.error:
                raise ValueError("Invalid PNG file")
        # handle JPEGs
        elif size >= 2 and head.startswith(b'\377\330'):
            try:
                fhandle.seek(0)  # Read 0xff next
                size = 2
                ftype = 0
                while not 0xc0 <= ftype <= 0xcf or ftype in [0xc4, 0xc8, 0xcc]:
                    fhandle.seek(size, 1)
                    byte = fhandle.read(1)
                    while ord(byte) == 0xff:
                        byte = fhandle.read(1)
                    ftype = ord(byte)
                    size = struct.unpack('>H', fhandle.read(2))[0] - 2
                # We are at a SOFn block
                fhandle.seek(1, 1)  # Skip `precision' byte.
                height, width = struct.unpack('>HH', fhandle.read(4))
            except (struct.error, TypeError):
                raise ValueError("Invalid JPEG file")
        # handle JPEG2000s
        elif size >= 12 and head.startswith(b'\x00\x00\x00\x0cjP  \r\n\x87\n'):
            fhandle.seek(48)
            try:
                height, width = struct.unpack('>LL', fhandle.read(8))
            except struct.error:
                raise ValueError("Invalid JPEG2000 file")
        # handle big endian TIFF
        elif size >= 8 and head.startswith(b"\x4d\x4d\x00\x2a"):
            offset = struct.unpack('>L', head[4:8])[0]
            fhandle.seek(offset)
            ifdsize = struct.unpack(">H", fhandle.read(2))[0]
            for i in range(ifdsize):
                tag, datatype, count, data = struct.unpack(">HHLL", fhandle.read(12))
                if tag == 256:
                    if datatype == 3:
                        width = int(data / 65536)
                    elif datatype == 4:
                        width = data
                    else:
                        raise ValueError("Invalid TIFF file: width column data type should be SHORT/LONG.")
                elif tag == 257:
                    if datatype == 3:
                        height = int(data / 65536)
                    elif datatype == 4:
                        height = data
                    else:
                        raise ValueError("Invalid TIFF file: height column data type should be SHORT/LONG.")
                if width != -1 and height != -1:
                    break
            if width == -1 or height == -1:
                raise ValueError("Invalid TIFF file: width and/or height IDS entries are missing.")
        elif size >= 8 and head.startswith(b"\x49\x49\x2a\x00"):
            offset = struct.unpack('<L', head[4:8])[0]
            fhandle.seek(offset)
            ifdsize = struct.unpack("<H", fhandle.read(2))[0]
            for i in range(ifdsize):
                tag, datatype, count, data = struct.unpack("<HHLL", fhandle.read(12))
                if tag == 256:
                    width = data
                elif tag == 257:
                    height = data
                if width != -1 and height != -1:
                    break
            if width == -1 or height == -1:
                raise ValueError("Invalid TIFF file: width and/or height IDS entries are missing.")
        # handle little endian BigTiff
        elif size >= 8 and head.startswith(b"\x49\x49\x2b\x00"):
            bytesize_offset = struct.unpack('<L', head[4:8])[0]
            if bytesize_offset != 8:
                raise ValueError('Invalid BigTIFF file: Expected offset to be 8, found {} instead.'.format(offset))
            offset = struct.unpack('<Q', head[8:16])[0]
            fhandle.seek(offset)
            ifdsize = struct.unpack("<Q", fhandle.read(8))[0]
            for i in range(ifdsize):
                tag, datatype, count, data = struct.unpack("<HHQQ", fhandle.read(20))
                if tag == 256:
                    width = data
                elif tag == 257:
                    height = data
                if width != -1 and height != -1:
                    break
            if width == -1 or height == -1:
                raise ValueError("Invalid BigTIFF file: width and/or height IDS entries are missing.")

        # handle SVGs
        elif size >= 5 and (head.startswith(b'<?xml') or head.startswith(b'<svg')):
            fhandle.seek(0)
            data = fhandle.read(1024)
            try:
                data = data.decode('utf-8')
                width = re.search(r'[^-]width="(.*?)"', data).group(1)
                height = re.search(r'[^-]height="(.*?)"', data).group(1)
            except Exception:
                raise ValueError("Invalid SVG file")
            width = _convertToPx(width)
            height = _convertToPx(height)

        # handle Netpbm
        elif head[:1] == b"P" and head[1:2] in b"123456":
            fhandle.seek(2)
            sizes = []

            while True:
                next_chr = fhandle.read(1)

                if next_chr.isspace():
                    continue

                if next_chr == b"":
                    raise ValueError("Invalid Netpbm file")

                if next_chr == b"#":
                    fhandle.readline()
                    continue

                if not next_chr.isdigit():
                    raise ValueError("Invalid character found on Netpbm file")

                size = next_chr
                next_chr = fhandle.read(1)

                while next_chr.isdigit():
                    size += next_chr
                    next_chr = fhandle.read(1)

                sizes.append(int(size))

                if len(sizes) == 2:
                    break

                fhandle.seek(-1, os.SEEK_CUR)
            width, height = sizes
        elif head.startswith(b"RIFF") and head[8:12] == b"WEBP":
            if head[12:16] == b"VP8 ":
                width, height = struct.unpack("<HH", head[26:30])
            elif head[12:16] == b"VP8X":
                width = struct.unpack("<I", head[24:27] + b"\0")[0]
                height = struct.unpack("<I", head[27:30] + b"\0")[0]
            elif head[12:16] == b"VP8L":
                b = head[21:25]
                width = (((b[1] & 63) << 8) | b[0]) + 1
                height = (((b[3] & 15) << 10) | (b[2] << 2) | ((b[1] & 192) >> 6)) + 1
            else:
                raise ValueError("Unsupported WebP file")

    finally:
        fhandle.close()

    return width, height


def getDPI(filepath):
    """
    Return (x DPI, y DPI) for a given img file content
    no requirements
    :type filepath: Union[bytes, str, pathlib.Path]
    :rtype Tuple[int, int]
    """
    xDPI = -1
    yDPI = -1

    if not isinstance(filepath, bytes):
        filepath = str(filepath)

    with open(filepath, 'rb') as fhandle:
        head = fhandle.read(24)
        size = len(head)
        # handle GIFs
        # GIFs doesn't have density
        if size >= 10 and head[:6] in (b'GIF87a', b'GIF89a'):
            pass
        # see png edition spec bytes are below chunk length then and finally the
        elif size >= 24 and head.startswith(b'\211PNG\r\n\032\n'):
            chunkOffset = 8
            chunk = head[8:]
            while True:
                chunkType = chunk[4:8]
                if chunkType == b'pHYs':
                    try:
                        xDensity, yDensity, unit = struct.unpack(">LLB", chunk[8:])
                    except struct.error:
                        raise ValueError("Invalid PNG file")
                    if unit:
                        xDPI = _convertToDPI(xDensity, _UNIT_1M)
                        yDPI = _convertToDPI(yDensity, _UNIT_1M)
                    else:  # no unit
                        xDPI = xDensity
                        yDPI = yDensity
                    break
                elif chunkType == b'IDAT':
                    break
                else:
                    try:
                        dataSize, = struct.unpack(">L", chunk[0:4])
                    except struct.error:
                        raise ValueError("Invalid PNG file")
                    chunkOffset += dataSize + 12
                    fhandle.seek(chunkOffset)
                    chunk = fhandle.read(17)
        # handle JPEGs
        elif size >= 2 and head.startswith(b'\377\330'):
            try:
                fhandle.seek(0)  # Read 0xff next
                size = 2
                ftype = 0
                while not 0xc0 <= ftype <= 0xcf:
                    if ftype == 0xe0:  # APP0 marker
                        fhandle.seek(7, 1)
                        unit, xDensity, yDensity = struct.unpack(">BHH", fhandle.read(5))
                        if unit == 1 or unit == 0:
                            xDPI = xDensity
                            yDPI = yDensity
                        elif unit == 2:
                            xDPI = _convertToDPI(xDensity, _UNIT_CM)
                            yDPI = _convertToDPI(yDensity, _UNIT_CM)
                        break
                    fhandle.seek(size, 1)
                    byte = fhandle.read(1)
                    while ord(byte) == 0xff:
                        byte = fhandle.read(1)
                    ftype = ord(byte)
                    size = struct.unpack('>H', fhandle.read(2))[0] - 2
            except struct.error:
                raise ValueError("Invalid JPEG file")
        # handle JPEG2000s
        elif size >= 12 and head.startswith(b'\x00\x00\x00\x0cjP  \r\n\x87\n'):
            fhandle.seek(32)
            # skip JP2 image header box
            headerSize = struct.unpack('>L', fhandle.read(4))[0] - 8
            fhandle.seek(4, 1)
            foundResBox = False
            try:
                while headerSize > 0:
                    boxHeader = fhandle.read(8)
                    boxType = boxHeader[4:]
                    if boxType == b'res ':  # find resolution super box
                        foundResBox = True
                        headerSize -= 8
                        break
                    boxSize, = struct.unpack('>L', boxHeader[:4])
                    fhandle.seek(boxSize - 8, 1)
                    headerSize -= boxSize
                if foundResBox:
                    while headerSize > 0:
                        boxHeader = fhandle.read(8)
                        boxType = boxHeader[4:]
                        if boxType == b'resd':  # Display resolution box
                            yDensity, xDensity, yUnit, xUnit = struct.unpack(">HHBB", fhandle.read(10))
                            xDPI = _convertToDPI(xDensity, xUnit)
                            yDPI = _convertToDPI(yDensity, yUnit)
                            break
                        boxSize, = struct.unpack('>L', boxHeader[:4])
                        fhandle.seek(boxSize - 8, 1)
                        headerSize -= boxSize
            except struct.error as e:
                raise ValueError("Invalid JPEG2000 file")
    return xDPI, yDPI

env_pyrecodes/lib/python3.9/site-packages/imagesize.py
NikolaBlagojevic/pyrecodes
"""
Compatibility layer with Python 3.8/3.9
"""
from typing import TYPE_CHECKING, Any, Optional

if TYPE_CHECKING:  # pragma: no cover
    # Prevent circular imports on runtime.
    from . import Distribution, EntryPoint
else:
    Distribution = EntryPoint = Any


def normalized_name(dist: Distribution) -> Optional[str]:
    """
    Honor name normalization for distributions that don't provide ``_normalized_name``.
    """
    try:
        return dist._normalized_name
    except AttributeError:
        from . import Prepared  # -> delay to prevent circular imports.

        return Prepared.normalize(getattr(dist, "name", None) or dist.metadata['Name'])


def ep_matches(ep: EntryPoint, **params) -> bool:
    """
    Workaround for ``EntryPoint`` objects without the ``matches`` method.
    """
    try:
        return ep.matches(**params)
    except AttributeError:
        from . import EntryPoint  # -> delay to prevent circular imports.

        # Reconstruct the EntryPoint object to make sure it is compatible.
        return EntryPoint(ep.name, ep.value, ep.group).matches(**params)

env_pyrecodes/lib/python3.9/site-packages/importlib_metadata/_py39compat.py
NikolaBlagojevic/pyrecodes
class NonDataProperty:
    """Much like the property builtin, but only implements __get__,
    making it a non-data property, and can be subsequently reset.

    See http://users.rcn.com/python/download/Descriptor.htm for more
    information.

    >>> class X(object):
    ...   @NonDataProperty
    ...   def foo(self):
    ...     return 3
    >>> x = X()
    >>> x.foo
    3
    >>> x.foo = 4
    >>> x.foo
    4

    '...' below should be 'jaraco.classes' but for pytest-dev/pytest#3396
    >>> X.foo
    <....properties.NonDataProperty object at ...>
    """

    def __init__(self, fget):
        assert fget is not None, "fget cannot be none"
        assert callable(fget), "fget must be callable"
        self.fget = fget

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        return self.fget(obj)


class classproperty:
    """
    Like @property but applies at the class level.


    >>> class X(metaclass=classproperty.Meta):
    ...   val = None
    ...   @classproperty
    ...   def foo(cls):
    ...     return cls.val
    ...   @foo.setter
    ...   def foo(cls, val):
    ...     cls.val = val
    >>> X.foo
    >>> X.foo = 3
    >>> X.foo
    3
    >>> x = X()
    >>> x.foo
    3
    >>> X.foo = 4
    >>> x.foo
    4

    Setting the property on an instance affects the class.

    >>> x.foo = 5
    >>> x.foo
    5
    >>> X.foo
    5
    >>> vars(x)
    {}
    >>> X().foo
    5

    Attempting to set an attribute where no setter was defined
    results in an AttributeError:

    >>> class GetOnly(metaclass=classproperty.Meta):
    ...   @classproperty
    ...   def foo(cls):
    ...     return 'bar'
    >>> GetOnly.foo = 3
    Traceback (most recent call last):
    ...
    AttributeError: can't set attribute

    It is also possible to wrap a classmethod or staticmethod in
    a classproperty.

    >>> class Static(metaclass=classproperty.Meta):
    ...   @classproperty
    ...   @classmethod
    ...   def foo(cls):
    ...     return 'foo'
    ...   @classproperty
    ...   @staticmethod
    ...   def bar():
    ...     return 'bar'
    >>> Static.foo
    'foo'
    >>> Static.bar
    'bar'

    *Legacy*

    For compatibility, if the metaclass isn't specified, the
    legacy behavior will be invoked.

    >>> class X:
    ...   val = None
    ...   @classproperty
    ...   def foo(cls):
    ...     return cls.val
    ...   @foo.setter
    ...   def foo(cls, val):
    ...     cls.val = val
    >>> X.foo
    >>> X.foo = 3
    >>> X.foo
    3
    >>> x = X()
    >>> x.foo
    3
    >>> X.foo = 4
    >>> x.foo
    4

    Note, because the metaclass was not specified, setting
    a value on an instance does not have the intended effect.

    >>> x.foo = 5
    >>> x.foo
    5
    >>> X.foo  # should be 5
    4
    >>> vars(x)  # should be empty
    {'foo': 5}
    >>> X().foo  # should be 5
    4
    """

    class Meta(type):
        def __setattr__(self, key, value):
            obj = self.__dict__.get(key, None)
            if type(obj) is classproperty:
                return obj.__set__(self, value)
            return super().__setattr__(key, value)

    def __init__(self, fget, fset=None):
        self.fget = self._ensure_method(fget)
        self.fset = fset
        fset and self.setter(fset)

    def __get__(self, instance, owner=None):
        return self.fget.__get__(None, owner)()

    def __set__(self, owner, value):
        if not self.fset:
            raise AttributeError("can't set attribute")
        if type(owner) is not classproperty.Meta:
            owner = type(owner)
        return self.fset.__get__(None, owner)(value)

    def setter(self, fset):
        self.fset = self._ensure_method(fset)
        return self

    @classmethod
    def _ensure_method(cls, fn):
        """
        Ensure fn is a classmethod or staticmethod.
        """
        needs_method = not isinstance(fn, (classmethod, staticmethod))
        return classmethod(fn) if needs_method else fn

env_pyrecodes/lib/python3.9/site-packages/jaraco/classes/properties.py
NikolaBlagojevic/pyrecodes
"""
The API basically only provides one class. You can create a :class:`Script` and
use its methods.

Additionally you can add a debug function with :func:`set_debug_function`.
Alternatively, if you don't need a custom function and are happy with printing
debug messages to stdout, simply call :func:`set_debug_function` without
arguments.
"""
import sys
from pathlib import Path

import parso
from parso.python import tree

from jedi.parser_utils import get_executable_nodes
from jedi import debug
from jedi import settings
from jedi import cache
from jedi.file_io import KnownContentFileIO
from jedi.api import classes
from jedi.api import interpreter
from jedi.api import helpers
from jedi.api.helpers import validate_line_column
from jedi.api.completion import Completion, search_in_module
from jedi.api.keywords import KeywordName
from jedi.api.environment import InterpreterEnvironment
from jedi.api.project import get_default_project, Project
from jedi.api.errors import parso_to_jedi_errors
from jedi.api import refactoring
from jedi.api.refactoring.extract import extract_function, extract_variable
from jedi.inference import InferenceState
from jedi.inference import imports
from jedi.inference.references import find_references
from jedi.inference.arguments import try_iter_content
from jedi.inference.helpers import infer_call_of_leaf
from jedi.inference.sys_path import transform_path_to_dotted
from jedi.inference.syntax_tree import tree_name_to_values
from jedi.inference.value import ModuleValue
from jedi.inference.base_value import ValueSet
from jedi.inference.value.iterable import unpack_tuple_to_dict
from jedi.inference.gradual.conversion import convert_names, convert_values
from jedi.inference.gradual.utils import load_proper_stub_module
from jedi.inference.utils import to_list

# Jedi uses lots and lots of recursion. By setting this a little bit higher, we
# can remove some "maximum recursion depth" errors.
sys.setrecursionlimit(3000)


class Script:
    """
    A Script is the base for completions, goto or whatever you want to do with
    Jedi. The counter part of this class is :class:`Interpreter`, which works
    with actual dictionaries and can work with a REPL. This class
    should be used when a user edits code in an editor.

    You can either use the ``code`` parameter or ``path`` to read a file.
    Usually you're going to want to use both of them (in an editor).

    The Script's ``sys.path`` is very customizable:

    - If `project` is provided with a ``sys_path``, that is going to be used.
    - If `environment` is provided, its ``sys.path`` will be used
      (see :func:`Environment.get_sys_path <jedi.api.environment.Environment.get_sys_path>`);
    - Otherwise ``sys.path`` will match that of the default environment of
      Jedi, which typically matches the sys path that was used at the time
      when Jedi was imported.

    Most methods have a ``line`` and a ``column`` parameter. Lines in Jedi are
    always 1-based and columns are always zero based. To avoid repetition they
    are not always documented. You can omit both line and column. Jedi will
    then just do whatever action you are calling at the end of the file. If you
    provide only the line, just will complete at the end of that line.

    .. warning:: By default :attr:`jedi.settings.fast_parser` is enabled, which means
        that parso reuses modules (i.e. they are not immutable). With this setting
        Jedi is **not thread safe** and it is also not safe to use multiple
        :class:`.Script` instances and its definitions at the same time.

        If you are a normal plugin developer this should not be an issue. It is
        an issue for people that do more complex stuff with Jedi.

        This is purely a performance optimization and works pretty well for all
        typical usages, however consider to turn the setting off if it causes
        you problems. See also
        `this discussion <https://github.com/davidhalter/jedi/issues/1240>`_.

    :param code: The source code of the current file, separated by newlines.
    :type code: str
    :param path: The path of the file in the file system, or ``''`` if
        it hasn't been saved yet.
    :type path: str or pathlib.Path or None
    :param Environment environment: Provide a predefined :ref:`Environment <environments>`
        to work with a specific Python version or virtualenv.
    :param Project project: Provide a :class:`.Project` to make sure finding
        references works well, because the right folder is searched. There are
        also ways to modify the sys path and other things.
    """
    def __init__(self, code=None, *, path=None, environment=None, project=None):
        self._orig_path = path
        if isinstance(path, str):
            path = Path(path)

        self.path = path.absolute() if path else None

        if code is None:
            if path is None:
                raise ValueError("Must provide at least one of code or path")

            # TODO add a better warning than the traceback!
            with open(path, 'rb') as f:
                code = f.read()

        if project is None:
            # Load the Python grammar of the current interpreter.
            project = get_default_project(None if self.path is None else self.path.parent)

        self._inference_state = InferenceState(
            project, environment=environment, script_path=self.path
        )
        debug.speed('init')
        self._module_node, code = self._inference_state.parse_and_get_code(
            code=code,
            path=self.path,
            use_latest_grammar=path and path.suffix == '.pyi',
            cache=False,  # No disk cache, because the current script often changes.
            diff_cache=settings.fast_parser,
            cache_path=settings.cache_directory,
        )
        debug.speed('parsed')
        self._code_lines = parso.split_lines(code, keepends=True)
        self._code = code

        cache.clear_time_caches()
        debug.reset_time()

    # Cache the module, this is mostly useful for testing, since this shouldn't
    # be called multiple times.
    @cache.memoize_method
    def _get_module(self):
        names = None
        is_package = False
        if self.path is not None:
            import_names, is_p = transform_path_to_dotted(
                self._inference_state.get_sys_path(add_parent_paths=False),
                self.path
            )
            if import_names is not None:
                names = import_names
                is_package = is_p

        if self.path is None:
            file_io = None
        else:
            file_io = KnownContentFileIO(self.path, self._code)
        if self.path is not None and self.path.suffix == '.pyi':
            # We are in a stub file. Try to load the stub properly.
            stub_module = load_proper_stub_module(
                self._inference_state,
                self._inference_state.latest_grammar,
                file_io,
                names,
                self._module_node
            )
            if stub_module is not None:
                return stub_module

        if names is None:
            names = ('__main__',)

        module = ModuleValue(
            self._inference_state, self._module_node,
            file_io=file_io,
            string_names=names,
            code_lines=self._code_lines,
            is_package=is_package,
        )
        if names[0] not in ('builtins', 'typing'):
            # These modules are essential for Jedi, so don't overwrite them.
            self._inference_state.module_cache.add(names, ValueSet([module]))
        return module

    def _get_module_context(self):
        return self._get_module().as_context()

    def __repr__(self):
        return '<%s: %s %r>' % (
            self.__class__.__name__,
            repr(self._orig_path),
            self._inference_state.environment,
        )

    @validate_line_column
    def complete(self, line=None, column=None, *, fuzzy=False):
        """
        Completes objects under the cursor.

        Those objects contain information about the completions, more than just
        names.

        :param fuzzy: Default False. Will return fuzzy completions, which means
            that e.g. ``ooa`` will match ``foobar``.
        :return: Completion objects, sorted by name. Normal names appear
            before "private" names that start with ``_`` and those appear
            before magic methods and name mangled names that start with ``__``.
        :rtype: list of :class:`.Completion`
        """
        with debug.increase_indent_cm('complete'):
            completion = Completion(
                self._inference_state, self._get_module_context(), self._code_lines,
                (line, column), self.get_signatures, fuzzy=fuzzy,
            )
            return completion.complete()

    @validate_line_column
    def infer(self, line=None, column=None, *, only_stubs=False, prefer_stubs=False):
        """
        Return the definitions of under the cursor. It is basically a wrapper
        around Jedi's type inference.

        This method follows complicated paths and returns the end, not the
        first definition. The big difference between :meth:`goto` and
        :meth:`infer` is that :meth:`goto` doesn't
        follow imports and statements. Multiple objects may be returned,
        because depending on an option you can have two different versions of a
        function.

        :param only_stubs: Only return stubs for this method.
        :param prefer_stubs: Prefer stubs to Python objects for this method.
        :rtype: list of :class:`.Name`
        """
        pos = line, column
        leaf = self._module_node.get_name_of_position(pos)
        if leaf is None:
            leaf = self._module_node.get_leaf_for_position(pos)
            if leaf is None or leaf.type == 'string':
                return []
            if leaf.end_pos == (line, column) and leaf.type == 'operator':
                next_ = leaf.get_next_leaf()
                if next_.start_pos == leaf.end_pos \
                        and next_.type in ('number', 'string', 'keyword'):
                    leaf = next_

        context = self._get_module_context().create_context(leaf)

        values = helpers.infer(self._inference_state, context, leaf)
        values = convert_values(
            values,
            only_stubs=only_stubs,
            prefer_stubs=prefer_stubs,
        )

        defs = [classes.Name(self._inference_state, c.name) for c in values]
        # The additional set here allows the definitions to become unique in an
        # API sense. In the internals we want to separate more things than in
        # the API.
        return helpers.sorted_definitions(set(defs))

    @validate_line_column
    def goto(self, line=None, column=None, *, follow_imports=False, follow_builtin_imports=False,
             only_stubs=False, prefer_stubs=False):
        """
        Goes to the name that defined the object under the cursor. Optionally
        you can follow imports.
        Multiple objects may be returned, depending on an if you can have two
        different versions of a function.

        :param follow_imports: The method will follow imports.
        :param follow_builtin_imports: If ``follow_imports`` is True will try
            to look up names in builtins (i.e. compiled or extension modules).
        :param only_stubs: Only return stubs for this method.
        :param prefer_stubs: Prefer stubs to Python objects for this method.
        :rtype: list of :class:`.Name`
        """
        tree_name = self._module_node.get_name_of_position((line, column))
        if tree_name is None:
            # Without a name we really just want to jump to the result e.g.
            # executed by `foo()`, if we the cursor is after `)`.
            return self.infer(line, column, only_stubs=only_stubs, prefer_stubs=prefer_stubs)
        name = self._get_module_context().create_name(tree_name)

        # Make it possible to goto the super class function/attribute
        # definitions, when they are overwritten.
        names = []
        if name.tree_name.is_definition() and name.parent_context.is_class():
            class_node = name.parent_context.tree_node
            class_value = self._get_module_context().create_value(class_node)
            mro = class_value.py__mro__()
            next(mro)  # Ignore the first entry, because it's the class itself.
            for cls in mro:
                names = cls.goto(tree_name.value)
                if names:
                    break

        if not names:
            names = list(name.goto())

        if follow_imports:
            names = helpers.filter_follow_imports(names, follow_builtin_imports)
        names = convert_names(
            names,
            only_stubs=only_stubs,
            prefer_stubs=prefer_stubs,
        )

        defs = [classes.Name(self._inference_state, d) for d in set(names)]
        # Avoid duplicates
        return list(set(helpers.sorted_definitions(defs)))

    def search(self, string, *, all_scopes=False):
        """
        Searches a name in the current file. For a description of how the
        search string should look like, please have a look at
        :meth:`.Project.search`.

        :param bool all_scopes: Default False; searches not only for
            definitions on the top level of a module level, but also in
            functions and classes.
        :yields: :class:`.Name`
        """
        return self._search_func(string, all_scopes=all_scopes)

    @to_list
    def _search_func(self, string, all_scopes=False, complete=False, fuzzy=False):
        names = self._names(all_scopes=all_scopes)
        wanted_type, wanted_names = helpers.split_search_string(string)
        return search_in_module(
            self._inference_state,
            self._get_module_context(),
            names=names,
            wanted_type=wanted_type,
            wanted_names=wanted_names,
            complete=complete,
            fuzzy=fuzzy,
        )

    def complete_search(self, string, **kwargs):
        """
        Like :meth:`.Script.search`, but completes that string. If you want to
        have all possible definitions in a file you can also provide an empty
        string.

        :param bool all_scopes: Default False; searches not only for
            definitions on the top level of a module level, but also in
            functions and classes.
        :param fuzzy: Default False. Will return fuzzy completions, which means
            that e.g. ``ooa`` will match ``foobar``.
        :yields: :class:`.Completion`
        """
        return self._search_func(string, complete=True, **kwargs)

    @validate_line_column
    def help(self, line=None, column=None):
        """
        Used to display a help window to users.  Uses :meth:`.Script.goto` and
        returns additional definitions for keywords and operators.

        Typically you will want to display :meth:`.BaseName.docstring` to the
        user for all the returned definitions.

        The additional definitions are ``Name(...).type == 'keyword'``.
        These definitions do not have a lot of value apart from their docstring
        attribute, which contains the output of Python's :func:`help` function.

        :rtype: list of :class:`.Name`
        """
        definitions = self.goto(line, column, follow_imports=True)
        if definitions:
            return definitions
        leaf = self._module_node.get_leaf_for_position((line, column))
        if leaf is not None and leaf.type in ('keyword', 'operator', 'error_leaf'):
            def need_pydoc():
                if leaf.value in ('(', ')', '[', ']'):
                    if leaf.parent.type == 'trailer':
                        return False
                    if leaf.parent.type == 'atom':
                        return False
                grammar = self._inference_state.grammar
                # This parso stuff is not public, but since I control it, this
                # is fine :-) ~dave
                reserved = grammar._pgen_grammar.reserved_syntax_strings.keys()
                return leaf.value in reserved

            if need_pydoc():
                name = KeywordName(self._inference_state, leaf.value)
                return [classes.Name(self._inference_state, name)]
        return []

    @validate_line_column
    def get_references(self, line=None, column=None, **kwargs):
        """
        Lists all references of a variable in a project. Since this can be
        quite hard to do for Jedi, if it is too complicated, Jedi will stop
        searching.

        :param include_builtins: Default ``True``. If ``False``, checks if a definition
            is a builtin (e.g. ``sys``) and in that case does not return it.
        :param scope: Default ``'project'``. If ``'file'``, include references in
            the current module only.
        :rtype: list of :class:`.Name`
        """

        def _references(include_builtins=True, scope='project'):
            if scope not in ('project', 'file'):
                raise ValueError('Only the scopes "file" and "project" are allowed')
            tree_name = self._module_node.get_name_of_position((line, column))
            if tree_name is None:
                # Must be syntax
                return []

            names = find_references(self._get_module_context(), tree_name, scope == 'file')

            definitions = [classes.Name(self._inference_state, n) for n in names]
            if not include_builtins or scope == 'file':
                definitions = [d for d in definitions if not d.in_builtin_module()]
            return helpers.sorted_definitions(definitions)
        return _references(**kwargs)

    @validate_line_column
    def get_signatures(self, line=None, column=None):
        """
        Return the function object of the call under the cursor.

        E.g. if the cursor is here::

            abs(# <-- cursor is here

        This would return the ``abs`` function. On the other hand::

            abs()# <-- cursor is here

        This would return an empty list..

        :rtype: list of :class:`.Signature`
        """
        pos = line, column
        call_details = helpers.get_signature_details(self._module_node, pos)
        if call_details is None:
            return []

        context = self._get_module_context().create_context(call_details.bracket_leaf)
        definitions = helpers.cache_signatures(
            self._inference_state,
            context,
            call_details.bracket_leaf,
            self._code_lines,
            pos
        )
        debug.speed('func_call followed')

        # TODO here we use stubs instead of the actual values. We should use
        # the signatures from stubs, but the actual values, probably?!
        return [classes.Signature(self._inference_state, signature, call_details)
                for signature in definitions.get_signatures()]

    @validate_line_column
    def get_context(self, line=None, column=None):
        """
        Returns the scope context under the cursor. This basically means the
        function, class or module where the cursor is at.

        :rtype: :class:`.Name`
        """
        pos = (line, column)
        leaf = self._module_node.get_leaf_for_position(pos, include_prefixes=True)
        if leaf.start_pos > pos or leaf.type == 'endmarker':
            previous_leaf = leaf.get_previous_leaf()
            if previous_leaf is not None:
                leaf = previous_leaf

        module_context = self._get_module_context()

        n = tree.search_ancestor(leaf, 'funcdef', 'classdef')
        if n is not None and n.start_pos < pos <= n.children[-1].start_pos:
            # This is a bit of a special case. The context of a function/class
            # name/param/keyword is always it's parent context, not the
            # function itself. Catch all the cases here where we are before the
            # suite object, but still in the function.
            context = module_context.create_value(n).as_context()
        else:
            context = module_context.create_context(leaf)

        while context.name is None:
            context = context.parent_context  # comprehensions

        definition = classes.Name(self._inference_state, context.name)
        while definition.type != 'module':
            name = definition._name  # TODO private access
            tree_name = name.tree_name
            if tree_name is not None:  # Happens with lambdas.
                scope = tree_name.get_definition()
                if scope.start_pos[1] < column:
                    break
            definition = definition.parent()
        return definition

    def _analysis(self):
        self._inference_state.is_analysis = True
        self._inference_state.analysis_modules = [self._module_node]
        module = self._get_module_context()
        try:
            for node in get_executable_nodes(self._module_node):
                context = module.create_context(node)
                if node.type in ('funcdef', 'classdef'):
                    # Resolve the decorators.
                    tree_name_to_values(self._inference_state, context, node.children[1])
                elif isinstance(node, tree.Import):
                    import_names = set(node.get_defined_names())
                    if node.is_nested():
                        import_names |= set(path[-1] for path in node.get_paths())
                    for n in import_names:
                        imports.infer_import(context, n)
                elif node.type == 'expr_stmt':
                    types = context.infer_node(node)
                    for testlist in node.children[:-1:2]:
                        # Iterate tuples.
                        unpack_tuple_to_dict(context, types, testlist)
                else:
                    if node.type == 'name':
                        defs = self._inference_state.infer(context, node)
                    else:
                        defs = infer_call_of_leaf(context, node)
                    try_iter_content(defs)
                self._inference_state.reset_recursion_limitations()

            ana = [a for a in self._inference_state.analysis if self.path == a.path]
            return sorted(set(ana), key=lambda x: x.line)
        finally:
            self._inference_state.is_analysis = False

    def get_names(self, **kwargs):
        """
        Returns names defined in the current file.

        :param all_scopes: If True lists the names of all scopes instead of
            only the module namespace.
        :param definitions: If True lists the names that have been defined by a
            class, function or a statement (``a = b`` returns ``a``).
        :param references: If True lists all the names that are not listed by
            ``definitions=True``. E.g. ``a = b`` returns ``b``.
        :rtype: list of :class:`.Name`
        """
        names = self._names(**kwargs)
        return [classes.Name(self._inference_state, n) for n in names]

    def get_syntax_errors(self):
        """
        Lists all syntax errors in the current file.

        :rtype: list of :class:`.SyntaxError`
        """
        return parso_to_jedi_errors(self._inference_state.grammar, self._module_node)

    def _names(self, all_scopes=False, definitions=True, references=False):
        # Set line/column to a random position, because they don't matter.
        module_context = self._get_module_context()
        defs = [
            module_context.create_name(name)
            for name in helpers.get_module_names(
                self._module_node,
                all_scopes=all_scopes,
                definitions=definitions,
                references=references,
            )
        ]
        return sorted(defs, key=lambda x: x.start_pos)

    def rename(self, line=None, column=None, *, new_name):
        """
        Renames all references of the variable under the cursor.

        :param new_name: The variable under the cursor will be renamed to this
            string.
        :raises: :exc:`.RefactoringError`
        :rtype: :class:`.Refactoring`
        """
        definitions = self.get_references(line, column, include_builtins=False)
        return refactoring.rename(self._inference_state, definitions, new_name)

    @validate_line_column
    def extract_variable(self, line, column, *, new_name, until_line=None, until_column=None):
        """
        Moves an expression to a new statement.

        For example if you have the cursor on ``foo`` and provide a
        ``new_name`` called ``bar``::

            foo = 3.1
            x = int(foo + 1)

        the code above will become::

            foo = 3.1
            bar = foo + 1
            x = int(bar)

        :param new_name: The expression under the cursor will be renamed to
            this string.
        :param int until_line: The the selection range ends at this line, when
            omitted, Jedi will be clever and try to define the range itself.
        :param int until_column: The the selection range ends at this column, when
            omitted, Jedi will be clever and try to define the range itself.
        :raises: :exc:`.RefactoringError`
        :rtype: :class:`.Refactoring`
        """
        if until_line is None and until_column is None:
            until_pos = None
        else:
            if until_line is None:
                until_line = line
            if until_column is None:
                until_column = len(self._code_lines[until_line - 1])
            until_pos = until_line, until_column
        return extract_variable(
            self._inference_state, self.path, self._module_node,
            new_name, (line, column), until_pos
        )

    @validate_line_column
    def extract_function(self, line, column, *, new_name, until_line=None, until_column=None):
        """
        Moves an expression to a new function.

        For example if you have the cursor on ``foo`` and provide a
        ``new_name`` called ``bar``::

            global_var = 3

            def x():
                foo = 3.1
                x = int(foo + 1 + global_var)

        the code above will become::

            global_var = 3

            def bar(foo):
                return int(foo + 1 + global_var)

            def x():
                foo = 3.1
                x = bar(foo)

        :param new_name: The expression under the cursor will be replaced with
            a function with this name.
        :param int until_line: The the selection range ends at this line, when
            omitted, Jedi will be clever and try to define the range itself.
        :param int until_column: The the selection range ends at this column, when
            omitted, Jedi will be clever and try to define the range itself.
        :raises: :exc:`.RefactoringError`
        :rtype: :class:`.Refactoring`
        """
        if until_line is None and until_column is None:
            until_pos = None
        else:
            if until_line is None:
                until_line = line
            if until_column is None:
                until_column = len(self._code_lines[until_line - 1])
            until_pos = until_line, until_column
        return extract_function(
            self._inference_state, self.path, self._get_module_context(),
            new_name, (line, column), until_pos
        )

    def inline(self, line=None, column=None):
        """
        Inlines a variable under the cursor. This is basically the opposite of
        extracting a variable. For example with the cursor on bar::

            foo = 3.1
            bar = foo + 1
            x = int(bar)

        the code above will become::

            foo = 3.1
            x = int(foo + 1)

        :raises: :exc:`.RefactoringError`
        :rtype: :class:`.Refactoring`
        """
        names = [d._name for d in self.get_references(line, column, include_builtins=True)]
        return refactoring.inline(self._inference_state, names)


class Interpreter(Script):
    """
    Jedi's API for Python REPLs.

    Implements all of the methods that are present in :class:`.Script` as well.

    In addition to completions that normal REPL completion does like
    ``str.upper``, Jedi also supports code completion based on static code
    analysis. For example Jedi will complete ``str().upper``.

    >>> from os.path import join
    >>> namespace = locals()
    >>> script = Interpreter('join("").up', [namespace])
    >>> print(script.complete()[0].name)
    upper

    All keyword arguments are same as the arguments for :class:`.Script`.

    :param str code: Code to parse.
    :type namespaces: typing.List[dict]
    :param namespaces: A list of namespace dictionaries such as the one
        returned by :func:`globals` and :func:`locals`.
    """
    _allow_descriptor_getattr_default = True

    def __init__(self, code, namespaces, *, project=None, **kwds):
        try:
            namespaces = [dict(n) for n in namespaces]
        except Exception:
            raise TypeError("namespaces must be a non-empty list of dicts.")

        environment = kwds.get('environment', None)
        if environment is None:
            environment = InterpreterEnvironment()
        else:
            if not isinstance(environment, InterpreterEnvironment):
                raise TypeError("The environment needs to be an InterpreterEnvironment subclass.")

        if project is None:
            project = Project(Path.cwd())

        super().__init__(code, environment=environment, project=project, **kwds)

        self.namespaces = namespaces
        self._inference_state.allow_descriptor_getattr = self._allow_descriptor_getattr_default

    @cache.memoize_method
    def _get_module_context(self):
        if self.path is None:
            file_io = None
        else:
            file_io = KnownContentFileIO(self.path, self._code)
        tree_module_value = ModuleValue(
            self._inference_state, self._module_node,
            file_io=file_io,
            string_names=('__main__',),
            code_lines=self._code_lines,
        )
        return interpreter.MixedModuleContext(
            tree_module_value,
            self.namespaces,
        )


def preload_module(*modules):
    """
    Preloading modules tells Jedi to load a module now, instead of lazy parsing
    of modules. This can be useful for IDEs, to control which modules to load
    on startup.

    :param modules: different module names, list of string.
    """
    for m in modules:
        s = "import %s as x; x." % m
        Script(s).complete(1, len(s))


def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,
                       notices=True, speed=True):
    """
    Define a callback debug function to get all the debug messages.

    If you don't specify any arguments, debug messages will be printed to stdout.

    :param func_cb: The callback function for debug messages.
    """
    debug.debug_function = func_cb
    debug.enable_warning = warnings
    debug.enable_notice = notices
    debug.enable_speed = speed

env_pyrecodes/lib/python3.9/site-packages/jedi/api/__init__.py
NikolaBlagojevic/pyrecodes
import difflib
from pathlib import Path
from typing import Dict, Iterable, Tuple

from parso import split_lines

from jedi.api.exceptions import RefactoringError

EXPRESSION_PARTS = (
    'or_test and_test not_test comparison '
    'expr xor_expr and_expr shift_expr arith_expr term factor power atom_expr'
).split()


class ChangedFile:
    def __init__(self, inference_state, from_path, to_path,
                 module_node, node_to_str_map):
        self._inference_state = inference_state
        self._from_path = from_path
        self._to_path = to_path
        self._module_node = module_node
        self._node_to_str_map = node_to_str_map

    def get_diff(self):
        old_lines = split_lines(self._module_node.get_code(), keepends=True)
        new_lines = split_lines(self.get_new_code(), keepends=True)

        # Add a newline at the end if it's missing. Otherwise the diff will be
        # very weird. A `diff -u file1 file2` would show the string:
        #
        #     \ No newline at end of file
        #
        # This is not necessary IMO, because Jedi does not really play with
        # newlines and the ending newline does not really matter in Python
        # files. ~dave
        if old_lines[-1] != '':
            old_lines[-1] += '\n'
        if new_lines[-1] != '':
            new_lines[-1] += '\n'

        project_path = self._inference_state.project.path
        if self._from_path is None:
            from_p = ''
        else:
            try:
                from_p = self._from_path.relative_to(project_path)
            except ValueError:  # Happens it the path is not on th project_path
                from_p = self._from_path
        if self._to_path is None:
            to_p = ''
        else:
            try:
                to_p = self._to_path.relative_to(project_path)
            except ValueError:
                to_p = self._to_path
        diff = difflib.unified_diff(
            old_lines, new_lines,
            fromfile=str(from_p),
            tofile=str(to_p),
        )
        # Apparently there's a space at the end of the diff - for whatever
        # reason.
        return ''.join(diff).rstrip(' ')

    def get_new_code(self):
        return self._inference_state.grammar.refactor(self._module_node, self._node_to_str_map)

    def apply(self):
        if self._from_path is None:
            raise RefactoringError(
                'Cannot apply a refactoring on a Script with path=None'
            )

        with open(self._from_path, 'w', newline='') as f:
            f.write(self.get_new_code())

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, self._from_path)


class Refactoring:
    def __init__(self, inference_state, file_to_node_changes, renames=()):
        self._inference_state = inference_state
        self._renames = renames
        self._file_to_node_changes = file_to_node_changes

    def get_changed_files(self) -> Dict[Path, ChangedFile]:
        def calculate_to_path(p):
            if p is None:
                return p
            p = str(p)
            for from_, to in renames:
                if p.startswith(str(from_)):
                    p = str(to) + p[len(str(from_)):]
            return Path(p)

        renames = self.get_renames()
        return {
            path: ChangedFile(
                self._inference_state,
                from_path=path,
                to_path=calculate_to_path(path),
                module_node=next(iter(map_)).get_root_node(),
                node_to_str_map=map_
            ) for path, map_ in sorted(self._file_to_node_changes.items())
        }

    def get_renames(self) -> Iterable[Tuple[Path, Path]]:
        """
        Files can be renamed in a refactoring.
        """
        return sorted(self._renames)

    def get_diff(self):
        text = ''
        project_path = self._inference_state.project.path
        for from_, to in self.get_renames():
            text += 'rename from %s\nrename to %s\n' \
                % (from_.relative_to(project_path), to.relative_to(project_path))

        return text + ''.join(f.get_diff() for f in self.get_changed_files().values())

    def apply(self):
        """
        Applies the whole refactoring to the files, which includes renames.
        """
        for f in self.get_changed_files().values():
            f.apply()

        for old, new in self.get_renames():
            old.rename(new)


def _calculate_rename(path, new_name):
    dir_ = path.parent
    if path.name in ('__init__.py', '__init__.pyi'):
        return dir_, dir_.parent.joinpath(new_name)
    return path, dir_.joinpath(new_name + path.suffix)


def rename(inference_state, definitions, new_name):
    file_renames = set()
    file_tree_name_map = {}

    if not definitions:
        raise RefactoringError("There is no name under the cursor")

    for d in definitions:
        tree_name = d._name.tree_name
        if d.type == 'module' and tree_name is None:
            p = None if d.module_path is None else Path(d.module_path)
            file_renames.add(_calculate_rename(p, new_name))
        else:
            # This private access is ok in a way. It's not public to
            # protect Jedi users from seeing it.
            if tree_name is not None:
                fmap = file_tree_name_map.setdefault(d.module_path, {})
                fmap[tree_name] = tree_name.prefix + new_name
    return Refactoring(inference_state, file_tree_name_map, file_renames)


def inline(inference_state, names):
    if not names:
        raise RefactoringError("There is no name under the cursor")
    if any(n.api_type in ('module', 'namespace') for n in names):
        raise RefactoringError("Cannot inline imports, modules or namespaces")
    if any(n.tree_name is None for n in names):
        raise RefactoringError("Cannot inline builtins/extensions")

    definitions = [n for n in names if n.tree_name.is_definition()]
    if len(definitions) == 0:
        raise RefactoringError("No definition found to inline")
    if len(definitions) > 1:
        raise RefactoringError("Cannot inline a name with multiple definitions")
    if len(names) == 1:
        raise RefactoringError("There are no references to this name")

    tree_name = definitions[0].tree_name

    expr_stmt = tree_name.get_definition()
    if expr_stmt.type != 'expr_stmt':
        type_ = dict(
            funcdef='function',
            classdef='class',
        ).get(expr_stmt.type, expr_stmt.type)
        raise RefactoringError("Cannot inline a %s" % type_)

    if len(expr_stmt.get_defined_names(include_setitem=True)) > 1:
        raise RefactoringError("Cannot inline a statement with multiple definitions")
    first_child = expr_stmt.children[1]
    if first_child.type == 'annassign' and len(first_child.children) == 4:
        first_child = first_child.children[2]
    if first_child != '=':
        if first_child.type == 'annassign':
            raise RefactoringError(
                'Cannot inline a statement that is defined by an annotation'
            )
        else:
            raise RefactoringError(
                'Cannot inline a statement with "%s"'
                % first_child.get_code(include_prefix=False)
            )

    rhs = expr_stmt.get_rhs()
    replace_code = rhs.get_code(include_prefix=False)

    references = [n for n in names if not n.tree_name.is_definition()]
    file_to_node_changes = {}
    for name in references:
        tree_name = name.tree_name
        path = name.get_root_context().py__file__()
        s = replace_code
        if rhs.type == 'testlist_star_expr' \
                or tree_name.parent.type in EXPRESSION_PARTS \
                or tree_name.parent.type == 'trailer' \
                and tree_name.parent.get_next_sibling() is not None:
            s = '(' + replace_code + ')'

        of_path = file_to_node_changes.setdefault(path, {})

        n = tree_name
        prefix = n.prefix
        par = n.parent
        if par.type == 'trailer' and par.children[0] == '.':
            prefix = par.parent.children[0].prefix
            n = par
            for some_node in par.parent.children[:par.parent.children.index(par)]:
                of_path[some_node] = ''
        of_path[n] = prefix + s

    path = definitions[0].get_root_context().py__file__()
    changes = file_to_node_changes.setdefault(path, {})
    changes[expr_stmt] = _remove_indent_of_prefix(expr_stmt.get_first_leaf().prefix)
    next_leaf = expr_stmt.get_next_leaf()

    # Most of the time we have to remove the newline at the end of the
    # statement, but if there's a comment we might not need to.
    if next_leaf.prefix.strip(' \t') == '' \
            and (next_leaf.type == 'newline' or next_leaf == ';'):
        changes[next_leaf] = ''
    return Refactoring(inference_state, file_to_node_changes)


def _remove_indent_of_prefix(prefix):
    r"""
    Removes the last indentation of a prefix, e.g. " \n \n " becomes " \n \n".
    """
    return ''.join(split_lines(prefix, keepends=True)[:-1])

env_pyrecodes/lib/python3.9/site-packages/jedi/api/refactoring/__init__.py
NikolaBlagojevic/pyrecodes
"""
To use Jedi completion in Python interpreter, add the following in your shell
setup (e.g., ``.bashrc``). This works only on Linux/Mac, because readline is
not available on Windows. If you still want Jedi autocompletion in your REPL,
just use IPython instead::

    export PYTHONSTARTUP="$(python -m jedi repl)"

Then you will be able to use Jedi completer in your Python interpreter::

    $ python
    Python 3.9.2+ (default, Jul 20 2020, 22:15:08)
    [GCC 4.6.1] on linux2
    Type "help", "copyright", "credits" or "license" for more information.
    >>> import os
    >>> os.path.join('a', 'b').split().in<TAB>            # doctest: +SKIP
    ..dex   ..sert

"""
import jedi.utils
from jedi import __version__ as __jedi_version__

print('REPL completion using Jedi %s' % __jedi_version__)
jedi.utils.setup_readline(fuzzy=False)

del jedi

# Note: try not to do many things here, as it will contaminate global
# namespace of the interpreter.

env_pyrecodes/lib/python3.9/site-packages/jedi/api/replstartup.py
NikolaBlagojevic/pyrecodes
"""
Used only for REPL Completion.
"""

import inspect
from pathlib import Path

from jedi.parser_utils import get_cached_code_lines

from jedi import settings
from jedi.cache import memoize_method
from jedi.inference import compiled
from jedi.file_io import FileIO
from jedi.inference.names import NameWrapper
from jedi.inference.base_value import ValueSet, ValueWrapper, NO_VALUES
from jedi.inference.value import ModuleValue
from jedi.inference.cache import inference_state_function_cache, \
    inference_state_method_cache
from jedi.inference.compiled.access import ALLOWED_GETITEM_TYPES, get_api_type
from jedi.inference.gradual.conversion import to_stub
from jedi.inference.context import CompiledContext, CompiledModuleContext, \
    TreeContextMixin

_sentinel = object()


class MixedObject(ValueWrapper):
    """
    A ``MixedObject`` is used in two ways:

    1. It uses the default logic of ``parser.python.tree`` objects,
    2. except for getattr calls and signatures. The names dicts are generated
       in a fashion like ``CompiledValue``.

    This combined logic makes it possible to provide more powerful REPL
    completion. It allows side effects that are not noticable with the default
    parser structure to still be completable.

    The biggest difference from CompiledValue to MixedObject is that we are
    generally dealing with Python code and not with C code. This will generate
    fewer special cases, because we in Python you don't have the same freedoms
    to modify the runtime.
    """
    def __init__(self, compiled_value, tree_value):
        super().__init__(tree_value)
        self.compiled_value = compiled_value
        self.access_handle = compiled_value.access_handle

    def get_filters(self, *args, **kwargs):
        yield MixedObjectFilter(
            self.inference_state, self.compiled_value, self._wrapped_value)

    def get_signatures(self):
        # Prefer `inspect.signature` over somehow analyzing Python code. It
        # should be very precise, especially for stuff like `partial`.
        return self.compiled_value.get_signatures()

    @inference_state_method_cache(default=NO_VALUES)
    def py__call__(self, arguments):
        # Fallback to the wrapped value if to stub returns no values.
        values = to_stub(self._wrapped_value)
        if not values:
            values = self._wrapped_value
        return values.py__call__(arguments)

    def get_safe_value(self, default=_sentinel):
        if default is _sentinel:
            return self.compiled_value.get_safe_value()
        else:
            return self.compiled_value.get_safe_value(default)

    @property
    def array_type(self):
        return self.compiled_value.array_type

    def get_key_values(self):
        return self.compiled_value.get_key_values()

    def py__simple_getitem__(self, index):
        python_object = self.compiled_value.access_handle.access._obj
        if type(python_object) in ALLOWED_GETITEM_TYPES:
            return self.compiled_value.py__simple_getitem__(index)
        return self._wrapped_value.py__simple_getitem__(index)

    def negate(self):
        return self.compiled_value.negate()

    def _as_context(self):
        if self.parent_context is None:
            return MixedModuleContext(self)
        return MixedContext(self)

    def __repr__(self):
        return '<%s: %s; %s>' % (
            type(self).__name__,
            self.access_handle.get_repr(),
            self._wrapped_value,
        )


class MixedContext(CompiledContext, TreeContextMixin):
    @property
    def compiled_value(self):
        return self._value.compiled_value


class MixedModuleContext(CompiledModuleContext, MixedContext):
    pass


class MixedName(NameWrapper):
    """
    The ``CompiledName._compiled_value`` is our MixedObject.
    """
    def __init__(self, wrapped_name, parent_tree_value):
        super().__init__(wrapped_name)
        self._parent_tree_value = parent_tree_value

    @property
    def start_pos(self):
        values = list(self.infer())
        if not values:
            # This means a start_pos that doesn't exist (compiled objects).
            return 0, 0
        return values[0].name.start_pos

    @memoize_method
    def infer(self):
        compiled_value = self._wrapped_name.infer_compiled_value()
        tree_value = self._parent_tree_value
        if tree_value.is_instance() or tree_value.is_class():
            tree_values = tree_value.py__getattribute__(self.string_name)
            if compiled_value.is_function():
                return ValueSet({MixedObject(compiled_value, v) for v in tree_values})

        module_context = tree_value.get_root_context()
        return _create(self._inference_state, compiled_value, module_context)


class MixedObjectFilter(compiled.CompiledValueFilter):
    def __init__(self, inference_state, compiled_value, tree_value):
        super().__init__(inference_state, compiled_value)
        self._tree_value = tree_value

    def _create_name(self, name):
        return MixedName(
            super()._create_name(name),
            self._tree_value,
        )


@inference_state_function_cache()
def _load_module(inference_state, path):
    return inference_state.parse(
        path=path,
        cache=True,
        diff_cache=settings.fast_parser,
        cache_path=settings.cache_directory
    ).get_root_node()


def _get_object_to_check(python_object):
    """Check if inspect.getfile has a chance to find the source."""
    try:
        python_object = inspect.unwrap(python_object)
    except ValueError:
        # Can return a ValueError when it wraps around
        pass

    if (inspect.ismodule(python_object)
            or inspect.isclass(python_object)
            or inspect.ismethod(python_object)
            or inspect.isfunction(python_object)
            or inspect.istraceback(python_object)
            or inspect.isframe(python_object)
            or inspect.iscode(python_object)):
        return python_object

    try:
        return python_object.__class__
    except AttributeError:
        raise TypeError  # Prevents computation of `repr` within inspect.


def _find_syntax_node_name(inference_state, python_object):
    original_object = python_object
    try:
        python_object = _get_object_to_check(python_object)
        path = inspect.getsourcefile(python_object)
    except (OSError, TypeError):
        # The type might not be known (e.g. class_with_dict.__weakref__)
        return None
    path = None if path is None else Path(path)
    try:
        if path is None or not path.exists():
            # The path might not exist or be e.g. <stdin>.
            return None
    except OSError:
        # Might raise an OSError on Windows:
        #
        #     [WinError 123] The filename, directory name, or volume label
        #     syntax is incorrect: '<string>'
        return None

    file_io = FileIO(path)
    module_node = _load_module(inference_state, path)

    if inspect.ismodule(python_object):
        # We don't need to check names for modules, because there's not really
        # a way to write a module in a module in Python (and also __name__ can
        # be something like ``email.utils``).
        code_lines = get_cached_code_lines(inference_state.grammar, path)
        return module_node, module_node, file_io, code_lines

    try:
        name_str = python_object.__name__
    except AttributeError:
        # Stuff like python_function.__code__.
        return None

    if name_str == '<lambda>':
        return None  # It's too hard to find lambdas.

    # Doesn't always work (e.g. os.stat_result)
    names = module_node.get_used_names().get(name_str, [])
    # Only functions and classes are relevant. If a name e.g. points to an
    # import, it's probably a builtin (like collections.deque) and needs to be
    # ignored.
    names = [
        n for n in names
        if n.parent.type in ('funcdef', 'classdef') and n.parent.name == n
    ]
    if not names:
        return None

    try:
        code = python_object.__code__
        # By using the line number of a code object we make the lookup in a
        # file pretty easy. There's still a possibility of people defining
        # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people
        # do so we just don't care.
        line_nr = code.co_firstlineno
    except AttributeError:
        pass
    else:
        line_names = [name for name in names if name.start_pos[0] == line_nr]
        # There's a chance that the object is not available anymore, because
        # the code has changed in the background.
        if line_names:
            names = line_names

    code_lines = get_cached_code_lines(inference_state.grammar, path)
    # It's really hard to actually get the right definition, here as a last
    # resort we just return the last one. This chance might lead to odd
    # completions at some points but will lead to mostly correct type
    # inference, because people tend to define a public name in a module only
    # once.
    tree_node = names[-1].parent
    if tree_node.type == 'funcdef' and get_api_type(original_object) == 'instance':
        # If an instance is given and we're landing on a function (e.g.
        # partial in 3.5), something is completely wrong and we should not
        # return that.
        return None
    return module_node, tree_node, file_io, code_lines


@inference_state_function_cache()
def _create(inference_state, compiled_value, module_context):
    # TODO accessing this is bad, but it probably doesn't matter that much,
    # because we're working with interpreters only here.
    python_object = compiled_value.access_handle.access._obj
    result = _find_syntax_node_name(inference_state, python_object)
    if result is None:
        # TODO Care about generics from stuff like `[1]` and don't return like this.
        if type(python_object) in (dict, list, tuple):
            return ValueSet({compiled_value})

        tree_values = to_stub(compiled_value)
        if not tree_values:
            return ValueSet({compiled_value})
    else:
        module_node, tree_node, file_io, code_lines = result

        if module_context is None or module_context.tree_node != module_node:
            root_compiled_value = compiled_value.get_root_context().get_value()
            # TODO this __name__ might be wrong.
            name = root_compiled_value.py__name__()
            string_names = tuple(name.split('.'))
            module_value = ModuleValue(
                inference_state, module_node,
                file_io=file_io,
                string_names=string_names,
                code_lines=code_lines,
                is_package=root_compiled_value.is_package(),
            )
            if name is not None:
                inference_state.module_cache.add(string_names, ValueSet([module_value]))
            module_context = module_value.as_context()

        tree_values = ValueSet({module_context.create_value(tree_node)})
        if tree_node.type == 'classdef':
            if not compiled_value.is_class():
                # Is an instance, not a class.
                tree_values = tree_values.execute_with_values()

    return ValueSet(
        MixedObject(compiled_value, tree_value=tree_value)
        for tree_value in tree_values
    )

env_pyrecodes/lib/python3.9/site-packages/jedi/inference/compiled/mixed.py
NikolaBlagojevic/pyrecodes
import sys
import os
import inspect
import importlib
import warnings
from pathlib import Path
from zipfile import ZipFile
from zipimport import zipimporter, ZipImportError
from importlib.machinery import all_suffixes

from jedi.inference.compiled import access
from jedi import debug
from jedi import parser_utils
from jedi.file_io import KnownContentFileIO, ZipFileIO


def get_sys_path():
    return sys.path


def load_module(inference_state, **kwargs):
    return access.load_module(inference_state, **kwargs)


def get_compiled_method_return(inference_state, id, attribute, *args, **kwargs):
    handle = inference_state.compiled_subprocess.get_access_handle(id)
    return getattr(handle.access, attribute)(*args, **kwargs)


def create_simple_object(inference_state, obj):
    return access.create_access_path(inference_state, obj)


def get_module_info(inference_state, sys_path=None, full_name=None, **kwargs):
    """
    Returns Tuple[Union[NamespaceInfo, FileIO, None], Optional[bool]]
    """
    if sys_path is not None:
        sys.path, temp = sys_path, sys.path
    try:
        return _find_module(full_name=full_name, **kwargs)
    except ImportError:
        return None, None
    finally:
        if sys_path is not None:
            sys.path = temp


def get_builtin_module_names(inference_state):
    return sys.builtin_module_names


def _test_raise_error(inference_state, exception_type):
    """
    Raise an error to simulate certain problems for unit tests.
    """
    raise exception_type


def _test_print(inference_state, stderr=None, stdout=None):
    """
    Force some prints in the subprocesses. This exists for unit tests.
    """
    if stderr is not None:
        print(stderr, file=sys.stderr)
        sys.stderr.flush()
    if stdout is not None:
        print(stdout)
        sys.stdout.flush()


def _get_init_path(directory_path):
    """
    The __init__ file can be searched in a directory. If found return it, else
    None.
    """
    for suffix in all_suffixes():
        path = os.path.join(directory_path, '__init__' + suffix)
        if os.path.exists(path):
            return path
    return None


def safe_literal_eval(inference_state, value):
    return parser_utils.safe_literal_eval(value)


def iter_module_names(*args, **kwargs):
    return list(_iter_module_names(*args, **kwargs))


def _iter_module_names(inference_state, paths):
    # Python modules/packages
    for path in paths:
        try:
            dir_entries = ((entry.name, entry.is_dir()) for entry in os.scandir(path))
        except OSError:
            try:
                zip_import_info = zipimporter(path)
                # Unfortunately, there is no public way to access zipimporter's
                # private _files member. We therefore have to use a
                # custom function to iterate over the files.
                dir_entries = _zip_list_subdirectory(
                    zip_import_info.archive, zip_import_info.prefix)
            except ZipImportError:
                # The file might not exist or reading it might lead to an error.
                debug.warning("Not possible to list directory: %s", path)
                continue
        for name, is_dir in dir_entries:
            # First Namespaces then modules/stubs
            if is_dir:
                # pycache is obviously not an interesting namespace. Also the
                # name must be a valid identifier.
                if name != '__pycache__' and name.isidentifier():
                    yield name
            else:
                if name.endswith('.pyi'):  # Stub files
                    modname = name[:-4]
                else:
                    modname = inspect.getmodulename(name)

                if modname and '.' not in modname:
                    if modname != '__init__':
                        yield modname


def _find_module(string, path=None, full_name=None, is_global_search=True):
    """
    Provides information about a module.

    This function isolates the differences in importing libraries introduced with
    python 3.3 on; it gets a module name and optionally a path. It will return a
    tuple containin an open file for the module (if not builtin), the filename
    or the name of the module if it is a builtin one and a boolean indicating
    if the module is contained in a package.
    """
    spec = None
    loader = None

    for finder in sys.meta_path:
        if is_global_search and finder != importlib.machinery.PathFinder:
            p = None
        else:
            p = path
        try:
            find_spec = finder.find_spec
        except AttributeError:
            # These are old-school clases that still have a different API, just
            # ignore those.
            continue

        spec = find_spec(string, p)
        if spec is not None:
            if spec.origin == "frozen":
                continue

            loader = spec.loader

            if loader is None and not spec.has_location:
                # This is a namespace package.
                full_name = string if not path else full_name
                implicit_ns_info = ImplicitNSInfo(full_name, spec.submodule_search_locations._path)
                return implicit_ns_info, True
            break

    return _find_module_py33(string, path, loader)


def _find_module_py33(string, path=None, loader=None, full_name=None, is_global_search=True):
    loader = loader or importlib.machinery.PathFinder.find_module(string, path)

    if loader is None and path is None:  # Fallback to find builtins
        try:
            with warnings.catch_warnings(record=True):
                # Mute "DeprecationWarning: Use importlib.util.find_spec()
                # instead." While we should replace that in the future, it's
                # probably good to wait until we deprecate Python 3.3, since
                # it was added in Python 3.4 and find_loader hasn't been
                # removed in 3.6.
                loader = importlib.find_loader(string)
        except ValueError as e:
            # See #491. Importlib might raise a ValueError, to avoid this, we
            # just raise an ImportError to fix the issue.
            raise ImportError("Originally  " + repr(e))

    if loader is None:
        raise ImportError("Couldn't find a loader for {}".format(string))

    return _from_loader(loader, string)


def _from_loader(loader, string):
    try:
        is_package_method = loader.is_package
    except AttributeError:
        is_package = False
    else:
        is_package = is_package_method(string)
    try:
        get_filename = loader.get_filename
    except AttributeError:
        return None, is_package
    else:
        module_path = get_filename(string)

    # To avoid unicode and read bytes, "overwrite" loader.get_source if
    # possible.
    try:
        f = type(loader).get_source
    except AttributeError:
        raise ImportError("get_source was not defined on loader")

    if f is not importlib.machinery.SourceFileLoader.get_source:
        # Unfortunately we are reading unicode here, not bytes.
        # It seems hard to get bytes, because the zip importer
        # logic just unpacks the zip file and returns a file descriptor
        # that we cannot as easily access. Therefore we just read it as
        # a string in the cases where get_source was overwritten.
        code = loader.get_source(string)
    else:
        code = _get_source(loader, string)

    if code is None:
        return None, is_package
    if isinstance(loader, zipimporter):
        return ZipFileIO(module_path, code, Path(loader.archive)), is_package

    return KnownContentFileIO(module_path, code), is_package


def _get_source(loader, fullname):
    """
    This method is here as a replacement for SourceLoader.get_source. That
    method returns unicode, but we prefer bytes.
    """
    path = loader.get_filename(fullname)
    try:
        return loader.get_data(path)
    except OSError:
        raise ImportError('source not available through get_data()',
                          name=fullname)


def _zip_list_subdirectory(zip_path, zip_subdir_path):
    zip_file = ZipFile(zip_path)
    zip_subdir_path = Path(zip_subdir_path)
    zip_content_file_paths = zip_file.namelist()
    for raw_file_name in zip_content_file_paths:
        file_path = Path(raw_file_name)
        if file_path.parent == zip_subdir_path:
            file_path = file_path.relative_to(zip_subdir_path)
            yield file_path.name, raw_file_name.endswith("/")


class ImplicitNSInfo:
    """Stores information returned from an implicit namespace spec"""
    def __init__(self, name, paths):
        self.name = name
        self.paths = paths

env_pyrecodes/lib/python3.9/site-packages/jedi/inference/compiled/subprocess/functions.py
NikolaBlagojevic/pyrecodes
"""
We need to somehow work with the typing objects. Since the typing objects are
pretty bare we need to add all the Jedi customizations to make them work as
values.

This file deals with all the typing.py cases.
"""
import itertools

from jedi import debug
from jedi.inference.compiled import builtin_from_name, create_simple_object
from jedi.inference.base_value import ValueSet, NO_VALUES, Value, \
    LazyValueWrapper, ValueWrapper
from jedi.inference.lazy_value import LazyKnownValues
from jedi.inference.arguments import repack_with_argument_clinic
from jedi.inference.filters import FilterWrapper
from jedi.inference.names import NameWrapper, ValueName
from jedi.inference.value.klass import ClassMixin
from jedi.inference.gradual.base import BaseTypingValue, \
    BaseTypingClassWithGenerics, BaseTypingInstance
from jedi.inference.gradual.type_var import TypeVarClass
from jedi.inference.gradual.generics import LazyGenericManager, TupleGenericManager

_PROXY_CLASS_TYPES = 'Tuple Generic Protocol Callable Type'.split()
_TYPE_ALIAS_TYPES = {
    'List': 'builtins.list',
    'Dict': 'builtins.dict',
    'Set': 'builtins.set',
    'FrozenSet': 'builtins.frozenset',
    'ChainMap': 'collections.ChainMap',
    'Counter': 'collections.Counter',
    'DefaultDict': 'collections.defaultdict',
    'Deque': 'collections.deque',
}
_PROXY_TYPES = 'Optional Union ClassVar'.split()


class TypingModuleName(NameWrapper):
    def infer(self):
        return ValueSet(self._remap())

    def _remap(self):
        name = self.string_name
        inference_state = self.parent_context.inference_state
        try:
            actual = _TYPE_ALIAS_TYPES[name]
        except KeyError:
            pass
        else:
            yield TypeAlias.create_cached(
                inference_state, self.parent_context, self.tree_name, actual)
            return

        if name in _PROXY_CLASS_TYPES:
            yield ProxyTypingClassValue.create_cached(
                inference_state, self.parent_context, self.tree_name)
        elif name in _PROXY_TYPES:
            yield ProxyTypingValue.create_cached(
                inference_state, self.parent_context, self.tree_name)
        elif name == 'runtime':
            # We don't want anything here, not sure what this function is
            # supposed to do, since it just appears in the stubs and shouldn't
            # have any effects there (because it's never executed).
            return
        elif name == 'TypeVar':
            cls, = self._wrapped_name.infer()
            yield TypeVarClass.create_cached(inference_state, cls)
        elif name == 'Any':
            yield AnyClass.create_cached(
                inference_state, self.parent_context, self.tree_name)
        elif name == 'TYPE_CHECKING':
            # This is needed for e.g. imports that are only available for type
            # checking or are in cycles. The user can then check this variable.
            yield builtin_from_name(inference_state, 'True')
        elif name == 'overload':
            yield OverloadFunction.create_cached(
                inference_state, self.parent_context, self.tree_name)
        elif name == 'NewType':
            v, = self._wrapped_name.infer()
            yield NewTypeFunction.create_cached(inference_state, v)
        elif name == 'cast':
            cast_fn, = self._wrapped_name.infer()
            yield CastFunction.create_cached(inference_state, cast_fn)
        elif name == 'TypedDict':
            # TODO doesn't even exist in typeshed/typing.py, yet. But will be
            # added soon.
            yield TypedDictClass.create_cached(
                inference_state, self.parent_context, self.tree_name)
        else:
            # Not necessary, as long as we are not doing type checking:
            # no_type_check & no_type_check_decorator
            # Everything else shouldn't be relevant...
            yield from self._wrapped_name.infer()


class TypingModuleFilterWrapper(FilterWrapper):
    name_wrapper_class = TypingModuleName


class ProxyWithGenerics(BaseTypingClassWithGenerics):
    def execute_annotation(self):
        string_name = self._tree_name.value

        if string_name == 'Union':
            # This is kind of a special case, because we have Unions (in Jedi
            # ValueSets).
            return self.gather_annotation_classes().execute_annotation()
        elif string_name == 'Optional':
            # Optional is basically just saying it's either None or the actual
            # type.
            return self.gather_annotation_classes().execute_annotation() \
                | ValueSet([builtin_from_name(self.inference_state, 'None')])
        elif string_name == 'Type':
            # The type is actually already given in the index_value
            return self._generics_manager[0]
        elif string_name == 'ClassVar':
            # For now don't do anything here, ClassVars are always used.
            return self._generics_manager[0].execute_annotation()

        mapped = {
            'Tuple': Tuple,
            'Generic': Generic,
            'Protocol': Protocol,
            'Callable': Callable,
        }
        cls = mapped[string_name]
        return ValueSet([cls(
            self.parent_context,
            self,
            self._tree_name,
            generics_manager=self._generics_manager,
        )])

    def gather_annotation_classes(self):
        return ValueSet.from_sets(self._generics_manager.to_tuple())

    def _create_instance_with_generics(self, generics_manager):
        return ProxyWithGenerics(
            self.parent_context,
            self._tree_name,
            generics_manager
        )

    def infer_type_vars(self, value_set):
        annotation_generics = self.get_generics()

        if not annotation_generics:
            return {}

        annotation_name = self.py__name__()
        if annotation_name == 'Optional':
            # Optional[T] is equivalent to Union[T, None]. In Jedi unions
            # are represented by members within a ValueSet, so we extract
            # the T from the Optional[T] by removing the None value.
            none = builtin_from_name(self.inference_state, 'None')
            return annotation_generics[0].infer_type_vars(
                value_set.filter(lambda x: x != none),
            )

        return {}


class ProxyTypingValue(BaseTypingValue):
    index_class = ProxyWithGenerics

    def with_generics(self, generics_tuple):
        return self.index_class.create_cached(
            self.inference_state,
            self.parent_context,
            self._tree_name,
            generics_manager=TupleGenericManager(generics_tuple)
        )

    def py__getitem__(self, index_value_set, contextualized_node):
        return ValueSet(
            self.index_class.create_cached(
                self.inference_state,
                self.parent_context,
                self._tree_name,
                generics_manager=LazyGenericManager(
                    context_of_index=contextualized_node.context,
                    index_value=index_value,
                )
            ) for index_value in index_value_set
        )


class _TypingClassMixin(ClassMixin):
    def py__bases__(self):
        return [LazyKnownValues(
            self.inference_state.builtins_module.py__getattribute__('object')
        )]

    def get_metaclasses(self):
        return []

    @property
    def name(self):
        return ValueName(self, self._tree_name)


class TypingClassWithGenerics(ProxyWithGenerics, _TypingClassMixin):
    def infer_type_vars(self, value_set):
        type_var_dict = {}
        annotation_generics = self.get_generics()

        if not annotation_generics:
            return type_var_dict

        annotation_name = self.py__name__()
        if annotation_name == 'Type':
            return annotation_generics[0].infer_type_vars(
                # This is basically a trick to avoid extra code: We execute the
                # incoming classes to be able to use the normal code for type
                # var inference.
                value_set.execute_annotation(),
            )

        elif annotation_name == 'Callable':
            if len(annotation_generics) == 2:
                return annotation_generics[1].infer_type_vars(
                    value_set.execute_annotation(),
                )

        elif annotation_name == 'Tuple':
            tuple_annotation, = self.execute_annotation()
            return tuple_annotation.infer_type_vars(value_set)

        return type_var_dict

    def _create_instance_with_generics(self, generics_manager):
        return TypingClassWithGenerics(
            self.parent_context,
            self._tree_name,
            generics_manager
        )


class ProxyTypingClassValue(ProxyTypingValue, _TypingClassMixin):
    index_class = TypingClassWithGenerics


class TypeAlias(LazyValueWrapper):
    def __init__(self, parent_context, origin_tree_name, actual):
        self.inference_state = parent_context.inference_state
        self.parent_context = parent_context
        self._origin_tree_name = origin_tree_name
        self._actual = actual  # e.g. builtins.list

    @property
    def name(self):
        return ValueName(self, self._origin_tree_name)

    def py__name__(self):
        return self.name.string_name

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, self._actual)

    def _get_wrapped_value(self):
        module_name, class_name = self._actual.split('.')

        # TODO use inference_state.import_module?
        from jedi.inference.imports import Importer
        module, = Importer(
            self.inference_state, [module_name], self.inference_state.builtins_module
        ).follow()
        classes = module.py__getattribute__(class_name)
        # There should only be one, because it's code that we control.
        assert len(classes) == 1, classes
        cls = next(iter(classes))
        return cls

    def gather_annotation_classes(self):
        return ValueSet([self._get_wrapped_value()])

    def get_signatures(self):
        return []


class Callable(BaseTypingInstance):
    def py__call__(self, arguments):
        """
            def x() -> Callable[[Callable[..., _T]], _T]: ...
        """
        # The 0th index are the arguments.
        try:
            param_values = self._generics_manager[0]
            result_values = self._generics_manager[1]
        except IndexError:
            debug.warning('Callable[...] defined without two arguments')
            return NO_VALUES
        else:
            from jedi.inference.gradual.annotation import infer_return_for_callable
            return infer_return_for_callable(arguments, param_values, result_values)

    def py__get__(self, instance, class_value):
        return ValueSet([self])


class Tuple(BaseTypingInstance):
    def _is_homogenous(self):
        # To specify a variable-length tuple of homogeneous type, Tuple[T, ...]
        # is used.
        return self._generics_manager.is_homogenous_tuple()

    def py__simple_getitem__(self, index):
        if self._is_homogenous():
            return self._generics_manager.get_index_and_execute(0)
        else:
            if isinstance(index, int):
                return self._generics_manager.get_index_and_execute(index)

            debug.dbg('The getitem type on Tuple was %s' % index)
            return NO_VALUES

    def py__iter__(self, contextualized_node=None):
        if self._is_homogenous():
            yield LazyKnownValues(self._generics_manager.get_index_and_execute(0))
        else:
            for v in self._generics_manager.to_tuple():
                yield LazyKnownValues(v.execute_annotation())

    def py__getitem__(self, index_value_set, contextualized_node):
        if self._is_homogenous():
            return self._generics_manager.get_index_and_execute(0)

        return ValueSet.from_sets(
            self._generics_manager.to_tuple()
        ).execute_annotation()

    def _get_wrapped_value(self):
        tuple_, = self.inference_state.builtins_module \
            .py__getattribute__('tuple').execute_annotation()
        return tuple_

    @property
    def name(self):
        return self._wrapped_value.name

    def infer_type_vars(self, value_set):
        # Circular
        from jedi.inference.gradual.annotation import merge_pairwise_generics, merge_type_var_dicts

        value_set = value_set.filter(
            lambda x: x.py__name__().lower() == 'tuple',
        )

        if self._is_homogenous():
            # The parameter annotation is of the form `Tuple[T, ...]`,
            # so we treat the incoming tuple like a iterable sequence
            # rather than a positional container of elements.
            return self._class_value.get_generics()[0].infer_type_vars(
                value_set.merge_types_of_iterate(),
            )

        else:
            # The parameter annotation has only explicit type parameters
            # (e.g: `Tuple[T]`, `Tuple[T, U]`, `Tuple[T, U, V]`, etc.) so we
            # treat the incoming values as needing to match the annotation
            # exactly, just as we would for non-tuple annotations.

            type_var_dict = {}
            for element in value_set:
                try:
                    method = element.get_annotated_class_object
                except AttributeError:
                    # This might still happen, because the tuple name matching
                    # above is not 100% correct, so just catch the remaining
                    # cases here.
                    continue

                py_class = method()
                merge_type_var_dicts(
                    type_var_dict,
                    merge_pairwise_generics(self._class_value, py_class),
                )

            return type_var_dict


class Generic(BaseTypingInstance):
    pass


class Protocol(BaseTypingInstance):
    pass


class AnyClass(BaseTypingValue):
    def execute_annotation(self):
        debug.warning('Used Any - returned no results')
        return NO_VALUES


class OverloadFunction(BaseTypingValue):
    @repack_with_argument_clinic('func, /')
    def py__call__(self, func_value_set):
        # Just pass arguments through.
        return func_value_set


class NewTypeFunction(ValueWrapper):
    def py__call__(self, arguments):
        ordered_args = arguments.unpack()
        next(ordered_args, (None, None))
        _, second_arg = next(ordered_args, (None, None))
        if second_arg is None:
            return NO_VALUES
        return ValueSet(
            NewType(
                self.inference_state,
                contextualized_node.context,
                contextualized_node.node,
                second_arg.infer(),
            ) for contextualized_node in arguments.get_calling_nodes())


class NewType(Value):
    def __init__(self, inference_state, parent_context, tree_node, type_value_set):
        super().__init__(inference_state, parent_context)
        self._type_value_set = type_value_set
        self.tree_node = tree_node

    def py__class__(self):
        c, = self._type_value_set.py__class__()
        return c

    def py__call__(self, arguments):
        return self._type_value_set.execute_annotation()

    @property
    def name(self):
        from jedi.inference.compiled.value import CompiledValueName
        return CompiledValueName(self, 'NewType')

    def __repr__(self) -> str:
        return '<NewType: %s>%s' % (self.tree_node, self._type_value_set)


class CastFunction(ValueWrapper):
    @repack_with_argument_clinic('type, object, /')
    def py__call__(self, type_value_set, object_value_set):
        return type_value_set.execute_annotation()


class TypedDictClass(BaseTypingValue):
    """
    This class has no responsibilities and is just here to make sure that typed
    dicts can be identified.
    """


class TypedDict(LazyValueWrapper):
    """Represents the instance version of ``TypedDictClass``."""
    def __init__(self, definition_class):
        self.inference_state = definition_class.inference_state
        self.parent_context = definition_class.parent_context
        self.tree_node = definition_class.tree_node
        self._definition_class = definition_class

    @property
    def name(self):
        return ValueName(self, self.tree_node.name)

    def py__simple_getitem__(self, index):
        if isinstance(index, str):
            return ValueSet.from_sets(
                name.infer()
                for filter in self._definition_class.get_filters(is_instance=True)
                for name in filter.get(index)
            )
        return NO_VALUES

    def get_key_values(self):
        filtered_values = itertools.chain.from_iterable((
            f.values()
            for f in self._definition_class.get_filters(is_instance=True)
        ))
        return ValueSet({
            create_simple_object(self.inference_state, v.string_name)
            for v in filtered_values
        })

    def _get_wrapped_value(self):
        d, = self.inference_state.builtins_module.py__getattribute__('dict')
        result, = d.execute_with_values()
        return result

env_pyrecodes/lib/python3.9/site-packages/jedi/inference/gradual/typing.py
NikolaBlagojevic/pyrecodes
from pathlib import Path

from parso.tree import search_ancestor
from jedi.inference.cache import inference_state_method_cache
from jedi.inference.imports import goto_import, load_module_from_path
from jedi.inference.filters import ParserTreeFilter
from jedi.inference.base_value import NO_VALUES, ValueSet
from jedi.inference.helpers import infer_call_of_leaf

_PYTEST_FIXTURE_MODULES = [
    ('_pytest', 'monkeypatch'),
    ('_pytest', 'capture'),
    ('_pytest', 'logging'),
    ('_pytest', 'tmpdir'),
    ('_pytest', 'pytester'),
]


def execute(callback):
    def wrapper(value, arguments):
        # This might not be necessary anymore in pytest 4/5, definitely needed
        # for pytest 3.
        if value.py__name__() == 'fixture' \
                and value.parent_context.py__name__() == '_pytest.fixtures':
            return NO_VALUES

        return callback(value, arguments)
    return wrapper


def infer_anonymous_param(func):
    def get_returns(value):
        if value.tree_node.annotation is not None:
            result = value.execute_with_values()
            if any(v.name.get_qualified_names(include_module_names=True)
                   == ('typing', 'Generator')
                   for v in result):
                return ValueSet.from_sets(
                    v.py__getattribute__('__next__').execute_annotation()
                    for v in result
                )
            return result

        # In pytest we need to differentiate between generators and normal
        # returns.
        # Parameters still need to be anonymous, .as_context() ensures that.
        function_context = value.as_context()
        if function_context.is_generator():
            return function_context.merge_yield_values()
        else:
            return function_context.get_return_values()

    def wrapper(param_name):
        # parameters with an annotation do not need special handling
        if param_name.annotation_node:
            return func(param_name)
        is_pytest_param, param_name_is_function_name = \
            _is_a_pytest_param_and_inherited(param_name)
        if is_pytest_param:
            module = param_name.get_root_context()
            fixtures = _goto_pytest_fixture(
                module,
                param_name.string_name,
                # This skips the current module, because we are basically
                # inheriting a fixture from somewhere else.
                skip_own_module=param_name_is_function_name,
            )
            if fixtures:
                return ValueSet.from_sets(
                    get_returns(value)
                    for fixture in fixtures
                    for value in fixture.infer()
                )
        return func(param_name)
    return wrapper


def goto_anonymous_param(func):
    def wrapper(param_name):
        is_pytest_param, param_name_is_function_name = \
            _is_a_pytest_param_and_inherited(param_name)
        if is_pytest_param:
            names = _goto_pytest_fixture(
                param_name.get_root_context(),
                param_name.string_name,
                skip_own_module=param_name_is_function_name,
            )
            if names:
                return names
        return func(param_name)
    return wrapper


def complete_param_names(func):
    def wrapper(context, func_name, decorator_nodes):
        module_context = context.get_root_context()
        if _is_pytest_func(func_name, decorator_nodes):
            names = []
            for module_context in _iter_pytest_modules(module_context):
                names += FixtureFilter(module_context).values()
            if names:
                return names
        return func(context, func_name, decorator_nodes)
    return wrapper


def _goto_pytest_fixture(module_context, name, skip_own_module):
    for module_context in _iter_pytest_modules(module_context, skip_own_module=skip_own_module):
        names = FixtureFilter(module_context).get(name)
        if names:
            return names


def _is_a_pytest_param_and_inherited(param_name):
    """
    Pytest params are either in a `test_*` function or have a pytest fixture
    with the decorator @pytest.fixture.

    This is a heuristic and will work in most cases.
    """
    funcdef = search_ancestor(param_name.tree_name, 'funcdef')
    if funcdef is None:  # A lambda
        return False, False
    decorators = funcdef.get_decorators()
    return _is_pytest_func(funcdef.name.value, decorators), \
        funcdef.name.value == param_name.string_name


def _is_pytest_func(func_name, decorator_nodes):
    return func_name.startswith('test') \
        or any('fixture' in n.get_code() for n in decorator_nodes)


def _find_pytest_plugin_modules():
    """
    Finds pytest plugin modules hooked by setuptools entry points

    See https://docs.pytest.org/en/stable/how-to/writing_plugins.html#setuptools-entry-points
    """
    from pkg_resources import iter_entry_points

    return [ep.module_name.split(".") for ep in iter_entry_points(group="pytest11")]


@inference_state_method_cache()
def _iter_pytest_modules(module_context, skip_own_module=False):
    if not skip_own_module:
        yield module_context

    file_io = module_context.get_value().file_io
    if file_io is not None:
        folder = file_io.get_parent_folder()
        sys_path = module_context.inference_state.get_sys_path()

        # prevent an infinite loop when reaching the root of the current drive
        last_folder = None

        while any(folder.path.startswith(p) for p in sys_path):
            file_io = folder.get_file_io('conftest.py')
            if Path(file_io.path) != module_context.py__file__():
                try:
                    m = load_module_from_path(module_context.inference_state, file_io)
                    yield m.as_context()
                except FileNotFoundError:
                    pass
            folder = folder.get_parent_folder()

            # prevent an infinite for loop if the same parent folder is return twice
            if last_folder is not None and folder.path == last_folder.path:
                break
            last_folder = folder  # keep track of the last found parent name

    for names in _PYTEST_FIXTURE_MODULES + _find_pytest_plugin_modules():
        for module_value in module_context.inference_state.import_module(names):
            yield module_value.as_context()


class FixtureFilter(ParserTreeFilter):
    def _filter(self, names):
        for name in super()._filter(names):
            # look for fixture definitions of imported names
            if name.parent.type == "import_from":
                imported_names = goto_import(self.parent_context, name)
                if any(
                    self._is_fixture(iname.parent_context, iname.tree_name)
                    for iname in imported_names
                    # discard imports of whole modules, that have no tree_name
                    if iname.tree_name
                ):
                    yield name

            elif self._is_fixture(self.parent_context, name):
                yield name

    def _is_fixture(self, context, name):
        funcdef = name.parent
        # Class fixtures are not supported
        if funcdef.type != "funcdef":
            return False
        decorated = funcdef.parent
        if decorated.type != "decorated":
            return False
        decorators = decorated.children[0]
        if decorators.type == 'decorators':
            decorators = decorators.children
        else:
            decorators = [decorators]
        for decorator in decorators:
            dotted_name = decorator.children[1]
            # A heuristic, this makes it faster.
            if 'fixture' in dotted_name.get_code():
                if dotted_name.type == 'atom_expr':
                    # Since Python3.9 a decorator does not have dotted names
                    # anymore.
                    last_trailer = dotted_name.children[-1]
                    last_leaf = last_trailer.get_last_leaf()
                    if last_leaf == ')':
                        values = infer_call_of_leaf(
                            context, last_leaf, cut_own_trailer=True
                        )
                    else:
                        values = context.infer_node(dotted_name)
                else:
                    values = context.infer_node(dotted_name)
                for value in values:
                    if value.name.get_qualified_names(include_module_names=True) \
                            == ('_pytest', 'fixtures', 'fixture'):
                        return True
        return False

env_pyrecodes/lib/python3.9/site-packages/jedi/plugins/pytest.py
NikolaBlagojevic/pyrecodes
"""
Implementations of standard library functions, because it's not possible to
understand them with Jedi.

To add a new implementation, create a function and add it to the
``_implemented`` dict at the bottom of this module.

Note that this module exists only to implement very specific functionality in
the standard library. The usual way to understand the standard library is the
compiled module that returns the types for C-builtins.
"""
import parso
import os
from inspect import Parameter

from jedi import debug
from jedi.inference.utils import safe_property
from jedi.inference.helpers import get_str_or_none
from jedi.inference.arguments import iterate_argument_clinic, ParamIssue, \
    repack_with_argument_clinic, AbstractArguments, TreeArgumentsWrapper
from jedi.inference import analysis
from jedi.inference import compiled
from jedi.inference.value.instance import \
    AnonymousMethodExecutionContext, MethodExecutionContext
from jedi.inference.base_value import ContextualizedNode, \
    NO_VALUES, ValueSet, ValueWrapper, LazyValueWrapper
from jedi.inference.value import ClassValue, ModuleValue
from jedi.inference.value.klass import ClassMixin
from jedi.inference.value.function import FunctionMixin
from jedi.inference.value import iterable
from jedi.inference.lazy_value import LazyTreeValue, LazyKnownValue, \
    LazyKnownValues
from jedi.inference.names import ValueName, BaseTreeParamName
from jedi.inference.filters import AttributeOverwrite, publish_method, \
    ParserTreeFilter, DictFilter
from jedi.inference.signature import AbstractSignature, SignatureWrapper


# Copied from Python 3.6's stdlib.
_NAMEDTUPLE_CLASS_TEMPLATE = """\
_property = property
_tuple = tuple
from operator import itemgetter as _itemgetter
from collections import OrderedDict

class {typename}(tuple):
    __slots__ = ()

    _fields = {field_names!r}

    def __new__(_cls, {arg_list}):
        'Create new instance of {typename}({arg_list})'
        return _tuple.__new__(_cls, ({arg_list}))

    @classmethod
    def _make(cls, iterable, new=tuple.__new__, len=len):
        'Make a new {typename} object from a sequence or iterable'
        result = new(cls, iterable)
        if len(result) != {num_fields:d}:
            raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result))
        return result

    def _replace(_self, **kwds):
        'Return a new {typename} object replacing specified fields with new values'
        result = _self._make(map(kwds.pop, {field_names!r}, _self))
        if kwds:
            raise ValueError('Got unexpected field names: %r' % list(kwds))
        return result

    def __repr__(self):
        'Return a nicely formatted representation string'
        return self.__class__.__name__ + '({repr_fmt})' % self

    def _asdict(self):
        'Return a new OrderedDict which maps field names to their values.'
        return OrderedDict(zip(self._fields, self))

    def __getnewargs__(self):
        'Return self as a plain tuple.  Used by copy and pickle.'
        return tuple(self)

    # These methods were added by Jedi.
    # __new__ doesn't really work with Jedi. So adding this to nametuples seems
    # like the easiest way.
    def __init__(self, {arg_list}):
        'A helper function for namedtuple.'
        self.__iterable = ({arg_list})

    def __iter__(self):
        for i in self.__iterable:
            yield i

    def __getitem__(self, y):
        return self.__iterable[y]

{field_defs}
"""

_NAMEDTUPLE_FIELD_TEMPLATE = '''\
    {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}')
'''


def execute(callback):
    def wrapper(value, arguments):
        def call():
            return callback(value, arguments=arguments)

        try:
            obj_name = value.name.string_name
        except AttributeError:
            pass
        else:
            p = value.parent_context
            if p is not None and p.is_builtins_module():
                module_name = 'builtins'
            elif p is not None and p.is_module():
                module_name = p.py__name__()
            else:
                return call()

            if value.is_bound_method() or value.is_instance():
                # value can be an instance for example if it is a partial
                # object.
                return call()

            # for now we just support builtin functions.
            try:
                func = _implemented[module_name][obj_name]
            except KeyError:
                pass
            else:
                return func(value, arguments=arguments, callback=call)
        return call()

    return wrapper


def _follow_param(inference_state, arguments, index):
    try:
        key, lazy_value = list(arguments.unpack())[index]
    except IndexError:
        return NO_VALUES
    else:
        return lazy_value.infer()


def argument_clinic(clinic_string, want_value=False, want_context=False,
                    want_arguments=False, want_inference_state=False,
                    want_callback=False):
    """
    Works like Argument Clinic (PEP 436), to validate function params.
    """

    def f(func):
        def wrapper(value, arguments, callback):
            try:
                args = tuple(iterate_argument_clinic(
                    value.inference_state, arguments, clinic_string))
            except ParamIssue:
                return NO_VALUES

            debug.dbg('builtin start %s' % value, color='MAGENTA')
            kwargs = {}
            if want_context:
                kwargs['context'] = arguments.context
            if want_value:
                kwargs['value'] = value
            if want_inference_state:
                kwargs['inference_state'] = value.inference_state
            if want_arguments:
                kwargs['arguments'] = arguments
            if want_callback:
                kwargs['callback'] = callback
            result = func(*args, **kwargs)
            debug.dbg('builtin end: %s', result, color='MAGENTA')
            return result

        return wrapper
    return f


@argument_clinic('iterator[, default], /', want_inference_state=True)
def builtins_next(iterators, defaults, inference_state):
    # TODO theoretically we have to check here if something is an iterator.
    # That is probably done by checking if it's not a class.
    return defaults | iterators.py__getattribute__('__next__').execute_with_values()


@argument_clinic('iterator[, default], /')
def builtins_iter(iterators_or_callables, defaults):
    # TODO implement this if it's a callable.
    return iterators_or_callables.py__getattribute__('__iter__').execute_with_values()


@argument_clinic('object, name[, default], /')
def builtins_getattr(objects, names, defaults=None):
    # follow the first param
    for value in objects:
        for name in names:
            string = get_str_or_none(name)
            if string is None:
                debug.warning('getattr called without str')
                continue
            else:
                return value.py__getattribute__(string)
    return NO_VALUES


@argument_clinic('object[, bases, dict], /')
def builtins_type(objects, bases, dicts):
    if bases or dicts:
        # It's a type creation... maybe someday...
        return NO_VALUES
    else:
        return objects.py__class__()


class SuperInstance(LazyValueWrapper):
    """To be used like the object ``super`` returns."""
    def __init__(self, inference_state, instance):
        self.inference_state = inference_state
        self._instance = instance  # Corresponds to super().__self__

    def _get_bases(self):
        return self._instance.py__class__().py__bases__()

    def _get_wrapped_value(self):
        objs = self._get_bases()[0].infer().execute_with_values()
        if not objs:
            # This is just a fallback and will only be used, if it's not
            # possible to find a class
            return self._instance
        return next(iter(objs))

    def get_filters(self, origin_scope=None):
        for b in self._get_bases():
            for value in b.infer().execute_with_values():
                for f in value.get_filters():
                    yield f


@argument_clinic('[type[, value]], /', want_context=True)
def builtins_super(types, objects, context):
    instance = None
    if isinstance(context, AnonymousMethodExecutionContext):
        instance = context.instance
    elif isinstance(context, MethodExecutionContext):
        instance = context.instance
    if instance is None:
        return NO_VALUES
    return ValueSet({SuperInstance(instance.inference_state, instance)})


class ReversedObject(AttributeOverwrite):
    def __init__(self, reversed_obj, iter_list):
        super().__init__(reversed_obj)
        self._iter_list = iter_list

    def py__iter__(self, contextualized_node=None):
        return self._iter_list

    @publish_method('__next__')
    def _next(self, arguments):
        return ValueSet.from_sets(
            lazy_value.infer() for lazy_value in self._iter_list
        )


@argument_clinic('sequence, /', want_value=True, want_arguments=True)
def builtins_reversed(sequences, value, arguments):
    # While we could do without this variable (just by using sequences), we
    # want static analysis to work well. Therefore we need to generated the
    # values again.
    key, lazy_value = next(arguments.unpack())
    cn = None
    if isinstance(lazy_value, LazyTreeValue):
        cn = ContextualizedNode(lazy_value.context, lazy_value.data)
    ordered = list(sequences.iterate(cn))

    # Repack iterator values and then run it the normal way. This is
    # necessary, because `reversed` is a function and autocompletion
    # would fail in certain cases like `reversed(x).__iter__` if we
    # just returned the result directly.
    seq, = value.inference_state.typing_module.py__getattribute__('Iterator').execute_with_values()
    return ValueSet([ReversedObject(seq, list(reversed(ordered)))])


@argument_clinic('value, type, /', want_arguments=True, want_inference_state=True)
def builtins_isinstance(objects, types, arguments, inference_state):
    bool_results = set()
    for o in objects:
        cls = o.py__class__()
        try:
            cls.py__bases__
        except AttributeError:
            # This is temporary. Everything should have a class attribute in
            # Python?! Maybe we'll leave it here, because some numpy objects or
            # whatever might not.
            bool_results = set([True, False])
            break

        mro = list(cls.py__mro__())

        for cls_or_tup in types:
            if cls_or_tup.is_class():
                bool_results.add(cls_or_tup in mro)
            elif cls_or_tup.name.string_name == 'tuple' \
                    and cls_or_tup.get_root_context().is_builtins_module():
                # Check for tuples.
                classes = ValueSet.from_sets(
                    lazy_value.infer()
                    for lazy_value in cls_or_tup.iterate()
                )
                bool_results.add(any(cls in mro for cls in classes))
            else:
                _, lazy_value = list(arguments.unpack())[1]
                if isinstance(lazy_value, LazyTreeValue):
                    node = lazy_value.data
                    message = 'TypeError: isinstance() arg 2 must be a ' \
                              'class, type, or tuple of classes and types, ' \
                              'not %s.' % cls_or_tup
                    analysis.add(lazy_value.context, 'type-error-isinstance', node, message)

    return ValueSet(
        compiled.builtin_from_name(inference_state, str(b))
        for b in bool_results
    )


class StaticMethodObject(ValueWrapper):
    def py__get__(self, instance, class_value):
        return ValueSet([self._wrapped_value])


@argument_clinic('sequence, /')
def builtins_staticmethod(functions):
    return ValueSet(StaticMethodObject(f) for f in functions)


class ClassMethodObject(ValueWrapper):
    def __init__(self, class_method_obj, function):
        super().__init__(class_method_obj)
        self._function = function

    def py__get__(self, instance, class_value):
        return ValueSet([
            ClassMethodGet(__get__, class_value, self._function)
            for __get__ in self._wrapped_value.py__getattribute__('__get__')
        ])


class ClassMethodGet(ValueWrapper):
    def __init__(self, get_method, klass, function):
        super().__init__(get_method)
        self._class = klass
        self._function = function

    def get_signatures(self):
        return [sig.bind(self._function) for sig in self._function.get_signatures()]

    def py__call__(self, arguments):
        return self._function.execute(ClassMethodArguments(self._class, arguments))


class ClassMethodArguments(TreeArgumentsWrapper):
    def __init__(self, klass, arguments):
        super().__init__(arguments)
        self._class = klass

    def unpack(self, func=None):
        yield None, LazyKnownValue(self._class)
        for values in self._wrapped_arguments.unpack(func):
            yield values


@argument_clinic('sequence, /', want_value=True, want_arguments=True)
def builtins_classmethod(functions, value, arguments):
    return ValueSet(
        ClassMethodObject(class_method_object, function)
        for class_method_object in value.py__call__(arguments=arguments)
        for function in functions
    )


class PropertyObject(AttributeOverwrite, ValueWrapper):
    api_type = 'property'

    def __init__(self, property_obj, function):
        super().__init__(property_obj)
        self._function = function

    def py__get__(self, instance, class_value):
        if instance is None:
            return ValueSet([self])
        return self._function.execute_with_values(instance)

    @publish_method('deleter')
    @publish_method('getter')
    @publish_method('setter')
    def _return_self(self, arguments):
        return ValueSet({self})


@argument_clinic('func, /', want_callback=True)
def builtins_property(functions, callback):
    return ValueSet(
        PropertyObject(property_value, function)
        for property_value in callback()
        for function in functions
    )


def collections_namedtuple(value, arguments, callback):
    """
    Implementation of the namedtuple function.

    This has to be done by processing the namedtuple class template and
    inferring the result.

    """
    inference_state = value.inference_state

    # Process arguments
    name = 'jedi_unknown_namedtuple'
    for c in _follow_param(inference_state, arguments, 0):
        x = get_str_or_none(c)
        if x is not None:
            name = x
            break

    # TODO here we only use one of the types, we should use all.
    param_values = _follow_param(inference_state, arguments, 1)
    if not param_values:
        return NO_VALUES
    _fields = list(param_values)[0]
    string = get_str_or_none(_fields)
    if string is not None:
        fields = string.replace(',', ' ').split()
    elif isinstance(_fields, iterable.Sequence):
        fields = [
            get_str_or_none(v)
            for lazy_value in _fields.py__iter__()
            for v in lazy_value.infer()
        ]
        fields = [f for f in fields if f is not None]
    else:
        return NO_VALUES

    # Build source code
    code = _NAMEDTUPLE_CLASS_TEMPLATE.format(
        typename=name,
        field_names=tuple(fields),
        num_fields=len(fields),
        arg_list=repr(tuple(fields)).replace("'", "")[1:-1],
        repr_fmt='',
        field_defs='\n'.join(_NAMEDTUPLE_FIELD_TEMPLATE.format(index=index, name=name)
                             for index, name in enumerate(fields))
    )

    # Parse source code
    module = inference_state.grammar.parse(code)
    generated_class = next(module.iter_classdefs())
    parent_context = ModuleValue(
        inference_state, module,
        code_lines=parso.split_lines(code, keepends=True),
    ).as_context()

    return ValueSet([ClassValue(inference_state, parent_context, generated_class)])


class PartialObject(ValueWrapper):
    def __init__(self, actual_value, arguments, instance=None):
        super().__init__(actual_value)
        self._arguments = arguments
        self._instance = instance

    def _get_functions(self, unpacked_arguments):
        key, lazy_value = next(unpacked_arguments, (None, None))
        if key is not None or lazy_value is None:
            debug.warning("Partial should have a proper function %s", self._arguments)
            return None
        return lazy_value.infer()

    def get_signatures(self):
        unpacked_arguments = self._arguments.unpack()
        funcs = self._get_functions(unpacked_arguments)
        if funcs is None:
            return []

        arg_count = 0
        if self._instance is not None:
            arg_count = 1
        keys = set()
        for key, _ in unpacked_arguments:
            if key is None:
                arg_count += 1
            else:
                keys.add(key)
        return [PartialSignature(s, arg_count, keys) for s in funcs.get_signatures()]

    def py__call__(self, arguments):
        funcs = self._get_functions(self._arguments.unpack())
        if funcs is None:
            return NO_VALUES

        return funcs.execute(
            MergedPartialArguments(self._arguments, arguments, self._instance)
        )

    def py__doc__(self):
        """
        In CPython partial does not replace the docstring. However we are still
        imitating it here, because we want this docstring to be worth something
        for the user.
        """
        callables = self._get_functions(self._arguments.unpack())
        if callables is None:
            return ''
        for callable_ in callables:
            return callable_.py__doc__()
        return ''

    def py__get__(self, instance, class_value):
        return ValueSet([self])


class PartialMethodObject(PartialObject):
    def py__get__(self, instance, class_value):
        if instance is None:
            return ValueSet([self])
        return ValueSet([PartialObject(self._wrapped_value, self._arguments, instance)])


class PartialSignature(SignatureWrapper):
    def __init__(self, wrapped_signature, skipped_arg_count, skipped_arg_set):
        super().__init__(wrapped_signature)
        self._skipped_arg_count = skipped_arg_count
        self._skipped_arg_set = skipped_arg_set

    def get_param_names(self, resolve_stars=False):
        names = self._wrapped_signature.get_param_names()[self._skipped_arg_count:]
        return [n for n in names if n.string_name not in self._skipped_arg_set]


class MergedPartialArguments(AbstractArguments):
    def __init__(self, partial_arguments, call_arguments, instance=None):
        self._partial_arguments = partial_arguments
        self._call_arguments = call_arguments
        self._instance = instance

    def unpack(self, funcdef=None):
        unpacked = self._partial_arguments.unpack(funcdef)
        # Ignore this one, it's the function. It was checked before that it's
        # there.
        next(unpacked, None)
        if self._instance is not None:
            yield None, LazyKnownValue(self._instance)
        for key_lazy_value in unpacked:
            yield key_lazy_value
        for key_lazy_value in self._call_arguments.unpack(funcdef):
            yield key_lazy_value


def functools_partial(value, arguments, callback):
    return ValueSet(
        PartialObject(instance, arguments)
        for instance in value.py__call__(arguments)
    )


def functools_partialmethod(value, arguments, callback):
    return ValueSet(
        PartialMethodObject(instance, arguments)
        for instance in value.py__call__(arguments)
    )


@argument_clinic('first, /')
def _return_first_param(firsts):
    return firsts


@argument_clinic('seq')
def _random_choice(sequences):
    return ValueSet.from_sets(
        lazy_value.infer()
        for sequence in sequences
        for lazy_value in sequence.py__iter__()
    )


def _dataclass(value, arguments, callback):
    for c in _follow_param(value.inference_state, arguments, 0):
        if c.is_class():
            return ValueSet([DataclassWrapper(c)])
        else:
            return ValueSet([value])
    return NO_VALUES


class DataclassWrapper(ValueWrapper, ClassMixin):
    def get_signatures(self):
        param_names = []
        for cls in reversed(list(self.py__mro__())):
            if isinstance(cls, DataclassWrapper):
                filter_ = cls.as_context().get_global_filter()
                # .values ordering is not guaranteed, at least not in
                # Python < 3.6, when dicts where not ordered, which is an
                # implementation detail anyway.
                for name in sorted(filter_.values(), key=lambda name: name.start_pos):
                    d = name.tree_name.get_definition()
                    annassign = d.children[1]
                    if d.type == 'expr_stmt' and annassign.type == 'annassign':
                        if len(annassign.children) < 4:
                            default = None
                        else:
                            default = annassign.children[3]
                        param_names.append(DataclassParamName(
                            parent_context=cls.parent_context,
                            tree_name=name.tree_name,
                            annotation_node=annassign.children[1],
                            default_node=default,
                        ))
        return [DataclassSignature(cls, param_names)]


class DataclassSignature(AbstractSignature):
    def __init__(self, value, param_names):
        super().__init__(value)
        self._param_names = param_names

    def get_param_names(self, resolve_stars=False):
        return self._param_names


class DataclassParamName(BaseTreeParamName):
    def __init__(self, parent_context, tree_name, annotation_node, default_node):
        super().__init__(parent_context, tree_name)
        self.annotation_node = annotation_node
        self.default_node = default_node

    def get_kind(self):
        return Parameter.POSITIONAL_OR_KEYWORD

    def infer(self):
        if self.annotation_node is None:
            return NO_VALUES
        else:
            return self.parent_context.infer_node(self.annotation_node)


class ItemGetterCallable(ValueWrapper):
    def __init__(self, instance, args_value_set):
        super().__init__(instance)
        self._args_value_set = args_value_set

    @repack_with_argument_clinic('item, /')
    def py__call__(self, item_value_set):
        value_set = NO_VALUES
        for args_value in self._args_value_set:
            lazy_values = list(args_value.py__iter__())
            if len(lazy_values) == 1:
                # TODO we need to add the contextualized value.
                value_set |= item_value_set.get_item(lazy_values[0].infer(), None)
            else:
                value_set |= ValueSet([iterable.FakeList(
                    self._wrapped_value.inference_state,
                    [
                        LazyKnownValues(item_value_set.get_item(lazy_value.infer(), None))
                        for lazy_value in lazy_values
                    ],
                )])
        return value_set


@argument_clinic('func, /')
def _functools_wraps(funcs):
    return ValueSet(WrapsCallable(func) for func in funcs)


class WrapsCallable(ValueWrapper):
    # XXX this is not the correct wrapped value, it should be a weird
    #     partials object, but it doesn't matter, because it's always used as a
    #     decorator anyway.
    @repack_with_argument_clinic('func, /')
    def py__call__(self, funcs):
        return ValueSet({Wrapped(func, self._wrapped_value) for func in funcs})


class Wrapped(ValueWrapper, FunctionMixin):
    def __init__(self, func, original_function):
        super().__init__(func)
        self._original_function = original_function

    @property
    def name(self):
        return self._original_function.name

    def get_signature_functions(self):
        return [self]


@argument_clinic('*args, /', want_value=True, want_arguments=True)
def _operator_itemgetter(args_value_set, value, arguments):
    return ValueSet([
        ItemGetterCallable(instance, args_value_set)
        for instance in value.py__call__(arguments)
    ])


def _create_string_input_function(func):
    @argument_clinic('string, /', want_value=True, want_arguments=True)
    def wrapper(strings, value, arguments):
        def iterate():
            for value in strings:
                s = get_str_or_none(value)
                if s is not None:
                    s = func(s)
                    yield compiled.create_simple_object(value.inference_state, s)
        values = ValueSet(iterate())
        if values:
            return values
        return value.py__call__(arguments)
    return wrapper


@argument_clinic('*args, /', want_callback=True)
def _os_path_join(args_set, callback):
    if len(args_set) == 1:
        string = ''
        sequence, = args_set
        is_first = True
        for lazy_value in sequence.py__iter__():
            string_values = lazy_value.infer()
            if len(string_values) != 1:
                break
            s = get_str_or_none(next(iter(string_values)))
            if s is None:
                break
            if not is_first:
                string += os.path.sep
            string += s
            is_first = False
        else:
            return ValueSet([compiled.create_simple_object(sequence.inference_state, string)])
    return callback()


_implemented = {
    'builtins': {
        'getattr': builtins_getattr,
        'type': builtins_type,
        'super': builtins_super,
        'reversed': builtins_reversed,
        'isinstance': builtins_isinstance,
        'next': builtins_next,
        'iter': builtins_iter,
        'staticmethod': builtins_staticmethod,
        'classmethod': builtins_classmethod,
        'property': builtins_property,
    },
    'copy': {
        'copy': _return_first_param,
        'deepcopy': _return_first_param,
    },
    'json': {
        'load': lambda value, arguments, callback: NO_VALUES,
        'loads': lambda value, arguments, callback: NO_VALUES,
    },
    'collections': {
        'namedtuple': collections_namedtuple,
    },
    'functools': {
        'partial': functools_partial,
        'partialmethod': functools_partialmethod,
        'wraps': _functools_wraps,
    },
    '_weakref': {
        'proxy': _return_first_param,
    },
    'random': {
        'choice': _random_choice,
    },
    'operator': {
        'itemgetter': _operator_itemgetter,
    },
    'abc': {
        # Not sure if this is necessary, but it's used a lot in typeshed and
        # it's for now easier to just pass the function.
        'abstractmethod': _return_first_param,
    },
    'typing': {
        # The _alias function just leads to some annoying type inference.
        # Therefore, just make it return nothing, which leads to the stubs
        # being used instead. This only matters for 3.7+.
        '_alias': lambda value, arguments, callback: NO_VALUES,
        # runtime_checkable doesn't really change anything and is just
        # adding logs for infering stuff, so we can safely ignore it.
        'runtime_checkable': lambda value, arguments, callback: NO_VALUES,
    },
    'dataclasses': {
        # For now this works at least better than Jedi trying to understand it.
        'dataclass': _dataclass
    },
    # attrs exposes declaration interface roughly compatible with dataclasses
    # via attrs.define, attrs.frozen and attrs.mutable
    # https://www.attrs.org/en/stable/names.html
    'attr': {
        'define': _dataclass,
    },
    'attrs': {
        'define': _dataclass,
    },
    'os.path': {
        'dirname': _create_string_input_function(os.path.dirname),
        'abspath': _create_string_input_function(os.path.abspath),
        'relpath': _create_string_input_function(os.path.relpath),
        'join': _os_path_join,
    }
}


def get_metaclass_filters(func):
    def wrapper(cls, metaclasses, is_instance):
        for metaclass in metaclasses:
            if metaclass.py__name__() == 'EnumMeta' \
                    and metaclass.get_root_context().py__name__() == 'enum':
                filter_ = ParserTreeFilter(parent_context=cls.as_context())
                return [DictFilter({
                    name.string_name: EnumInstance(cls, name).name
                    for name in filter_.values()
                })]
        return func(cls, metaclasses, is_instance)
    return wrapper


class EnumInstance(LazyValueWrapper):
    def __init__(self, cls, name):
        self.inference_state = cls.inference_state
        self._cls = cls  # Corresponds to super().__self__
        self._name = name
        self.tree_node = self._name.tree_name

    @safe_property
    def name(self):
        return ValueName(self, self._name.tree_name)

    def _get_wrapped_value(self):
        n = self._name.string_name
        if n.startswith('__') and n.endswith('__') or self._name.api_type == 'function':
            inferred = self._name.infer()
            if inferred:
                return next(iter(inferred))
            o, = self.inference_state.builtins_module.py__getattribute__('object')
            return o

        value, = self._cls.execute_with_values()
        return value

    def get_filters(self, origin_scope=None):
        yield DictFilter(dict(
            name=compiled.create_simple_object(self.inference_state, self._name.string_name).name,
            value=self._name,
        ))
        for f in self._get_wrapped_value().get_filters():
            yield f


def tree_name_to_values(func):
    def wrapper(inference_state, context, tree_name):
        if tree_name.value == 'sep' and context.is_module() and context.py__name__() == 'os.path':
            return ValueSet({
                compiled.create_simple_object(inference_state, os.path.sep),
            })
        return func(inference_state, context, tree_name)
    return wrapper

env_pyrecodes/lib/python3.9/site-packages/jedi/plugins/stdlib.py
NikolaBlagojevic/pyrecodes
"""
Creation and extension of validators, with implementations for existing drafts.
"""
from __future__ import annotations

from collections import deque
from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from operator import methodcaller
from urllib.parse import unquote, urldefrag, urljoin, urlsplit
from urllib.request import urlopen
from warnings import warn
import contextlib
import json
import reprlib
import warnings

from attrs import define, field, fields
from jsonschema_specifications import REGISTRY as SPECIFICATIONS
from rpds import HashTrieMap
import referencing.exceptions
import referencing.jsonschema

from jsonschema import (
    _format,
    _legacy_validators,
    _types,
    _typing,
    _utils,
    _validators,
    exceptions,
)
from jsonschema.protocols import Validator

_UNSET = _utils.Unset()

_VALIDATORS: dict[str, Validator] = {}
_META_SCHEMAS = _utils.URIDict()


def __getattr__(name):
    if name == "ErrorTree":
        warnings.warn(
            "Importing ErrorTree from jsonschema.validators is deprecated. "
            "Instead import it from jsonschema.exceptions.",
            DeprecationWarning,
            stacklevel=2,
        )
        from jsonschema.exceptions import ErrorTree
        return ErrorTree
    elif name == "validators":
        warnings.warn(
            "Accessing jsonschema.validators.validators is deprecated. "
            "Use jsonschema.validators.validator_for with a given schema.",
            DeprecationWarning,
            stacklevel=2,
        )
        return _VALIDATORS
    elif name == "meta_schemas":
        warnings.warn(
            "Accessing jsonschema.validators.meta_schemas is deprecated. "
            "Use jsonschema.validators.validator_for with a given schema.",
            DeprecationWarning,
            stacklevel=2,
        )
        return _META_SCHEMAS
    elif name == "RefResolver":
        warnings.warn(
            _RefResolver._DEPRECATION_MESSAGE,
            DeprecationWarning,
            stacklevel=2,
        )
        return _RefResolver
    raise AttributeError(f"module {__name__} has no attribute {name}")


def validates(version):
    """
    Register the decorated validator for a ``version`` of the specification.

    Registered validators and their meta schemas will be considered when
    parsing :kw:`$schema` keywords' URIs.

    Arguments:

        version (str):

            An identifier to use as the version's name

    Returns:

        collections.abc.Callable:

            a class decorator to decorate the validator with the version
    """

    def _validates(cls):
        _VALIDATORS[version] = cls
        meta_schema_id = cls.ID_OF(cls.META_SCHEMA)
        _META_SCHEMAS[meta_schema_id] = cls
        return cls
    return _validates


def _warn_for_remote_retrieve(uri: str):
    from urllib.request import Request, urlopen
    headers = {"User-Agent": "python-jsonschema (deprecated $ref resolution)"}
    request = Request(uri, headers=headers)
    with urlopen(request) as response:
        warnings.warn(
            "Automatically retrieving remote references can be a security "
            "vulnerability and is discouraged by the JSON Schema "
            "specifications. Relying on this behavior is deprecated "
            "and will shortly become an error. If you are sure you want to "
            "remotely retrieve your reference and that it is safe to do so, "
            "you can find instructions for doing so via referencing.Registry "
            "in the referencing documentation "
            "(https://referencing.readthedocs.org).",
            DeprecationWarning,
            stacklevel=9,  # Ha ha ha ha magic numbers :/
        )
        return referencing.Resource.from_contents(
            json.load(response),
            default_specification=referencing.jsonschema.DRAFT202012,
        )


_REMOTE_WARNING_REGISTRY = SPECIFICATIONS.combine(
    referencing.Registry(retrieve=_warn_for_remote_retrieve),  # type: ignore[call-arg]  # noqa: E501
)


def create(
    meta_schema: referencing.jsonschema.ObjectSchema,
    validators: (
        Mapping[str, _typing.SchemaKeywordValidator]
        | Iterable[tuple[str, _typing.SchemaKeywordValidator]]
    ) = (),
    version: str | None = None,
    type_checker: _types.TypeChecker = _types.draft202012_type_checker,
    format_checker: _format.FormatChecker = _format.draft202012_format_checker,
    id_of: _typing.id_of = referencing.jsonschema.DRAFT202012.id_of,
    applicable_validators: _typing.ApplicableValidators = methodcaller(
        "items",
    ),
):
    """
    Create a new validator class.

    Arguments:

        meta_schema:

            the meta schema for the new validator class

        validators:

            a mapping from names to callables, where each callable will
            validate the schema property with the given name.

            Each callable should take 4 arguments:

                1. a validator instance,
                2. the value of the property being validated within the
                   instance
                3. the instance
                4. the schema

        version:

            an identifier for the version that this validator class will
            validate. If provided, the returned validator class will
            have its ``__name__`` set to include the version, and also
            will have `jsonschema.validators.validates` automatically
            called for the given version.

        type_checker:

            a type checker, used when applying the :kw:`type` keyword.

            If unprovided, a `jsonschema.TypeChecker` will be created
            with a set of default types typical of JSON Schema drafts.

        format_checker:

            a format checker, used when applying the :kw:`format` keyword.

            If unprovided, a `jsonschema.FormatChecker` will be created
            with a set of default formats typical of JSON Schema drafts.

        id_of:

            A function that given a schema, returns its ID.

        applicable_validators:

            A function that, given a schema, returns the list of
            applicable schema keywords and associated values
            which will be used to validate the instance.
            This is mostly used to support pre-draft 7 versions of JSON Schema
            which specified behavior around ignoring keywords if they were
            siblings of a ``$ref`` keyword. If you're not attempting to
            implement similar behavior, you can typically ignore this argument
            and leave it at its default.

    Returns:

        a new `jsonschema.protocols.Validator` class
    """
    # preemptively don't shadow the `Validator.format_checker` local
    format_checker_arg = format_checker

    specification = referencing.jsonschema.specification_with(
        dialect_id=id_of(meta_schema) or "urn:unknown-dialect",
        default=referencing.Specification.OPAQUE,
    )

    @define
    class Validator:

        VALIDATORS = dict(validators)  # noqa: RUF012
        META_SCHEMA = dict(meta_schema)  # noqa: RUF012
        TYPE_CHECKER = type_checker
        FORMAT_CHECKER = format_checker_arg
        ID_OF = staticmethod(id_of)

        _APPLICABLE_VALIDATORS = applicable_validators

        schema: referencing.jsonschema.Schema = field(repr=reprlib.repr)
        _ref_resolver = field(default=None, repr=False, alias="resolver")
        format_checker: _format.FormatChecker | None = field(default=None)
        # TODO: include new meta-schemas added at runtime
        _registry: referencing.jsonschema.SchemaRegistry = field(
            default=_REMOTE_WARNING_REGISTRY,
            kw_only=True,
            repr=False,
        )
        _resolver = field(
            alias="_resolver",
            default=None,
            kw_only=True,
            repr=False,
        )

        def __init_subclass__(cls):
            warnings.warn(
                (
                    "Subclassing validator classes is not intended to "
                    "be part of their public API. A future version "
                    "will make doing so an error, as the behavior of "
                    "subclasses isn't guaranteed to stay the same "
                    "between releases of jsonschema. Instead, prefer "
                    "composition of validators, wrapping them in an object "
                    "owned entirely by the downstream library."
                ),
                DeprecationWarning,
                stacklevel=2,
            )

            def evolve(self, **changes):
                cls = self.__class__
                schema = changes.setdefault("schema", self.schema)
                NewValidator = validator_for(schema, default=cls)

                for field in fields(cls):  # noqa: F402
                    if not field.init:
                        continue
                    attr_name = field.name
                    init_name = field.alias
                    if init_name not in changes:
                        changes[init_name] = getattr(self, attr_name)

                return NewValidator(**changes)

            cls.evolve = evolve

        def __attrs_post_init__(self):
            if self._resolver is None:
                registry = self._registry
                if registry is not _REMOTE_WARNING_REGISTRY:
                    registry = SPECIFICATIONS.combine(registry)
                resource = specification.create_resource(self.schema)
                self._resolver = registry.resolver_with_root(resource)

            # REMOVEME: Legacy ref resolution state management.
            push_scope = getattr(self._ref_resolver, "push_scope", None)
            if push_scope is not None:
                id = id_of(self.schema)
                if id is not None:
                    push_scope(id)

        @classmethod
        def check_schema(cls, schema, format_checker=_UNSET):
            Validator = validator_for(cls.META_SCHEMA, default=cls)
            if format_checker is _UNSET:
                format_checker = Validator.FORMAT_CHECKER
            validator = Validator(
                schema=cls.META_SCHEMA,
                format_checker=format_checker,
            )
            for error in validator.iter_errors(schema):
                raise exceptions.SchemaError.create_from(error)

        @property
        def resolver(self):
            warnings.warn(
                (
                    f"Accessing {self.__class__.__name__}.resolver is "
                    "deprecated as of v4.18.0, in favor of the "
                    "https://github.com/python-jsonschema/referencing "
                    "library, which provides more compliant referencing "
                    "behavior as well as more flexible APIs for "
                    "customization."
                ),
                DeprecationWarning,
                stacklevel=2,
            )
            if self._ref_resolver is None:
                self._ref_resolver = _RefResolver.from_schema(
                    self.schema,
                    id_of=id_of,
                )
            return self._ref_resolver

        def evolve(self, **changes):
            schema = changes.setdefault("schema", self.schema)
            NewValidator = validator_for(schema, default=self.__class__)

            for (attr_name, init_name) in evolve_fields:
                if init_name not in changes:
                    changes[init_name] = getattr(self, attr_name)

            return NewValidator(**changes)

        def iter_errors(self, instance, _schema=None):
            if _schema is not None:
                warnings.warn(
                    (
                        "Passing a schema to Validator.iter_errors "
                        "is deprecated and will be removed in a future "
                        "release. Call validator.evolve(schema=new_schema)."
                        "iter_errors(...) instead."
                    ),
                    DeprecationWarning,
                    stacklevel=2,
                )
            else:
                _schema = self.schema

            if _schema is True:
                return
            elif _schema is False:
                yield exceptions.ValidationError(
                    f"False schema does not allow {instance!r}",
                    validator=None,
                    validator_value=None,
                    instance=instance,
                    schema=_schema,
                )
                return

            for k, v in applicable_validators(_schema):
                validator = self.VALIDATORS.get(k)
                if validator is None:
                    continue

                errors = validator(self, v, instance, _schema) or ()
                for error in errors:
                    # set details if not already set by the called fn
                    error._set(
                        validator=k,
                        validator_value=v,
                        instance=instance,
                        schema=_schema,
                        type_checker=self.TYPE_CHECKER,
                    )
                    if k not in {"if", "$ref"}:
                        error.schema_path.appendleft(k)
                    yield error

        def descend(
            self,
            instance,
            schema,
            path=None,
            schema_path=None,
            resolver=None,
        ):
            if schema is True:
                return
            elif schema is False:
                yield exceptions.ValidationError(
                    f"False schema does not allow {instance!r}",
                    validator=None,
                    validator_value=None,
                    instance=instance,
                    schema=schema,
                )
                return

            if self._ref_resolver is not None:
                evolved = self.evolve(schema=schema)
            else:
                if resolver is None:
                    resolver = self._resolver.in_subresource(
                        specification.create_resource(schema),
                    )
                evolved = self.evolve(schema=schema, _resolver=resolver)

            for k, v in applicable_validators(schema):
                validator = evolved.VALIDATORS.get(k)
                if validator is None:
                    continue

                errors = validator(evolved, v, instance, schema) or ()
                for error in errors:
                    # set details if not already set by the called fn
                    error._set(
                        validator=k,
                        validator_value=v,
                        instance=instance,
                        schema=schema,
                        type_checker=evolved.TYPE_CHECKER,
                    )
                    if k not in {"if", "$ref"}:
                        error.schema_path.appendleft(k)
                    if path is not None:
                        error.path.appendleft(path)
                    if schema_path is not None:
                        error.schema_path.appendleft(schema_path)
                    yield error

        def validate(self, *args, **kwargs):
            for error in self.iter_errors(*args, **kwargs):
                raise error

        def is_type(self, instance, type):
            try:
                return self.TYPE_CHECKER.is_type(instance, type)
            except exceptions.UndefinedTypeCheck:
                raise exceptions.UnknownType(type, instance, self.schema)

        def _validate_reference(self, ref, instance):
            if self._ref_resolver is None:
                try:
                    resolved = self._resolver.lookup(ref)
                except referencing.exceptions.Unresolvable as err:
                    raise exceptions._WrappedReferencingError(err)

                return self.descend(
                    instance,
                    resolved.contents,
                    resolver=resolved.resolver,
                )
            else:
                resolve = getattr(self._ref_resolver, "resolve", None)
                if resolve is None:
                    with self._ref_resolver.resolving(ref) as resolved:
                        return self.descend(instance, resolved)
                else:
                    scope, resolved = resolve(ref)
                    self._ref_resolver.push_scope(scope)

                    try:
                        return list(self.descend(instance, resolved))
                    finally:
                        self._ref_resolver.pop_scope()

        def is_valid(self, instance, _schema=None):
            if _schema is not None:
                warnings.warn(
                    (
                        "Passing a schema to Validator.is_valid is deprecated "
                        "and will be removed in a future release. Call "
                        "validator.evolve(schema=new_schema).is_valid(...) "
                        "instead."
                    ),
                    DeprecationWarning,
                    stacklevel=2,
                )
                self = self.evolve(schema=_schema)

            error = next(self.iter_errors(instance), None)
            return error is None

    evolve_fields = [
        (field.name, field.alias)
        for field in fields(Validator)
        if field.init
    ]

    if version is not None:
        safe = version.title().replace(" ", "").replace("-", "")
        Validator.__name__ = Validator.__qualname__ = f"{safe}Validator"
        Validator = validates(version)(Validator)  # type: ignore[misc]

    return Validator


def extend(
    validator,
    validators=(),
    version=None,
    type_checker=None,
    format_checker=None,
):
    """
    Create a new validator class by extending an existing one.

    Arguments:

        validator (jsonschema.protocols.Validator):

            an existing validator class

        validators (collections.abc.Mapping):

            a mapping of new validator callables to extend with, whose
            structure is as in `create`.

            .. note::

                Any validator callables with the same name as an
                existing one will (silently) replace the old validator
                callable entirely, effectively overriding any validation
                done in the "parent" validator class.

                If you wish to instead extend the behavior of a parent's
                validator callable, delegate and call it directly in
                the new validator function by retrieving it using
                ``OldValidator.VALIDATORS["validation_keyword_name"]``.

        version (str):

            a version for the new validator class

        type_checker (jsonschema.TypeChecker):

            a type checker, used when applying the :kw:`type` keyword.

            If unprovided, the type checker of the extended
            `jsonschema.protocols.Validator` will be carried along.

        format_checker (jsonschema.FormatChecker):

            a format checker, used when applying the :kw:`format` keyword.

            If unprovided, the format checker of the extended
            `jsonschema.protocols.Validator` will be carried along.

    Returns:

        a new `jsonschema.protocols.Validator` class extending the one
        provided

    .. note:: Meta Schemas

        The new validator class will have its parent's meta schema.

        If you wish to change or extend the meta schema in the new
        validator class, modify ``META_SCHEMA`` directly on the returned
        class. Note that no implicit copying is done, so a copy should
        likely be made before modifying it, in order to not affect the
        old validator.
    """
    all_validators = dict(validator.VALIDATORS)
    all_validators.update(validators)

    if type_checker is None:
        type_checker = validator.TYPE_CHECKER
    if format_checker is None:
        format_checker = validator.FORMAT_CHECKER
    return create(
        meta_schema=validator.META_SCHEMA,
        validators=all_validators,
        version=version,
        type_checker=type_checker,
        format_checker=format_checker,
        id_of=validator.ID_OF,
        applicable_validators=validator._APPLICABLE_VALIDATORS,
    )


Draft3Validator = create(
    meta_schema=SPECIFICATIONS.contents(
        "http://json-schema.org/draft-03/schema#",
    ),
    validators={
        "$ref": _validators.ref,
        "additionalItems": _validators.additionalItems,
        "additionalProperties": _validators.additionalProperties,
        "dependencies": _legacy_validators.dependencies_draft3,
        "disallow": _legacy_validators.disallow_draft3,
        "divisibleBy": _validators.multipleOf,
        "enum": _validators.enum,
        "extends": _legacy_validators.extends_draft3,
        "format": _validators.format,
        "items": _legacy_validators.items_draft3_draft4,
        "maxItems": _validators.maxItems,
        "maxLength": _validators.maxLength,
        "maximum": _legacy_validators.maximum_draft3_draft4,
        "minItems": _validators.minItems,
        "minLength": _validators.minLength,
        "minimum": _legacy_validators.minimum_draft3_draft4,
        "pattern": _validators.pattern,
        "patternProperties": _validators.patternProperties,
        "properties": _legacy_validators.properties_draft3,
        "type": _legacy_validators.type_draft3,
        "uniqueItems": _validators.uniqueItems,
    },
    type_checker=_types.draft3_type_checker,
    format_checker=_format.draft3_format_checker,
    version="draft3",
    id_of=referencing.jsonschema.DRAFT3.id_of,
    applicable_validators=_legacy_validators.ignore_ref_siblings,
)

Draft4Validator = create(
    meta_schema=SPECIFICATIONS.contents(
        "http://json-schema.org/draft-04/schema#",
    ),
    validators={
        "$ref": _validators.ref,
        "additionalItems": _validators.additionalItems,
        "additionalProperties": _validators.additionalProperties,
        "allOf": _validators.allOf,
        "anyOf": _validators.anyOf,
        "dependencies": _legacy_validators.dependencies_draft4_draft6_draft7,
        "enum": _validators.enum,
        "format": _validators.format,
        "items": _legacy_validators.items_draft3_draft4,
        "maxItems": _validators.maxItems,
        "maxLength": _validators.maxLength,
        "maxProperties": _validators.maxProperties,
        "maximum": _legacy_validators.maximum_draft3_draft4,
        "minItems": _validators.minItems,
        "minLength": _validators.minLength,
        "minProperties": _validators.minProperties,
        "minimum": _legacy_validators.minimum_draft3_draft4,
        "multipleOf": _validators.multipleOf,
        "not": _validators.not_,
        "oneOf": _validators.oneOf,
        "pattern": _validators.pattern,
        "patternProperties": _validators.patternProperties,
        "properties": _validators.properties,
        "required": _validators.required,
        "type": _validators.type,
        "uniqueItems": _validators.uniqueItems,
    },
    type_checker=_types.draft4_type_checker,
    format_checker=_format.draft4_format_checker,
    version="draft4",
    id_of=referencing.jsonschema.DRAFT4.id_of,
    applicable_validators=_legacy_validators.ignore_ref_siblings,
)

Draft6Validator = create(
    meta_schema=SPECIFICATIONS.contents(
        "http://json-schema.org/draft-06/schema#",
    ),
    validators={
        "$ref": _validators.ref,
        "additionalItems": _validators.additionalItems,
        "additionalProperties": _validators.additionalProperties,
        "allOf": _validators.allOf,
        "anyOf": _validators.anyOf,
        "const": _validators.const,
        "contains": _legacy_validators.contains_draft6_draft7,
        "dependencies": _legacy_validators.dependencies_draft4_draft6_draft7,
        "enum": _validators.enum,
        "exclusiveMaximum": _validators.exclusiveMaximum,
        "exclusiveMinimum": _validators.exclusiveMinimum,
        "format": _validators.format,
        "items": _legacy_validators.items_draft6_draft7_draft201909,
        "maxItems": _validators.maxItems,
        "maxLength": _validators.maxLength,
        "maxProperties": _validators.maxProperties,
        "maximum": _validators.maximum,
        "minItems": _validators.minItems,
        "minLength": _validators.minLength,
        "minProperties": _validators.minProperties,
        "minimum": _validators.minimum,
        "multipleOf": _validators.multipleOf,
        "not": _validators.not_,
        "oneOf": _validators.oneOf,
        "pattern": _validators.pattern,
        "patternProperties": _validators.patternProperties,
        "properties": _validators.properties,
        "propertyNames": _validators.propertyNames,
        "required": _validators.required,
        "type": _validators.type,
        "uniqueItems": _validators.uniqueItems,
    },
    type_checker=_types.draft6_type_checker,
    format_checker=_format.draft6_format_checker,
    version="draft6",
    id_of=referencing.jsonschema.DRAFT6.id_of,
    applicable_validators=_legacy_validators.ignore_ref_siblings,
)

Draft7Validator = create(
    meta_schema=SPECIFICATIONS.contents(
        "http://json-schema.org/draft-07/schema#",
    ),
    validators={
        "$ref": _validators.ref,
        "additionalItems": _validators.additionalItems,
        "additionalProperties": _validators.additionalProperties,
        "allOf": _validators.allOf,
        "anyOf": _validators.anyOf,
        "const": _validators.const,
        "contains": _legacy_validators.contains_draft6_draft7,
        "dependencies": _legacy_validators.dependencies_draft4_draft6_draft7,
        "enum": _validators.enum,
        "exclusiveMaximum": _validators.exclusiveMaximum,
        "exclusiveMinimum": _validators.exclusiveMinimum,
        "format": _validators.format,
        "if": _validators.if_,
        "items": _legacy_validators.items_draft6_draft7_draft201909,
        "maxItems": _validators.maxItems,
        "maxLength": _validators.maxLength,
        "maxProperties": _validators.maxProperties,
        "maximum": _validators.maximum,
        "minItems": _validators.minItems,
        "minLength": _validators.minLength,
        "minProperties": _validators.minProperties,
        "minimum": _validators.minimum,
        "multipleOf": _validators.multipleOf,
        "not": _validators.not_,
        "oneOf": _validators.oneOf,
        "pattern": _validators.pattern,
        "patternProperties": _validators.patternProperties,
        "properties": _validators.properties,
        "propertyNames": _validators.propertyNames,
        "required": _validators.required,
        "type": _validators.type,
        "uniqueItems": _validators.uniqueItems,
    },
    type_checker=_types.draft7_type_checker,
    format_checker=_format.draft7_format_checker,
    version="draft7",
    id_of=referencing.jsonschema.DRAFT7.id_of,
    applicable_validators=_legacy_validators.ignore_ref_siblings,
)

Draft201909Validator = create(
    meta_schema=SPECIFICATIONS.contents(
        "https://json-schema.org/draft/2019-09/schema",
    ),
    validators={
        "$recursiveRef": _legacy_validators.recursiveRef,
        "$ref": _validators.ref,
        "additionalItems": _validators.additionalItems,
        "additionalProperties": _validators.additionalProperties,
        "allOf": _validators.allOf,
        "anyOf": _validators.anyOf,
        "const": _validators.const,
        "contains": _validators.contains,
        "dependentRequired": _validators.dependentRequired,
        "dependentSchemas": _validators.dependentSchemas,
        "enum": _validators.enum,
        "exclusiveMaximum": _validators.exclusiveMaximum,
        "exclusiveMinimum": _validators.exclusiveMinimum,
        "format": _validators.format,
        "if": _validators.if_,
        "items": _legacy_validators.items_draft6_draft7_draft201909,
        "maxItems": _validators.maxItems,
        "maxLength": _validators.maxLength,
        "maxProperties": _validators.maxProperties,
        "maximum": _validators.maximum,
        "minItems": _validators.minItems,
        "minLength": _validators.minLength,
        "minProperties": _validators.minProperties,
        "minimum": _validators.minimum,
        "multipleOf": _validators.multipleOf,
        "not": _validators.not_,
        "oneOf": _validators.oneOf,
        "pattern": _validators.pattern,
        "patternProperties": _validators.patternProperties,
        "properties": _validators.properties,
        "propertyNames": _validators.propertyNames,
        "required": _validators.required,
        "type": _validators.type,
        "unevaluatedItems": _legacy_validators.unevaluatedItems_draft2019,
        "unevaluatedProperties": _validators.unevaluatedProperties,
        "uniqueItems": _validators.uniqueItems,
    },
    type_checker=_types.draft201909_type_checker,
    format_checker=_format.draft201909_format_checker,
    version="draft2019-09",
)

Draft202012Validator = create(
    meta_schema=SPECIFICATIONS.contents(
        "https://json-schema.org/draft/2020-12/schema",
    ),
    validators={
        "$dynamicRef": _validators.dynamicRef,
        "$ref": _validators.ref,
        "additionalItems": _validators.additionalItems,
        "additionalProperties": _validators.additionalProperties,
        "allOf": _validators.allOf,
        "anyOf": _validators.anyOf,
        "const": _validators.const,
        "contains": _validators.contains,
        "dependentRequired": _validators.dependentRequired,
        "dependentSchemas": _validators.dependentSchemas,
        "enum": _validators.enum,
        "exclusiveMaximum": _validators.exclusiveMaximum,
        "exclusiveMinimum": _validators.exclusiveMinimum,
        "format": _validators.format,
        "if": _validators.if_,
        "items": _validators.items,
        "maxItems": _validators.maxItems,
        "maxLength": _validators.maxLength,
        "maxProperties": _validators.maxProperties,
        "maximum": _validators.maximum,
        "minItems": _validators.minItems,
        "minLength": _validators.minLength,
        "minProperties": _validators.minProperties,
        "minimum": _validators.minimum,
        "multipleOf": _validators.multipleOf,
        "not": _validators.not_,
        "oneOf": _validators.oneOf,
        "pattern": _validators.pattern,
        "patternProperties": _validators.patternProperties,
        "prefixItems": _validators.prefixItems,
        "properties": _validators.properties,
        "propertyNames": _validators.propertyNames,
        "required": _validators.required,
        "type": _validators.type,
        "unevaluatedItems": _validators.unevaluatedItems,
        "unevaluatedProperties": _validators.unevaluatedProperties,
        "uniqueItems": _validators.uniqueItems,
    },
    type_checker=_types.draft202012_type_checker,
    format_checker=_format.draft202012_format_checker,
    version="draft2020-12",
)

_LATEST_VERSION = Draft202012Validator


class _RefResolver:
    """
    Resolve JSON References.

    Arguments:

        base_uri (str):

            The URI of the referring document

        referrer:

            The actual referring document

        store (dict):

            A mapping from URIs to documents to cache

        cache_remote (bool):

            Whether remote refs should be cached after first resolution

        handlers (dict):

            A mapping from URI schemes to functions that should be used
            to retrieve them

        urljoin_cache (:func:`functools.lru_cache`):

            A cache that will be used for caching the results of joining
            the resolution scope to subscopes.

        remote_cache (:func:`functools.lru_cache`):

            A cache that will be used for caching the results of
            resolved remote URLs.

    Attributes:

        cache_remote (bool):

            Whether remote refs should be cached after first resolution

    .. deprecated:: v4.18.0

        ``RefResolver`` has been deprecated in favor of `referencing`.
    """

    _DEPRECATION_MESSAGE = (
        "jsonschema.RefResolver is deprecated as of v4.18.0, in favor of the "
        "https://github.com/python-jsonschema/referencing library, which "
        "provides more compliant referencing behavior as well as more "
        "flexible APIs for customization. A future release will remove "
        "RefResolver. Please file a feature request (on referencing) if you "
        "are missing an API for the kind of customization you need."
    )

    def __init__(
        self,
        base_uri,
        referrer,
        store=HashTrieMap(),
        cache_remote=True,
        handlers=(),
        urljoin_cache=None,
        remote_cache=None,
    ):
        if urljoin_cache is None:
            urljoin_cache = lru_cache(1024)(urljoin)
        if remote_cache is None:
            remote_cache = lru_cache(1024)(self.resolve_from_url)

        self.referrer = referrer
        self.cache_remote = cache_remote
        self.handlers = dict(handlers)

        self._scopes_stack = [base_uri]

        self.store = _utils.URIDict(
            (uri, each.contents) for uri, each in SPECIFICATIONS.items()
        )
        self.store.update(
            (id, each.META_SCHEMA) for id, each in _META_SCHEMAS.items()
        )
        self.store.update(store)
        self.store.update(
            (schema["$id"], schema)
            for schema in store.values()
            if isinstance(schema, Mapping) and "$id" in schema
        )
        self.store[base_uri] = referrer

        self._urljoin_cache = urljoin_cache
        self._remote_cache = remote_cache

    @classmethod
    def from_schema(  # noqa: D417
        cls,
        schema,
        id_of=referencing.jsonschema.DRAFT202012.id_of,
        *args,
        **kwargs,
    ):
        """
        Construct a resolver from a JSON schema object.

        Arguments:

            schema:

                the referring schema

        Returns:

            `_RefResolver`
        """
        return cls(base_uri=id_of(schema) or "", referrer=schema, *args, **kwargs)  # noqa: B026, E501

    def push_scope(self, scope):
        """
        Enter a given sub-scope.

        Treats further dereferences as being performed underneath the
        given scope.
        """
        self._scopes_stack.append(
            self._urljoin_cache(self.resolution_scope, scope),
        )

    def pop_scope(self):
        """
        Exit the most recent entered scope.

        Treats further dereferences as being performed underneath the
        original scope.

        Don't call this method more times than `push_scope` has been
        called.
        """
        try:
            self._scopes_stack.pop()
        except IndexError:
            raise exceptions._RefResolutionError(
                "Failed to pop the scope from an empty stack. "
                "`pop_scope()` should only be called once for every "
                "`push_scope()`",
            )

    @property
    def resolution_scope(self):
        """
        Retrieve the current resolution scope.
        """
        return self._scopes_stack[-1]

    @property
    def base_uri(self):
        """
        Retrieve the current base URI, not including any fragment.
        """
        uri, _ = urldefrag(self.resolution_scope)
        return uri

    @contextlib.contextmanager
    def in_scope(self, scope):
        """
        Temporarily enter the given scope for the duration of the context.

        .. deprecated:: v4.0.0
        """
        warnings.warn(
            "jsonschema.RefResolver.in_scope is deprecated and will be "
            "removed in a future release.",
            DeprecationWarning,
            stacklevel=3,
        )
        self.push_scope(scope)
        try:
            yield
        finally:
            self.pop_scope()

    @contextlib.contextmanager
    def resolving(self, ref):
        """
        Resolve the given ``ref`` and enter its resolution scope.

        Exits the scope on exit of this context manager.

        Arguments:

            ref (str):

                The reference to resolve
        """
        url, resolved = self.resolve(ref)
        self.push_scope(url)
        try:
            yield resolved
        finally:
            self.pop_scope()

    def _find_in_referrer(self, key):
        return self._get_subschemas_cache()[key]

    @lru_cache  # noqa: B019
    def _get_subschemas_cache(self):
        cache = {key: [] for key in _SUBSCHEMAS_KEYWORDS}
        for keyword, subschema in _search_schema(
            self.referrer, _match_subschema_keywords,
        ):
            cache[keyword].append(subschema)
        return cache

    @lru_cache  # noqa: B019
    def _find_in_subschemas(self, url):
        subschemas = self._get_subschemas_cache()["$id"]
        if not subschemas:
            return None
        uri, fragment = urldefrag(url)
        for subschema in subschemas:
            id = subschema["$id"]
            if not isinstance(id, str):
                continue
            target_uri = self._urljoin_cache(self.resolution_scope, id)
            if target_uri.rstrip("/") == uri.rstrip("/"):
                if fragment:
                    subschema = self.resolve_fragment(subschema, fragment)
                self.store[url] = subschema
                return url, subschema
        return None

    def resolve(self, ref):
        """
        Resolve the given reference.
        """
        url = self._urljoin_cache(self.resolution_scope, ref).rstrip("/")

        match = self._find_in_subschemas(url)
        if match is not None:
            return match

        return url, self._remote_cache(url)

    def resolve_from_url(self, url):
        """
        Resolve the given URL.
        """
        url, fragment = urldefrag(url)
        if not url:
            url = self.base_uri

        try:
            document = self.store[url]
        except KeyError:
            try:
                document = self.resolve_remote(url)
            except Exception as exc:
                raise exceptions._RefResolutionError(exc)

        return self.resolve_fragment(document, fragment)

    def resolve_fragment(self, document, fragment):
        """
        Resolve a ``fragment`` within the referenced ``document``.

        Arguments:

            document:

                The referent document

            fragment (str):

                a URI fragment to resolve within it
        """
        fragment = fragment.lstrip("/")

        if not fragment:
            return document

        if document is self.referrer:
            find = self._find_in_referrer
        else:

            def find(key):
                yield from _search_schema(document, _match_keyword(key))

        for keyword in ["$anchor", "$dynamicAnchor"]:
            for subschema in find(keyword):
                if fragment == subschema[keyword]:
                    return subschema
        for keyword in ["id", "$id"]:
            for subschema in find(keyword):
                if "#" + fragment == subschema[keyword]:
                    return subschema

        # Resolve via path
        parts = unquote(fragment).split("/") if fragment else []
        for part in parts:
            part = part.replace("~1", "/").replace("~0", "~")

            if isinstance(document, Sequence):
                try:  # noqa: SIM105
                    part = int(part)
                except ValueError:
                    pass
            try:
                document = document[part]
            except (TypeError, LookupError):
                raise exceptions._RefResolutionError(
                    f"Unresolvable JSON pointer: {fragment!r}",
                )

        return document

    def resolve_remote(self, uri):
        """
        Resolve a remote ``uri``.

        If called directly, does not check the store first, but after
        retrieving the document at the specified URI it will be saved in
        the store if :attr:`cache_remote` is True.

        .. note::

            If the requests_ library is present, ``jsonschema`` will use it to
            request the remote ``uri``, so that the correct encoding is
            detected and used.

            If it isn't, or if the scheme of the ``uri`` is not ``http`` or
            ``https``, UTF-8 is assumed.

        Arguments:

            uri (str):

                The URI to resolve

        Returns:

            The retrieved document

        .. _requests: https://pypi.org/project/requests/
        """
        try:
            import requests
        except ImportError:
            requests = None

        scheme = urlsplit(uri).scheme

        if scheme in self.handlers:
            result = self.handlers[scheme](uri)
        elif scheme in ["http", "https"] and requests:
            # Requests has support for detecting the correct encoding of
            # json over http
            result = requests.get(uri).json()
        else:
            # Otherwise, pass off to urllib and assume utf-8
            with urlopen(uri) as url:
                result = json.loads(url.read().decode("utf-8"))

        if self.cache_remote:
            self.store[uri] = result
        return result


_SUBSCHEMAS_KEYWORDS = ("$id", "id", "$anchor", "$dynamicAnchor")


def _match_keyword(keyword):

    def matcher(value):
        if keyword in value:
            yield value

    return matcher


def _match_subschema_keywords(value):
    for keyword in _SUBSCHEMAS_KEYWORDS:
        if keyword in value:
            yield keyword, value


def _search_schema(schema, matcher):
    """Breadth-first search routine."""
    values = deque([schema])
    while values:
        value = values.pop()
        if not isinstance(value, dict):
            continue
        yield from matcher(value)
        values.extendleft(value.values())


def validate(instance, schema, cls=None, *args, **kwargs):  # noqa: D417
    """
    Validate an instance under the given schema.

        >>> validate([2, 3, 4], {"maxItems": 2})
        Traceback (most recent call last):
            ...
        ValidationError: [2, 3, 4] is too long

    :func:`~jsonschema.validators.validate` will first verify that the
    provided schema is itself valid, since not doing so can lead to less
    obvious error messages and fail in less obvious or consistent ways.

    If you know you have a valid schema already, especially
    if you intend to validate multiple instances with
    the same schema, you likely would prefer using the
    `jsonschema.protocols.Validator.validate` method directly on a
    specific validator (e.g. ``Draft202012Validator.validate``).


    Arguments:

        instance:

            The instance to validate

        schema:

            The schema to validate with

        cls (jsonschema.protocols.Validator):

            The class that will be used to validate the instance.

    If the ``cls`` argument is not provided, two things will happen
    in accordance with the specification. First, if the schema has a
    :kw:`$schema` keyword containing a known meta-schema [#]_ then the
    proper validator will be used. The specification recommends that
    all schemas contain :kw:`$schema` properties for this reason. If no
    :kw:`$schema` property is found, the default validator class is the
    latest released draft.

    Any other provided positional and keyword arguments will be passed
    on when instantiating the ``cls``.

    Raises:

        `jsonschema.exceptions.ValidationError`:

            if the instance is invalid

        `jsonschema.exceptions.SchemaError`:

            if the schema itself is invalid

    .. rubric:: Footnotes
    .. [#] known by a validator registered with
        `jsonschema.validators.validates`
    """
    if cls is None:
        cls = validator_for(schema)

    cls.check_schema(schema)
    validator = cls(schema, *args, **kwargs)
    error = exceptions.best_match(validator.iter_errors(instance))
    if error is not None:
        raise error


def validator_for(schema, default=_UNSET):
    """
    Retrieve the validator class appropriate for validating the given schema.

    Uses the :kw:`$schema` keyword that should be present in the given
    schema to look up the appropriate validator class.

    Arguments:

        schema (collections.abc.Mapping or bool):

            the schema to look at

        default:

            the default to return if the appropriate validator class
            cannot be determined.

            If unprovided, the default is to return the latest supported
            draft.

    Examples:

        The :kw:`$schema` JSON Schema keyword will control which validator
        class is returned:

        >>> schema = {
        ...     "$schema": "https://json-schema.org/draft/2020-12/schema",
        ...     "type": "integer",
        ... }
        >>> jsonschema.validators.validator_for(schema)
        <class 'jsonschema.validators.Draft202012Validator'>


        Here, a draft 7 schema instead will return the draft 7 validator:

        >>> schema = {
        ...     "$schema": "http://json-schema.org/draft-07/schema#",
        ...     "type": "integer",
        ... }
        >>> jsonschema.validators.validator_for(schema)
        <class 'jsonschema.validators.Draft7Validator'>


        Schemas with no ``$schema`` keyword will fallback to the default
        argument:

        >>> schema = {"type": "integer"}
        >>> jsonschema.validators.validator_for(
        ...     schema, default=Draft7Validator,
        ... )
        <class 'jsonschema.validators.Draft7Validator'>

        or if none is provided, to the latest version supported.
        Always including the keyword when authoring schemas is highly
        recommended.

    """
    DefaultValidator = _LATEST_VERSION if default is _UNSET else default

    if schema is True or schema is False or "$schema" not in schema:
        return DefaultValidator
    if schema["$schema"] not in _META_SCHEMAS and default is _UNSET:
        warn(
            (
                "The metaschema specified by $schema was not found. "
                "Using the latest draft to validate, but this will raise "
                "an error in the future."
            ),
            DeprecationWarning,
            stacklevel=2,
        )
    return _META_SCHEMAS.get(schema["$schema"], DefaultValidator)

env_pyrecodes/lib/python3.9/site-packages/jsonschema/validators.py
NikolaBlagojevic/pyrecodes
"""
The JSON Schema meta-schemas and vocabularies, exposed as a Registry.
"""
from referencing import Registry as _Registry
from referencing.jsonschema import SchemaRegistry as _SchemaRegistry

from jsonschema_specifications._core import _schemas

#: A `referencing.jsonschema.SchemaRegistry` containing all of the official
#: meta-schemas and vocabularies.
REGISTRY: _SchemaRegistry = (_schemas() @ _Registry()).crawl()

__all__ = ["REGISTRY"]

env_pyrecodes/lib/python3.9/site-packages/jsonschema_specifications/__init__.py
NikolaBlagojevic/pyrecodes
"""
A `referencing.Registry` containing schemas from the JSON Schema specification.
"""

import json

try:
    from importlib.resources import files
except ImportError:
    from importlib_resources import files  # type: ignore

from referencing import Resource


def _schemas():
    """
    All schemas we ship.
    """
    # importlib.resources.abc.Traversal doesn't have nice ways to do this that
    # I'm aware of...
    #
    # It can't recurse arbitrarily, e.g. no ``.glob()``.
    #
    # So this takes some liberties given the real layout of what we ship
    # (only 2 levels of nesting, no directories within the second level).

    for version in files(__package__).joinpath("schemas").iterdir():
        for child in version.iterdir():
            children = [child] if child.is_file() else child.iterdir()
            for path in children:
                contents = json.loads(path.read_text(encoding="utf-8"))
                yield Resource.from_contents(contents)

env_pyrecodes/lib/python3.9/site-packages/jsonschema_specifications/_core.py
NikolaBlagojevic/pyrecodes
from collections.abc import Mapping

from jsonschema_specifications import REGISTRY


def test_it_contains_metaschemas():
    schema = REGISTRY.contents("http://json-schema.org/draft-07/schema#")
    assert isinstance(schema, Mapping)
    assert schema["$id"] == "http://json-schema.org/draft-07/schema#"
    assert schema["title"] == "Core schema meta-schema"

env_pyrecodes/lib/python3.9/site-packages/jsonschema_specifications/tests/test_jsonschema_specifications.py
NikolaBlagojevic/pyrecodes
"""
store the current version info of jupyter-events.
"""
import re
from typing import List

# Version string must appear intact for hatch versioning
__version__ = "0.7.0"

# Build up version_info tuple for backwards compatibility
pattern = r"(?P<major>\d+).(?P<minor>\d+).(?P<patch>\d+)(?P<rest>.*)"
match = re.match(pattern, __version__)
assert match is not None  # noqa
parts: List[object] = [int(match[part]) for part in ["major", "minor", "patch"]]
if match["rest"]:
    parts.append(match["rest"])
version_info = tuple(parts)

kernel_protocol_version_info = (5, 3)
kernel_protocol_version = "{}.{}".format(*kernel_protocol_version_info)

env_pyrecodes/lib/python3.9/site-packages/jupyter_events/_version.py
NikolaBlagojevic/pyrecodes
"""
Emit structured, discrete events when various actions happen.
"""
import asyncio
import copy
import inspect
import json
import logging
import warnings
from datetime import datetime, timezone
from typing import Any, Callable, Coroutine, Optional, Union

from jsonschema import ValidationError
from pythonjsonlogger import jsonlogger
from traitlets import Dict, Instance, Set, default
from traitlets.config import Config, LoggingConfigurable

from .schema import SchemaType
from .schema_registry import SchemaRegistry
from .traits import Handlers
from .validators import JUPYTER_EVENTS_CORE_VALIDATOR

# Increment this version when the metadata included with each event
# changes.
EVENTS_METADATA_VERSION = 1


class SchemaNotRegistered(Warning):
    """A warning to raise when an event is given to the logger
    but its schema has not be registered with the EventLogger
    """


class ModifierError(Exception):
    """An exception to raise when a modifier does not
    show the proper signature.
    """


class CoreMetadataError(Exception):
    """An exception raised when event core metadata is not valid."""


# Only show this warning on the first instance
# of each event type that fails to emit.
warnings.simplefilter("once", SchemaNotRegistered)


class ListenerError(Exception):
    """An exception to raise when a listener does not
    show the proper signature.
    """


class EventLogger(LoggingConfigurable):
    """
    An Event logger for emitting structured events.

    Event schemas must be registered with the
    EventLogger using the `register_schema` or
    `register_schema_file` methods. Every schema
    will be validated against Jupyter Event's metaschema.
    """

    handlers = Handlers(
        default_value=None,
        allow_none=True,
        help="""A list of logging.Handler instances to send events to.

        When set to None (the default), all events are discarded.
        """,
    ).tag(config=True)

    schemas = Instance(
        SchemaRegistry,
        help="""The SchemaRegistry for caching validated schemas
        and their jsonschema validators.
        """,
    )

    _modifiers = Dict({}, help="A mapping of schemas to their list of modifiers.")

    _modified_listeners = Dict({}, help="A mapping of schemas to the listeners of modified events.")

    _unmodified_listeners = Dict(
        {}, help="A mapping of schemas to the listeners of unmodified/raw events."
    )

    _active_listeners = Set()

    async def gather_listeners(self):
        """Gather all of the active listeners."""
        return await asyncio.gather(*self._active_listeners, return_exceptions=True)

    @default("schemas")
    def _default_schemas(self) -> SchemaRegistry:
        return SchemaRegistry()

    def __init__(self, *args, **kwargs):
        """Initialize the logger."""
        # We need to initialize the configurable before
        # adding the logging handlers.
        super().__init__(*args, **kwargs)
        # Use a unique name for the logger so that multiple instances of EventLog do not write
        # to each other's handlers.
        log_name = __name__ + "." + str(id(self))
        self._logger = logging.getLogger(log_name)
        # We don't want events to show up in the default logs
        self._logger.propagate = False
        # We will use log.info to emit
        self._logger.setLevel(logging.INFO)
        # Add each handler to the logger and format the handlers.
        if self.handlers:
            for handler in self.handlers:
                self.register_handler(handler)

    def _load_config(self, cfg, section_names=None, traits=None):
        """Load EventLogger traits from a Config object, patching the
        handlers trait in the Config object to avoid deepcopy errors.
        """
        my_cfg = self._find_my_config(cfg)
        handlers = my_cfg.pop("handlers", [])

        # Turn handlers list into a pickeable function
        def get_handlers():
            return handlers

        my_cfg["handlers"] = get_handlers

        # Build a new eventlog config object.
        eventlogger_cfg = Config({"EventLogger": my_cfg})
        super()._load_config(eventlogger_cfg, section_names=None, traits=None)

    def register_event_schema(self, schema: SchemaType) -> None:
        """Register this schema with the schema registry.

        Get this registered schema using the EventLogger.schema.get() method.
        """

        event_schema = self.schemas.register(schema)
        key = event_schema.id
        self._modifiers[key] = set()
        self._modified_listeners[key] = set()
        self._unmodified_listeners[key] = set()

    def register_handler(self, handler: logging.Handler) -> None:
        """Register a new logging handler to the Event Logger.

        All outgoing messages will be formatted as a JSON string.
        """

        def _handle_message_field(record, **kwargs):
            """Python's logger always emits the "message" field with
            the value as "null" unless it's present in the schema/data.
            Message happens to be a common field for event logs,
            so special case it here and only emit it if "message"
            is found the in the schema's property list.
            """
            schema = self.schemas.get(record["__schema__"])
            if "message" not in schema.properties:
                del record["message"]
            return json.dumps(record, **kwargs)

        formatter = jsonlogger.JsonFormatter(
            json_serializer=_handle_message_field,
        )
        handler.setFormatter(formatter)
        self._logger.addHandler(handler)
        if handler not in self.handlers:
            self.handlers.append(handler)

    def remove_handler(self, handler: logging.Handler) -> None:
        """Remove a logging handler from the logger and list of handlers."""
        self._logger.removeHandler(handler)
        if handler in self.handlers:
            self.handlers.remove(handler)

    def add_modifier(
        self,
        *,
        schema_id: Union[str, None] = None,
        modifier: Callable[[str, dict], dict],
    ) -> None:
        """Add a modifier (callable) to a registered event.

        Parameters
        ----------
        modifier: Callable
            A callable function/method that executes when the named event occurs.
            This method enforces a string signature for modifiers:

                (schema_id: str, data: dict) -> dict:
        """
        # Ensure that this is a callable function/method
        if not callable(modifier):
            msg = "`modifier` must be a callable"
            raise TypeError(msg)

        # Now let's verify the function signature.
        signature = inspect.signature(modifier)

        def modifier_signature(  # type:ignore[empty-body]
            schema_id: str, data: dict
        ) -> dict:
            """Signature to enforce"""
            ...

        expected_signature = inspect.signature(modifier_signature)
        # Assert this signature or raise an exception
        if signature == expected_signature:
            # If the schema ID and version is given, only add
            # this modifier to that schema
            if schema_id:
                self._modifiers[schema_id].add(modifier)
                return
            for id_ in self._modifiers:
                if schema_id is None or id_ == schema_id:
                    self._modifiers[id_].add(modifier)
        else:
            msg = (
                "Modifiers are required to follow an exact function/method "
                "signature. The signature should look like:"
                f"\n\n\tdef my_modifier{expected_signature}:\n\n"
                "Check that you are using type annotations for each argument "
                "and the return value."
            )
            raise ModifierError(msg)

    def remove_modifier(
        self, *, schema_id: Optional[str] = None, modifier: Callable[[str, dict], dict]
    ) -> None:
        """Remove a modifier from an event or all events.

        Parameters
        ----------
        schema_id: str
            If given, remove this modifier only for a specific event type.
        modifier: Callable[[str, dict], dict]

            The modifier to remove.
        """
        # If schema_id is given remove the modifier from this schema.
        if schema_id:
            self._modifiers[schema_id].discard(modifier)
        # If no schema_id is given, remove the modifier from all events.
        else:
            for schema_id in self.schemas.schema_ids:
                # Remove the modifier if it is found in the list.
                self._modifiers[schema_id].discard(modifier)
                self._modifiers[schema_id].discard(modifier)

    def add_listener(
        self,
        *,
        modified: bool = True,
        schema_id: Union[str, None] = None,
        listener: Callable[["EventLogger", str, dict], Coroutine[Any, Any, None]],
    ) -> None:
        """Add a listener (callable) to a registered event.

        Parameters
        ----------
        modified: bool
            If True (default), listens to the data after it has been mutated/modified
            by the list of modifiers.
        schema_id: str
            $id of the schema
        listener: Callable
            A callable function/method that executes when the named event occurs.
        """
        if not callable(listener):
            msg = "`listener` must be a callable"
            raise TypeError(msg)

        signature = inspect.signature(listener)

        async def listener_signature(logger: EventLogger, schema_id: str, data: dict) -> None:
            """An interface for a listener."""
            ...

        expected_signature = inspect.signature(listener_signature)
        # Assert this signature or raise an exception
        if signature == expected_signature:
            # If the schema ID and version is given, only add
            # this modifier to that schema
            if schema_id:
                if modified:
                    self._modified_listeners[schema_id].add(listener)
                    return
                self._unmodified_listeners[schema_id].add(listener)
            for id_ in self.schemas.schema_ids:
                if schema_id is None or id_ == schema_id:
                    if modified:
                        self._modified_listeners[id_].add(listener)
                    else:
                        self._unmodified_listeners[schema_id].add(listener)
        else:
            msg = (
                "Listeners are required to follow an exact function/method "
                "signature. The signature should look like:"
                f"\n\n\tasync def my_listener{expected_signature}:\n\n"
                "Check that you are using type annotations for each argument "
                "and the return value."
            )

            raise ListenerError(msg)

    def remove_listener(
        self,
        *,
        schema_id: Optional[str] = None,
        listener: Callable[["EventLogger", str, dict], Coroutine[Any, Any, None]],
    ) -> None:
        """Remove a listener from an event or all events.

        Parameters
        ----------
        schema_id: str
            If given, remove this modifier only for a specific event type.

        listener: Callable[[EventLogger, str, dict], dict]
            The modifier to remove.
        """
        # If schema_id is given remove the listener from this schema.
        if schema_id:
            self._modified_listeners[schema_id].discard(listener)
            self._unmodified_listeners[schema_id].discard(listener)
        # If no schema_id is given, remove the listener from all events.
        else:
            for schema_id in self.schemas.schema_ids:
                # Remove the listener if it is found in the list.
                self._modified_listeners[schema_id].discard(listener)
                self._unmodified_listeners[schema_id].discard(listener)

    def emit(
        self, *, schema_id: str, data: dict, timestamp_override: Optional[datetime] = None
    ) -> Optional[dict]:
        """
        Record given event with schema has occurred.

        Parameters
        ----------
        schema_id: str
            $id of the schema
        data: dict
            The event to record
        timestamp_override: datetime, optional
            Optionally override the event timestamp. By default it is set to the current timestamp.

        Returns
        -------
        dict
            The recorded event data
        """
        # If no handlers are routing these events, there's no need to proceed.
        if (
            not self.handlers
            and not self._modified_listeners[schema_id]
            and not self._unmodified_listeners[schema_id]
        ):
            return None

        # If the schema hasn't been registered, raise a warning to make sure
        # this was intended.
        if schema_id not in self.schemas:
            warnings.warn(
                f"{schema_id} has not been registered yet. If "
                "this was not intentional, please register the schema using the "
                "`register_event_schema` method.",
                SchemaNotRegistered,
                stacklevel=2,
            )
            return None

        schema = self.schemas.get(schema_id)

        # Deep copy the data and modify the copy.
        modified_data = copy.deepcopy(data)
        for modifier in self._modifiers[schema.id]:
            modified_data = modifier(schema_id=schema_id, data=modified_data)

        if self._unmodified_listeners[schema.id]:
            # Process this event, i.e. validate and modify (in place)
            self.schemas.validate_event(schema_id, data)

        # Validate the modified data.
        self.schemas.validate_event(schema_id, modified_data)

        # Generate the empty event capsule.
        timestamp = (
            datetime.now(tz=timezone.utc) if timestamp_override is None else timestamp_override
        )
        capsule = {
            "__timestamp__": timestamp.isoformat() + "Z",
            "__schema__": schema_id,
            "__schema_version__": schema.version,
            "__metadata_version__": EVENTS_METADATA_VERSION,
        }
        try:
            JUPYTER_EVENTS_CORE_VALIDATOR.validate(capsule)
        except ValidationError as err:
            raise CoreMetadataError from err

        capsule.update(modified_data)

        self._logger.info(capsule)

        # callback for removing from finished listeners
        # from active listeners set.
        def _listener_task_done(task: asyncio.Task) -> None:
            # If an exception happens, log it to the main
            # applications logger
            err = task.exception()
            if err:
                self.log.error(err)
            self._active_listeners.discard(task)

        # Loop over listeners and execute them.
        for listener in self._modified_listeners[schema_id]:
            # Schedule this listener as a task and add
            # it to the list of active listeners
            task = asyncio.create_task(
                listener(
                    logger=self,
                    schema_id=schema_id,
                    data=modified_data,
                )
            )
            self._active_listeners.add(task)

            # Adds the task and cleans it up later if needed.
            task.add_done_callback(_listener_task_done)

        for listener in self._unmodified_listeners[schema_id]:
            task = asyncio.create_task(listener(logger=self, schema_id=schema_id, data=data))
            self._active_listeners.add(task)

            # Remove task from active listeners once its finished.
            def _listener_task_done(task: asyncio.Task) -> None:
                # If an exception happens, log it to the main
                # applications logger
                err = task.exception()
                if err:
                    self.log.error(err)
                self._active_listeners.discard(task)

            # Adds the task and cleans it up later if needed.
            task.add_done_callback(_listener_task_done)

        return capsule

env_pyrecodes/lib/python3.9/site-packages/jupyter_events/logger.py
NikolaBlagojevic/pyrecodes
"""Fixtures for use with jupyter events."""
import io
import json
import logging

import pytest

from jupyter_events import EventLogger


@pytest.fixture
def jp_event_sink():
    """A stream for capture events."""
    return io.StringIO()


@pytest.fixture
def jp_event_handler(jp_event_sink):
    """A logging handler that captures any events emitted by the event handler"""
    return logging.StreamHandler(jp_event_sink)


@pytest.fixture
def jp_read_emitted_events(jp_event_handler, jp_event_sink):
    """Reads list of events since last time it was called."""

    def _read():
        jp_event_handler.flush()
        event_buf = jp_event_sink.getvalue().strip()
        output = [json.loads(item) for item in event_buf.split("\n")] if event_buf else None
        # Clear the sink.
        jp_event_sink.truncate(0)
        jp_event_sink.seek(0)
        return output

    return _read


@pytest.fixture
def jp_event_schemas():
    """A list of schema references.

    Each item should be one of the following:
    - string of serialized JSON/YAML content representing a schema
    - a pathlib.Path object pointing to a schema file on disk
    - a dictionary with the schema data.
    """
    return []


@pytest.fixture
def jp_event_logger(jp_event_handler, jp_event_schemas):
    """A pre-configured event logger for tests."""
    logger = EventLogger()
    for schema in jp_event_schemas:
        logger.register_event_schema(schema)
    logger.register_handler(handler=jp_event_handler)
    return logger

env_pyrecodes/lib/python3.9/site-packages/jupyter_events/pytest_plugin.py
NikolaBlagojevic/pyrecodes
"""Event schema objects."""
import json
from pathlib import Path, PurePath
from typing import Optional, Type, Union

from jsonschema import FormatChecker, validators
from referencing import Registry
from referencing.jsonschema import DRAFT7

try:
    from jsonschema.protocols import Validator
except ImportError:
    from typing import Any

    Validator = Any  # type:ignore

from . import yaml
from .validators import draft7_format_checker, validate_schema


class EventSchemaUnrecognized(Exception):  # noqa
    """An error for an unrecognized event schema."""

    pass


class EventSchemaLoadingError(Exception):
    """An error for an event schema loading error."""

    pass


class EventSchemaFileAbsent(Exception):  # noqa
    """An error for an absent event schema file."""

    pass


SchemaType = Union[dict, str, PurePath]


class EventSchema:
    """A validated schema that can be used.

    On instantiation, validate the schema against
    Jupyter Event's metaschema.

    Parameters
    ----------
    schema: dict or str
        JSON schema to validate against Jupyter Events.

    validator_class: jsonschema.validators
        The validator class from jsonschema used to validate instances
        of this event schema. The schema itself will be validated
        against Jupyter Event's metaschema to ensure that
        any schema registered here follows the expected form
        of Jupyter Events.

    registry:
        Registry for nested JSON schema references.
    """

    def __init__(
        self,
        schema: SchemaType,
        validator_class: Type[Validator] = validators.Draft7Validator,  # type:ignore[assignment]
        format_checker: FormatChecker = draft7_format_checker,
        registry: Optional[Registry] = None,
    ):
        """Initialize an event schema."""
        _schema = self._load_schema(schema)
        # Validate the schema against Jupyter Events metaschema.
        validate_schema(_schema)

        if registry is None:
            registry = DRAFT7.create_resource(_schema) @ Registry()

        # Create a validator for this schema
        self._validator = validator_class(_schema, registry=registry, format_checker=format_checker)  # type: ignore
        self._schema = _schema

    def __repr__(self):
        """A string repr for an event schema."""
        return json.dumps(self._schema, indent=2)

    @staticmethod
    def _ensure_yaml_loaded(schema: SchemaType, was_str: bool = False) -> None:
        """Ensures schema was correctly loaded into a dictionary. Raises
        EventSchemaLoadingError otherwise."""
        if isinstance(schema, dict):
            return

        error_msg = "Could not deserialize schema into a dictionary."

        def intended_as_path(schema: str) -> bool:
            path = Path(schema)
            return path.match("*.yml") or path.match("*.yaml") or path.match("*.json")

        # detect whether the user specified a string but intended a PurePath to
        # generate a more helpful error message
        if was_str and intended_as_path(schema):  # type:ignore[arg-type]
            error_msg += " Paths to schema files must be explicitly wrapped in a Pathlib object."
        else:
            error_msg += " Double check the schema and ensure it is in the proper form."

        raise EventSchemaLoadingError(error_msg)

    @staticmethod
    def _load_schema(schema: SchemaType) -> dict:
        """Load a JSON schema from different sources/data types.

        `schema` could be a dictionary or serialized string representing the
        schema itself or a Pathlib object representing a schema file on disk.

        Returns a dictionary with schema data.
        """

        # if schema is already a dictionary, return it
        if isinstance(schema, dict):
            return schema

        # if schema is PurePath, ensure file exists at path and then load from file
        if isinstance(schema, PurePath):
            if not Path(schema).exists():
                msg = f'Schema file not present at path "{schema}".'
                raise EventSchemaFileAbsent(msg)

            loaded_schema = yaml.load(schema)
            EventSchema._ensure_yaml_loaded(loaded_schema)
            return loaded_schema

        # finally, if schema is string, attempt to deserialize and return the output
        if isinstance(schema, str):
            # note the diff b/w load v.s. loads
            loaded_schema = yaml.loads(schema)
            EventSchema._ensure_yaml_loaded(loaded_schema, was_str=True)
            return loaded_schema

        msg = f"Expected a dictionary, string, or PurePath, but instead received {schema.__class__.__name__}."
        raise EventSchemaUnrecognized(msg)

    @property
    def id(self) -> str:  # noqa
        """Schema $id field."""
        return self._schema["$id"]

    @property
    def version(self) -> int:
        """Schema's version."""
        return self._schema["version"]

    @property
    def properties(self) -> dict:
        return self._schema["properties"]

    def validate(self, data: dict) -> None:
        """Validate an incoming instance of this event schema."""
        self._validator.validate(data)

env_pyrecodes/lib/python3.9/site-packages/jupyter_events/schema.py
NikolaBlagojevic/pyrecodes
"""Event validators."""
import pathlib

import jsonschema
from jsonschema import Draft7Validator, ValidationError
from referencing import Registry
from referencing.jsonschema import DRAFT7

from . import yaml

draft7_format_checker = (
    Draft7Validator.FORMAT_CHECKER
    if hasattr(Draft7Validator, "FORMAT_CHECKER")
    else jsonschema.draft7_format_checker
)


METASCHEMA_PATH = pathlib.Path(__file__).parent.joinpath("schemas")

EVENT_METASCHEMA_FILEPATH = METASCHEMA_PATH.joinpath("event-metaschema.yml")
EVENT_METASCHEMA = yaml.load(EVENT_METASCHEMA_FILEPATH)

EVENT_CORE_SCHEMA_FILEPATH = METASCHEMA_PATH.joinpath("event-core-schema.yml")
EVENT_CORE_SCHEMA = yaml.load(EVENT_CORE_SCHEMA_FILEPATH)

PROPERTY_METASCHEMA_FILEPATH = METASCHEMA_PATH.joinpath("property-metaschema.yml")
PROPERTY_METASCHEMA = yaml.load(PROPERTY_METASCHEMA_FILEPATH)

SCHEMA_STORE = {
    EVENT_METASCHEMA["$id"]: EVENT_METASCHEMA,
    PROPERTY_METASCHEMA["$id"]: PROPERTY_METASCHEMA,
    EVENT_CORE_SCHEMA["$id"]: EVENT_CORE_SCHEMA,
}

resources = [
    DRAFT7.create_resource(each)
    for each in (EVENT_METASCHEMA, PROPERTY_METASCHEMA, EVENT_CORE_SCHEMA)
]
METASCHEMA_REGISTRY: Registry = resources @ Registry()

JUPYTER_EVENTS_SCHEMA_VALIDATOR = Draft7Validator(  # type: ignore
    schema=EVENT_METASCHEMA,
    registry=METASCHEMA_REGISTRY,
    format_checker=draft7_format_checker,
)

JUPYTER_EVENTS_CORE_VALIDATOR = Draft7Validator(  # type: ignore
    schema=EVENT_CORE_SCHEMA,
    registry=METASCHEMA_REGISTRY,
    format_checker=draft7_format_checker,
)


def validate_schema(schema: dict) -> None:
    """Validate a schema dict."""
    try:
        # Validate the schema against Jupyter Events metaschema.
        JUPYTER_EVENTS_SCHEMA_VALIDATOR.validate(schema)
    except ValidationError as err:
        reserved_property_msg = " does not match '^(?!__.*)'"
        if reserved_property_msg in str(err):
            idx = str(err).find(reserved_property_msg)
            bad_property = str(err)[:idx].strip()
            msg = (
                f"{bad_property} is an invalid property name because it "
                "starts with `__`. Properties starting with 'dunder' "
                "are reserved as special meta-fields for Jupyter Events to use."
            )
            raise ValidationError(msg) from err
        raise err

env_pyrecodes/lib/python3.9/site-packages/jupyter_events/validators.py
NikolaBlagojevic/pyrecodes
""" A configurable frontend for stdio-based Language Servers
"""
import asyncio
import os
import sys
import traceback
from typing import Dict, Text, Tuple, cast

# See compatibility note on `group` keyword in
# https://docs.python.org/3/library/importlib.metadata.html#entry-points
if sys.version_info < (3, 10):  # pragma: no cover
    from importlib_metadata import entry_points
else:  # pragma: no cover
    from importlib.metadata import entry_points

from jupyter_core.paths import jupyter_config_path
from jupyter_server.services.config import ConfigManager

try:
    from jupyter_server.transutils import _i18n as _
except ImportError:  # pragma: no cover
    from jupyter_server.transutils import _

from traitlets import Bool
from traitlets import Dict as Dict_
from traitlets import Instance
from traitlets import List as List_
from traitlets import Unicode, default

from .constants import (
    APP_CONFIG_D_SECTIONS,
    EP_LISTENER_ALL_V1,
    EP_LISTENER_CLIENT_V1,
    EP_LISTENER_SERVER_V1,
    EP_SPEC_V1,
)
from .schema import LANGUAGE_SERVER_SPEC_MAP
from .session import LanguageServerSession
from .trait_types import LoadableCallable, Schema
from .types import (
    KeyedLanguageServerSpecs,
    LanguageServerManagerAPI,
    MessageScope,
    SpecBase,
    SpecMaker,
)


class LanguageServerManager(LanguageServerManagerAPI):
    """Manage language servers"""

    conf_d_language_servers = Schema(  # type:ignore[assignment]
        validator=LANGUAGE_SERVER_SPEC_MAP,
        help=_("extra language server specs, keyed by implementation, from conf.d"),
    )  # type: KeyedLanguageServerSpecs

    language_servers = Schema(  # type:ignore[assignment]
        validator=LANGUAGE_SERVER_SPEC_MAP,
        help=_("a dict of language server specs, keyed by implementation"),
    ).tag(
        config=True
    )  # type: KeyedLanguageServerSpecs

    autodetect = Bool(
        True, help=_("try to find known language servers in sys.prefix (and elsewhere)")
    ).tag(
        config=True
    )  # type: bool

    sessions = Dict_(  # type:ignore[assignment]
        trait=Instance(LanguageServerSession),
        default_value={},
        help="sessions keyed by language server name",
    )  # type: Dict[Tuple[Text], LanguageServerSession]

    virtual_documents_dir = Unicode(
        help="""Path to virtual documents relative to the content manager root
        directory.

        Its default value can be set with JP_LSP_VIRTUAL_DIR and fallback to
        '.virtual_documents'.
        """
    ).tag(config=True)

    _ready = Bool(
        help="""Whether the manager has been initialized""", default_value=False
    )

    all_listeners = List_(trait=LoadableCallable).tag(config=True)
    server_listeners = List_(trait=LoadableCallable).tag(config=True)
    client_listeners = List_(trait=LoadableCallable).tag(config=True)

    @default("language_servers")
    def _default_language_servers(self):
        return {}

    @default("virtual_documents_dir")
    def _default_virtual_documents_dir(self):
        return os.getenv("JP_LSP_VIRTUAL_DIR", ".virtual_documents")

    @default("conf_d_language_servers")
    def _default_conf_d_language_servers(self):
        language_servers = {}  # type: KeyedLanguageServerSpecs

        manager = ConfigManager(read_config_path=jupyter_config_path())

        for app in APP_CONFIG_D_SECTIONS:
            language_servers.update(
                **manager.get(f"jupyter{app}config")
                .get(self.__class__.__name__, {})
                .get("language_servers", {})
            )

        return language_servers

    def __init__(self, **kwargs):
        """Before starting, perform all necessary configuration"""
        self.all_language_servers: KeyedLanguageServerSpecs = {}
        self._language_servers_from_config = {}
        super().__init__(**kwargs)

    def initialize(self, *args, **kwargs):
        self.init_language_servers()
        self.init_listeners()
        self.init_sessions()
        self._ready = True

    async def ready(self):
        while not self._ready:  # pragma: no cover
            await asyncio.sleep(0.1)
        return True

    def init_language_servers(self) -> None:
        """determine the final language server configuration."""
        # copy the language servers before anybody monkeys with them
        self._language_servers_from_config = dict(self.language_servers)
        self.language_servers = self._collect_language_servers(only_installed=True)
        self.all_language_servers = self._collect_language_servers(only_installed=False)

    def _collect_language_servers(
        self, only_installed: bool
    ) -> KeyedLanguageServerSpecs:
        language_servers: KeyedLanguageServerSpecs = {}

        language_servers_from_config = dict(self._language_servers_from_config)
        language_servers_from_config.update(self.conf_d_language_servers)

        if self.autodetect:
            language_servers.update(
                self._autodetect_language_servers(only_installed=only_installed)
            )

        # restore config
        language_servers.update(language_servers_from_config)

        # coalesce the servers, allowing a user to opt-out by specifying `[]`
        return {key: spec for key, spec in language_servers.items() if spec.get("argv")}

    def init_sessions(self):
        """create, but do not initialize all sessions"""
        sessions = {}
        for language_server, spec in self.language_servers.items():
            sessions[language_server] = LanguageServerSession(
                language_server=language_server, spec=spec, parent=self
            )
        self.sessions = sessions

    def init_listeners(self):
        """register traitlets-configured listeners"""

        scopes = {
            MessageScope.ALL: [self.all_listeners, EP_LISTENER_ALL_V1],
            MessageScope.CLIENT: [self.client_listeners, EP_LISTENER_CLIENT_V1],
            MessageScope.SERVER: [self.server_listeners, EP_LISTENER_SERVER_V1],
        }
        for scope, trt_ep in scopes.items():
            listeners, entry_point = trt_ep

            for ept in entry_points(group=entry_point):  # pragma: no cover
                try:
                    listeners.append(ept.load())
                except Exception as err:
                    self.log.warning("Failed to load entry point %s: %s", ept.name, err)

            for listener in listeners:
                self.__class__.register_message_listener(scope=scope.value)(listener)

    def subscribe(self, handler):
        """subscribe a handler to session, or sta"""
        session = self.sessions.get(handler.language_server)

        if session is None:
            self.log.error(
                "[{}] no session: handler subscription failed".format(
                    handler.language_server
                )
            )
            return

        session.handlers = set([handler]) | session.handlers

    async def on_client_message(self, message, handler):
        await self.wait_for_listeners(
            MessageScope.CLIENT, message, handler.language_server
        )
        session = self.sessions.get(handler.language_server)

        if session is None:
            self.log.error(
                "[{}] no session: client message dropped".format(
                    handler.language_server
                )
            )
            return

        session.write(message)

    async def on_server_message(self, message, session):
        language_servers = [
            ls_key for ls_key, sess in self.sessions.items() if sess == session
        ]

        for language_servers in language_servers:
            await self.wait_for_listeners(
                MessageScope.SERVER, message, language_servers
            )

        for handler in session.handlers:
            handler.write_message(message)

    def unsubscribe(self, handler):
        session = self.sessions.get(handler.language_server)

        if session is None:
            self.log.error(
                "[{}] no session: handler unsubscription failed".format(
                    handler.language_server
                )
            )
            return

        session.handlers = [h for h in session.handlers if h != handler]

    def _autodetect_language_servers(self, only_installed: bool):
        _entry_points = None

        try:
            _entry_points = entry_points(group=EP_SPEC_V1)
        except Exception:  # pragma: no cover
            self.log.exception("Failed to load entry_points")

        skipped_servers = []

        for ep in _entry_points or []:
            try:
                spec_finder = ep.load()  # type: SpecMaker
            except Exception as err:  # pragma: no cover
                self.log.warning(
                    _("Failed to load language server spec finder `{}`: \n{}").format(
                        ep.name, err
                    )
                )
                continue

            try:
                if only_installed:
                    if hasattr(spec_finder, "is_installed"):
                        spec_finder_from_base = cast(SpecBase, spec_finder)
                        if not spec_finder_from_base.is_installed(self):
                            skipped_servers.append(ep.name)
                            continue
                specs = spec_finder(self) or {}
            except Exception as err:  # pragma: no cover
                self.log.warning(
                    _(
                        "Failed to fetch commands from language server spec finder"
                        " `{}`:\n{}"
                    ).format(ep.name, err)
                )
                traceback.print_exc()

                continue

            errors = list(LANGUAGE_SERVER_SPEC_MAP.iter_errors(specs))

            if errors:  # pragma: no cover
                self.log.warning(
                    _(
                        "Failed to validate commands from language server spec finder"
                        " `{}`:\n{}"
                    ).format(ep.name, errors)
                )
                continue

            for key, spec in specs.items():
                yield key, spec

        if skipped_servers:
            self.log.info(
                _("Skipped non-installed server(s): {}").format(
                    ", ".join(skipped_servers)
                )
            )


# the listener decorator
lsp_message_listener = LanguageServerManager.register_message_listener  # noqa

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/manager.py
NikolaBlagojevic/pyrecodes
""" add language server support to the running jupyter notebook application
"""
import json
from pathlib import Path

import traitlets

from .handlers import add_handlers
from .manager import LanguageServerManager
from .paths import normalized_uri


async def initialize(nbapp, virtual_documents_uri):  # pragma: no cover
    """Perform lazy initialization."""
    import concurrent.futures

    from .virtual_documents_shadow import setup_shadow_filesystem

    manager: LanguageServerManager = nbapp.language_server_manager

    with concurrent.futures.ThreadPoolExecutor() as pool:
        await nbapp.io_loop.run_in_executor(pool, manager.initialize)

    servers_requiring_disk_access = [
        server_id
        for server_id, server in manager.language_servers.items()
        if server.get("requires_documents_on_disk", True)
    ]

    if any(servers_requiring_disk_access):
        nbapp.log.debug(
            "[lsp] Servers that requested virtual documents on disk: %s",
            servers_requiring_disk_access,
        )
        setup_shadow_filesystem(virtual_documents_uri=virtual_documents_uri)
    else:
        nbapp.log.debug(
            "[lsp] None of the installed servers require virtual documents"
            " disabling shadow filesystem."
        )

    nbapp.log.debug(
        "[lsp] The following Language Servers will be available: {}".format(
            json.dumps(manager.language_servers, indent=2, sort_keys=True)
        )
    )


def load_jupyter_server_extension(nbapp):
    """create a LanguageServerManager and add handlers"""
    nbapp.add_traits(language_server_manager=traitlets.Instance(LanguageServerManager))
    manager = nbapp.language_server_manager = LanguageServerManager(parent=nbapp)

    contents = nbapp.contents_manager
    page_config = nbapp.web_app.settings.setdefault("page_config_data", {})

    root_uri = ""
    virtual_documents_uri = ""

    # try to set the rootUri from the contents manager path
    if hasattr(contents, "root_dir"):
        root_uri = normalized_uri(contents.root_dir)
        nbapp.log.debug("[lsp] rootUri will be %s", root_uri)
        virtual_documents_uri = normalized_uri(
            Path(contents.root_dir) / manager.virtual_documents_dir
        )
        nbapp.log.debug("[lsp] virtualDocumentsUri will be %s", virtual_documents_uri)
    else:  # pragma: no cover
        nbapp.log.warn(
            "[lsp] %s did not appear to have a root_dir, could not set rootUri",
            contents,
        )
    page_config.update(rootUri=root_uri, virtualDocumentsUri=virtual_documents_uri)

    add_handlers(nbapp)
    nbapp.io_loop.call_later(0, initialize, nbapp, virtual_documents_uri)

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/serverextension.py
NikolaBlagojevic/pyrecodes
from ..types import LanguageServerManagerAPI
from .config import load_config_schema
from .utils import NodeModuleSpec


class BashLanguageServer(NodeModuleSpec):
    node_module = key = "bash-language-server"
    script = ["out", "cli.js"]
    fallback_script = ["bin", "main.js"]
    args = ["start"]
    languages = ["bash", "sh"]
    spec = dict(
        display_name=key,
        mime_types=["text/x-sh", "application/x-sh"],
        urls=dict(
            home="https://github.com/bash-lsp/{}".format(key),
            issues="https://github.com/bash-lsp/{}/issues".format(key),
        ),
        install=dict(
            npm="npm install --save-dev {}".format(key),
            yarn="yarn add --dev {}".format(key),
            jlpm="jlpm add --dev {}".format(key),
        ),
        config_schema=load_config_schema(key),
    )

    def solve(self, mgr: LanguageServerManagerAPI):
        new_path = mgr.find_node_module(self.node_module, *self.script)
        if new_path:
            return new_path
        return mgr.find_node_module(
            self.node_module, *self.fallback_script
        )  # pragma: no cover

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/specs/bash_language_server.py
NikolaBlagojevic/pyrecodes
from .config import load_config_schema
from .utils import NodeModuleSpec


class PyrightLanguageServer(NodeModuleSpec):
    node_module = key = "pyright"
    script = ["langserver.index.js"]
    args = ["--stdio"]
    languages = ["python"]
    spec = dict(
        display_name=key,
        mime_types=["text/python", "text/x-ipython"],
        urls=dict(
            home="https://github.com/microsoft/pyright",
            issues="https://github.com/microsoft/pyright/issues",
        ),
        install=dict(
            npm="npm install --save-dev {}".format(key),
            yarn="yarn add --dev {}".format(key),
            jlpm="jlpm add --dev {}".format(key),
        ),
        config_schema=load_config_schema(key),
        requires_documents_on_disk=False,
    )

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/specs/pyright.py
NikolaBlagojevic/pyrecodes
import logging
from pathlib import Path
from types import SimpleNamespace
from typing import List

import pytest

from jupyter_lsp import LanguageServerManager

from ..virtual_documents_shadow import (
    EditableFile,
    ShadowFilesystemError,
    extract_or_none,
    setup_shadow_filesystem,
)


@pytest.mark.asyncio
async def test_read(tmp_path):
    path = tmp_path / "existing.py"
    path.write_text("a\ntest")

    editable_file = EditableFile(path)

    await editable_file.read()

    assert editable_file.lines == ["a", "test"]


@pytest.mark.asyncio
async def test_read_missing(tmp_path):
    path = tmp_path / "missing.py"
    missing_file = EditableFile(path)

    await missing_file.read()

    assert missing_file.lines == [""]


@pytest.mark.asyncio
async def test_apply_change(tmp_path):
    # inserting text
    path = tmp_path / "test.py"
    editable_file = EditableFile(path)
    await editable_file.read()

    editable_file.apply_change("new\ntext", **editable_file.full_range)
    assert editable_file.lines == ["new", "text"]

    # modifying a range
    editable_file.apply_change(
        "ves", start={"line": 1, "character": 0}, end={"line": 1, "character": 3}
    )
    assert editable_file.lines == ["new", "vest"]

    editable_file.apply_change("", **editable_file.full_range)
    assert editable_file.lines == [""]


def test_extract_or_none():
    obj = {"nested": {"value": 1}}
    assert extract_or_none(obj, ["nested"]) == {"value": 1}
    assert extract_or_none(obj, ["nested", "value"]) == 1
    assert extract_or_none(obj, ["missing", "value"]) is None


def did_open(uri, text):
    return {
        "method": "textDocument/didOpen",
        "params": {"textDocument": {"uri": uri, "text": text}},
    }


def did_change(uri, changes: List):
    return {
        "method": "textDocument/didChange",
        "params": {"textDocument": {"uri": uri}, "contentChanges": changes},
    }


def did_save_with_text(uri, text):
    return {
        "method": "textDocument/didSave",
        "params": {"textDocument": {"uri": uri, "text": text}},
    }


def did_save_without_text(uri):
    return {"method": "textDocument/didSave", "params": {"textDocument": {"uri": uri}}}


@pytest.fixture
def shadow_path(tmpdir):
    return str(tmpdir.mkdir(".virtual_documents"))


@pytest.fixture
def manager():
    manager = LanguageServerManager()
    manager.language_servers = {
        "python-lsp-server": {
            "requires_documents_on_disk": True,
            "argv": [],
            "languages": ["python"],
            "version": 2,
        }
    }
    return manager


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "message_func, content, expected_content",
    [
        [did_open, "content\nof\nopened\nfile", "content\nof\nopened\nfile"],
        [did_change, [{"text": "content after change"}], "content after change"],
        [did_save_with_text, "content at save", "content at save"],
    ],
)
async def test_shadow(shadow_path, message_func, content, expected_content, manager):
    shadow = setup_shadow_filesystem(Path(shadow_path).as_uri())
    ok_file_path = Path(shadow_path) / "test.py"

    message = message_func(ok_file_path.as_uri(), content)
    result = await shadow("client", message, "python-lsp-server", manager)
    assert isinstance(result, str)

    with open(ok_file_path) as f:
        assert f.read() == expected_content


@pytest.mark.asyncio
async def test_no_shadow_for_well_behaved_server(
    shadow_path,
):
    """We call server well behaved when it does not require a disk copy"""
    shadow_path_for_well = Path(shadow_path) / "well"
    shadow = setup_shadow_filesystem(Path(shadow_path_for_well).as_uri())
    ok_file_path = Path(shadow_path_for_well) / "test.py"

    manager = SimpleNamespace(
        language_servers={"python-lsp-server": {"requires_documents_on_disk": False}}
    )

    message = did_open(ok_file_path.as_uri(), "content\nof\nopened\nfile")
    result = await shadow("client", message, "python-lsp-server", manager)
    # should short-circuit for well behaved server
    assert result is None
    # should not create the directory
    assert not shadow_path_for_well.exists()


@pytest.mark.asyncio
async def test_shadow_created_for_ill_behaved_server(
    shadow_path,
):
    shadow_path_for_ill = Path(shadow_path) / "ill"
    shadow = setup_shadow_filesystem(shadow_path_for_ill.as_uri())
    ok_file_path = Path(shadow_path_for_ill) / "test.py"

    manager = SimpleNamespace(
        language_servers={"python-lsp-server": {"requires_documents_on_disk": True}}
    )

    message = did_open(ok_file_path.as_uri(), "content\nof\nopened\nfile")
    result = await shadow("client", message, "python-lsp-server", manager)
    assert result is not None
    # should create the directory at given path
    assert shadow_path_for_ill.exists()
    assert shadow_path_for_ill.is_dir()


@pytest.mark.asyncio
async def test_shadow_failures(shadow_path, manager):
    shadow = setup_shadow_filesystem(Path(shadow_path).as_uri())
    ok_file_uri = (Path(shadow_path) / "test.py").as_uri()

    def run_shadow(message):
        return shadow("client", message, "python-lsp-server", manager)

    # missing textDocument
    with pytest.raises(ShadowFilesystemError, match="Could not get textDocument from"):
        await run_shadow({"method": "textDocument/didChange"})

    # missing URI
    with pytest.raises(ShadowFilesystemError, match="Could not get URI from"):
        await run_shadow(
            {"method": "textDocument/didChange", "params": {"textDocument": {}}}
        )

    # should ignore other methods
    result = await run_shadow({"method": "textDocument/completion"})
    assert result is None

    # should NOT intercept (nor shadow) files from location other than shadow_path
    result = await run_shadow(did_open("file:///other/path.py", "content"))
    assert result is None

    # should fail silently on missing text in didSave
    result = await run_shadow(did_save_without_text(ok_file_uri))
    assert result is None

    # should raise on missing changes in didChange
    with pytest.raises(ShadowFilesystemError, match=".* is missing contentChanges"):
        await run_shadow(
            {
                "method": "textDocument/didChange",
                "params": {"textDocument": {"uri": ok_file_uri}},
            }
        )


@pytest.fixture
def forbidden_shadow_path(tmpdir):
    path = Path(tmpdir) / "no_permission_dir"
    path.mkdir()
    path.chmod(0o000)

    yield path

    # re-adjust the permissions, see https://github.com/pytest-dev/pytest/issues/7821
    path.chmod(0o755)


@pytest.mark.asyncio
async def test_io_failure(forbidden_shadow_path, manager, caplog):
    file_uri = (forbidden_shadow_path / "test.py").as_uri()

    shadow = setup_shadow_filesystem(forbidden_shadow_path.as_uri())

    def send_change():
        message = did_open(file_uri, "content")
        return shadow("client", message, "python-lsp-server", manager)

    with caplog.at_level(logging.WARNING):
        assert await send_change() is None
        assert await send_change() is None
    # no message should be emitted during the first two attempts
    assert caplog.text == ""

    # a wargning should be emitted on third failure
    with caplog.at_level(logging.WARNING):
        assert await send_change() is None
    assert "initialization of shadow filesystem failed three times" in caplog.text
    assert "PermissionError" in caplog.text
    caplog.clear()

    # no message should be emitted in subsequent attempts
    with caplog.at_level(logging.WARNING):
        assert await send_change() is None
    assert caplog.text == ""

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/tests/test_virtual_documents_shadow.py
NikolaBlagojevic/pyrecodes
""" API used by spec finders and manager
"""
import asyncio
import enum
import json
import pathlib
import re
import shutil
import subprocess
import sys
from functools import lru_cache
from typing import (
    TYPE_CHECKING,
    Any,
    Awaitable,
    Callable,
    Dict,
    List,
    Optional,
    Pattern,
    Text,
    Union,
    cast,
)

try:
    from jupyter_server.transutils import _i18n as _
except ImportError:  # pragma: no cover
    from jupyter_server.transutils import _

from traitlets import Instance
from traitlets import List as List_
from traitlets import Unicode, default
from traitlets.config import LoggingConfigurable

LanguageServerSpec = Dict[Text, Any]
LanguageServerMessage = Dict[Text, Any]
KeyedLanguageServerSpecs = Dict[Text, LanguageServerSpec]

if TYPE_CHECKING:  # pragma: no cover
    from typing_extensions import Protocol

    class HandlerListenerCallback(Protocol):
        def __call__(
            self,
            scope: Text,
            message: LanguageServerMessage,
            language_server: Text,
            manager: "LanguageServerManagerAPI",
        ) -> Awaitable[None]:
            ...


class SessionStatus(enum.Enum):
    """States in which a language server session can be"""

    NOT_STARTED = "not_started"
    STARTING = "starting"
    STARTED = "started"
    STOPPING = "stopping"
    STOPPED = "stopped"


class MessageScope(enum.Enum):
    """Scopes for message listeners"""

    ALL = "all"
    CLIENT = "client"
    SERVER = "server"


class MessageListener(object):
    """A base listener implementation"""

    listener = None  # type: HandlerListenerCallback
    language_server = None  # type: Optional[Pattern[Text]]
    method = None  # type: Optional[Pattern[Text]]

    def __init__(
        self,
        listener: "HandlerListenerCallback",
        language_server: Optional[Text],
        method: Optional[Text],
    ):
        self.listener = listener
        self.language_server = re.compile(language_server) if language_server else None
        self.method = re.compile(method) if method else None

    async def __call__(
        self,
        scope: Text,
        message: LanguageServerMessage,
        language_server: Text,
        manager: "LanguageServerManagerAPI",
    ) -> None:
        """actually dispatch the message to the listener and capture any errors"""
        try:
            await self.listener(
                scope=scope,
                message=message,
                language_server=language_server,
                manager=manager,
            )
        except Exception:  # pragma: no cover
            manager.log.warn(
                "[lsp] error in listener %s for message %s",
                self.listener,
                message,
                exc_info=True,
            )

    def wants(self, message: LanguageServerMessage, language_server: Text):
        """whether this listener wants a particular message

        `method` is currently the only message content discriminator, but not
        all messages will have a `method`
        """
        if self.method:
            method = message.get("method")

            if method is None or re.match(self.method, method) is None:
                return False
        return self.language_server is None or re.match(
            self.language_server, language_server
        )

    def __repr__(self):
        return (
            "<MessageListener"
            " listener={self.listener},"
            " method={self.method},"
            " language_server={self.language_server}>"
        ).format(self=self)


class HasListeners:
    _listeners = {
        str(scope.value): [] for scope in MessageScope
    }  # type: Dict[Text, List[MessageListener]]

    log: Any = Instance("logging.Logger")

    @classmethod
    def register_message_listener(
        cls,
        scope: Text,
        language_server: Optional[Text] = None,
        method: Optional[Text] = None,
    ):
        """register a listener for language server protocol messages"""

        def inner(listener: "HandlerListenerCallback") -> "HandlerListenerCallback":
            cls.unregister_message_listener(listener)
            cls._listeners[scope].append(
                MessageListener(
                    listener=listener, language_server=language_server, method=method
                )
            )
            return listener

        return inner

    @classmethod
    def unregister_message_listener(cls, listener: "HandlerListenerCallback"):
        """unregister a listener for language server protocol messages"""
        for scope in MessageScope:
            cls._listeners[str(scope.value)] = [
                lst
                for lst in cls._listeners[str(scope.value)]
                if lst.listener != listener
            ]

    async def wait_for_listeners(
        self, scope: MessageScope, message_str: Text, language_server: Text
    ) -> None:
        scope_val = str(scope.value)
        listeners = self._listeners[scope_val] + self._listeners[MessageScope.ALL.value]

        if listeners:
            message = json.loads(message_str)

            futures = [
                listener(
                    scope_val,
                    message=message,
                    language_server=language_server,
                    manager=cast("LanguageServerManagerAPI", self),
                )
                for listener in listeners
                if listener.wants(message, language_server)
            ]

            if futures:
                await asyncio.gather(*futures)


class LanguageServerManagerAPI(LoggingConfigurable, HasListeners):
    """Public API that can be used for python-based spec finders and listeners"""

    language_servers: KeyedLanguageServerSpecs

    nodejs = Unicode(help=_("path to nodejs executable")).tag(config=True)

    node_roots = List_([], help=_("absolute paths in which to seek node_modules")).tag(
        config=True
    )

    extra_node_roots = List_(
        [], help=_("additional absolute paths to seek node_modules first")
    ).tag(config=True)

    def find_node_module(self, *path_frag):
        """look through the node_module roots to find the given node module"""
        all_roots = self.extra_node_roots + self.node_roots
        found = None

        for candidate_root in all_roots:
            candidate = pathlib.Path(candidate_root, "node_modules", *path_frag)
            self.log.debug("Checking for %s", candidate)
            if candidate.exists():
                found = str(candidate)
                break

        if found is None:  # pragma: no cover
            self.log.debug(
                "{} not found in node_modules of {}".format(
                    pathlib.Path(*path_frag), all_roots
                )
            )

        return found

    @default("nodejs")
    def _default_nodejs(self):
        return (
            shutil.which("node") or shutil.which("nodejs") or shutil.which("nodejs.exe")
        )

    @lru_cache(maxsize=1)
    def _npm_prefix(self):
        try:
            return (
                subprocess.run(["npm", "prefix", "-g"], check=True, capture_output=True)
                .stdout.decode("utf-8")
                .strip()
            )
        except Exception as e:  # pragma: no cover
            self.log.warn(f"Could not determine npm prefix: {e}")

    @default("node_roots")
    def _default_node_roots(self):
        """get the "usual suspects" for where `node_modules` may be found

        - where this was launch (usually the same as NotebookApp.notebook_dir)
        - the JupyterLab staging folder (if available)
        - wherever conda puts it
        - wherever some other conventions put it
        """

        # check where the server was started first
        roots = [pathlib.Path.cwd()]

        # try jupyterlab staging next
        try:
            from jupyterlab import commands

            roots += [pathlib.Path(commands.get_app_dir()) / "staging"]
        except ImportError:  # pragma: no cover
            pass

        # conda puts stuff in $PREFIX/lib on POSIX systems
        roots += [pathlib.Path(sys.prefix) / "lib"]

        # ... but right in %PREFIX% on nt
        roots += [pathlib.Path(sys.prefix)]

        # check for custom npm prefix
        if shutil.which("npm"):
            prefix = self._npm_prefix()
            if prefix:
                roots += [  # pragma: no cover
                    pathlib.Path(prefix) / "lib",
                    pathlib.Path(prefix),
                ]

        return roots


SimpleSpecMaker = Callable[[LanguageServerManagerAPI], KeyedLanguageServerSpecs]

# String corresponding to a fragment of a shell command
# arguments list such as returned by `shlex.split`
Token = Text


class SpecBase:
    """Base for a spec finder that returns a spec for starting a language server"""

    key = ""
    languages: List[Text] = []
    args: List[Token] = []
    spec: LanguageServerSpec = {}

    def is_installed(self, mgr: LanguageServerManagerAPI) -> bool:  # pragma: no cover
        """Whether the language server is installed or not.

        This method may become abstract in the next major release."""
        return True

    def __call__(
        self, mgr: LanguageServerManagerAPI
    ) -> KeyedLanguageServerSpecs:  # pragma: no cover
        return {}


# Gotta be down here so it can by typed... really should have a IL
SpecMaker = Union[SpecBase, SimpleSpecMaker]

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/types.py
NikolaBlagojevic/pyrecodes
# flake8: noqa: W503
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from shutil import rmtree
from typing import List

from tornado.concurrent import run_on_executor
from tornado.gen import convert_yielded

from .manager import lsp_message_listener
from .paths import file_uri_to_path
from .types import LanguageServerManagerAPI

# TODO: make configurable
MAX_WORKERS = 4


def extract_or_none(obj, path):
    for crumb in path:
        try:
            obj = obj[crumb]
        except (KeyError, TypeError):
            return None
    return obj


class EditableFile:
    executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)

    def __init__(self, path):
        # Python 3.5 relict:
        self.path = Path(path) if isinstance(path, str) else path

    async def read(self):
        self.lines = await convert_yielded(self.read_lines())

    async def write(self):
        return await convert_yielded(self.write_lines())

    @run_on_executor
    def read_lines(self):
        # empty string required by the assumptions of the gluing algorithm
        lines = [""]
        try:
            # TODO: what to do about bad encoding reads?
            lines = self.path.read_text(encoding="utf-8").splitlines()
        except FileNotFoundError:
            pass
        return lines

    @run_on_executor
    def write_lines(self):
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self.path.write_text("\n".join(self.lines), encoding="utf-8")

    @staticmethod
    def trim(lines: list, character: int, side: int):
        needs_glue = False
        if lines:
            trimmed = lines[side][character:]
            if lines[side] != trimmed:
                needs_glue = True
            lines[side] = trimmed
        return needs_glue

    @staticmethod
    def join(left, right, glue: bool):
        if not glue:
            return []
        return [(left[-1] if left else "") + (right[0] if right else "")]

    def apply_change(self, text: str, start, end):
        before = self.lines[: start["line"]]
        after = self.lines[end["line"] :]

        needs_glue_left = self.trim(lines=before, character=start["character"], side=0)
        needs_glue_right = self.trim(lines=after, character=end["character"], side=-1)

        inner = text.split("\n")

        self.lines = (
            before[: -1 if needs_glue_left else None]
            + self.join(before, inner, needs_glue_left)
            + inner[1 if needs_glue_left else None : -1 if needs_glue_right else None]
            + self.join(inner, after, needs_glue_right)
            + after[1 if needs_glue_right else None :]
        ) or [""]

    @property
    def full_range(self):
        start = {"line": 0, "character": 0}
        end = {
            "line": len(self.lines),
            "character": len(self.lines[-1]) if self.lines else 0,
        }
        return {"start": start, "end": end}


WRITE_ONE = ["textDocument/didOpen", "textDocument/didChange", "textDocument/didSave"]


class ShadowFilesystemError(ValueError):
    """Error in the shadow file system."""


def setup_shadow_filesystem(virtual_documents_uri: str):

    if not virtual_documents_uri.startswith("file:/"):
        raise ShadowFilesystemError(  # pragma: no cover
            'Virtual documents URI has to start with "file:/", got '
            + virtual_documents_uri
        )

    initialized = False
    failures: List[Exception] = []

    shadow_filesystem = Path(file_uri_to_path(virtual_documents_uri))

    @lsp_message_listener("client")
    async def shadow_virtual_documents(scope, message, language_server, manager):
        """Intercept a message with document contents creating a shadow file for it.

        Only create the shadow file if the URI matches the virtual documents URI.
        Returns the path on filesystem where the content was stored.
        """
        nonlocal initialized

        # short-circut if language server does not require documents on disk
        server_spec = manager.language_servers[language_server]
        if not server_spec.get("requires_documents_on_disk", True):
            return

        if not message.get("method") in WRITE_ONE:
            return

        document = extract_or_none(message, ["params", "textDocument"])
        if document is None:
            raise ShadowFilesystemError(
                "Could not get textDocument from: {}".format(message)
            )

        uri = extract_or_none(document, ["uri"])
        if not uri:
            raise ShadowFilesystemError("Could not get URI from: {}".format(message))

        if not uri.startswith(virtual_documents_uri):
            return

        # initialization (/any file system operations) delayed until needed
        if not initialized:
            if len(failures) == 3:
                return
            try:
                # create if does no exist (so that removal does not raise)
                shadow_filesystem.mkdir(parents=True, exist_ok=True)
                # remove with contents
                rmtree(str(shadow_filesystem))
                # create again
                shadow_filesystem.mkdir(parents=True, exist_ok=True)
            except (OSError, PermissionError, FileNotFoundError) as e:
                failures.append(e)
                if len(failures) == 3:
                    manager.log.warn(
                        "[lsp] initialization of shadow filesystem failed three times"
                        " check if the path set by `LanguageServerManager.virtual_documents_dir`"
                        " or `JP_LSP_VIRTUAL_DIR` is correct; if this is happening with a server"
                        " for which which you control (or wish to override) jupyter-lsp specification"
                        " you can try switching `requires_documents_on_disk` off. The errors were: %s",
                        failures,
                    )
                return
            initialized = True

        path = file_uri_to_path(uri)
        editable_file = EditableFile(path)

        await editable_file.read()

        text = extract_or_none(document, ["text"])

        if text is not None:
            # didOpen and didSave may provide text within the document
            changes = [{"text": text}]
        else:
            # didChange is the only one which can also provide it in params (as contentChanges)
            if message["method"] != "textDocument/didChange":
                return
            if "contentChanges" not in message["params"]:
                raise ShadowFilesystemError(
                    "textDocument/didChange is missing contentChanges"
                )
            changes = message["params"]["contentChanges"]

        if len(changes) > 1:
            manager.log.warn(  # pragma: no cover
                "LSP warning: up to one change supported for textDocument/didChange"
            )

        for change in changes[:1]:
            change_range = change.get("range", editable_file.full_range)
            editable_file.apply_change(change["text"], **change_range)

        await editable_file.write()

        return path

    return shadow_virtual_documents

env_pyrecodes/lib/python3.9/site-packages/jupyter_lsp/virtual_documents_shadow.py
NikolaBlagojevic/pyrecodes
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.

"""Testing utils."""
import json
import os
import sys
from http.cookies import SimpleCookie
from pathlib import Path
from typing import Optional
from urllib.parse import parse_qs, urlparse

import tornado.httpclient
import tornado.web
from openapi_core import V30RequestValidator, V30ResponseValidator
from openapi_core.spec.paths import Spec
from openapi_core.validation.request.datatypes import RequestParameters
from tornado.httpclient import HTTPRequest, HTTPResponse
from werkzeug.datastructures import Headers, ImmutableMultiDict

from jupyterlab_server.spec import get_openapi_spec

HERE = Path(os.path.dirname(__file__)).resolve()

with open(HERE / "test_data" / "app-settings" / "overrides.json", encoding="utf-8") as fpt:
    big_unicode_string = json.load(fpt)["@jupyterlab/unicode-extension:plugin"]["comment"]


class TornadoOpenAPIRequest:
    """
    Converts a torando request to an OpenAPI one
    """

    def __init__(self, request: HTTPRequest, spec: Spec):
        """Initialize the request."""
        self.request = request
        self.spec = spec
        if request.url is None:
            msg = "Request URL is missing"
            raise RuntimeError(msg)
        self._url_parsed = urlparse(request.url)

        cookie: SimpleCookie = SimpleCookie()
        cookie.load(request.headers.get("Set-Cookie", ""))
        cookies = {}
        for key, morsel in cookie.items():
            cookies[key] = morsel.value

        # extract the path
        o = urlparse(request.url)

        # gets deduced by path finder against spec
        path: dict = {}

        self.parameters = RequestParameters(
            query=ImmutableMultiDict(parse_qs(o.query)),
            header=dict(request.headers),
            cookie=ImmutableMultiDict(cookies),
            path=path,
        )

    @property
    def host_url(self) -> str:
        url = self.request.url
        return url[: url.index('/lab')]

    @property
    def path(self) -> str:
        # extract the best matching url
        # work around lack of support for path parameters which can contain slashes
        # https://github.com/OAI/OpenAPI-Specification/issues/892
        url = None
        o = urlparse(self.request.url)
        for path_ in self.spec["paths"]:
            if url:
                continue
            has_arg = "{" in path_
            path = path_[: path_.index("{")] if has_arg else path_
            if path in o.path:
                u = o.path[o.path.index(path) :]
                if not has_arg and len(u) == len(path):
                    url = u
                if has_arg and not u.endswith("/"):
                    url = u[: len(path)] + r"foo"

        if url is None:
            msg = f"Could not find matching pattern for {o.path}"
            raise ValueError(msg)
        return url

    @property
    def method(self) -> str:
        method = self.request.method
        return method and method.lower() or ""

    @property
    def body(self) -> Optional[str]:
        if self.request.body is None:
            return None
        if not isinstance(self.request.body, bytes):
            msg = 'Request body is invalid'
            raise AssertionError(msg)
        return self.request.body.decode("utf-8")

    @property
    def mimetype(self) -> str:
        # Order matters because all tornado requests
        # include Accept */* which does not necessarily match the content type
        request = self.request
        return (
            request.headers.get("Content-Type")
            or request.headers.get("Accept")
            or "application/json"
        )


class TornadoOpenAPIResponse:
    """A tornado open API response."""

    def __init__(self, response: HTTPResponse):
        """Initialize the response."""
        self.response = response

    @property
    def data(self) -> str:
        if not isinstance(self.response.body, bytes):
            msg = 'Response body is invalid'
            raise AssertionError(msg)
        return self.response.body.decode("utf-8")

    @property
    def status_code(self) -> int:
        return int(self.response.code)

    @property
    def mimetype(self) -> str:
        return str(self.response.headers.get("Content-Type", "application/json"))

    @property
    def headers(self) -> Headers:
        return Headers(dict(self.response.headers))


def validate_request(response):
    """Validate an API request"""
    openapi_spec = get_openapi_spec()

    request = TornadoOpenAPIRequest(response.request, openapi_spec)
    V30RequestValidator(openapi_spec).validate(request)

    response = TornadoOpenAPIResponse(response)
    V30ResponseValidator(openapi_spec).validate(request, response)


def maybe_patch_ioloop():
    """a windows 3.8+ patch for the asyncio loop"""
    if (
        sys.platform.startswith("win")
        and tornado.version_info < (6, 1)
        and sys.version_info >= (3, 8)
    ):
        try:
            from asyncio import WindowsProactorEventLoopPolicy, WindowsSelectorEventLoopPolicy
        except ImportError:
            pass
            # not affected
        else:
            from asyncio import get_event_loop_policy, set_event_loop_policy

            if type(get_event_loop_policy()) is WindowsProactorEventLoopPolicy:
                # WindowsProactorEventLoopPolicy is not compatible with tornado 6
                # fallback to the pre-3.8 default of Selector
                set_event_loop_policy(WindowsSelectorEventLoopPolicy())


def expected_http_error(error, expected_code, expected_message=None):  # noqa
    """Check that the error matches the expected output error."""
    e = error.value
    if isinstance(e, tornado.web.HTTPError):
        if expected_code != e.status_code:
            return False
        if expected_message is not None and expected_message != str(e):
            return False
        return True
    elif any(
        [
            isinstance(e, tornado.httpclient.HTTPClientError),
            isinstance(e, tornado.httpclient.HTTPError),
        ]
    ):
        if expected_code != e.code:
            return False
        if expected_message:
            message = json.loads(e.response.body.decode())["message"]
            if expected_message != message:
                return False
        return True

    return False

env_pyrecodes/lib/python3.9/site-packages/jupyterlab_server/test_utils.py
NikolaBlagojevic/pyrecodes
from matplotlib.testing.conftest import (  # noqa
    mpl_test_settings, pytest_configure, pytest_unconfigure, pd, xr)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/conftest.py
NikolaBlagojevic/pyrecodes
import re

import numpy as np
import pytest

from matplotlib import _api


@pytest.mark.parametrize('target,test_shape',
                         [((None, ), (1, 3)),
                          ((None, 3), (1,)),
                          ((None, 3), (1, 2)),
                          ((1, 5), (1, 9)),
                          ((None, 2, None), (1, 3, 1))
                          ])
def test_check_shape(target, test_shape):
    error_pattern = (f"^'aardvark' must be {len(target)}D.*" +
                     re.escape(f'has shape {test_shape}'))
    data = np.zeros(test_shape)
    with pytest.raises(ValueError, match=error_pattern):
        _api.check_shape(target, aardvark=data)


def test_classproperty_deprecation():
    class A:
        @_api.deprecated("0.0.0")
        @_api.classproperty
        def f(cls):
            pass
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        A.f
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        a = A()
        a.f


def test_deprecate_privatize_attribute():
    class C:
        def __init__(self): self._attr = 1
        def _meth(self, arg): return arg
        attr = _api.deprecate_privatize_attribute("0.0")
        meth = _api.deprecate_privatize_attribute("0.0")

    c = C()
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        assert c.attr == 1
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        c.attr = 2
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        assert c.attr == 2
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        assert c.meth(42) == 42


def test_delete_parameter():
    @_api.delete_parameter("3.0", "foo")
    def func1(foo=None):
        pass

    @_api.delete_parameter("3.0", "foo")
    def func2(**kwargs):
        pass

    for func in [func1, func2]:
        func()  # No warning.
        with pytest.warns(_api.MatplotlibDeprecationWarning):
            func(foo="bar")

    def pyplot_wrapper(foo=_api.deprecation._deprecated_parameter):
        func1(foo)

    pyplot_wrapper()  # No warning.
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        func(foo="bar")


def test_make_keyword_only():
    @_api.make_keyword_only("3.0", "arg")
    def func(pre, arg, post=None):
        pass

    func(1, arg=2)  # Check that no warning is emitted.

    with pytest.warns(_api.MatplotlibDeprecationWarning):
        func(1, 2)
    with pytest.warns(_api.MatplotlibDeprecationWarning):
        func(1, 2, 3)


def test_deprecation_alternative():
    alternative = "`.f1`, `f2`, `f3(x) <.f3>` or `f4(x)<f4>`"
    @_api.deprecated("1", alternative=alternative)
    def f():
        pass
    assert alternative in f.__doc__


def test_empty_check_in_list():
    with pytest.raises(TypeError, match="No argument to check!"):
        _api.check_in_list(["a"])

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_api.py
NikolaBlagojevic/pyrecodes
from matplotlib import pyplot as plt

import pytest


pytest.importorskip("matplotlib.backends.backend_gtk3agg")


@pytest.mark.backend("gtk3agg", skip_on_importerror=True)
def test_correct_key():
    pytest.xfail("test_widget_send_event is not triggering key_press_event")

    from gi.repository import Gdk, Gtk
    fig = plt.figure()
    buf = []

    def send(event):
        for key, mod in [
                (Gdk.KEY_a, Gdk.ModifierType.SHIFT_MASK),
                (Gdk.KEY_a, 0),
                (Gdk.KEY_a, Gdk.ModifierType.CONTROL_MASK),
                (Gdk.KEY_agrave, 0),
                (Gdk.KEY_Control_L, Gdk.ModifierType.MOD1_MASK),
                (Gdk.KEY_Alt_L, Gdk.ModifierType.CONTROL_MASK),
                (Gdk.KEY_agrave,
                 Gdk.ModifierType.CONTROL_MASK
                 | Gdk.ModifierType.MOD1_MASK
                 | Gdk.ModifierType.MOD4_MASK),
                (0xfd16, 0),   # KEY_3270_Play.
                (Gdk.KEY_BackSpace, 0),
                (Gdk.KEY_BackSpace, Gdk.ModifierType.CONTROL_MASK),
        ]:
            # This is not actually really the right API: it depends on the
            # actual keymap (e.g. on Azerty, shift+agrave -> 0).
            Gtk.test_widget_send_key(fig.canvas, key, mod)

    def receive(event):
        buf.append(event.key)
        if buf == [
                "A", "a", "ctrl+a",
                "\N{LATIN SMALL LETTER A WITH GRAVE}",
                "alt+control", "ctrl+alt",
                "ctrl+alt+super+\N{LATIN SMALL LETTER A WITH GRAVE}",
                # (No entry for KEY_3270_Play.)
                "backspace", "ctrl+backspace",
        ]:
            plt.close(fig)

    fig.canvas.mpl_connect("draw_event", send)
    fig.canvas.mpl_connect("key_press_event", receive)
    plt.show()

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_gtk3.py
NikolaBlagojevic/pyrecodes
import os

import pytest

import matplotlib as mpl
import matplotlib.pyplot as plt
try:
    from matplotlib.backends import _macosx
except ImportError:
    pytest.skip("These are mac only tests", allow_module_level=True)


@pytest.mark.backend('macosx')
def test_cached_renderer():
    # Make sure that figures have an associated renderer after
    # a fig.canvas.draw() call
    fig = plt.figure(1)
    fig.canvas.draw()
    assert fig.canvas.get_renderer()._renderer is not None

    fig = plt.figure(2)
    fig.draw_without_rendering()
    assert fig.canvas.get_renderer()._renderer is not None


@pytest.mark.backend('macosx')
def test_savefig_rcparam(monkeypatch, tmp_path):

    def new_choose_save_file(title, directory, filename):
        # Replacement function instead of opening a GUI window
        # Make a new directory for testing the update of the rcParams
        assert directory == str(tmp_path)
        os.makedirs(f"{directory}/test")
        return f"{directory}/test/{filename}"

    monkeypatch.setattr(_macosx, "choose_save_file", new_choose_save_file)
    fig = plt.figure()
    with mpl.rc_context({"savefig.directory": tmp_path}):
        fig.canvas.toolbar.save_figure()
        # Check the saved location got created
        save_file = f"{tmp_path}/test/{fig.canvas.get_default_filename()}"
        assert os.path.exists(save_file)

        # Check the savefig.directory rcParam got updated because
        # we added a subdirectory "test"
        assert mpl.rcParams["savefig.directory"] == f"{tmp_path}/test"

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_macosx.py
NikolaBlagojevic/pyrecodes
import os
from pathlib import Path
import subprocess
from tempfile import TemporaryDirectory

import pytest

nbformat = pytest.importorskip('nbformat')
pytest.importorskip('nbconvert')
pytest.importorskip('ipykernel')

# From https://blog.thedataincubator.com/2016/06/testing-jupyter-notebooks/


def test_ipynb():
    nb_path = Path(__file__).parent / 'test_nbagg_01.ipynb'

    with TemporaryDirectory() as tmpdir:
        out_path = Path(tmpdir, "out.ipynb")
        subprocess.check_call(
            ["jupyter", "nbconvert", "--to", "notebook",
             "--execute", "--ExecutePreprocessor.timeout=500",
             "--output", str(out_path), str(nb_path)],
            env={**os.environ, "IPYTHONDIR": tmpdir})
        with out_path.open() as out:
            nb = nbformat.read(out, nbformat.current_nbformat)

    errors = [output for cell in nb.cells for output in cell.get("outputs", [])
              if output.output_type == "error"]
    assert not errors

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_nbagg.py
NikolaBlagojevic/pyrecodes
import datetime
import decimal
import io
import os
from pathlib import Path
from tempfile import NamedTemporaryFile

import numpy as np
import pytest

import matplotlib as mpl
from matplotlib import (
    pyplot as plt, rcParams, font_manager as fm
)
from matplotlib.cbook import _get_data_path
from matplotlib.ft2font import FT2Font
from matplotlib.font_manager import findfont, FontProperties
from matplotlib.backends._backend_pdf_ps import get_glyphs_subset
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.patches import Rectangle
from matplotlib.testing.decorators import check_figures_equal, image_comparison
from matplotlib.testing._markers import needs_usetex


@image_comparison(['pdf_use14corefonts.pdf'])
def test_use14corefonts():
    rcParams['pdf.use14corefonts'] = True
    rcParams['font.family'] = 'sans-serif'
    rcParams['font.size'] = 8
    rcParams['font.sans-serif'] = ['Helvetica']
    rcParams['pdf.compression'] = 0

    text = '''A three-line text positioned just above a blue line
and containing some French characters and the euro symbol:
"Merci pépé pour les 10 €"'''

    fig, ax = plt.subplots()
    ax.set_title('Test PDF backend with option use14corefonts=True')
    ax.text(0.5, 0.5, text, horizontalalignment='center',
            verticalalignment='bottom',
            fontsize=14)
    ax.axhline(0.5, linewidth=0.5)


@pytest.mark.parametrize('fontname, fontfile', [
    ('DejaVu Sans', 'DejaVuSans.ttf'),
    ('WenQuanYi Zen Hei', 'wqy-zenhei.ttc'),
])
@pytest.mark.parametrize('fonttype', [3, 42])
def test_embed_fonts(fontname, fontfile, fonttype):
    if Path(findfont(FontProperties(family=[fontname]))).name != fontfile:
        pytest.skip(f'Font {fontname!r} may be missing')

    rcParams['pdf.fonttype'] = fonttype
    fig, ax = plt.subplots()
    ax.plot([1, 2, 3])
    ax.set_title('Axes Title', font=fontname)
    fig.savefig(io.BytesIO(), format='pdf')


def test_multipage_pagecount():
    with PdfPages(io.BytesIO()) as pdf:
        assert pdf.get_pagecount() == 0
        fig, ax = plt.subplots()
        ax.plot([1, 2, 3])
        fig.savefig(pdf, format="pdf")
        assert pdf.get_pagecount() == 1
        pdf.savefig()
        assert pdf.get_pagecount() == 2


def test_multipage_properfinalize():
    pdfio = io.BytesIO()
    with PdfPages(pdfio) as pdf:
        for i in range(10):
            fig, ax = plt.subplots()
            ax.set_title('This is a long title')
            fig.savefig(pdf, format="pdf")
    s = pdfio.getvalue()
    assert s.count(b'startxref') == 1
    assert len(s) < 40000


def test_multipage_keep_empty():
    # test empty pdf files
    # test that an empty pdf is left behind with keep_empty=True (default)
    with NamedTemporaryFile(delete=False) as tmp:
        with PdfPages(tmp) as pdf:
            filename = pdf._file.fh.name
        assert os.path.exists(filename)
    os.remove(filename)
    # test if an empty pdf is deleting itself afterwards with keep_empty=False
    with PdfPages(filename, keep_empty=False) as pdf:
        pass
    assert not os.path.exists(filename)
    # test pdf files with content, they should never be deleted
    fig, ax = plt.subplots()
    ax.plot([1, 2, 3])
    # test that a non-empty pdf is left behind with keep_empty=True (default)
    with NamedTemporaryFile(delete=False) as tmp:
        with PdfPages(tmp) as pdf:
            filename = pdf._file.fh.name
            pdf.savefig()
        assert os.path.exists(filename)
    os.remove(filename)
    # test that a non-empty pdf is left behind with keep_empty=False
    with NamedTemporaryFile(delete=False) as tmp:
        with PdfPages(tmp, keep_empty=False) as pdf:
            filename = pdf._file.fh.name
            pdf.savefig()
        assert os.path.exists(filename)
    os.remove(filename)


def test_composite_image():
    # Test that figures can be saved with and without combining multiple images
    # (on a single set of axes) into a single composite image.
    X, Y = np.meshgrid(np.arange(-5, 5, 1), np.arange(-5, 5, 1))
    Z = np.sin(Y ** 2)
    fig, ax = plt.subplots()
    ax.set_xlim(0, 3)
    ax.imshow(Z, extent=[0, 1, 0, 1])
    ax.imshow(Z[::-1], extent=[2, 3, 0, 1])
    plt.rcParams['image.composite_image'] = True
    with PdfPages(io.BytesIO()) as pdf:
        fig.savefig(pdf, format="pdf")
        assert len(pdf._file._images) == 1
    plt.rcParams['image.composite_image'] = False
    with PdfPages(io.BytesIO()) as pdf:
        fig.savefig(pdf, format="pdf")
        assert len(pdf._file._images) == 2


def test_savefig_metadata(monkeypatch):
    pikepdf = pytest.importorskip('pikepdf')
    monkeypatch.setenv('SOURCE_DATE_EPOCH', '0')

    fig, ax = plt.subplots()
    ax.plot(range(5))

    md = {
        'Author': 'me',
        'Title': 'Multipage PDF',
        'Subject': 'Test page',
        'Keywords': 'test,pdf,multipage',
        'ModDate': datetime.datetime(
            1968, 8, 1, tzinfo=datetime.timezone(datetime.timedelta(0))),
        'Trapped': 'True'
    }
    buf = io.BytesIO()
    fig.savefig(buf, metadata=md, format='pdf')

    with pikepdf.Pdf.open(buf) as pdf:
        info = {k: str(v) for k, v in pdf.docinfo.items()}

    assert info == {
        '/Author': 'me',
        '/CreationDate': 'D:19700101000000Z',
        '/Creator': f'Matplotlib v{mpl.__version__}, https://matplotlib.org',
        '/Keywords': 'test,pdf,multipage',
        '/ModDate': 'D:19680801000000Z',
        '/Producer': f'Matplotlib pdf backend v{mpl.__version__}',
        '/Subject': 'Test page',
        '/Title': 'Multipage PDF',
        '/Trapped': '/True',
    }


def test_invalid_metadata():
    fig, ax = plt.subplots()

    with pytest.warns(UserWarning,
                      match="Unknown infodict keyword: 'foobar'."):
        fig.savefig(io.BytesIO(), format='pdf', metadata={'foobar': 'invalid'})

    with pytest.warns(UserWarning,
                      match='not an instance of datetime.datetime.'):
        fig.savefig(io.BytesIO(), format='pdf',
                    metadata={'ModDate': '1968-08-01'})

    with pytest.warns(UserWarning,
                      match='not one of {"True", "False", "Unknown"}'):
        fig.savefig(io.BytesIO(), format='pdf', metadata={'Trapped': 'foo'})

    with pytest.warns(UserWarning, match='not an instance of str.'):
        fig.savefig(io.BytesIO(), format='pdf', metadata={'Title': 1234})


def test_multipage_metadata(monkeypatch):
    pikepdf = pytest.importorskip('pikepdf')
    monkeypatch.setenv('SOURCE_DATE_EPOCH', '0')

    fig, ax = plt.subplots()
    ax.plot(range(5))

    md = {
        'Author': 'me',
        'Title': 'Multipage PDF',
        'Subject': 'Test page',
        'Keywords': 'test,pdf,multipage',
        'ModDate': datetime.datetime(
            1968, 8, 1, tzinfo=datetime.timezone(datetime.timedelta(0))),
        'Trapped': 'True'
    }
    buf = io.BytesIO()
    with PdfPages(buf, metadata=md) as pdf:
        pdf.savefig(fig)
        pdf.savefig(fig)

    with pikepdf.Pdf.open(buf) as pdf:
        info = {k: str(v) for k, v in pdf.docinfo.items()}

    assert info == {
        '/Author': 'me',
        '/CreationDate': 'D:19700101000000Z',
        '/Creator': f'Matplotlib v{mpl.__version__}, https://matplotlib.org',
        '/Keywords': 'test,pdf,multipage',
        '/ModDate': 'D:19680801000000Z',
        '/Producer': f'Matplotlib pdf backend v{mpl.__version__}',
        '/Subject': 'Test page',
        '/Title': 'Multipage PDF',
        '/Trapped': '/True',
    }


def test_text_urls():
    pikepdf = pytest.importorskip('pikepdf')

    test_url = 'https://test_text_urls.matplotlib.org/'

    fig = plt.figure(figsize=(2, 1))
    fig.text(0.1, 0.1, 'test plain 123', url=f'{test_url}plain')
    fig.text(0.1, 0.4, 'test mathtext $123$', url=f'{test_url}mathtext')

    with io.BytesIO() as fd:
        fig.savefig(fd, format='pdf')

        with pikepdf.Pdf.open(fd) as pdf:
            annots = pdf.pages[0].Annots

            # Iteration over Annots must occur within the context manager,
            # otherwise it may fail depending on the pdf structure.
            for y, fragment in [('0.1', 'plain'), ('0.4', 'mathtext')]:
                annot = next(
                    (a for a in annots if a.A.URI == f'{test_url}{fragment}'),
                    None)
                assert annot is not None
                assert getattr(annot, 'QuadPoints', None) is None
                # Positions in points (72 per inch.)
                assert annot.Rect[1] == decimal.Decimal(y) * 72


def test_text_rotated_urls():
    pikepdf = pytest.importorskip('pikepdf')

    test_url = 'https://test_text_urls.matplotlib.org/'

    fig = plt.figure(figsize=(1, 1))
    fig.text(0.1, 0.1, 'N', rotation=45, url=f'{test_url}')

    with io.BytesIO() as fd:
        fig.savefig(fd, format='pdf')

        with pikepdf.Pdf.open(fd) as pdf:
            annots = pdf.pages[0].Annots

            # Iteration over Annots must occur within the context manager,
            # otherwise it may fail depending on the pdf structure.
            annot = next(
                (a for a in annots if a.A.URI == f'{test_url}'),
                None)
            assert annot is not None
            assert getattr(annot, 'QuadPoints', None) is not None
            # Positions in points (72 per inch)
            assert annot.Rect[0] == \
               annot.QuadPoints[6] - decimal.Decimal('0.00001')


@needs_usetex
def test_text_urls_tex():
    pikepdf = pytest.importorskip('pikepdf')

    test_url = 'https://test_text_urls.matplotlib.org/'

    fig = plt.figure(figsize=(2, 1))
    fig.text(0.1, 0.7, 'test tex $123$', usetex=True, url=f'{test_url}tex')

    with io.BytesIO() as fd:
        fig.savefig(fd, format='pdf')

        with pikepdf.Pdf.open(fd) as pdf:
            annots = pdf.pages[0].Annots

            # Iteration over Annots must occur within the context manager,
            # otherwise it may fail depending on the pdf structure.
            annot = next(
                (a for a in annots if a.A.URI == f'{test_url}tex'),
                None)
            assert annot is not None
            # Positions in points (72 per inch.)
            assert annot.Rect[1] == decimal.Decimal('0.7') * 72


def test_pdfpages_fspath():
    with PdfPages(Path(os.devnull)) as pdf:
        pdf.savefig(plt.figure())


@image_comparison(['hatching_legend.pdf'])
def test_hatching_legend():
    """Test for correct hatching on patches in legend"""
    fig = plt.figure(figsize=(1, 2))

    a = Rectangle([0, 0], 0, 0, facecolor="green", hatch="XXXX")
    b = Rectangle([0, 0], 0, 0, facecolor="blue", hatch="XXXX")

    fig.legend([a, b, a, b], ["", "", "", ""])


@image_comparison(['grayscale_alpha.pdf'])
def test_grayscale_alpha():
    """Masking images with NaN did not work for grayscale images"""
    x, y = np.ogrid[-2:2:.1, -2:2:.1]
    dd = np.exp(-(x**2 + y**2))
    dd[dd < .1] = np.nan
    fig, ax = plt.subplots()
    ax.imshow(dd, interpolation='none', cmap='gray_r')
    ax.set_xticks([])
    ax.set_yticks([])


@mpl.style.context('default')
@check_figures_equal(extensions=["pdf", "eps"])
def test_pdf_eps_savefig_when_color_is_none(fig_test, fig_ref):
    ax_test = fig_test.add_subplot()
    ax_test.set_axis_off()
    ax_test.plot(np.sin(np.linspace(-5, 5, 100)), "v", c="none")
    ax_ref = fig_ref.add_subplot()
    ax_ref.set_axis_off()


@needs_usetex
def test_failing_latex():
    """Test failing latex subprocess call"""
    plt.xlabel("$22_2_2$", usetex=True)  # This fails with "Double subscript"
    with pytest.raises(RuntimeError):
        plt.savefig(io.BytesIO(), format="pdf")


def test_empty_rasterized():
    # Check that empty figures that are rasterised save to pdf files fine
    fig, ax = plt.subplots()
    ax.plot([], [], rasterized=True)
    fig.savefig(io.BytesIO(), format="pdf")


@image_comparison(['kerning.pdf'])
def test_kerning():
    fig = plt.figure()
    s = "AVAVAVAVAVAVAVAV€AAVV"
    fig.text(0, .25, s, size=5)
    fig.text(0, .75, s, size=20)


def test_glyphs_subset():
    fpath = str(_get_data_path("fonts/ttf/DejaVuSerif.ttf"))
    chars = "these should be subsetted! 1234567890"

    # non-subsetted FT2Font
    nosubfont = FT2Font(fpath)
    nosubfont.set_text(chars)

    # subsetted FT2Font
    subfont = FT2Font(get_glyphs_subset(fpath, chars))
    subfont.set_text(chars)

    nosubcmap = nosubfont.get_charmap()
    subcmap = subfont.get_charmap()

    # all unique chars must be available in subsetted font
    assert set(chars) == set(chr(key) for key in subcmap.keys())

    # subsetted font's charmap should have less entries
    assert len(subcmap) < len(nosubcmap)

    # since both objects are assigned same characters
    assert subfont.get_num_glyphs() == nosubfont.get_num_glyphs()


@image_comparison(["multi_font_type3.pdf"], tol=4.6)
def test_multi_font_type3():
    fp = fm.FontProperties(family=["WenQuanYi Zen Hei"])
    if Path(fm.findfont(fp)).name != "wqy-zenhei.ttc":
        pytest.skip("Font may be missing")

    plt.rc('font', family=['DejaVu Sans', 'WenQuanYi Zen Hei'], size=27)
    plt.rc('pdf', fonttype=3)

    fig = plt.figure()
    fig.text(0.15, 0.475, "There are 几个汉字 in between!")


@image_comparison(["multi_font_type42.pdf"], tol=2.2)
def test_multi_font_type42():
    fp = fm.FontProperties(family=["WenQuanYi Zen Hei"])
    if Path(fm.findfont(fp)).name != "wqy-zenhei.ttc":
        pytest.skip("Font may be missing")

    plt.rc('font', family=['DejaVu Sans', 'WenQuanYi Zen Hei'], size=27)
    plt.rc('pdf', fonttype=42)

    fig = plt.figure()
    fig.text(0.15, 0.475, "There are 几个汉字 in between!")

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_pdf.py
NikolaBlagojevic/pyrecodes
import datetime
from io import BytesIO
import os
import shutil

import numpy as np
from packaging.version import parse as parse_version
import pytest

import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.testing import _has_tex_package, _check_for_pgf
from matplotlib.testing.compare import compare_images, ImageComparisonFailure
from matplotlib.backends.backend_pgf import PdfPages, _tex_escape
from matplotlib.testing.decorators import (
    _image_directories, check_figures_equal, image_comparison)
from matplotlib.testing._markers import (
    needs_ghostscript, needs_pgf_lualatex, needs_pgf_pdflatex,
    needs_pgf_xelatex)


baseline_dir, result_dir = _image_directories(lambda: 'dummy func')


def compare_figure(fname, savefig_kwargs={}, tol=0):
    actual = os.path.join(result_dir, fname)
    plt.savefig(actual, **savefig_kwargs)

    expected = os.path.join(result_dir, "expected_%s" % fname)
    shutil.copyfile(os.path.join(baseline_dir, fname), expected)
    err = compare_images(expected, actual, tol=tol)
    if err:
        raise ImageComparisonFailure(err)


@pytest.mark.parametrize('plain_text, escaped_text', [
    (r'quad_sum: $\sum x_i^2$', r'quad_sum: \(\displaystyle \sum x_i^2\)'),
    ('% not a comment', r'\% not a comment'),
    ('^not', r'\^not'),
])
def test_tex_escape(plain_text, escaped_text):
    assert _tex_escape(plain_text) == escaped_text


@needs_pgf_xelatex
@pytest.mark.backend('pgf')
def test_tex_special_chars(tmp_path):
    fig = plt.figure()
    fig.text(.5, .5, "_^ $a_b^c$")
    fig.savefig(tmp_path / "test.pdf")  # Should not error.


def create_figure():
    plt.figure()
    x = np.linspace(0, 1, 15)

    # line plot
    plt.plot(x, x ** 2, "b-")

    # marker
    plt.plot(x, 1 - x**2, "g>")

    # filled paths and patterns
    plt.fill_between([0., .4], [.4, 0.], hatch='//', facecolor="lightgray",
                     edgecolor="red")
    plt.fill([3, 3, .8, .8, 3], [2, -2, -2, 0, 2], "b")

    # text and typesetting
    plt.plot([0.9], [0.5], "ro", markersize=3)
    plt.text(0.9, 0.5, 'unicode (ü, °, µ) and math ($\\mu_i = x_i^2$)',
             ha='right', fontsize=20)
    plt.ylabel('sans-serif, blue, $\\frac{\\sqrt{x}}{y^2}$..',
               family='sans-serif', color='blue')

    plt.xlim(0, 1)
    plt.ylim(0, 1)


# test compiling a figure to pdf with xelatex
@needs_pgf_xelatex
@pytest.mark.backend('pgf')
@image_comparison(['pgf_xelatex.pdf'], style='default')
def test_xelatex():
    rc_xelatex = {'font.family': 'serif',
                  'pgf.rcfonts': False}
    mpl.rcParams.update(rc_xelatex)
    create_figure()


try:
    _old_gs_version = \
        mpl._get_executable_info('gs').version < parse_version('9.50')
except mpl.ExecutableNotFoundError:
    _old_gs_version = True


# test compiling a figure to pdf with pdflatex
@needs_pgf_pdflatex
@pytest.mark.skipif(not _has_tex_package('ucs'), reason='needs ucs.sty')
@pytest.mark.backend('pgf')
@image_comparison(['pgf_pdflatex.pdf'], style='default',
                  tol=11.7 if _old_gs_version else 0)
def test_pdflatex():
    if os.environ.get('APPVEYOR'):
        pytest.xfail("pdflatex test does not work on appveyor due to missing "
                     "LaTeX fonts")

    rc_pdflatex = {'font.family': 'serif',
                   'pgf.rcfonts': False,
                   'pgf.texsystem': 'pdflatex',
                   'pgf.preamble': ('\\usepackage[utf8x]{inputenc}'
                                    '\\usepackage[T1]{fontenc}')}
    mpl.rcParams.update(rc_pdflatex)
    create_figure()


# test updating the rc parameters for each figure
@needs_pgf_xelatex
@needs_pgf_pdflatex
@mpl.style.context('default')
@pytest.mark.backend('pgf')
def test_rcupdate():
    rc_sets = [{'font.family': 'sans-serif',
                'font.size': 30,
                'figure.subplot.left': .2,
                'lines.markersize': 10,
                'pgf.rcfonts': False,
                'pgf.texsystem': 'xelatex'},
               {'font.family': 'monospace',
                'font.size': 10,
                'figure.subplot.left': .1,
                'lines.markersize': 20,
                'pgf.rcfonts': False,
                'pgf.texsystem': 'pdflatex',
                'pgf.preamble': ('\\usepackage[utf8x]{inputenc}'
                                 '\\usepackage[T1]{fontenc}'
                                 '\\usepackage{sfmath}')}]
    tol = [0, 13.2] if _old_gs_version else [0, 0]
    for i, rc_set in enumerate(rc_sets):
        with mpl.rc_context(rc_set):
            for substring, pkg in [('sfmath', 'sfmath'), ('utf8x', 'ucs')]:
                if (substring in mpl.rcParams['pgf.preamble']
                        and not _has_tex_package(pkg)):
                    pytest.skip(f'needs {pkg}.sty')
            create_figure()
            compare_figure(f'pgf_rcupdate{i + 1}.pdf', tol=tol[i])


# test backend-side clipping, since large numbers are not supported by TeX
@needs_pgf_xelatex
@mpl.style.context('default')
@pytest.mark.backend('pgf')
def test_pathclip():
    np.random.seed(19680801)
    mpl.rcParams.update({'font.family': 'serif', 'pgf.rcfonts': False})
    fig, axs = plt.subplots(1, 2)

    axs[0].plot([0., 1e100], [0., 1e100])
    axs[0].set_xlim(0, 1)
    axs[0].set_ylim(0, 1)

    axs[1].scatter([0, 1], [1, 1])
    axs[1].hist(np.random.normal(size=1000), bins=20, range=[-10, 10])
    axs[1].set_xscale('log')

    fig.savefig(BytesIO(), format="pdf")  # No image comparison.


# test mixed mode rendering
@needs_pgf_xelatex
@pytest.mark.backend('pgf')
@image_comparison(['pgf_mixedmode.pdf'], style='default')
def test_mixedmode():
    mpl.rcParams.update({'font.family': 'serif', 'pgf.rcfonts': False})
    Y, X = np.ogrid[-1:1:40j, -1:1:40j]
    plt.pcolor(X**2 + Y**2).set_rasterized(True)


# test bbox_inches clipping
@needs_pgf_xelatex
@mpl.style.context('default')
@pytest.mark.backend('pgf')
def test_bbox_inches():
    mpl.rcParams.update({'font.family': 'serif', 'pgf.rcfonts': False})
    fig, (ax1, ax2) = plt.subplots(1, 2)
    ax1.plot(range(5))
    ax2.plot(range(5))
    plt.tight_layout()
    bbox = ax1.get_window_extent().transformed(fig.dpi_scale_trans.inverted())
    compare_figure('pgf_bbox_inches.pdf', savefig_kwargs={'bbox_inches': bbox},
                   tol=0)


@mpl.style.context('default')
@pytest.mark.backend('pgf')
@pytest.mark.parametrize('system', [
    pytest.param('lualatex', marks=[needs_pgf_lualatex]),
    pytest.param('pdflatex', marks=[needs_pgf_pdflatex]),
    pytest.param('xelatex', marks=[needs_pgf_xelatex]),
])
def test_pdf_pages(system):
    rc_pdflatex = {
        'font.family': 'serif',
        'pgf.rcfonts': False,
        'pgf.texsystem': system,
    }
    mpl.rcParams.update(rc_pdflatex)

    fig1, ax1 = plt.subplots()
    ax1.plot(range(5))
    fig1.tight_layout()

    fig2, ax2 = plt.subplots(figsize=(3, 2))
    ax2.plot(range(5))
    fig2.tight_layout()

    path = os.path.join(result_dir, f'pdfpages_{system}.pdf')
    md = {
        'Author': 'me',
        'Title': 'Multipage PDF with pgf',
        'Subject': 'Test page',
        'Keywords': 'test,pdf,multipage',
        'ModDate': datetime.datetime(
            1968, 8, 1, tzinfo=datetime.timezone(datetime.timedelta(0))),
        'Trapped': 'Unknown'
    }

    with PdfPages(path, metadata=md) as pdf:
        pdf.savefig(fig1)
        pdf.savefig(fig2)
        pdf.savefig(fig1)

        assert pdf.get_pagecount() == 3


@mpl.style.context('default')
@pytest.mark.backend('pgf')
@pytest.mark.parametrize('system', [
    pytest.param('lualatex', marks=[needs_pgf_lualatex]),
    pytest.param('pdflatex', marks=[needs_pgf_pdflatex]),
    pytest.param('xelatex', marks=[needs_pgf_xelatex]),
])
def test_pdf_pages_metadata_check(monkeypatch, system):
    # Basically the same as test_pdf_pages, but we keep it separate to leave
    # pikepdf as an optional dependency.
    pikepdf = pytest.importorskip('pikepdf')
    monkeypatch.setenv('SOURCE_DATE_EPOCH', '0')

    mpl.rcParams.update({'pgf.texsystem': system})

    fig, ax = plt.subplots()
    ax.plot(range(5))

    md = {
        'Author': 'me',
        'Title': 'Multipage PDF with pgf',
        'Subject': 'Test page',
        'Keywords': 'test,pdf,multipage',
        'ModDate': datetime.datetime(
            1968, 8, 1, tzinfo=datetime.timezone(datetime.timedelta(0))),
        'Trapped': 'True'
    }
    path = os.path.join(result_dir, f'pdfpages_meta_check_{system}.pdf')
    with PdfPages(path, metadata=md) as pdf:
        pdf.savefig(fig)

    with pikepdf.Pdf.open(path) as pdf:
        info = {k: str(v) for k, v in pdf.docinfo.items()}

    # Not set by us, so don't bother checking.
    if '/PTEX.FullBanner' in info:
        del info['/PTEX.FullBanner']
    if '/PTEX.Fullbanner' in info:
        del info['/PTEX.Fullbanner']

    # Some LaTeX engines ignore this setting, and state themselves as producer.
    producer = info.pop('/Producer')
    assert producer == f'Matplotlib pgf backend v{mpl.__version__}' or (
            system == 'lualatex' and 'LuaTeX' in producer)

    assert info == {
        '/Author': 'me',
        '/CreationDate': 'D:19700101000000Z',
        '/Creator': f'Matplotlib v{mpl.__version__}, https://matplotlib.org',
        '/Keywords': 'test,pdf,multipage',
        '/ModDate': 'D:19680801000000Z',
        '/Subject': 'Test page',
        '/Title': 'Multipage PDF with pgf',
        '/Trapped': '/True',
    }


@needs_pgf_xelatex
def test_tex_restart_after_error():
    fig = plt.figure()
    fig.suptitle(r"\oops")
    with pytest.raises(ValueError):
        fig.savefig(BytesIO(), format="pgf")

    fig = plt.figure()  # start from scratch
    fig.suptitle(r"this is ok")
    fig.savefig(BytesIO(), format="pgf")


@needs_pgf_xelatex
def test_bbox_inches_tight():
    fig, ax = plt.subplots()
    ax.imshow([[0, 1], [2, 3]])
    fig.savefig(BytesIO(), format="pdf", backend="pgf", bbox_inches="tight")


@needs_pgf_xelatex
@needs_ghostscript
def test_png_transparency():  # Actually, also just testing that png works.
    buf = BytesIO()
    plt.figure().savefig(buf, format="png", backend="pgf", transparent=True)
    buf.seek(0)
    t = plt.imread(buf)
    assert (t[..., 3] == 0).all()  # fully transparent.


@needs_pgf_xelatex
def test_unknown_font(caplog):
    with caplog.at_level("WARNING"):
        mpl.rcParams["font.family"] = "this-font-does-not-exist"
        plt.figtext(.5, .5, "hello, world")
        plt.savefig(BytesIO(), format="pgf")
    assert "Ignoring unknown font: this-font-does-not-exist" in [
        r.getMessage() for r in caplog.records]


@check_figures_equal(extensions=["pdf"])
@pytest.mark.parametrize("texsystem", ("pdflatex", "xelatex", "lualatex"))
@pytest.mark.backend("pgf")
def test_minus_signs_with_tex(fig_test, fig_ref, texsystem):
    if not _check_for_pgf(texsystem):
        pytest.skip(texsystem + ' + pgf is required')
    mpl.rcParams["pgf.texsystem"] = texsystem
    fig_test.text(.5, .5, "$-1$")
    fig_ref.text(.5, .5, "$\N{MINUS SIGN}1$")


@pytest.mark.backend("pgf")
def test_sketch_params():
    fig, ax = plt.subplots(figsize=(3, 3))
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_frame_on(False)
    handle, = ax.plot([0, 1])
    handle.set_sketch_params(scale=5, length=30, randomness=42)

    with BytesIO() as fd:
        fig.savefig(fd, format='pgf')
        buf = fd.getvalue().decode()

    baseline = r"""\pgfpathmoveto{\pgfqpoint{0.375000in}{0.300000in}}%
\pgfpathlineto{\pgfqpoint{2.700000in}{2.700000in}}%
\usepgfmodule{decorations}%
\usepgflibrary{decorations.pathmorphing}%
\pgfkeys{/pgf/decoration/.cd, """ \
    r"""segment length = 0.150000in, amplitude = 0.100000in}%
\pgfmathsetseed{42}%
\pgfdecoratecurrentpath{random steps}%
\pgfusepath{stroke}%"""
    # \pgfdecoratecurrentpath must be after the path definition and before the
    # path is used (\pgfusepath)
    assert baseline in buf

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_pgf.py
NikolaBlagojevic/pyrecodes
"""
Backend-loading machinery tests, using variations on the template backend.
"""

import sys
from types import SimpleNamespace
from unittest.mock import MagicMock

import matplotlib as mpl
from matplotlib import pyplot as plt
from matplotlib.backends import backend_template
from matplotlib.backends.backend_template import (
    FigureCanvasTemplate, FigureManagerTemplate)


def test_load_template():
    mpl.use("template")
    assert type(plt.figure().canvas) == FigureCanvasTemplate


def test_load_old_api(monkeypatch):
    mpl_test_backend = SimpleNamespace(**vars(backend_template))
    mpl_test_backend.new_figure_manager = (
        lambda num, *args, FigureClass=mpl.figure.Figure, **kwargs:
        FigureManagerTemplate(
            FigureCanvasTemplate(FigureClass(*args, **kwargs)), num))
    monkeypatch.setitem(sys.modules, "mpl_test_backend", mpl_test_backend)
    mpl.use("module://mpl_test_backend")
    assert type(plt.figure().canvas) == FigureCanvasTemplate
    plt.draw_if_interactive()


def test_show(monkeypatch):
    mpl_test_backend = SimpleNamespace(**vars(backend_template))
    mock_show = MagicMock()
    monkeypatch.setattr(
        mpl_test_backend.FigureManagerTemplate, "pyplot_show", mock_show)
    monkeypatch.setitem(sys.modules, "mpl_test_backend", mpl_test_backend)
    mpl.use("module://mpl_test_backend")
    plt.show()
    mock_show.assert_called_with()


def test_show_old_global_api(monkeypatch):
    mpl_test_backend = SimpleNamespace(**vars(backend_template))
    mock_show = MagicMock()
    monkeypatch.setattr(mpl_test_backend, "show", mock_show, raising=False)
    monkeypatch.setitem(sys.modules, "mpl_test_backend", mpl_test_backend)
    mpl.use("module://mpl_test_backend")
    plt.show()
    mock_show.assert_called_with()

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_template.py
NikolaBlagojevic/pyrecodes
import subprocess
import os
import sys
import pytest
import matplotlib.backends.backend_webagg_core


@pytest.mark.parametrize("backend", ["webagg", "nbagg"])
def test_webagg_fallback(backend):
    pytest.importorskip("tornado")
    if backend == "nbagg":
        pytest.importorskip("IPython")
    env = dict(os.environ)
    if os.name != "nt":
        env["DISPLAY"] = ""

    env["MPLBACKEND"] = backend

    test_code = (
        "import os;"
        + f"assert os.environ['MPLBACKEND'] == '{backend}';"
        + "import matplotlib.pyplot as plt; "
        + "print(plt.get_backend());"
        f"assert '{backend}' == plt.get_backend().lower();"
    )
    ret = subprocess.call([sys.executable, "-c", test_code], env=env)

    assert ret == 0


def test_webagg_core_no_toolbar():
    fm = matplotlib.backends.backend_webagg_core.FigureManagerWebAgg
    assert fm._toolbar2_class is None

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_backend_webagg.py
NikolaBlagojevic/pyrecodes
"""Catch all for categorical functions"""
import pytest
import numpy as np

import matplotlib as mpl
from matplotlib.axes import Axes
import matplotlib.pyplot as plt
import matplotlib.category as cat
from matplotlib.testing.decorators import check_figures_equal


class TestUnitData:
    test_cases = [('single', (["hello world"], [0])),
                  ('unicode', (["Здравствуйте мир"], [0])),
                  ('mixed', (['A', "np.nan", 'B', "3.14", "мир"],
                             [0, 1, 2, 3, 4]))]
    ids, data = zip(*test_cases)

    @pytest.mark.parametrize("data, locs", data, ids=ids)
    def test_unit(self, data, locs):
        unit = cat.UnitData(data)
        assert list(unit._mapping.keys()) == data
        assert list(unit._mapping.values()) == locs

    def test_update(self):
        data = ['a', 'd']
        locs = [0, 1]

        data_update = ['b', 'd', 'e']
        unique_data = ['a', 'd', 'b', 'e']
        updated_locs = [0, 1, 2, 3]

        unit = cat.UnitData(data)
        assert list(unit._mapping.keys()) == data
        assert list(unit._mapping.values()) == locs

        unit.update(data_update)
        assert list(unit._mapping.keys()) == unique_data
        assert list(unit._mapping.values()) == updated_locs

    failing_test_cases = [("number", 3.14), ("nan", np.nan),
                          ("list", [3.14, 12]), ("mixed type", ["A", 2])]

    fids, fdata = zip(*test_cases)

    @pytest.mark.parametrize("fdata", fdata, ids=fids)
    def test_non_string_fails(self, fdata):
        with pytest.raises(TypeError):
            cat.UnitData(fdata)

    @pytest.mark.parametrize("fdata", fdata, ids=fids)
    def test_non_string_update_fails(self, fdata):
        unitdata = cat.UnitData()
        with pytest.raises(TypeError):
            unitdata.update(fdata)


class FakeAxis:
    def __init__(self, units):
        self.units = units


class TestStrCategoryConverter:
    """
    Based on the pandas conversion and factorization tests:

    ref: /pandas/tseries/tests/test_converter.py
         /pandas/tests/test_algos.py:TestFactorize
    """
    test_cases = [("unicode", ["Здравствуйте мир"]),
                  ("ascii", ["hello world"]),
                  ("single", ['a', 'b', 'c']),
                  ("integer string", ["1", "2"]),
                  ("single + values>10", ["A", "B", "C", "D", "E", "F", "G",
                                          "H", "I", "J", "K", "L", "M", "N",
                                          "O", "P", "Q", "R", "S", "T", "U",
                                          "V", "W", "X", "Y", "Z"])]

    ids, values = zip(*test_cases)

    failing_test_cases = [("mixed", [3.14, 'A', np.inf]),
                          ("string integer", ['42', 42])]

    fids, fvalues = zip(*failing_test_cases)

    @pytest.fixture(autouse=True)
    def mock_axis(self, request):
        self.cc = cat.StrCategoryConverter()
        # self.unit should be probably be replaced with real mock unit
        self.unit = cat.UnitData()
        self.ax = FakeAxis(self.unit)

    @pytest.mark.parametrize("vals", values, ids=ids)
    def test_convert(self, vals):
        np.testing.assert_allclose(self.cc.convert(vals, self.ax.units,
                                                   self.ax),
                                   range(len(vals)))

    @pytest.mark.parametrize("value", ["hi", "мир"], ids=["ascii", "unicode"])
    def test_convert_one_string(self, value):
        assert self.cc.convert(value, self.unit, self.ax) == 0

    @pytest.mark.parametrize("fvals", fvalues, ids=fids)
    def test_convert_fail(self, fvals):
        with pytest.raises(TypeError):
            self.cc.convert(fvals, self.unit, self.ax)

    def test_axisinfo(self):
        axis = self.cc.axisinfo(self.unit, self.ax)
        assert isinstance(axis.majloc, cat.StrCategoryLocator)
        assert isinstance(axis.majfmt, cat.StrCategoryFormatter)

    def test_default_units(self):
        assert isinstance(self.cc.default_units(["a"], self.ax), cat.UnitData)


PLOT_LIST = [Axes.scatter, Axes.plot, Axes.bar]
PLOT_IDS = ["scatter", "plot", "bar"]


class TestStrCategoryLocator:
    def test_StrCategoryLocator(self):
        locs = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
        unit = cat.UnitData([str(j) for j in locs])
        ticks = cat.StrCategoryLocator(unit._mapping)
        np.testing.assert_array_equal(ticks.tick_values(None, None), locs)

    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_StrCategoryLocatorPlot(self, plotter):
        ax = plt.figure().subplots()
        plotter(ax, [1, 2, 3], ["a", "b", "c"])
        np.testing.assert_array_equal(ax.yaxis.major.locator(), range(3))


class TestStrCategoryFormatter:
    test_cases = [("ascii", ["hello", "world", "hi"]),
                  ("unicode", ["Здравствуйте", "привет"])]

    ids, cases = zip(*test_cases)

    @pytest.mark.parametrize("ydata", cases, ids=ids)
    def test_StrCategoryFormatter(self, ydata):
        unit = cat.UnitData(ydata)
        labels = cat.StrCategoryFormatter(unit._mapping)
        for i, d in enumerate(ydata):
            assert labels(i, i) == d
            assert labels(i, None) == d

    @pytest.mark.parametrize("ydata", cases, ids=ids)
    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_StrCategoryFormatterPlot(self, ydata, plotter):
        ax = plt.figure().subplots()
        plotter(ax, range(len(ydata)), ydata)
        for i, d in enumerate(ydata):
            assert ax.yaxis.major.formatter(i) == d
        assert ax.yaxis.major.formatter(i+1) == ""


def axis_test(axis, labels):
    ticks = list(range(len(labels)))
    np.testing.assert_array_equal(axis.get_majorticklocs(), ticks)
    graph_labels = [axis.major.formatter(i, i) for i in ticks]
    # _text also decodes bytes as utf-8.
    assert graph_labels == [cat.StrCategoryFormatter._text(l) for l in labels]
    assert list(axis.units._mapping.keys()) == [l for l in labels]
    assert list(axis.units._mapping.values()) == ticks


class TestPlotBytes:
    bytes_cases = [('string list', ['a', 'b', 'c']),
                   ('bytes list', [b'a', b'b', b'c']),
                   ('bytes ndarray', np.array([b'a', b'b', b'c']))]

    bytes_ids, bytes_data = zip(*bytes_cases)

    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    @pytest.mark.parametrize("bdata", bytes_data, ids=bytes_ids)
    def test_plot_bytes(self, plotter, bdata):
        ax = plt.figure().subplots()
        counts = np.array([4, 6, 5])
        plotter(ax, bdata, counts)
        axis_test(ax.xaxis, bdata)


class TestPlotNumlike:
    numlike_cases = [('string list', ['1', '11', '3']),
                     ('string ndarray', np.array(['1', '11', '3'])),
                     ('bytes list', [b'1', b'11', b'3']),
                     ('bytes ndarray', np.array([b'1', b'11', b'3']))]
    numlike_ids, numlike_data = zip(*numlike_cases)

    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    @pytest.mark.parametrize("ndata", numlike_data, ids=numlike_ids)
    def test_plot_numlike(self, plotter, ndata):
        ax = plt.figure().subplots()
        counts = np.array([4, 6, 5])
        plotter(ax, ndata, counts)
        axis_test(ax.xaxis, ndata)


class TestPlotTypes:
    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_plot_unicode(self, plotter):
        ax = plt.figure().subplots()
        words = ['Здравствуйте', 'привет']
        plotter(ax, words, [0, 1])
        axis_test(ax.xaxis, words)

    @pytest.fixture
    def test_data(self):
        self.x = ["hello", "happy", "world"]
        self.xy = [2, 6, 3]
        self.y = ["Python", "is", "fun"]
        self.yx = [3, 4, 5]

    @pytest.mark.usefixtures("test_data")
    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_plot_xaxis(self, test_data, plotter):
        ax = plt.figure().subplots()
        plotter(ax, self.x, self.xy)
        axis_test(ax.xaxis, self.x)

    @pytest.mark.usefixtures("test_data")
    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_plot_yaxis(self, test_data, plotter):
        ax = plt.figure().subplots()
        plotter(ax, self.yx, self.y)
        axis_test(ax.yaxis, self.y)

    @pytest.mark.usefixtures("test_data")
    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_plot_xyaxis(self, test_data, plotter):
        ax = plt.figure().subplots()
        plotter(ax, self.x, self.y)
        axis_test(ax.xaxis, self.x)
        axis_test(ax.yaxis, self.y)

    @pytest.mark.parametrize("plotter", PLOT_LIST, ids=PLOT_IDS)
    def test_update_plot(self, plotter):
        ax = plt.figure().subplots()
        plotter(ax, ['a', 'b'], ['e', 'g'])
        plotter(ax, ['a', 'b', 'd'], ['f', 'a', 'b'])
        plotter(ax, ['b', 'c', 'd'], ['g', 'e', 'd'])
        axis_test(ax.xaxis, ['a', 'b', 'd', 'c'])
        axis_test(ax.yaxis, ['e', 'g', 'f', 'a', 'b', 'd'])

    failing_test_cases = [("mixed", ['A', 3.14]),
                          ("number integer", ['1', 1]),
                          ("string integer", ['42', 42]),
                          ("missing", ['12', np.nan])]

    fids, fvalues = zip(*failing_test_cases)

    plotters = [Axes.scatter, Axes.bar,
                pytest.param(Axes.plot, marks=pytest.mark.xfail)]

    @pytest.mark.parametrize("plotter", plotters)
    @pytest.mark.parametrize("xdata", fvalues, ids=fids)
    def test_mixed_type_exception(self, plotter, xdata):
        ax = plt.figure().subplots()
        with pytest.raises(TypeError):
            plotter(ax, xdata, [1, 2])

    @pytest.mark.parametrize("plotter", plotters)
    @pytest.mark.parametrize("xdata", fvalues, ids=fids)
    def test_mixed_type_update_exception(self, plotter, xdata):
        ax = plt.figure().subplots()
        with pytest.raises(TypeError):
            plotter(ax, [0, 3], [1, 3])
            plotter(ax, xdata, [1, 2])


@mpl.style.context('default')
@check_figures_equal(extensions=["png"])
def test_overriding_units_in_plot(fig_test, fig_ref):
    from datetime import datetime

    t0 = datetime(2018, 3, 1)
    t1 = datetime(2018, 3, 2)
    t2 = datetime(2018, 3, 3)
    t3 = datetime(2018, 3, 4)

    ax_test = fig_test.subplots()
    ax_ref = fig_ref.subplots()
    for ax, kwargs in zip([ax_test, ax_ref],
                          ({}, dict(xunits=None, yunits=None))):
        # First call works
        ax.plot([t0, t1], ["V1", "V2"], **kwargs)
        x_units = ax.xaxis.units
        y_units = ax.yaxis.units
        # this should not raise
        ax.plot([t2, t3], ["V1", "V2"], **kwargs)
        # assert that we have not re-set the units attribute at all
        assert x_units is ax.xaxis.units
        assert y_units is ax.yaxis.units


def test_no_deprecation_on_empty_data():
    """
    Smoke test to check that no deprecation warning is emitted. See #22640.
    """
    f, ax = plt.subplots()
    ax.xaxis.update_units(["a", "b"])
    ax.plot([], [])


def test_hist():
    fig, ax = plt.subplots()
    n, bins, patches = ax.hist(['a', 'b', 'a', 'c', 'ff'])
    assert n.shape == (10,)
    np.testing.assert_allclose(n, [2., 0., 0., 1., 0., 0., 1., 0., 0., 1.])

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_category.py
NikolaBlagojevic/pyrecodes
from pathlib import Path
import shutil

import pytest
from pytest import approx

from matplotlib.testing.compare import compare_images
from matplotlib.testing.decorators import _image_directories


# Tests of the image comparison algorithm.
@pytest.mark.parametrize(
    'im1, im2, tol, expect_rms',
    [
        # Comparison of an image and the same image with minor differences.
        # This expects the images to compare equal under normal tolerance, and
        # have a small RMS.
        ('basn3p02.png', 'basn3p02-minorchange.png', 10, None),
        # Now test with no tolerance.
        ('basn3p02.png', 'basn3p02-minorchange.png', 0, 6.50646),
        # Comparison with an image that is shifted by 1px in the X axis.
        ('basn3p02.png', 'basn3p02-1px-offset.png', 0, 90.15611),
        # Comparison with an image with half the pixels shifted by 1px in the X
        # axis.
        ('basn3p02.png', 'basn3p02-half-1px-offset.png', 0, 63.75),
        # Comparison of an image and the same image scrambled.
        # This expects the images to compare completely different, with a very
        # large RMS.
        # Note: The image has been scrambled in a specific way, by having
        # each color component of each pixel randomly placed somewhere in the
        # image. It contains exactly the same number of pixels of each color
        # value of R, G and B, but in a totally different position.
        # Test with no tolerance to make sure that we pick up even a very small
        # RMS error.
        ('basn3p02.png', 'basn3p02-scrambled.png', 0, 172.63582),
        # Comparison of an image and a slightly brighter image.
        # The two images are solid color, with the second image being exactly 1
        # color value brighter.
        # This expects the images to compare equal under normal tolerance, and
        # have an RMS of exactly 1.
        ('all127.png', 'all128.png', 0, 1),
        # Now test the reverse comparison.
        ('all128.png', 'all127.png', 0, 1),
    ])
def test_image_comparison_expect_rms(im1, im2, tol, expect_rms, tmp_path,
                                     monkeypatch):
    """
    Compare two images, expecting a particular RMS error.

    im1 and im2 are filenames relative to the baseline_dir directory.

    tol is the tolerance to pass to compare_images.

    expect_rms is the expected RMS value, or None. If None, the test will
    succeed if compare_images succeeds. Otherwise, the test will succeed if
    compare_images fails and returns an RMS error almost equal to this value.
    """
    # Change the working directory using monkeypatch to use a temporary
    # test specific directory
    monkeypatch.chdir(tmp_path)
    baseline_dir, result_dir = map(Path, _image_directories(lambda: "dummy"))
    # Copy "test" image to result_dir, so that compare_images writes
    # the diff to result_dir, rather than to the source tree
    result_im2 = result_dir / im1
    shutil.copyfile(baseline_dir / im2, result_im2)
    results = compare_images(
        baseline_dir / im1, result_im2, tol=tol, in_decorator=True)

    if expect_rms is None:
        assert results is None
    else:
        assert results is not None
        assert results['rms'] == approx(expect_rms, abs=1e-4)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_compare_images.py
NikolaBlagojevic/pyrecodes
"""
Test output reproducibility.
"""

import os
import subprocess
import sys

import pytest

import matplotlib as mpl
import matplotlib.testing.compare
from matplotlib import pyplot as plt
from matplotlib.testing._markers import needs_ghostscript, needs_usetex


def _save_figure(objects='mhi', fmt="pdf", usetex=False):
    mpl.use(fmt)
    mpl.rcParams.update({'svg.hashsalt': 'asdf', 'text.usetex': usetex})

    fig = plt.figure()

    if 'm' in objects:
        # use different markers...
        ax1 = fig.add_subplot(1, 6, 1)
        x = range(10)
        ax1.plot(x, [1] * 10, marker='D')
        ax1.plot(x, [2] * 10, marker='x')
        ax1.plot(x, [3] * 10, marker='^')
        ax1.plot(x, [4] * 10, marker='H')
        ax1.plot(x, [5] * 10, marker='v')

    if 'h' in objects:
        # also use different hatch patterns
        ax2 = fig.add_subplot(1, 6, 2)
        bars = (ax2.bar(range(1, 5), range(1, 5)) +
                ax2.bar(range(1, 5), [6] * 4, bottom=range(1, 5)))
        ax2.set_xticks([1.5, 2.5, 3.5, 4.5])

        patterns = ('-', '+', 'x', '\\', '*', 'o', 'O', '.')
        for bar, pattern in zip(bars, patterns):
            bar.set_hatch(pattern)

    if 'i' in objects:
        # also use different images
        A = [[1, 2, 3], [2, 3, 1], [3, 1, 2]]
        fig.add_subplot(1, 6, 3).imshow(A, interpolation='nearest')
        A = [[1, 3, 2], [1, 2, 3], [3, 1, 2]]
        fig.add_subplot(1, 6, 4).imshow(A, interpolation='bilinear')
        A = [[2, 3, 1], [1, 2, 3], [2, 1, 3]]
        fig.add_subplot(1, 6, 5).imshow(A, interpolation='bicubic')

    x = range(5)
    ax = fig.add_subplot(1, 6, 6)
    ax.plot(x, x)
    ax.set_title('A string $1+2+\\sigma$')
    ax.set_xlabel('A string $1+2+\\sigma$')
    ax.set_ylabel('A string $1+2+\\sigma$')

    stdout = getattr(sys.stdout, 'buffer', sys.stdout)
    fig.savefig(stdout, format=fmt)


@pytest.mark.parametrize(
    "objects, fmt, usetex", [
        ("", "pdf", False),
        ("m", "pdf", False),
        ("h", "pdf", False),
        ("i", "pdf", False),
        ("mhi", "pdf", False),
        ("mhi", "ps", False),
        pytest.param(
            "mhi", "ps", True, marks=[needs_usetex, needs_ghostscript]),
        ("mhi", "svg", False),
        pytest.param("mhi", "svg", True, marks=needs_usetex),
    ]
)
def test_determinism_check(objects, fmt, usetex):
    """
    Output three times the same graphs and checks that the outputs are exactly
    the same.

    Parameters
    ----------
    objects : str
        Objects to be included in the test document: 'm' for markers, 'h' for
        hatch patterns, 'i' for images.
    fmt : {"pdf", "ps", "svg"}
        Output format.
    """
    plots = [
        subprocess.check_output(
            [sys.executable, "-R", "-c",
             f"from matplotlib.tests.test_determinism import _save_figure;"
             f"_save_figure({objects!r}, {fmt!r}, {usetex})"],
            env={**os.environ, "SOURCE_DATE_EPOCH": "946684800",
                 "MPLBACKEND": "Agg"})
        for _ in range(3)
    ]
    for p in plots[1:]:
        if fmt == "ps" and usetex:
            if p != plots[0]:
                pytest.skip("failed, maybe due to ghostscript timestamps")
        else:
            assert p == plots[0]


@pytest.mark.parametrize(
    "fmt, string", [
        ("pdf", b"/CreationDate (D:20000101000000Z)"),
        # SOURCE_DATE_EPOCH support is not tested with text.usetex,
        # because the produced timestamp comes from ghostscript:
        # %%CreationDate: D:20000101000000Z00\'00\', and this could change
        # with another ghostscript version.
        ("ps", b"%%CreationDate: Sat Jan 01 00:00:00 2000"),
    ]
)
def test_determinism_source_date_epoch(fmt, string):
    """
    Test SOURCE_DATE_EPOCH support. Output a document with the environment
    variable SOURCE_DATE_EPOCH set to 2000-01-01 00:00 UTC and check that the
    document contains the timestamp that corresponds to this date (given as an
    argument).

    Parameters
    ----------
    fmt : {"pdf", "ps", "svg"}
        Output format.
    string : bytes
        Timestamp string for 2000-01-01 00:00 UTC.
    """
    buf = subprocess.check_output(
        [sys.executable, "-R", "-c",
         f"from matplotlib.tests.test_determinism import _save_figure; "
         f"_save_figure('', {fmt!r})"],
        env={**os.environ, "SOURCE_DATE_EPOCH": "946684800",
             "MPLBACKEND": "Agg"})
    assert string in buf

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_determinism.py
NikolaBlagojevic/pyrecodes
import pytest


def test_sphinx_gallery_example_header():
    """
    We have copied EXAMPLE_HEADER and modified it to include meta keywords.
    This test monitors that the version we have copied is still the same as
    the EXAMPLE_HEADER in sphinx-gallery. If sphinx-gallery changes its
    EXAMPLE_HEADER, this test will start to fail. In that case, please update
    the monkey-patching of EXAMPLE_HEADER in conf.py.
    """
    gen_rst = pytest.importorskip('sphinx_gallery.gen_rst')

    EXAMPLE_HEADER = """
.. DO NOT EDIT.
.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
.. "{0}"
.. LINE NUMBERS ARE GIVEN BELOW.

.. only:: html

    .. note::
        :class: sphx-glr-download-link-note

        Click :ref:`here <sphx_glr_download_{1}>`
        to download the full example code{2}

.. rst-class:: sphx-glr-example-title

.. _sphx_glr_{1}:

"""
    assert gen_rst.EXAMPLE_HEADER == EXAMPLE_HEADER

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_doc.py
NikolaBlagojevic/pyrecodes
import pytest

from matplotlib.font_manager import FontProperties


# Attributes on FontProperties object to check for consistency
keys = [
    "get_family",
    "get_style",
    "get_variant",
    "get_weight",
    "get_size",
    ]


def test_fontconfig_pattern():
    """Test converting a FontProperties to string then back."""

    # Defaults
    test = "defaults "
    f1 = FontProperties()
    s = str(f1)

    f2 = FontProperties(s)
    for k in keys:
        assert getattr(f1, k)() == getattr(f2, k)(), test + k

    # Basic inputs
    test = "basic "
    f1 = FontProperties(family="serif", size=20, style="italic")
    s = str(f1)

    f2 = FontProperties(s)
    for k in keys:
        assert getattr(f1, k)() == getattr(f2, k)(), test + k

    # Full set of inputs.
    test = "full "
    f1 = FontProperties(family="sans-serif", size=24, weight="bold",
                        style="oblique", variant="small-caps",
                        stretch="expanded")
    s = str(f1)

    f2 = FontProperties(s)
    for k in keys:
        assert getattr(f1, k)() == getattr(f2, k)(), test + k


def test_fontconfig_str():
    """Test FontProperties string conversions for correctness."""

    # Known good strings taken from actual font config specs on a linux box
    # and modified for MPL defaults.

    # Default values found by inspection.
    test = "defaults "
    s = ("sans\\-serif:style=normal:variant=normal:weight=normal"
         ":stretch=normal:size=12.0")
    font = FontProperties(s)
    right = FontProperties()
    for k in keys:
        assert getattr(font, k)() == getattr(right, k)(), test + k

    test = "full "
    s = ("serif-24:style=oblique:variant=small-caps:weight=bold"
         ":stretch=expanded")
    font = FontProperties(s)
    right = FontProperties(family="serif", size=24, weight="bold",
                           style="oblique", variant="small-caps",
                           stretch="expanded")
    for k in keys:
        assert getattr(font, k)() == getattr(right, k)(), test + k


def test_fontconfig_unknown_constant():
    with pytest.warns(DeprecationWarning):
        FontProperties(":unknown")

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_fontconfig_pattern.py
NikolaBlagojevic/pyrecodes
from pathlib import Path
import io

import pytest

from matplotlib import ft2font
from matplotlib.testing.decorators import check_figures_equal
import matplotlib.font_manager as fm
import matplotlib.pyplot as plt


def test_fallback_errors():
    file_name = fm.findfont('DejaVu Sans')

    with pytest.raises(TypeError, match="Fallback list must be a list"):
        # failing to be a list will fail before the 0
        ft2font.FT2Font(file_name, _fallback_list=(0,))

    with pytest.raises(
            TypeError, match="Fallback fonts must be FT2Font objects."
    ):
        ft2font.FT2Font(file_name, _fallback_list=[0])


def test_ft2font_positive_hinting_factor():
    file_name = fm.findfont('DejaVu Sans')
    with pytest.raises(
            ValueError, match="hinting_factor must be greater than 0"
    ):
        ft2font.FT2Font(file_name, 0)


def test_fallback_smoke():
    fp = fm.FontProperties(family=["WenQuanYi Zen Hei"])
    if Path(fm.findfont(fp)).name != "wqy-zenhei.ttc":
        pytest.skip("Font wqy-zenhei.ttc may be missing")

    fp = fm.FontProperties(family=["Noto Sans CJK JP"])
    if Path(fm.findfont(fp)).name != "NotoSansCJK-Regular.ttc":
        pytest.skip("Noto Sans CJK JP font may be missing.")

    plt.rcParams['font.size'] = 20
    fig = plt.figure(figsize=(4.75, 1.85))
    fig.text(0.05, 0.45, "There are 几个汉字 in between!",
             family=['DejaVu Sans', "Noto Sans CJK JP"])
    fig.text(0.05, 0.25, "There are 几个汉字 in between!",
             family=['DejaVu Sans', "WenQuanYi Zen Hei"])
    fig.text(0.05, 0.65, "There are 几个汉字 in between!",
             family=["Noto Sans CJK JP"])
    fig.text(0.05, 0.85, "There are 几个汉字 in between!",
             family=["WenQuanYi Zen Hei"])

    # TODO enable fallback for other backends!
    for fmt in ['png', 'raw']:  # ["svg", "pdf", "ps"]:
        fig.savefig(io.BytesIO(), format=fmt)


@pytest.mark.parametrize('family_name, file_name',
                         [("WenQuanYi Zen Hei",  "wqy-zenhei"),
                          ("Noto Sans CJK JP", "NotoSansCJK")]
                         )
@check_figures_equal(extensions=["png", "pdf", "eps", "svg"])
def test_font_fallback_chinese(fig_test, fig_ref, family_name, file_name):
    fp = fm.FontProperties(family=[family_name])
    if file_name not in Path(fm.findfont(fp)).name:
        pytest.skip(f"Font {family_name} ({file_name}) is missing")

    text = ["There are", "几个汉字", "in between!"]

    plt.rcParams["font.size"] = 20
    test_fonts = [["DejaVu Sans", family_name]] * 3
    ref_fonts = [["DejaVu Sans"], [family_name], ["DejaVu Sans"]]

    for j, (txt, test_font, ref_font) in enumerate(
            zip(text, test_fonts, ref_fonts)
    ):
        fig_ref.text(0.05, .85 - 0.15*j, txt, family=ref_font)
        fig_test.text(0.05, .85 - 0.15*j, txt, family=test_font)


@pytest.mark.parametrize(
    "family_name, file_name",
    [
        ("WenQuanYi Zen Hei", "wqy-zenhei"),
        ("Noto Sans CJK JP", "NotoSansCJK"),
    ],
)
def test__get_fontmap(family_name, file_name):
    fp = fm.FontProperties(family=[family_name])
    found_file_name = Path(fm.findfont(fp)).name
    if file_name not in found_file_name:
        pytest.skip(f"Font {family_name} ({file_name}) is missing")

    text = "There are 几个汉字 in between!"
    ft = fm.get_font(
        fm.fontManager._find_fonts_by_props(
            fm.FontProperties(family=["DejaVu Sans", family_name])
        )
    )

    fontmap = ft._get_fontmap(text)
    for char, font in fontmap.items():
        if ord(char) > 127:
            assert Path(font.fname).name == found_file_name
        else:
            assert Path(font.fname).name == "DejaVuSans.ttf"

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_ft2font.py
NikolaBlagojevic/pyrecodes
"""
Tests specific to the patches module.
"""
import numpy as np
from numpy.testing import assert_almost_equal, assert_array_equal
import pytest

import matplotlib as mpl
from matplotlib.patches import (Annulus, Ellipse, Patch, Polygon, Rectangle,
                                FancyArrowPatch, FancyArrow, BoxStyle, Arc)
from matplotlib.testing.decorators import image_comparison, check_figures_equal
from matplotlib.transforms import Bbox
import matplotlib.pyplot as plt
from matplotlib import (
    collections as mcollections, colors as mcolors, patches as mpatches,
    path as mpath, transforms as mtransforms, rcParams)

import sys
on_win = (sys.platform == 'win32')


def test_Polygon_close():
    #: GitHub issue #1018 identified a bug in the Polygon handling
    #: of the closed attribute; the path was not getting closed
    #: when set_xy was used to set the vertices.

    # open set of vertices:
    xy = [[0, 0], [0, 1], [1, 1]]
    # closed set:
    xyclosed = xy + [[0, 0]]

    # start with open path and close it:
    p = Polygon(xy, closed=True)
    assert p.get_closed()
    assert_array_equal(p.get_xy(), xyclosed)
    p.set_xy(xy)
    assert_array_equal(p.get_xy(), xyclosed)

    # start with closed path and open it:
    p = Polygon(xyclosed, closed=False)
    assert_array_equal(p.get_xy(), xy)
    p.set_xy(xyclosed)
    assert_array_equal(p.get_xy(), xy)

    # start with open path and leave it open:
    p = Polygon(xy, closed=False)
    assert not p.get_closed()
    assert_array_equal(p.get_xy(), xy)
    p.set_xy(xy)
    assert_array_equal(p.get_xy(), xy)

    # start with closed path and leave it closed:
    p = Polygon(xyclosed, closed=True)
    assert_array_equal(p.get_xy(), xyclosed)
    p.set_xy(xyclosed)
    assert_array_equal(p.get_xy(), xyclosed)


def test_corner_center():
    loc = [10, 20]
    width = 1
    height = 2

    # Rectangle
    # No rotation
    corners = ((10, 20), (11, 20), (11, 22), (10, 22))
    rect = Rectangle(loc, width, height)
    assert_array_equal(rect.get_corners(), corners)
    assert_array_equal(rect.get_center(), (10.5, 21))

    # 90 deg rotation
    corners_rot = ((10, 20), (10, 21), (8, 21), (8, 20))
    rect.set_angle(90)
    assert_array_equal(rect.get_corners(), corners_rot)
    assert_array_equal(rect.get_center(), (9, 20.5))

    # Rotation not a multiple of 90 deg
    theta = 33
    t = mtransforms.Affine2D().rotate_around(*loc, np.deg2rad(theta))
    corners_rot = t.transform(corners)
    rect.set_angle(theta)
    assert_almost_equal(rect.get_corners(), corners_rot)

    # Ellipse
    loc = [loc[0] + width / 2,
           loc[1] + height / 2]
    ellipse = Ellipse(loc, width, height)

    # No rotation
    assert_array_equal(ellipse.get_corners(), corners)

    # 90 deg rotation
    corners_rot = ((11.5, 20.5), (11.5, 21.5), (9.5, 21.5), (9.5, 20.5))
    ellipse.set_angle(90)
    assert_array_equal(ellipse.get_corners(), corners_rot)
    # Rotation shouldn't change ellipse center
    assert_array_equal(ellipse.get_center(), loc)

    # Rotation not a multiple of 90 deg
    theta = 33
    t = mtransforms.Affine2D().rotate_around(*loc, np.deg2rad(theta))
    corners_rot = t.transform(corners)
    ellipse.set_angle(theta)
    assert_almost_equal(ellipse.get_corners(), corners_rot)


def test_rotate_rect():
    loc = np.asarray([1.0, 2.0])
    width = 2
    height = 3
    angle = 30.0

    # A rotated rectangle
    rect1 = Rectangle(loc, width, height, angle=angle)

    # A non-rotated rectangle
    rect2 = Rectangle(loc, width, height)

    # Set up an explicit rotation matrix (in radians)
    angle_rad = np.pi * angle / 180.0
    rotation_matrix = np.array([[np.cos(angle_rad), -np.sin(angle_rad)],
                                [np.sin(angle_rad),  np.cos(angle_rad)]])

    # Translate to origin, rotate each vertex, and then translate back
    new_verts = np.inner(rotation_matrix, rect2.get_verts() - loc).T + loc

    # They should be the same
    assert_almost_equal(rect1.get_verts(), new_verts)


@check_figures_equal(extensions=['png'])
def test_rotate_rect_draw(fig_test, fig_ref):
    ax_test = fig_test.add_subplot()
    ax_ref = fig_ref.add_subplot()

    loc = (0, 0)
    width, height = (1, 1)
    angle = 30
    rect_ref = Rectangle(loc, width, height, angle=angle)
    ax_ref.add_patch(rect_ref)
    assert rect_ref.get_angle() == angle

    # Check that when the angle is updated after adding to an Axes, that the
    # patch is marked stale and redrawn in the correct location
    rect_test = Rectangle(loc, width, height)
    assert rect_test.get_angle() == 0
    ax_test.add_patch(rect_test)
    rect_test.set_angle(angle)
    assert rect_test.get_angle() == angle


@check_figures_equal(extensions=['png'])
def test_dash_offset_patch_draw(fig_test, fig_ref):
    ax_test = fig_test.add_subplot()
    ax_ref = fig_ref.add_subplot()

    loc = (0.1, 0.1)
    width, height = (0.8, 0.8)
    rect_ref = Rectangle(loc, width, height, linewidth=3, edgecolor='b',
                                                linestyle=(0, [6, 6]))
    # fill the line gaps using a linestyle (0, [0, 6, 6, 0]), which is
    # equivalent to (6, [6, 6]) but has 0 dash offset
    rect_ref2 = Rectangle(loc, width, height, linewidth=3, edgecolor='r',
                                            linestyle=(0, [0, 6, 6, 0]))
    assert rect_ref.get_linestyle() == (0, [6, 6])
    assert rect_ref2.get_linestyle() == (0, [0, 6, 6, 0])

    ax_ref.add_patch(rect_ref)
    ax_ref.add_patch(rect_ref2)

    # Check that the dash offset of the rect is the same if we pass it in the
    # init method and if we create two rects with appropriate onoff sequence
    # of linestyle.

    rect_test = Rectangle(loc, width, height, linewidth=3, edgecolor='b',
                                                    linestyle=(0, [6, 6]))
    rect_test2 = Rectangle(loc, width, height, linewidth=3, edgecolor='r',
                                                    linestyle=(6, [6, 6]))
    assert rect_test.get_linestyle() == (0, [6, 6])
    assert rect_test2.get_linestyle() == (6, [6, 6])

    ax_test.add_patch(rect_test)
    ax_test.add_patch(rect_test2)


def test_negative_rect():
    # These two rectangles have the same vertices, but starting from a
    # different point.  (We also drop the last vertex, which is a duplicate.)
    pos_vertices = Rectangle((-3, -2), 3, 2).get_verts()[:-1]
    neg_vertices = Rectangle((0, 0), -3, -2).get_verts()[:-1]
    assert_array_equal(np.roll(neg_vertices, 2, 0), pos_vertices)


@image_comparison(['clip_to_bbox'])
def test_clip_to_bbox():
    fig, ax = plt.subplots()
    ax.set_xlim([-18, 20])
    ax.set_ylim([-150, 100])

    path = mpath.Path.unit_regular_star(8).deepcopy()
    path.vertices *= [10, 100]
    path.vertices -= [5, 25]

    path2 = mpath.Path.unit_circle().deepcopy()
    path2.vertices *= [10, 100]
    path2.vertices += [10, -25]

    combined = mpath.Path.make_compound_path(path, path2)

    patch = mpatches.PathPatch(
        combined, alpha=0.5, facecolor='coral', edgecolor='none')
    ax.add_patch(patch)

    bbox = mtransforms.Bbox([[-12, -77.5], [50, -110]])
    result_path = combined.clip_to_bbox(bbox)
    result_patch = mpatches.PathPatch(
        result_path, alpha=0.5, facecolor='green', lw=4, edgecolor='black')

    ax.add_patch(result_patch)


@image_comparison(['patch_alpha_coloring'], remove_text=True)
def test_patch_alpha_coloring():
    """
    Test checks that the patch and collection are rendered with the specified
    alpha values in their facecolor and edgecolor.
    """
    star = mpath.Path.unit_regular_star(6)
    circle = mpath.Path.unit_circle()
    # concatenate the star with an internal cutout of the circle
    verts = np.concatenate([circle.vertices, star.vertices[::-1]])
    codes = np.concatenate([circle.codes, star.codes])
    cut_star1 = mpath.Path(verts, codes)
    cut_star2 = mpath.Path(verts + 1, codes)

    ax = plt.axes()
    col = mcollections.PathCollection([cut_star2],
                                      linewidth=5, linestyles='dashdot',
                                      facecolor=(1, 0, 0, 0.5),
                                      edgecolor=(0, 0, 1, 0.75))
    ax.add_collection(col)

    patch = mpatches.PathPatch(cut_star1,
                               linewidth=5, linestyle='dashdot',
                               facecolor=(1, 0, 0, 0.5),
                               edgecolor=(0, 0, 1, 0.75))
    ax.add_patch(patch)

    ax.set_xlim([-1, 2])
    ax.set_ylim([-1, 2])


@image_comparison(['patch_alpha_override'], remove_text=True)
def test_patch_alpha_override():
    #: Test checks that specifying an alpha attribute for a patch or
    #: collection will override any alpha component of the facecolor
    #: or edgecolor.
    star = mpath.Path.unit_regular_star(6)
    circle = mpath.Path.unit_circle()
    # concatenate the star with an internal cutout of the circle
    verts = np.concatenate([circle.vertices, star.vertices[::-1]])
    codes = np.concatenate([circle.codes, star.codes])
    cut_star1 = mpath.Path(verts, codes)
    cut_star2 = mpath.Path(verts + 1, codes)

    ax = plt.axes()
    col = mcollections.PathCollection([cut_star2],
                                      linewidth=5, linestyles='dashdot',
                                      alpha=0.25,
                                      facecolor=(1, 0, 0, 0.5),
                                      edgecolor=(0, 0, 1, 0.75))
    ax.add_collection(col)

    patch = mpatches.PathPatch(cut_star1,
                               linewidth=5, linestyle='dashdot',
                               alpha=0.25,
                               facecolor=(1, 0, 0, 0.5),
                               edgecolor=(0, 0, 1, 0.75))
    ax.add_patch(patch)

    ax.set_xlim([-1, 2])
    ax.set_ylim([-1, 2])


@mpl.style.context('default')
def test_patch_color_none():
    # Make sure the alpha kwarg does not override 'none' facecolor.
    # Addresses issue #7478.
    c = plt.Circle((0, 0), 1, facecolor='none', alpha=1)
    assert c.get_facecolor()[0] == 0


@image_comparison(['patch_custom_linestyle'], remove_text=True)
def test_patch_custom_linestyle():
    #: A test to check that patches and collections accept custom dash
    #: patterns as linestyle and that they display correctly.
    star = mpath.Path.unit_regular_star(6)
    circle = mpath.Path.unit_circle()
    # concatenate the star with an internal cutout of the circle
    verts = np.concatenate([circle.vertices, star.vertices[::-1]])
    codes = np.concatenate([circle.codes, star.codes])
    cut_star1 = mpath.Path(verts, codes)
    cut_star2 = mpath.Path(verts + 1, codes)

    ax = plt.axes()
    col = mcollections.PathCollection(
        [cut_star2],
        linewidth=5, linestyles=[(0, (5, 7, 10, 7))],
        facecolor=(1, 0, 0), edgecolor=(0, 0, 1))
    ax.add_collection(col)

    patch = mpatches.PathPatch(
        cut_star1,
        linewidth=5, linestyle=(0, (5, 7, 10, 7)),
        facecolor=(1, 0, 0), edgecolor=(0, 0, 1))
    ax.add_patch(patch)

    ax.set_xlim([-1, 2])
    ax.set_ylim([-1, 2])


def test_patch_linestyle_accents():
    #: Test if linestyle can also be specified with short mnemonics like "--"
    #: c.f. GitHub issue #2136
    star = mpath.Path.unit_regular_star(6)
    circle = mpath.Path.unit_circle()
    # concatenate the star with an internal cutout of the circle
    verts = np.concatenate([circle.vertices, star.vertices[::-1]])
    codes = np.concatenate([circle.codes, star.codes])

    linestyles = ["-", "--", "-.", ":",
                  "solid", "dashed", "dashdot", "dotted"]

    fig, ax = plt.subplots()
    for i, ls in enumerate(linestyles):
        star = mpath.Path(verts + i, codes)
        patch = mpatches.PathPatch(star,
                                   linewidth=3, linestyle=ls,
                                   facecolor=(1, 0, 0),
                                   edgecolor=(0, 0, 1))
        ax.add_patch(patch)

    ax.set_xlim([-1, i + 1])
    ax.set_ylim([-1, i + 1])
    fig.canvas.draw()


@check_figures_equal(extensions=['png'])
def test_patch_linestyle_none(fig_test, fig_ref):
    circle = mpath.Path.unit_circle()

    ax_test = fig_test.add_subplot()
    ax_ref = fig_ref.add_subplot()
    for i, ls in enumerate(['none', 'None', ' ', '']):
        path = mpath.Path(circle.vertices + i, circle.codes)
        patch = mpatches.PathPatch(path,
                                   linewidth=3, linestyle=ls,
                                   facecolor=(1, 0, 0),
                                   edgecolor=(0, 0, 1))
        ax_test.add_patch(patch)

        patch = mpatches.PathPatch(path,
                                   linewidth=3, linestyle='-',
                                   facecolor=(1, 0, 0),
                                   edgecolor='none')
        ax_ref.add_patch(patch)

    ax_test.set_xlim([-1, i + 1])
    ax_test.set_ylim([-1, i + 1])
    ax_ref.set_xlim([-1, i + 1])
    ax_ref.set_ylim([-1, i + 1])


def test_wedge_movement():
    param_dict = {'center': ((0, 0), (1, 1), 'set_center'),
                  'r': (5, 8, 'set_radius'),
                  'width': (2, 3, 'set_width'),
                  'theta1': (0, 30, 'set_theta1'),
                  'theta2': (45, 50, 'set_theta2')}

    init_args = {k: v[0] for k, v in param_dict.items()}

    w = mpatches.Wedge(**init_args)
    for attr, (old_v, new_v, func) in param_dict.items():
        assert getattr(w, attr) == old_v
        getattr(w, func)(new_v)
        assert getattr(w, attr) == new_v


# png needs tol>=0.06, pdf tol>=1.617
@image_comparison(['wedge_range'], remove_text=True, tol=1.65 if on_win else 0)
def test_wedge_range():
    ax = plt.axes()

    t1 = 2.313869244286224

    args = [[52.31386924, 232.31386924],
            [52.313869244286224, 232.31386924428622],
            [t1, t1 + 180.0],
            [0, 360],
            [90, 90 + 360],
            [-180, 180],
            [0, 380],
            [45, 46],
            [46, 45]]

    for i, (theta1, theta2) in enumerate(args):
        x = i % 3
        y = i // 3

        wedge = mpatches.Wedge((x * 3, y * 3), 1, theta1, theta2,
                               facecolor='none', edgecolor='k', lw=3)

        ax.add_artist(wedge)

    ax.set_xlim([-2, 8])
    ax.set_ylim([-2, 9])


def test_patch_str():
    """
    Check that patches have nice and working `str` representation.

    Note that the logic is that `__str__` is defined such that:
    str(eval(str(p))) == str(p)
    """
    p = mpatches.Circle(xy=(1, 2), radius=3)
    assert str(p) == 'Circle(xy=(1, 2), radius=3)'

    p = mpatches.Ellipse(xy=(1, 2), width=3, height=4, angle=5)
    assert str(p) == 'Ellipse(xy=(1, 2), width=3, height=4, angle=5)'

    p = mpatches.Rectangle(xy=(1, 2), width=3, height=4, angle=5)
    assert str(p) == 'Rectangle(xy=(1, 2), width=3, height=4, angle=5)'

    p = mpatches.Wedge(center=(1, 2), r=3, theta1=4, theta2=5, width=6)
    assert str(p) == 'Wedge(center=(1, 2), r=3, theta1=4, theta2=5, width=6)'

    p = mpatches.Arc(xy=(1, 2), width=3, height=4, angle=5, theta1=6, theta2=7)
    expected = 'Arc(xy=(1, 2), width=3, height=4, angle=5, theta1=6, theta2=7)'
    assert str(p) == expected

    p = mpatches.Annulus(xy=(1, 2), r=(3, 4), width=1, angle=2)
    expected = "Annulus(xy=(1, 2), r=(3, 4), width=1, angle=2)"
    assert str(p) == expected

    p = mpatches.RegularPolygon((1, 2), 20, radius=5)
    assert str(p) == "RegularPolygon((1, 2), 20, radius=5, orientation=0)"

    p = mpatches.CirclePolygon(xy=(1, 2), radius=5, resolution=20)
    assert str(p) == "CirclePolygon((1, 2), radius=5, resolution=20)"

    p = mpatches.FancyBboxPatch((1, 2), width=3, height=4)
    assert str(p) == "FancyBboxPatch((1, 2), width=3, height=4)"

    # Further nice __str__ which cannot be `eval`uated:
    path = mpath.Path([(1, 2), (2, 2), (1, 2)], closed=True)
    p = mpatches.PathPatch(path)
    assert str(p) == "PathPatch3((1, 2) ...)"

    p = mpatches.Polygon(np.empty((0, 2)))
    assert str(p) == "Polygon0()"

    data = [[1, 2], [2, 2], [1, 2]]
    p = mpatches.Polygon(data)
    assert str(p) == "Polygon3((1, 2) ...)"

    p = mpatches.FancyArrowPatch(path=path)
    assert str(p)[:27] == "FancyArrowPatch(Path(array("

    p = mpatches.FancyArrowPatch((1, 2), (3, 4))
    assert str(p) == "FancyArrowPatch((1, 2)->(3, 4))"

    p = mpatches.ConnectionPatch((1, 2), (3, 4), 'data')
    assert str(p) == "ConnectionPatch((1, 2), (3, 4))"

    s = mpatches.Shadow(p, 1, 1)
    assert str(s) == "Shadow(ConnectionPatch((1, 2), (3, 4)))"

    # Not testing Arrow, FancyArrow here
    # because they seem to exist only for historical reasons.


@image_comparison(['multi_color_hatch'], remove_text=True, style='default')
def test_multi_color_hatch():
    fig, ax = plt.subplots()

    rects = ax.bar(range(5), range(1, 6))
    for i, rect in enumerate(rects):
        rect.set_facecolor('none')
        rect.set_edgecolor('C{}'.format(i))
        rect.set_hatch('/')

    ax.autoscale_view()
    ax.autoscale(False)

    for i in range(5):
        with mpl.style.context({'hatch.color': 'C{}'.format(i)}):
            r = Rectangle((i - .8 / 2, 5), .8, 1, hatch='//', fc='none')
        ax.add_patch(r)


@image_comparison(['units_rectangle.png'])
def test_units_rectangle():
    import matplotlib.testing.jpl_units as U
    U.register()

    p = mpatches.Rectangle((5*U.km, 6*U.km), 1*U.km, 2*U.km)

    fig, ax = plt.subplots()
    ax.add_patch(p)
    ax.set_xlim([4*U.km, 7*U.km])
    ax.set_ylim([5*U.km, 9*U.km])


@image_comparison(['connection_patch.png'], style='mpl20', remove_text=True)
def test_connection_patch():
    fig, (ax1, ax2) = plt.subplots(1, 2)

    con = mpatches.ConnectionPatch(xyA=(0.1, 0.1), xyB=(0.9, 0.9),
                                   coordsA='data', coordsB='data',
                                   axesA=ax2, axesB=ax1,
                                   arrowstyle="->")
    ax2.add_artist(con)

    xyA = (0.6, 1.0)  # in axes coordinates
    xyB = (0.0, 0.2)  # x in axes coordinates, y in data coordinates
    coordsA = "axes fraction"
    coordsB = ax2.get_yaxis_transform()
    con = mpatches.ConnectionPatch(xyA=xyA, xyB=xyB, coordsA=coordsA,
                                   coordsB=coordsB, arrowstyle="-")
    ax2.add_artist(con)


@check_figures_equal(extensions=["png"])
def test_connection_patch_fig(fig_test, fig_ref):
    # Test that connection patch can be added as figure artist, and that figure
    # pixels count negative values from the top right corner (this API may be
    # changed in the future).
    ax1, ax2 = fig_test.subplots(1, 2)
    con = mpatches.ConnectionPatch(
        xyA=(.3, .2), coordsA="data", axesA=ax1,
        xyB=(-30, -20), coordsB="figure pixels",
        arrowstyle="->", shrinkB=5)
    fig_test.add_artist(con)

    ax1, ax2 = fig_ref.subplots(1, 2)
    bb = fig_ref.bbox
    # Necessary so that pixel counts match on both sides.
    plt.rcParams["savefig.dpi"] = plt.rcParams["figure.dpi"]
    con = mpatches.ConnectionPatch(
        xyA=(.3, .2), coordsA="data", axesA=ax1,
        xyB=(bb.width - 30, bb.height - 20), coordsB="figure pixels",
        arrowstyle="->", shrinkB=5)
    fig_ref.add_artist(con)


def test_datetime_rectangle():
    # Check that creating a rectangle with timedeltas doesn't fail
    from datetime import datetime, timedelta

    start = datetime(2017, 1, 1, 0, 0, 0)
    delta = timedelta(seconds=16)
    patch = mpatches.Rectangle((start, 0), delta, 1)

    fig, ax = plt.subplots()
    ax.add_patch(patch)


def test_datetime_datetime_fails():
    from datetime import datetime

    start = datetime(2017, 1, 1, 0, 0, 0)
    dt_delta = datetime(1970, 1, 5)  # Will be 5 days if units are done wrong.

    with pytest.raises(TypeError):
        mpatches.Rectangle((start, 0), dt_delta, 1)

    with pytest.raises(TypeError):
        mpatches.Rectangle((0, start), 1, dt_delta)


def test_contains_point():
    ell = mpatches.Ellipse((0.5, 0.5), 0.5, 1.0)
    points = [(0.0, 0.5), (0.2, 0.5), (0.25, 0.5), (0.5, 0.5)]
    path = ell.get_path()
    transform = ell.get_transform()
    radius = ell._process_radius(None)
    expected = np.array([path.contains_point(point,
                                             transform,
                                             radius) for point in points])
    result = np.array([ell.contains_point(point) for point in points])
    assert np.all(result == expected)


def test_contains_points():
    ell = mpatches.Ellipse((0.5, 0.5), 0.5, 1.0)
    points = [(0.0, 0.5), (0.2, 0.5), (0.25, 0.5), (0.5, 0.5)]
    path = ell.get_path()
    transform = ell.get_transform()
    radius = ell._process_radius(None)
    expected = path.contains_points(points, transform, radius)
    result = ell.contains_points(points)
    assert np.all(result == expected)


# Currently fails with pdf/svg, probably because some parts assume a dpi of 72.
@check_figures_equal(extensions=["png"])
def test_shadow(fig_test, fig_ref):
    xy = np.array([.2, .3])
    dxy = np.array([.1, .2])
    # We need to work around the nonsensical (dpi-dependent) interpretation of
    # offsets by the Shadow class...
    plt.rcParams["savefig.dpi"] = "figure"
    # Test image.
    a1 = fig_test.subplots()
    rect = mpatches.Rectangle(xy=xy, width=.5, height=.5)
    shadow = mpatches.Shadow(rect, ox=dxy[0], oy=dxy[1])
    a1.add_patch(rect)
    a1.add_patch(shadow)
    # Reference image.
    a2 = fig_ref.subplots()
    rect = mpatches.Rectangle(xy=xy, width=.5, height=.5)
    shadow = mpatches.Rectangle(
        xy=xy + fig_ref.dpi / 72 * dxy, width=.5, height=.5,
        fc=np.asarray(mcolors.to_rgb(rect.get_facecolor())) * .3,
        ec=np.asarray(mcolors.to_rgb(rect.get_facecolor())) * .3,
        alpha=.5)
    a2.add_patch(shadow)
    a2.add_patch(rect)


def test_fancyarrow_units():
    from datetime import datetime
    # Smoke test to check that FancyArrowPatch works with units
    dtime = datetime(2000, 1, 1)
    fig, ax = plt.subplots()
    arrow = FancyArrowPatch((0, dtime), (0.01, dtime))


def test_fancyarrow_setdata():
    fig, ax = plt.subplots()
    arrow = ax.arrow(0, 0, 10, 10, head_length=5, head_width=1, width=.5)
    expected1 = np.array(
      [[13.54, 13.54],
       [10.35,  9.65],
       [10.18,  9.82],
       [0.18, -0.18],
       [-0.18,  0.18],
       [9.82, 10.18],
       [9.65, 10.35],
       [13.54, 13.54]]
    )
    assert np.allclose(expected1, np.round(arrow.verts, 2))

    expected2 = np.array(
      [[16.71, 16.71],
       [16.71, 15.29],
       [16.71, 15.29],
       [1.71,  0.29],
       [0.29,  1.71],
       [15.29, 16.71],
       [15.29, 16.71],
       [16.71, 16.71]]
    )
    arrow.set_data(
        x=1, y=1, dx=15, dy=15, width=2, head_width=2, head_length=1
    )
    assert np.allclose(expected2, np.round(arrow.verts, 2))


@image_comparison(["large_arc.svg"], style="mpl20")
def test_large_arc():
    fig, (ax1, ax2) = plt.subplots(1, 2)
    x = 210
    y = -2115
    diameter = 4261
    for ax in [ax1, ax2]:
        a = Arc((x, y), diameter, diameter, lw=2, color='k')
        ax.add_patch(a)
        ax.set_axis_off()
        ax.set_aspect('equal')
    # force the high accuracy case
    ax1.set_xlim(7, 8)
    ax1.set_ylim(5, 6)

    # force the low accuracy case
    ax2.set_xlim(-25000, 18000)
    ax2.set_ylim(-20000, 6600)


@image_comparison(["all_quadrants_arcs.svg"], style="mpl20")
def test_rotated_arcs():
    fig, ax_arr = plt.subplots(2, 2, squeeze=False, figsize=(10, 10))

    scale = 10_000_000
    diag_centers = ((-1, -1), (-1, 1), (1, 1), (1, -1))
    on_axis_centers = ((0, 1), (1, 0), (0, -1), (-1, 0))
    skews = ((2, 2), (2, 1/10), (2,  1/100), (2, 1/1000))

    for ax, (sx, sy) in zip(ax_arr.ravel(), skews):
        k = 0
        for prescale, centers in zip((1 - .0001, (1 - .0001) / np.sqrt(2)),
                                      (on_axis_centers, diag_centers)):
            for j, (x_sign, y_sign) in enumerate(centers, start=k):
                a = Arc(
                    (x_sign * scale * prescale,
                     y_sign * scale * prescale),
                    scale * sx,
                    scale * sy,
                    lw=4,
                    color=f"C{j}",
                    zorder=1 + j,
                    angle=np.rad2deg(np.arctan2(y_sign, x_sign)) % 360,
                    label=f'big {j}',
                    gid=f'big {j}'
                )
                ax.add_patch(a)

            k = j+1
        ax.set_xlim(-scale / 4000, scale / 4000)
        ax.set_ylim(-scale / 4000, scale / 4000)
        ax.axhline(0, color="k")
        ax.axvline(0, color="k")
        ax.set_axis_off()
        ax.set_aspect("equal")


def test_fancyarrow_shape_error():
    with pytest.raises(ValueError, match="Got unknown shape: 'foo'"):
        FancyArrow(0, 0, 0.2, 0.2, shape='foo')


@pytest.mark.parametrize('fmt, match', (
    ("foo", "Unknown style: 'foo'"),
    ("Round,foo", "Incorrect style argument: 'Round,foo'"),
))
def test_boxstyle_errors(fmt, match):
    with pytest.raises(ValueError, match=match):
        BoxStyle(fmt)


@image_comparison(baseline_images=['annulus'], extensions=['png'])
def test_annulus():

    fig, ax = plt.subplots()
    cir = Annulus((0.5, 0.5), 0.2, 0.05, fc='g')        # circular annulus
    ell = Annulus((0.5, 0.5), (0.5, 0.3), 0.1, 45,      # elliptical
                  fc='m', ec='b', alpha=0.5, hatch='xxx')
    ax.add_patch(cir)
    ax.add_patch(ell)
    ax.set_aspect('equal')


@image_comparison(baseline_images=['annulus'], extensions=['png'])
def test_annulus_setters():

    fig, ax = plt.subplots()
    cir = Annulus((0., 0.), 0.2, 0.01, fc='g')   # circular annulus
    ell = Annulus((0., 0.), (1, 2), 0.1, 0,      # elliptical
                  fc='m', ec='b', alpha=0.5, hatch='xxx')
    ax.add_patch(cir)
    ax.add_patch(ell)
    ax.set_aspect('equal')

    cir.center = (0.5, 0.5)
    cir.radii = 0.2
    cir.width = 0.05

    ell.center = (0.5, 0.5)
    ell.radii = (0.5, 0.3)
    ell.width = 0.1
    ell.angle = 45


@image_comparison(baseline_images=['annulus'], extensions=['png'])
def test_annulus_setters2():

    fig, ax = plt.subplots()
    cir = Annulus((0., 0.), 0.2, 0.01, fc='g')   # circular annulus
    ell = Annulus((0., 0.), (1, 2), 0.1, 0,      # elliptical
                  fc='m', ec='b', alpha=0.5, hatch='xxx')
    ax.add_patch(cir)
    ax.add_patch(ell)
    ax.set_aspect('equal')

    cir.center = (0.5, 0.5)
    cir.set_semimajor(0.2)
    cir.set_semiminor(0.2)
    assert cir.radii == (0.2, 0.2)
    cir.width = 0.05

    ell.center = (0.5, 0.5)
    ell.set_semimajor(0.5)
    ell.set_semiminor(0.3)
    assert ell.radii == (0.5, 0.3)
    ell.width = 0.1
    ell.angle = 45


def test_degenerate_polygon():
    point = [0, 0]
    correct_extents = Bbox([point, point]).extents
    assert np.all(Polygon([point]).get_extents().extents == correct_extents)


@pytest.mark.parametrize('kwarg', ('edgecolor', 'facecolor'))
def test_color_override_warning(kwarg):
    with pytest.warns(UserWarning,
                      match="Setting the 'color' property will override "
                            "the edgecolor or facecolor properties."):
        Patch(color='black', **{kwarg: 'black'})


def test_empty_verts():
    poly = Polygon(np.zeros((0, 2)))
    assert poly.get_verts() == []


def test_default_antialiased():
    patch = Patch()

    patch.set_antialiased(not rcParams['patch.antialiased'])
    assert patch.get_antialiased() == (not rcParams['patch.antialiased'])
    # Check that None resets the state
    patch.set_antialiased(None)
    assert patch.get_antialiased() == rcParams['patch.antialiased']


def test_default_linestyle():
    patch = Patch()
    patch.set_linestyle('--')
    patch.set_linestyle(None)
    assert patch.get_linestyle() == 'solid'


def test_default_capstyle():
    patch = Patch()
    assert patch.get_capstyle() == 'butt'


def test_default_joinstyle():
    patch = Patch()
    assert patch.get_joinstyle() == 'miter'


@image_comparison(["autoscale_arc"], extensions=['png', 'svg'],
                  style="mpl20", remove_text=True)
def test_autoscale_arc():
    fig, axs = plt.subplots(1, 3, figsize=(4, 1))
    arc_lists = (
        [Arc((0, 0), 1, 1, theta1=0, theta2=90)],
        [Arc((0.5, 0.5), 1.5, 0.5, theta1=10, theta2=20)],
        [Arc((0.5, 0.5), 1.5, 0.5, theta1=10, theta2=20),
         Arc((0.5, 0.5), 2.5, 0.5, theta1=110, theta2=120),
         Arc((0.5, 0.5), 3.5, 0.5, theta1=210, theta2=220),
         Arc((0.5, 0.5), 4.5, 0.5, theta1=310, theta2=320)])

    for ax, arcs in zip(axs, arc_lists):
        for arc in arcs:
            ax.add_patch(arc)
        ax.autoscale()


@check_figures_equal(extensions=["png", 'svg', 'pdf', 'eps'])
def test_arc_in_collection(fig_test, fig_ref):
    arc1 = Arc([.5, .5], .5, 1, theta1=0, theta2=60, angle=20)
    arc2 = Arc([.5, .5], .5, 1, theta1=0, theta2=60, angle=20)
    col = mcollections.PatchCollection(patches=[arc2], facecolors='none',
                                       edgecolors='k')
    fig_ref.subplots().add_patch(arc1)
    fig_test.subplots().add_collection(col)


@check_figures_equal(extensions=["png", 'svg', 'pdf', 'eps'])
def test_modifying_arc(fig_test, fig_ref):
    arc1 = Arc([.5, .5], .5, 1, theta1=0, theta2=60, angle=20)
    arc2 = Arc([.5, .5], 1.5, 1, theta1=0, theta2=60, angle=10)
    fig_ref.subplots().add_patch(arc1)
    fig_test.subplots().add_patch(arc2)
    arc2.set_width(.5)
    arc2.set_angle(20)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_patches.py
NikolaBlagojevic/pyrecodes
import re

import numpy as np

from numpy.testing import assert_array_equal
import pytest

from matplotlib import patches
from matplotlib.path import Path
from matplotlib.patches import Polygon
from matplotlib.testing.decorators import image_comparison
import matplotlib.pyplot as plt
from matplotlib import transforms
from matplotlib.backend_bases import MouseEvent


def test_empty_closed_path():
    path = Path(np.zeros((0, 2)), closed=True)
    assert path.vertices.shape == (0, 2)
    assert path.codes is None
    assert_array_equal(path.get_extents().extents,
                       transforms.Bbox.null().extents)


def test_readonly_path():
    path = Path.unit_circle()

    def modify_vertices():
        path.vertices = path.vertices * 2.0

    with pytest.raises(AttributeError):
        modify_vertices()


def test_path_exceptions():
    bad_verts1 = np.arange(12).reshape(4, 3)
    with pytest.raises(ValueError,
                       match=re.escape(f'has shape {bad_verts1.shape}')):
        Path(bad_verts1)

    bad_verts2 = np.arange(12).reshape(2, 3, 2)
    with pytest.raises(ValueError,
                       match=re.escape(f'has shape {bad_verts2.shape}')):
        Path(bad_verts2)

    good_verts = np.arange(12).reshape(6, 2)
    bad_codes = np.arange(2)
    msg = re.escape(f"Your vertices have shape {good_verts.shape} "
                    f"but your codes have shape {bad_codes.shape}")
    with pytest.raises(ValueError, match=msg):
        Path(good_verts, bad_codes)


def test_point_in_path():
    # Test #1787
    path = Path._create_closed([(0, 0), (0, 1), (1, 1), (1, 0)])
    points = [(0.5, 0.5), (1.5, 0.5)]
    ret = path.contains_points(points)
    assert ret.dtype == 'bool'
    np.testing.assert_equal(ret, [True, False])


def test_contains_points_negative_radius():
    path = Path.unit_circle()

    points = [(0.0, 0.0), (1.25, 0.0), (0.9, 0.9)]
    result = path.contains_points(points, radius=-0.5)
    np.testing.assert_equal(result, [True, False, False])


_test_paths = [
    # interior extrema determine extents and degenerate derivative
    Path([[0, 0], [1, 0], [1, 1], [0, 1]],
           [Path.MOVETO, Path.CURVE4, Path.CURVE4, Path.CURVE4]),
    # a quadratic curve
    Path([[0, 0], [0, 1], [1, 0]], [Path.MOVETO, Path.CURVE3, Path.CURVE3]),
    # a linear curve, degenerate vertically
    Path([[0, 1], [1, 1]], [Path.MOVETO, Path.LINETO]),
    # a point
    Path([[1, 2]], [Path.MOVETO]),
]


_test_path_extents = [(0., 0., 0.75, 1.), (0., 0., 1., 0.5), (0., 1., 1., 1.),
                      (1., 2., 1., 2.)]


@pytest.mark.parametrize('path, extents', zip(_test_paths, _test_path_extents))
def test_exact_extents(path, extents):
    # notice that if we just looked at the control points to get the bounding
    # box of each curve, we would get the wrong answers. For example, for
    # hard_curve = Path([[0, 0], [1, 0], [1, 1], [0, 1]],
    #                   [Path.MOVETO, Path.CURVE4, Path.CURVE4, Path.CURVE4])
    # we would get that the extents area (0, 0, 1, 1). This code takes into
    # account the curved part of the path, which does not typically extend all
    # the way out to the control points.
    # Note that counterintuitively, path.get_extents() returns a Bbox, so we
    # have to get that Bbox's `.extents`.
    assert np.all(path.get_extents().extents == extents)


@pytest.mark.parametrize('ignored_code', [Path.CLOSEPOLY, Path.STOP])
def test_extents_with_ignored_codes(ignored_code):
    # Check that STOP and CLOSEPOLY points are ignored when calculating extents
    # of a path with only straight lines
    path = Path([[0, 0],
                 [1, 1],
                 [2, 2]], [Path.MOVETO, Path.MOVETO, ignored_code])
    assert np.all(path.get_extents().extents == (0., 0., 1., 1.))


def test_point_in_path_nan():
    box = np.array([[0, 0], [1, 0], [1, 1], [0, 1], [0, 0]])
    p = Path(box)
    test = np.array([[np.nan, 0.5]])
    contains = p.contains_points(test)
    assert len(contains) == 1
    assert not contains[0]


def test_nonlinear_containment():
    fig, ax = plt.subplots()
    ax.set(xscale="log", ylim=(0, 1))
    polygon = ax.axvspan(1, 10)
    assert polygon.get_path().contains_point(
        ax.transData.transform((5, .5)), ax.transData)
    assert not polygon.get_path().contains_point(
        ax.transData.transform((.5, .5)), ax.transData)
    assert not polygon.get_path().contains_point(
        ax.transData.transform((50, .5)), ax.transData)


@image_comparison(['arrow_contains_point.png'],
                  remove_text=True, style='mpl20')
def test_arrow_contains_point():
    # fix bug (#8384)
    fig, ax = plt.subplots()
    ax.set_xlim((0, 2))
    ax.set_ylim((0, 2))

    # create an arrow with Curve style
    arrow = patches.FancyArrowPatch((0.5, 0.25), (1.5, 0.75),
                                    arrowstyle='->',
                                    mutation_scale=40)
    ax.add_patch(arrow)
    # create an arrow with Bracket style
    arrow1 = patches.FancyArrowPatch((0.5, 1), (1.5, 1.25),
                                     arrowstyle=']-[',
                                     mutation_scale=40)
    ax.add_patch(arrow1)
    # create an arrow with other arrow style
    arrow2 = patches.FancyArrowPatch((0.5, 1.5), (1.5, 1.75),
                                     arrowstyle='fancy',
                                     fill=False,
                                     mutation_scale=40)
    ax.add_patch(arrow2)
    patches_list = [arrow, arrow1, arrow2]

    # generate some points
    X, Y = np.meshgrid(np.arange(0, 2, 0.1),
                       np.arange(0, 2, 0.1))
    for k, (x, y) in enumerate(zip(X.ravel(), Y.ravel())):
        xdisp, ydisp = ax.transData.transform([x, y])
        event = MouseEvent('button_press_event', fig.canvas, xdisp, ydisp)
        for m, patch in enumerate(patches_list):
            # set the points to red only if the arrow contains the point
            inside, res = patch.contains(event)
            if inside:
                ax.scatter(x, y, s=5, c="r")


@image_comparison(['path_clipping.svg'], remove_text=True)
def test_path_clipping():
    fig = plt.figure(figsize=(6.0, 6.2))

    for i, xy in enumerate([
            [(200, 200), (200, 350), (400, 350), (400, 200)],
            [(200, 200), (200, 350), (400, 350), (400, 100)],
            [(200, 100), (200, 350), (400, 350), (400, 100)],
            [(200, 100), (200, 415), (400, 350), (400, 100)],
            [(200, 100), (200, 415), (400, 415), (400, 100)],
            [(200, 415), (400, 415), (400, 100), (200, 100)],
            [(400, 415), (400, 100), (200, 100), (200, 415)]]):
        ax = fig.add_subplot(4, 2, i+1)
        bbox = [0, 140, 640, 260]
        ax.set_xlim(bbox[0], bbox[0] + bbox[2])
        ax.set_ylim(bbox[1], bbox[1] + bbox[3])
        ax.add_patch(Polygon(
            xy, facecolor='none', edgecolor='red', closed=True))


@image_comparison(['semi_log_with_zero.png'], style='mpl20')
def test_log_transform_with_zero():
    x = np.arange(-10, 10)
    y = (1.0 - 1.0/(x**2+1))**20

    fig, ax = plt.subplots()
    ax.semilogy(x, y, "-o", lw=15, markeredgecolor='k')
    ax.set_ylim(1e-7, 1)
    ax.grid(True)


def test_make_compound_path_empty():
    # We should be able to make a compound path with no arguments.
    # This makes it easier to write generic path based code.
    r = Path.make_compound_path()
    assert r.vertices.shape == (0, 2)


def test_make_compound_path_stops():
    zero = [0, 0]
    paths = 3*[Path([zero, zero], [Path.MOVETO, Path.STOP])]
    compound_path = Path.make_compound_path(*paths)
    # the choice to not preserve the terminal STOP is arbitrary, but
    # documented, so we test that it is in fact respected here
    assert np.sum(compound_path.codes == Path.STOP) == 0


@image_comparison(['xkcd.png'], remove_text=True)
def test_xkcd():
    np.random.seed(0)

    x = np.linspace(0, 2 * np.pi, 100)
    y = np.sin(x)

    with plt.xkcd():
        fig, ax = plt.subplots()
        ax.plot(x, y)


@image_comparison(['xkcd_marker.png'], remove_text=True)
def test_xkcd_marker():
    np.random.seed(0)

    x = np.linspace(0, 5, 8)
    y1 = x
    y2 = 5 - x
    y3 = 2.5 * np.ones(8)

    with plt.xkcd():
        fig, ax = plt.subplots()
        ax.plot(x, y1, '+', ms=10)
        ax.plot(x, y2, 'o', ms=10)
        ax.plot(x, y3, '^', ms=10)


@image_comparison(['marker_paths.pdf'], remove_text=True)
def test_marker_paths_pdf():
    N = 7

    plt.errorbar(np.arange(N),
                 np.ones(N) + 4,
                 np.ones(N))
    plt.xlim(-1, N)
    plt.ylim(-1, 7)


@image_comparison(['nan_path'], style='default', remove_text=True,
                  extensions=['pdf', 'svg', 'eps', 'png'])
def test_nan_isolated_points():

    y0 = [0, np.nan, 2, np.nan, 4, 5, 6]
    y1 = [np.nan, 7, np.nan, 9, 10, np.nan, 12]

    fig, ax = plt.subplots()

    ax.plot(y0, '-o')
    ax.plot(y1, '-o')


def test_path_no_doubled_point_in_to_polygon():
    hand = np.array(
        [[1.64516129, 1.16145833],
         [1.64516129, 1.59375],
         [1.35080645, 1.921875],
         [1.375, 2.18229167],
         [1.68548387, 1.9375],
         [1.60887097, 2.55208333],
         [1.68548387, 2.69791667],
         [1.76209677, 2.56770833],
         [1.83064516, 1.97395833],
         [1.89516129, 2.75],
         [1.9516129, 2.84895833],
         [2.01209677, 2.76041667],
         [1.99193548, 1.99479167],
         [2.11290323, 2.63020833],
         [2.2016129, 2.734375],
         [2.25403226, 2.60416667],
         [2.14919355, 1.953125],
         [2.30645161, 2.36979167],
         [2.39112903, 2.36979167],
         [2.41532258, 2.1875],
         [2.1733871, 1.703125],
         [2.07782258, 1.16666667]])

    (r0, c0, r1, c1) = (1.0, 1.5, 2.1, 2.5)

    poly = Path(np.vstack((hand[:, 1], hand[:, 0])).T, closed=True)
    clip_rect = transforms.Bbox([[r0, c0], [r1, c1]])
    poly_clipped = poly.clip_to_bbox(clip_rect).to_polygons()[0]

    assert np.all(poly_clipped[-2] != poly_clipped[-1])
    assert np.all(poly_clipped[-1] == poly_clipped[0])


def test_path_to_polygons():
    data = [[10, 10], [20, 20]]
    p = Path(data)

    assert_array_equal(p.to_polygons(width=40, height=40), [])
    assert_array_equal(p.to_polygons(width=40, height=40, closed_only=False),
                       [data])
    assert_array_equal(p.to_polygons(), [])
    assert_array_equal(p.to_polygons(closed_only=False), [data])

    data = [[10, 10], [20, 20], [30, 30]]
    closed_data = [[10, 10], [20, 20], [30, 30], [10, 10]]
    p = Path(data)

    assert_array_equal(p.to_polygons(width=40, height=40), [closed_data])
    assert_array_equal(p.to_polygons(width=40, height=40, closed_only=False),
                       [data])
    assert_array_equal(p.to_polygons(), [closed_data])
    assert_array_equal(p.to_polygons(closed_only=False), [data])


def test_path_deepcopy():
    # Should not raise any error
    verts = [[0, 0], [1, 1]]
    codes = [Path.MOVETO, Path.LINETO]
    path1 = Path(verts)
    path2 = Path(verts, codes)
    path1_copy = path1.deepcopy()
    path2_copy = path2.deepcopy()
    assert path1 is not path1_copy
    assert path1.vertices is not path1_copy.vertices
    assert path2 is not path2_copy
    assert path2.vertices is not path2_copy.vertices
    assert path2.codes is not path2_copy.codes


def test_path_shallowcopy():
    # Should not raise any error
    verts = [[0, 0], [1, 1]]
    codes = [Path.MOVETO, Path.LINETO]
    path1 = Path(verts)
    path2 = Path(verts, codes)
    path1_copy = path1.copy()
    path2_copy = path2.copy()
    assert path1 is not path1_copy
    assert path1.vertices is path1_copy.vertices
    assert path2 is not path2_copy
    assert path2.vertices is path2_copy.vertices
    assert path2.codes is path2_copy.codes


@pytest.mark.parametrize('phi', np.concatenate([
    np.array([0, 15, 30, 45, 60, 75, 90, 105, 120, 135]) + delta
    for delta in [-1, 0, 1]]))
def test_path_intersect_path(phi):
    # test for the range of intersection angles
    eps_array = [1e-5, 1e-8, 1e-10, 1e-12]

    transform = transforms.Affine2D().rotate(np.deg2rad(phi))

    # a and b intersect at angle phi
    a = Path([(-2, 0), (2, 0)])
    b = transform.transform_path(a)
    assert a.intersects_path(b) and b.intersects_path(a)

    # a and b touch at angle phi at (0, 0)
    a = Path([(0, 0), (2, 0)])
    b = transform.transform_path(a)
    assert a.intersects_path(b) and b.intersects_path(a)

    # a and b are orthogonal and intersect at (0, 3)
    a = transform.transform_path(Path([(0, 1), (0, 3)]))
    b = transform.transform_path(Path([(1, 3), (0, 3)]))
    assert a.intersects_path(b) and b.intersects_path(a)

    # a and b are collinear and intersect at (0, 3)
    a = transform.transform_path(Path([(0, 1), (0, 3)]))
    b = transform.transform_path(Path([(0, 5), (0, 3)]))
    assert a.intersects_path(b) and b.intersects_path(a)

    # self-intersect
    assert a.intersects_path(a)

    # a contains b
    a = transform.transform_path(Path([(0, 0), (5, 5)]))
    b = transform.transform_path(Path([(1, 1), (3, 3)]))
    assert a.intersects_path(b) and b.intersects_path(a)

    # a and b are collinear but do not intersect
    a = transform.transform_path(Path([(0, 1), (0, 5)]))
    b = transform.transform_path(Path([(3, 0), (3, 3)]))
    assert not a.intersects_path(b) and not b.intersects_path(a)

    # a and b are on the same line but do not intersect
    a = transform.transform_path(Path([(0, 1), (0, 5)]))
    b = transform.transform_path(Path([(0, 6), (0, 7)]))
    assert not a.intersects_path(b) and not b.intersects_path(a)

    # Note: 1e-13 is the absolute tolerance error used for
    # `isclose` function from src/_path.h

    # a and b are parallel but do not touch
    for eps in eps_array:
        a = transform.transform_path(Path([(0, 1), (0, 5)]))
        b = transform.transform_path(Path([(0 + eps, 1), (0 + eps, 5)]))
        assert not a.intersects_path(b) and not b.intersects_path(a)

    # a and b are on the same line but do not intersect (really close)
    for eps in eps_array:
        a = transform.transform_path(Path([(0, 1), (0, 5)]))
        b = transform.transform_path(Path([(0, 5 + eps), (0, 7)]))
        assert not a.intersects_path(b) and not b.intersects_path(a)

    # a and b are on the same line and intersect (really close)
    for eps in eps_array:
        a = transform.transform_path(Path([(0, 1), (0, 5)]))
        b = transform.transform_path(Path([(0, 5 - eps), (0, 7)]))
        assert a.intersects_path(b) and b.intersects_path(a)

    # b is the same as a but with an extra point
    a = transform.transform_path(Path([(0, 1), (0, 5)]))
    b = transform.transform_path(Path([(0, 1), (0, 2), (0, 5)]))
    assert a.intersects_path(b) and b.intersects_path(a)

    # a and b are collinear but do not intersect
    a = transform.transform_path(Path([(1, -1), (0, -1)]))
    b = transform.transform_path(Path([(0, 1), (0.9, 1)]))
    assert not a.intersects_path(b) and not b.intersects_path(a)

    # a and b are collinear but do not intersect
    a = transform.transform_path(Path([(0., -5.), (1., -5.)]))
    b = transform.transform_path(Path([(1., 5.), (0., 5.)]))
    assert not a.intersects_path(b) and not b.intersects_path(a)


@pytest.mark.parametrize('offset', range(-720, 361, 45))
def test_full_arc(offset):
    low = offset
    high = 360 + offset

    path = Path.arc(low, high)
    mins = np.min(path.vertices, axis=0)
    maxs = np.max(path.vertices, axis=0)
    np.testing.assert_allclose(mins, -1)
    np.testing.assert_allclose(maxs, 1)


def test_disjoint_zero_length_segment():
    this_path = Path(
        np.array([
            [824.85064295, 2056.26489203],
            [861.69033931, 2041.00539016],
            [868.57864109, 2057.63522175],
            [831.73894473, 2072.89472361],
            [824.85064295, 2056.26489203]]),
        np.array([1, 2, 2, 2, 79], dtype=Path.code_type))

    outline_path = Path(
        np.array([
            [859.91051028, 2165.38461538],
            [859.06772495, 2149.30331334],
            [859.06772495, 2181.46591743],
            [859.91051028, 2165.38461538],
            [859.91051028, 2165.38461538]]),
        np.array([1, 2, 2, 2, 2],
                 dtype=Path.code_type))

    assert not outline_path.intersects_path(this_path)
    assert not this_path.intersects_path(outline_path)


def test_intersect_zero_length_segment():
    this_path = Path(
        np.array([
            [0, 0],
            [1, 1],
        ]))

    outline_path = Path(
        np.array([
            [1, 0],
            [.5, .5],
            [.5, .5],
            [0, 1],
        ]))

    assert outline_path.intersects_path(this_path)
    assert this_path.intersects_path(outline_path)


def test_cleanup_closepoly():
    # if the first connected component of a Path ends in a CLOSEPOLY, but that
    # component contains a NaN, then Path.cleaned should ignore not just the
    # control points but also the CLOSEPOLY, since it has nowhere valid to
    # point.
    paths = [
        Path([[np.nan, np.nan], [np.nan, np.nan]],
             [Path.MOVETO, Path.CLOSEPOLY]),
        # we trigger a different path in the C++ code if we don't pass any
        # codes explicitly, so we must also make sure that this works
        Path([[np.nan, np.nan], [np.nan, np.nan]]),
        # we should also make sure that this cleanup works if there's some
        # multi-vertex curves
        Path([[np.nan, np.nan], [np.nan, np.nan], [np.nan, np.nan],
              [np.nan, np.nan]],
             [Path.MOVETO, Path.CURVE3, Path.CURVE3, Path.CLOSEPOLY])
    ]
    for p in paths:
        cleaned = p.cleaned(remove_nans=True)
        assert len(cleaned) == 1
        assert cleaned.codes[0] == Path.STOP

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_path.py
NikolaBlagojevic/pyrecodes
import numpy as np

from matplotlib.testing.decorators import image_comparison
import matplotlib.pyplot as plt
import matplotlib.patheffects as path_effects
from matplotlib.path import Path
import matplotlib.patches as patches


@image_comparison(['patheffect1'], remove_text=True)
def test_patheffect1():
    ax1 = plt.subplot()
    ax1.imshow([[1, 2], [2, 3]])
    txt = ax1.annotate("test", (1., 1.), (0., 0),
                       arrowprops=dict(arrowstyle="->",
                                       connectionstyle="angle3", lw=2),
                       size=20, ha="center",
                       path_effects=[path_effects.withStroke(linewidth=3,
                                                             foreground="w")])
    txt.arrow_patch.set_path_effects([path_effects.Stroke(linewidth=5,
                                                          foreground="w"),
                                      path_effects.Normal()])

    pe = [path_effects.withStroke(linewidth=3, foreground="w")]
    ax1.grid(True, linestyle="-", path_effects=pe)


@image_comparison(['patheffect2'], remove_text=True, style='mpl20')
def test_patheffect2():

    ax2 = plt.subplot()
    arr = np.arange(25).reshape((5, 5))
    ax2.imshow(arr, interpolation='nearest')
    cntr = ax2.contour(arr, colors="k")

    plt.setp(cntr.collections,
             path_effects=[path_effects.withStroke(linewidth=3,
                                                   foreground="w")])

    clbls = ax2.clabel(cntr, fmt="%2.0f", use_clabeltext=True)
    plt.setp(clbls,
             path_effects=[path_effects.withStroke(linewidth=3,
                                                   foreground="w")])


@image_comparison(['patheffect3'])
def test_patheffect3():
    p1, = plt.plot([1, 3, 5, 4, 3], 'o-b', lw=4)
    p1.set_path_effects([path_effects.SimpleLineShadow(),
                         path_effects.Normal()])
    plt.title(
        r'testing$^{123}$',
        path_effects=[path_effects.withStroke(linewidth=1, foreground="r")])
    leg = plt.legend([p1], [r'Line 1$^2$'], fancybox=True, loc='upper left')
    leg.legendPatch.set_path_effects([path_effects.withSimplePatchShadow()])

    text = plt.text(2, 3, 'Drop test', color='white',
                    bbox={'boxstyle': 'circle,pad=0.1', 'color': 'red'})
    pe = [path_effects.Stroke(linewidth=3.75, foreground='k'),
          path_effects.withSimplePatchShadow((6, -3), shadow_rgbFace='blue')]
    text.set_path_effects(pe)
    text.get_bbox_patch().set_path_effects(pe)

    pe = [path_effects.PathPatchEffect(offset=(4, -4), hatch='xxxx',
                                       facecolor='gray'),
          path_effects.PathPatchEffect(edgecolor='white', facecolor='black',
                                       lw=1.1)]

    t = plt.gcf().text(0.02, 0.1, 'Hatch shadow', fontsize=75, weight=1000,
                       va='center')
    t.set_path_effects(pe)


@image_comparison(['stroked_text.png'])
def test_patheffects_stroked_text():
    text_chunks = [
        'A B C D E F G H I J K L',
        'M N O P Q R S T U V W',
        'X Y Z a b c d e f g h i j',
        'k l m n o p q r s t u v',
        'w x y z 0123456789',
        r"!@#$%^&*()-=_+[]\;'",
        ',./{}|:"<>?'
    ]
    font_size = 50

    ax = plt.axes([0, 0, 1, 1])
    for i, chunk in enumerate(text_chunks):
        text = ax.text(x=0.01, y=(0.9 - i * 0.13), s=chunk,
                       fontdict={'ha': 'left', 'va': 'center',
                                 'size': font_size, 'color': 'white'})

        text.set_path_effects([path_effects.Stroke(linewidth=font_size / 10,
                                                   foreground='black'),
                               path_effects.Normal()])

    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1)
    ax.axis('off')


def test_PathEffect_points_to_pixels():
    fig = plt.figure(dpi=150)
    p1, = plt.plot(range(10))
    p1.set_path_effects([path_effects.SimpleLineShadow(),
                         path_effects.Normal()])
    renderer = fig.canvas.get_renderer()
    pe_renderer = path_effects.PathEffectRenderer(
        p1.get_path_effects(), renderer)
    # Confirm that using a path effects renderer maintains point sizes
    # appropriately. Otherwise rendered font would be the wrong size.
    assert renderer.points_to_pixels(15) == pe_renderer.points_to_pixels(15)


def test_SimplePatchShadow_offset():
    pe = path_effects.SimplePatchShadow(offset=(4, 5))
    assert pe._offset == (4, 5)


@image_comparison(['collection'], tol=0.03, style='mpl20')
def test_collection():
    x, y = np.meshgrid(np.linspace(0, 10, 150), np.linspace(-5, 5, 100))
    data = np.sin(x) + np.cos(y)
    cs = plt.contour(data)
    pe = [path_effects.PathPatchEffect(edgecolor='black', facecolor='none',
                                       linewidth=12),
          path_effects.Stroke(linewidth=5)]

    for collection in cs.collections:
        collection.set_path_effects(pe)

    for text in plt.clabel(cs, colors='white'):
        text.set_path_effects([path_effects.withStroke(foreground='k',
                                                       linewidth=3)])
        text.set_bbox({'boxstyle': 'sawtooth', 'facecolor': 'none',
                       'edgecolor': 'blue'})


@image_comparison(['tickedstroke'], remove_text=True, extensions=['png'],
                  tol=0.22)  # Increased tolerance due to fixed clipping.
def test_tickedstroke():
    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(12, 4))
    path = Path.unit_circle()
    patch = patches.PathPatch(path, facecolor='none', lw=2, path_effects=[
        path_effects.withTickedStroke(angle=-90, spacing=10,
                                      length=1)])

    ax1.add_patch(patch)
    ax1.axis('equal')
    ax1.set_xlim(-2, 2)
    ax1.set_ylim(-2, 2)

    ax2.plot([0, 1], [0, 1], label=' ',
             path_effects=[path_effects.withTickedStroke(spacing=7,
                                                         angle=135)])
    nx = 101
    x = np.linspace(0.0, 1.0, nx)
    y = 0.3 * np.sin(x * 8) + 0.4
    ax2.plot(x, y, label=' ', path_effects=[path_effects.withTickedStroke()])

    ax2.legend()

    nx = 101
    ny = 105

    # Set up survey vectors
    xvec = np.linspace(0.001, 4.0, nx)
    yvec = np.linspace(0.001, 4.0, ny)

    # Set up survey matrices.  Design disk loading and gear ratio.
    x1, x2 = np.meshgrid(xvec, yvec)

    # Evaluate some stuff to plot
    g1 = -(3 * x1 + x2 - 5.5)
    g2 = -(x1 + 2 * x2 - 4)
    g3 = .8 + x1 ** -3 - x2

    cg1 = ax3.contour(x1, x2, g1, [0], colors=('k',))
    plt.setp(cg1.collections,
             path_effects=[path_effects.withTickedStroke(angle=135)])

    cg2 = ax3.contour(x1, x2, g2, [0], colors=('r',))
    plt.setp(cg2.collections,
             path_effects=[path_effects.withTickedStroke(angle=60, length=2)])

    cg3 = ax3.contour(x1, x2, g3, [0], colors=('b',))
    plt.setp(cg3.collections,
             path_effects=[path_effects.withTickedStroke(spacing=7)])

    ax3.set_xlim(0, 4)
    ax3.set_ylim(0, 4)


@image_comparison(['spaces_and_newlines.png'], remove_text=True)
def test_patheffects_spaces_and_newlines():
    ax = plt.subplot()
    s1 = "         "
    s2 = "\nNewline also causes problems"
    text1 = ax.text(0.5, 0.75, s1, ha='center', va='center', size=20,
                    bbox={'color': 'salmon'})
    text2 = ax.text(0.5, 0.25, s2, ha='center', va='center', size=20,
                    bbox={'color': 'thistle'})
    text1.set_path_effects([path_effects.Normal()])
    text2.set_path_effects([path_effects.Normal()])

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_patheffects.py
NikolaBlagojevic/pyrecodes
import re
import subprocess
import sys

import numpy as np
import pytest

from matplotlib import _preprocess_data
from matplotlib.axes import Axes
from matplotlib.testing.decorators import check_figures_equal

# Notes on testing the plotting functions itself
# *   the individual decorated plotting functions are tested in 'test_axes.py'
# *   that pyplot functions accept a data kwarg is only tested in
#     test_axes.test_pie_linewidth_0


# this gets used in multiple tests, so define it here
@_preprocess_data(replace_names=["x", "y"], label_namer="y")
def plot_func(ax, x, y, ls="x", label=None, w="xyz"):
    return ("x: %s, y: %s, ls: %s, w: %s, label: %s" % (
        list(x), list(y), ls, w, label))


all_funcs = [plot_func]
all_func_ids = ['plot_func']


def test_compiletime_checks():
    """Test decorator invocations -> no replacements."""

    def func(ax, x, y): pass
    def func_args(ax, x, y, *args): pass
    def func_kwargs(ax, x, y, **kwargs): pass
    def func_no_ax_args(*args, **kwargs): pass

    # this is ok
    _preprocess_data(replace_names=["x", "y"])(func)
    _preprocess_data(replace_names=["x", "y"])(func_kwargs)
    # this has "enough" information to do all the replaces
    _preprocess_data(replace_names=["x", "y"])(func_args)

    # no positional_parameter_names but needed due to replaces
    with pytest.raises(AssertionError):
        # z is unknown
        _preprocess_data(replace_names=["x", "y", "z"])(func_args)

    # no replacements at all -> all ok...
    _preprocess_data(replace_names=[], label_namer=None)(func)
    _preprocess_data(replace_names=[], label_namer=None)(func_args)
    _preprocess_data(replace_names=[], label_namer=None)(func_kwargs)
    _preprocess_data(replace_names=[], label_namer=None)(func_no_ax_args)

    # label namer is unknown
    with pytest.raises(AssertionError):
        _preprocess_data(label_namer="z")(func)

    with pytest.raises(AssertionError):
        _preprocess_data(label_namer="z")(func_args)


@pytest.mark.parametrize('func', all_funcs, ids=all_func_ids)
def test_function_call_without_data(func):
    """Test without data -> no replacements."""
    assert (func(None, "x", "y") ==
            "x: ['x'], y: ['y'], ls: x, w: xyz, label: None")
    assert (func(None, x="x", y="y") ==
            "x: ['x'], y: ['y'], ls: x, w: xyz, label: None")
    assert (func(None, "x", "y", label="") ==
            "x: ['x'], y: ['y'], ls: x, w: xyz, label: ")
    assert (func(None, "x", "y", label="text") ==
            "x: ['x'], y: ['y'], ls: x, w: xyz, label: text")
    assert (func(None, x="x", y="y", label="") ==
            "x: ['x'], y: ['y'], ls: x, w: xyz, label: ")
    assert (func(None, x="x", y="y", label="text") ==
            "x: ['x'], y: ['y'], ls: x, w: xyz, label: text")


@pytest.mark.parametrize('func', all_funcs, ids=all_func_ids)
def test_function_call_with_dict_input(func):
    """Tests with dict input, unpacking via preprocess_pipeline"""
    data = {'a': 1, 'b': 2}
    assert (func(None, data.keys(), data.values()) ==
            "x: ['a', 'b'], y: [1, 2], ls: x, w: xyz, label: None")


@pytest.mark.parametrize('func', all_funcs, ids=all_func_ids)
def test_function_call_with_dict_data(func):
    """Test with dict data -> label comes from the value of 'x' parameter."""
    data = {"a": [1, 2], "b": [8, 9], "w": "NOT"}
    assert (func(None, "a", "b", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: b")
    assert (func(None, x="a", y="b", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: b")
    assert (func(None, "a", "b", label="", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (func(None, "a", "b", label="text", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")
    assert (func(None, x="a", y="b", label="", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (func(None, x="a", y="b", label="text", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")


@pytest.mark.parametrize('func', all_funcs, ids=all_func_ids)
def test_function_call_with_dict_data_not_in_data(func):
    """Test the case that one var is not in data -> half replaces, half kept"""
    data = {"a": [1, 2], "w": "NOT"}
    assert (func(None, "a", "b", data=data) ==
            "x: [1, 2], y: ['b'], ls: x, w: xyz, label: b")
    assert (func(None, x="a", y="b", data=data) ==
            "x: [1, 2], y: ['b'], ls: x, w: xyz, label: b")
    assert (func(None, "a", "b", label="", data=data) ==
            "x: [1, 2], y: ['b'], ls: x, w: xyz, label: ")
    assert (func(None, "a", "b", label="text", data=data) ==
            "x: [1, 2], y: ['b'], ls: x, w: xyz, label: text")
    assert (func(None, x="a", y="b", label="", data=data) ==
            "x: [1, 2], y: ['b'], ls: x, w: xyz, label: ")
    assert (func(None, x="a", y="b", label="text", data=data) ==
            "x: [1, 2], y: ['b'], ls: x, w: xyz, label: text")


@pytest.mark.parametrize('func', all_funcs, ids=all_func_ids)
def test_function_call_with_pandas_data(func, pd):
    """Test with pandas dataframe -> label comes from ``data["col"].name``."""
    data = pd.DataFrame({"a": np.array([1, 2], dtype=np.int32),
                         "b": np.array([8, 9], dtype=np.int32),
                         "w": ["NOT", "NOT"]})

    assert (func(None, "a", "b", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: b")
    assert (func(None, x="a", y="b", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: b")
    assert (func(None, "a", "b", label="", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (func(None, "a", "b", label="text", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")
    assert (func(None, x="a", y="b", label="", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (func(None, x="a", y="b", label="text", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")


def test_function_call_replace_all():
    """Test without a "replace_names" argument, all vars should be replaced."""
    data = {"a": [1, 2], "b": [8, 9], "x": "xyz"}

    @_preprocess_data(label_namer="y")
    def func_replace_all(ax, x, y, ls="x", label=None, w="NOT"):
        return "x: %s, y: %s, ls: %s, w: %s, label: %s" % (
            list(x), list(y), ls, w, label)

    assert (func_replace_all(None, "a", "b", w="x", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: b")
    assert (func_replace_all(None, x="a", y="b", w="x", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: b")
    assert (func_replace_all(None, "a", "b", w="x", label="", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (
        func_replace_all(None, "a", "b", w="x", label="text", data=data) ==
        "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")
    assert (
        func_replace_all(None, x="a", y="b", w="x", label="", data=data) ==
        "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (
        func_replace_all(None, x="a", y="b", w="x", label="text", data=data) ==
        "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")


def test_no_label_replacements():
    """Test with "label_namer=None" -> no label replacement at all."""

    @_preprocess_data(replace_names=["x", "y"], label_namer=None)
    def func_no_label(ax, x, y, ls="x", label=None, w="xyz"):
        return "x: %s, y: %s, ls: %s, w: %s, label: %s" % (
            list(x), list(y), ls, w, label)

    data = {"a": [1, 2], "b": [8, 9], "w": "NOT"}
    assert (func_no_label(None, "a", "b", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: None")
    assert (func_no_label(None, x="a", y="b", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: None")
    assert (func_no_label(None, "a", "b", label="", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: ")
    assert (func_no_label(None, "a", "b", label="text", data=data) ==
            "x: [1, 2], y: [8, 9], ls: x, w: xyz, label: text")


def test_more_args_than_pos_parameter():
    @_preprocess_data(replace_names=["x", "y"], label_namer="y")
    def func(ax, x, y, z=1):
        pass

    data = {"a": [1, 2], "b": [8, 9], "w": "NOT"}
    with pytest.raises(TypeError):
        func(None, "a", "b", "z", "z", data=data)


def test_docstring_addition():
    @_preprocess_data()
    def funcy(ax, *args, **kwargs):
        """
        Parameters
        ----------
        data : indexable object, optional
            DATA_PARAMETER_PLACEHOLDER
        """

    assert re.search(r"all parameters also accept a string", funcy.__doc__)
    assert not re.search(r"the following parameters", funcy.__doc__)

    @_preprocess_data(replace_names=[])
    def funcy(ax, x, y, z, bar=None):
        """
        Parameters
        ----------
        data : indexable object, optional
            DATA_PARAMETER_PLACEHOLDER
        """

    assert not re.search(r"all parameters also accept a string", funcy.__doc__)
    assert not re.search(r"the following parameters", funcy.__doc__)

    @_preprocess_data(replace_names=["bar"])
    def funcy(ax, x, y, z, bar=None):
        """
        Parameters
        ----------
        data : indexable object, optional
            DATA_PARAMETER_PLACEHOLDER
        """

    assert not re.search(r"all parameters also accept a string", funcy.__doc__)
    assert not re.search(r"the following parameters .*: \*bar\*\.",
                         funcy.__doc__)

    @_preprocess_data(replace_names=["x", "t"])
    def funcy(ax, x, y, z, t=None):
        """
        Parameters
        ----------
        data : indexable object, optional
            DATA_PARAMETER_PLACEHOLDER
        """

    assert not re.search(r"all parameters also accept a string", funcy.__doc__)
    assert not re.search(r"the following parameters .*: \*x\*, \*t\*\.",
                         funcy.__doc__)


def test_data_parameter_replacement():
    """
    Test that the docstring contains the correct *data* parameter stub
    for all methods that we run _preprocess_data() on.
    """
    program = (
        "import logging; "
        "logging.basicConfig(level=logging.DEBUG); "
        "import matplotlib.pyplot as plt"
    )
    cmd = [sys.executable, "-c", program]
    completed_proc = subprocess.run(cmd, text=True, capture_output=True)
    assert 'data parameter docstring error' not in completed_proc.stderr


class TestPlotTypes:

    plotters = [Axes.scatter, Axes.bar, Axes.plot]

    @pytest.mark.parametrize('plotter', plotters)
    @check_figures_equal(extensions=['png'])
    def test_dict_unpack(self, plotter, fig_test, fig_ref):
        x = [1, 2, 3]
        y = [4, 5, 6]
        ddict = dict(zip(x, y))

        plotter(fig_test.subplots(),
                ddict.keys(), ddict.values())
        plotter(fig_ref.subplots(), x, y)

    @pytest.mark.parametrize('plotter', plotters)
    @check_figures_equal(extensions=['png'])
    def test_data_kwarg(self, plotter, fig_test, fig_ref):
        x = [1, 2, 3]
        y = [4, 5, 6]

        plotter(fig_test.subplots(), 'xval', 'yval',
                data={'xval': x, 'yval': y})
        plotter(fig_ref.subplots(), x, y)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_preprocess_data.py
NikolaBlagojevic/pyrecodes
import platform
import sys

import numpy as np
import pytest

from matplotlib import pyplot as plt
from matplotlib.testing.decorators import image_comparison


def draw_quiver(ax, **kwargs):
    X, Y = np.meshgrid(np.arange(0, 2 * np.pi, 1),
                       np.arange(0, 2 * np.pi, 1))
    U = np.cos(X)
    V = np.sin(Y)

    Q = ax.quiver(U, V, **kwargs)
    return Q


@pytest.mark.skipif(platform.python_implementation() != 'CPython',
                    reason='Requires CPython')
def test_quiver_memory_leak():
    fig, ax = plt.subplots()

    Q = draw_quiver(ax)
    ttX = Q.X
    Q.remove()

    del Q

    assert sys.getrefcount(ttX) == 2


@pytest.mark.skipif(platform.python_implementation() != 'CPython',
                    reason='Requires CPython')
def test_quiver_key_memory_leak():
    fig, ax = plt.subplots()

    Q = draw_quiver(ax)

    qk = ax.quiverkey(Q, 0.5, 0.92, 2, r'$2 \frac{m}{s}$',
                      labelpos='W',
                      fontproperties={'weight': 'bold'})
    assert sys.getrefcount(qk) == 3
    qk.remove()
    assert sys.getrefcount(qk) == 2


def test_quiver_number_of_args():
    X = [1, 2]
    with pytest.raises(
            TypeError,
            match='takes from 2 to 5 positional arguments but 1 were given'):
        plt.quiver(X)
    with pytest.raises(
            TypeError,
            match='takes from 2 to 5 positional arguments but 6 were given'):
        plt.quiver(X, X, X, X, X, X)


def test_quiver_arg_sizes():
    X2 = [1, 2]
    X3 = [1, 2, 3]
    with pytest.raises(
            ValueError, match=('X and Y must be the same size, but '
                               'X.size is 2 and Y.size is 3.')):
        plt.quiver(X2, X3, X2, X2)
    with pytest.raises(
            ValueError, match=('Argument U has a size 3 which does not match '
                               '2, the number of arrow positions')):
        plt.quiver(X2, X2, X3, X2)
    with pytest.raises(
            ValueError, match=('Argument V has a size 3 which does not match '
                               '2, the number of arrow positions')):
        plt.quiver(X2, X2, X2, X3)
    with pytest.raises(
            ValueError, match=('Argument C has a size 3 which does not match '
                               '2, the number of arrow positions')):
        plt.quiver(X2, X2, X2, X2, X3)


def test_no_warnings():
    fig, ax = plt.subplots()
    X, Y = np.meshgrid(np.arange(15), np.arange(10))
    U = V = np.ones_like(X)
    phi = (np.random.rand(15, 10) - .5) * 150
    ax.quiver(X, Y, U, V, angles=phi)
    fig.canvas.draw()  # Check that no warning is emitted.


def test_zero_headlength():
    # Based on report by Doug McNeil:
    # https://discourse.matplotlib.org/t/quiver-warnings/16722
    fig, ax = plt.subplots()
    X, Y = np.meshgrid(np.arange(10), np.arange(10))
    U, V = np.cos(X), np.sin(Y)
    ax.quiver(U, V, headlength=0, headaxislength=0)
    fig.canvas.draw()  # Check that no warning is emitted.


@image_comparison(['quiver_animated_test_image.png'])
def test_quiver_animate():
    # Tests fix for #2616
    fig, ax = plt.subplots()
    Q = draw_quiver(ax, animated=True)
    ax.quiverkey(Q, 0.5, 0.92, 2, r'$2 \frac{m}{s}$',
                 labelpos='W', fontproperties={'weight': 'bold'})


@image_comparison(['quiver_with_key_test_image.png'])
def test_quiver_with_key():
    fig, ax = plt.subplots()
    ax.margins(0.1)
    Q = draw_quiver(ax)
    ax.quiverkey(Q, 0.5, 0.95, 2,
                 r'$2\, \mathrm{m}\, \mathrm{s}^{-1}$',
                 angle=-10,
                 coordinates='figure',
                 labelpos='W',
                 fontproperties={'weight': 'bold', 'size': 'large'})


@image_comparison(['quiver_single_test_image.png'], remove_text=True)
def test_quiver_single():
    fig, ax = plt.subplots()
    ax.margins(0.1)
    ax.quiver([1], [1], [2], [2])


def test_quiver_copy():
    fig, ax = plt.subplots()
    uv = dict(u=np.array([1.1]), v=np.array([2.0]))
    q0 = ax.quiver([1], [1], uv['u'], uv['v'])
    uv['v'][0] = 0
    assert q0.V[0] == 2.0


@image_comparison(['quiver_key_pivot.png'], remove_text=True)
def test_quiver_key_pivot():
    fig, ax = plt.subplots()

    u, v = np.mgrid[0:2*np.pi:10j, 0:2*np.pi:10j]

    q = ax.quiver(np.sin(u), np.cos(v))
    ax.set_xlim(-2, 11)
    ax.set_ylim(-2, 11)
    ax.quiverkey(q, 0.5, 1, 1, 'N', labelpos='N')
    ax.quiverkey(q, 1, 0.5, 1, 'E', labelpos='E')
    ax.quiverkey(q, 0.5, 0, 1, 'S', labelpos='S')
    ax.quiverkey(q, 0, 0.5, 1, 'W', labelpos='W')


@image_comparison(['quiver_key_xy.png'], remove_text=True)
def test_quiver_key_xy():
    # With scale_units='xy', ensure quiverkey still matches its quiver.
    # Note that the quiver and quiverkey lengths depend on the axes aspect
    # ratio, and that with angles='xy' their angles also depend on the axes
    # aspect ratio.
    X = np.arange(8)
    Y = np.zeros(8)
    angles = X * (np.pi / 4)
    uv = np.exp(1j * angles)
    U = uv.real
    V = uv.imag
    fig, axs = plt.subplots(2)
    for ax, angle_str in zip(axs, ('uv', 'xy')):
        ax.set_xlim(-1, 8)
        ax.set_ylim(-0.2, 0.2)
        q = ax.quiver(X, Y, U, V, pivot='middle',
                      units='xy', width=0.05,
                      scale=2, scale_units='xy',
                      angles=angle_str)
        for x, angle in zip((0.2, 0.5, 0.8), (0, 45, 90)):
            ax.quiverkey(q, X=x, Y=0.8, U=1, angle=angle, label='', color='b')


@image_comparison(['barbs_test_image.png'], remove_text=True)
def test_barbs():
    x = np.linspace(-5, 5, 5)
    X, Y = np.meshgrid(x, x)
    U, V = 12*X, 12*Y
    fig, ax = plt.subplots()
    ax.barbs(X, Y, U, V, np.hypot(U, V), fill_empty=True, rounding=False,
             sizes=dict(emptybarb=0.25, spacing=0.2, height=0.3),
             cmap='viridis')


@image_comparison(['barbs_pivot_test_image.png'], remove_text=True)
def test_barbs_pivot():
    x = np.linspace(-5, 5, 5)
    X, Y = np.meshgrid(x, x)
    U, V = 12*X, 12*Y
    fig, ax = plt.subplots()
    ax.barbs(X, Y, U, V, fill_empty=True, rounding=False, pivot=1.7,
             sizes=dict(emptybarb=0.25, spacing=0.2, height=0.3))
    ax.scatter(X, Y, s=49, c='black')


@image_comparison(['barbs_test_flip.png'], remove_text=True)
def test_barbs_flip():
    """Test barbs with an array for flip_barb."""
    x = np.linspace(-5, 5, 5)
    X, Y = np.meshgrid(x, x)
    U, V = 12*X, 12*Y
    fig, ax = plt.subplots()
    ax.barbs(X, Y, U, V, fill_empty=True, rounding=False, pivot=1.7,
             sizes=dict(emptybarb=0.25, spacing=0.2, height=0.3),
             flip_barb=Y < 0)


def test_barb_copy():
    fig, ax = plt.subplots()
    u = np.array([1.1])
    v = np.array([2.2])
    b0 = ax.barbs([1], [1], u, v)
    u[0] = 0
    assert b0.u[0] == 1.1
    v[0] = 0
    assert b0.v[0] == 2.2


def test_bad_masked_sizes():
    """Test error handling when given differing sized masked arrays."""
    x = np.arange(3)
    y = np.arange(3)
    u = np.ma.array(15. * np.ones((4,)))
    v = np.ma.array(15. * np.ones_like(u))
    u[1] = np.ma.masked
    v[1] = np.ma.masked
    fig, ax = plt.subplots()
    with pytest.raises(ValueError):
        ax.barbs(x, y, u, v)


def test_angles_and_scale():
    # angles array + scale_units kwarg
    fig, ax = plt.subplots()
    X, Y = np.meshgrid(np.arange(15), np.arange(10))
    U = V = np.ones_like(X)
    phi = (np.random.rand(15, 10) - .5) * 150
    ax.quiver(X, Y, U, V, angles=phi, scale_units='xy')


@image_comparison(['quiver_xy.png'], remove_text=True)
def test_quiver_xy():
    # simple arrow pointing from SW to NE
    fig, ax = plt.subplots(subplot_kw=dict(aspect='equal'))
    ax.quiver(0, 0, 1, 1, angles='xy', scale_units='xy', scale=1)
    ax.set_xlim(0, 1.1)
    ax.set_ylim(0, 1.1)
    ax.grid()


def test_quiverkey_angles():
    # Check that only a single arrow is plotted for a quiverkey when an array
    # of angles is given to the original quiver plot
    fig, ax = plt.subplots()

    X, Y = np.meshgrid(np.arange(2), np.arange(2))
    U = V = angles = np.ones_like(X)

    q = ax.quiver(X, Y, U, V, angles=angles)
    qk = ax.quiverkey(q, 1, 1, 2, 'Label')
    # The arrows are only created when the key is drawn
    fig.canvas.draw()
    assert len(qk.verts) == 1


def test_quiver_setuvc_numbers():
    """Check that it is possible to set all arrow UVC to the same numbers"""

    fig, ax = plt.subplots()

    X, Y = np.meshgrid(np.arange(2), np.arange(2))
    U = V = np.ones_like(X)

    q = ax.quiver(X, Y, U, V)
    q.set_UVC(0, 1)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_quiver.py
NikolaBlagojevic/pyrecodes
import pytest
from numpy.testing import assert_allclose, assert_array_equal

from matplotlib.sankey import Sankey
from matplotlib.testing.decorators import check_figures_equal


def test_sankey():
    # lets just create a sankey instance and check the code runs
    sankey = Sankey()
    sankey.add()


def test_label():
    s = Sankey(flows=[0.25], labels=['First'], orientations=[-1])
    assert s.diagrams[0].texts[0].get_text() == 'First\n0.25'


def test_format_using_callable():
    # test using callable by slightly incrementing above label example

    def show_three_decimal_places(value):
        return f'{value:.3f}'

    s = Sankey(flows=[0.25], labels=['First'], orientations=[-1],
               format=show_three_decimal_places)

    assert s.diagrams[0].texts[0].get_text() == 'First\n0.250'


@pytest.mark.parametrize('kwargs, msg', (
    ({'gap': -1}, "'gap' is negative"),
    ({'gap': 1, 'radius': 2}, "'radius' is greater than 'gap'"),
    ({'head_angle': -1}, "'head_angle' is negative"),
    ({'tolerance': -1}, "'tolerance' is negative"),
    ({'flows': [1, -1], 'orientations': [-1, 0, 1]},
     r"The shapes of 'flows' \(2,\) and 'orientations'"),
    ({'flows': [1, -1], 'labels': ['a', 'b', 'c']},
     r"The shapes of 'flows' \(2,\) and 'labels'"),
    ))
def test_sankey_errors(kwargs, msg):
    with pytest.raises(ValueError, match=msg):
        Sankey(**kwargs)


@pytest.mark.parametrize('kwargs, msg', (
    ({'trunklength': -1}, "'trunklength' is negative"),
    ({'flows': [0.2, 0.3], 'prior': 0}, "The scaled sum of the connected"),
    ({'prior': -1}, "The index of the prior diagram is negative"),
    ({'prior': 1}, "The index of the prior diagram is 1"),
    ({'connect': (-1, 1), 'prior': 0}, "At least one of the connection"),
    ({'connect': (2, 1), 'prior': 0}, "The connection index to the source"),
    ({'connect': (1, 3), 'prior': 0}, "The connection index to this dia"),
    ({'connect': (1, 1), 'prior': 0, 'flows': [-0.2, 0.2],
      'orientations': [2]}, "The value of orientations"),
    ({'connect': (1, 1), 'prior': 0, 'flows': [-0.2, 0.2],
      'pathlengths': [2]}, "The lengths of 'flows'"),
    ))
def test_sankey_add_errors(kwargs, msg):
    sankey = Sankey()
    with pytest.raises(ValueError, match=msg):
        sankey.add(flows=[0.2, -0.2])
        sankey.add(**kwargs)


def test_sankey2():
    s = Sankey(flows=[0.25, -0.25, 0.5, -0.5], labels=['Foo'],
               orientations=[-1], unit='Bar')
    sf = s.finish()
    assert_array_equal(sf[0].flows, [0.25, -0.25, 0.5, -0.5])
    assert sf[0].angles == [1, 3, 1, 3]
    assert all([text.get_text()[0:3] == 'Foo' for text in sf[0].texts])
    assert all([text.get_text()[-3:] == 'Bar' for text in sf[0].texts])
    assert sf[0].text.get_text() == ''
    assert_allclose(sf[0].tips,
                    [(-1.375, -0.52011255),
                     (1.375, -0.75506044),
                     (-0.75, -0.41522509),
                     (0.75, -0.8599479)])

    s = Sankey(flows=[0.25, -0.25, 0, 0.5, -0.5], labels=['Foo'],
               orientations=[-1], unit='Bar')
    sf = s.finish()
    assert_array_equal(sf[0].flows, [0.25, -0.25, 0, 0.5, -0.5])
    assert sf[0].angles == [1, 3, None, 1, 3]
    assert_allclose(sf[0].tips,
                    [(-1.375, -0.52011255),
                     (1.375, -0.75506044),
                     (0, 0),
                     (-0.75, -0.41522509),
                     (0.75, -0.8599479)])


@check_figures_equal(extensions=['png'])
def test_sankey3(fig_test, fig_ref):
    ax_test = fig_test.gca()
    s_test = Sankey(ax=ax_test, flows=[0.25, -0.25, -0.25, 0.25, 0.5, -0.5],
                    orientations=[1, -1, 1, -1, 0, 0])
    s_test.finish()

    ax_ref = fig_ref.gca()
    s_ref = Sankey(ax=ax_ref)
    s_ref.add(flows=[0.25, -0.25, -0.25, 0.25, 0.5, -0.5],
              orientations=[1, -1, 1, -1, 0, 0])
    s_ref.finish()

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_sankey.py
NikolaBlagojevic/pyrecodes
import base64
import io

import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal

import pytest

from matplotlib.testing.decorators import (
    check_figures_equal, image_comparison, remove_ticks_and_titles)
import matplotlib.pyplot as plt

from matplotlib import patches, transforms
from matplotlib.path import Path


# NOTE: All of these tests assume that path.simplify is set to True
# (the default)

@image_comparison(['clipping'], remove_text=True)
def test_clipping():
    t = np.arange(0.0, 2.0, 0.01)
    s = np.sin(2*np.pi*t)

    fig, ax = plt.subplots()
    ax.plot(t, s, linewidth=1.0)
    ax.set_ylim((-0.20, -0.28))


@image_comparison(['overflow'], remove_text=True)
def test_overflow():
    x = np.array([1.0, 2.0, 3.0, 2.0e5])
    y = np.arange(len(x))

    fig, ax = plt.subplots()
    ax.plot(x, y)
    ax.set_xlim(2, 6)


@image_comparison(['clipping_diamond'], remove_text=True)
def test_diamond():
    x = np.array([0.0, 1.0, 0.0, -1.0, 0.0])
    y = np.array([1.0, 0.0, -1.0, 0.0, 1.0])

    fig, ax = plt.subplots()
    ax.plot(x, y)
    ax.set_xlim(-0.6, 0.6)
    ax.set_ylim(-0.6, 0.6)


def test_clipping_out_of_bounds():
    # Should work on a Path *without* codes.
    path = Path([(0, 0), (1, 2), (2, 1)])
    simplified = path.cleaned(clip=(10, 10, 20, 20))
    assert_array_equal(simplified.vertices, [(0, 0)])
    assert simplified.codes == [Path.STOP]

    # Should work on a Path *with* codes, and no curves.
    path = Path([(0, 0), (1, 2), (2, 1)],
                [Path.MOVETO, Path.LINETO, Path.LINETO])
    simplified = path.cleaned(clip=(10, 10, 20, 20))
    assert_array_equal(simplified.vertices, [(0, 0)])
    assert simplified.codes == [Path.STOP]

    # A Path with curves does not do any clipping yet.
    path = Path([(0, 0), (1, 2), (2, 3)],
                [Path.MOVETO, Path.CURVE3, Path.CURVE3])
    simplified = path.cleaned()
    simplified_clipped = path.cleaned(clip=(10, 10, 20, 20))
    assert_array_equal(simplified.vertices, simplified_clipped.vertices)
    assert_array_equal(simplified.codes, simplified_clipped.codes)


def test_noise():
    np.random.seed(0)
    x = np.random.uniform(size=50000) * 50

    fig, ax = plt.subplots()
    p1 = ax.plot(x, solid_joinstyle='round', linewidth=2.0)

    # Ensure that the path's transform takes the new axes limits into account.
    fig.canvas.draw()
    path = p1[0].get_path()
    transform = p1[0].get_transform()
    path = transform.transform_path(path)
    simplified = path.cleaned(simplify=True)

    assert simplified.vertices.size == 25512


def test_antiparallel_simplification():
    def _get_simplified(x, y):
        fig, ax = plt.subplots()
        p1 = ax.plot(x, y)

        path = p1[0].get_path()
        transform = p1[0].get_transform()
        path = transform.transform_path(path)
        simplified = path.cleaned(simplify=True)
        simplified = transform.inverted().transform_path(simplified)

        return simplified

    # test ending on a maximum
    x = [0, 0, 0, 0, 0, 1]
    y = [.5, 1, -1, 1, 2, .5]

    simplified = _get_simplified(x, y)

    assert_array_almost_equal([[0., 0.5],
                               [0., -1.],
                               [0., 2.],
                               [1., 0.5]],
                              simplified.vertices[:-2, :])

    # test ending on a minimum
    x = [0, 0,  0, 0, 0, 1]
    y = [.5, 1, -1, 1, -2, .5]

    simplified = _get_simplified(x, y)

    assert_array_almost_equal([[0., 0.5],
                               [0., 1.],
                               [0., -2.],
                               [1., 0.5]],
                              simplified.vertices[:-2, :])

    # test ending in between
    x = [0, 0, 0, 0, 0, 1]
    y = [.5, 1, -1, 1, 0, .5]

    simplified = _get_simplified(x, y)

    assert_array_almost_equal([[0., 0.5],
                               [0., 1.],
                               [0., -1.],
                               [0., 0.],
                               [1., 0.5]],
                              simplified.vertices[:-2, :])

    # test no anti-parallel ending at max
    x = [0, 0, 0, 0, 0, 1]
    y = [.5, 1, 2, 1, 3, .5]

    simplified = _get_simplified(x, y)

    assert_array_almost_equal([[0., 0.5],
                               [0., 3.],
                               [1., 0.5]],
                              simplified.vertices[:-2, :])

    # test no anti-parallel ending in middle
    x = [0, 0, 0, 0, 0, 1]
    y = [.5, 1, 2, 1, 1, .5]

    simplified = _get_simplified(x, y)

    assert_array_almost_equal([[0., 0.5],
                               [0., 2.],
                               [0., 1.],
                               [1., 0.5]],
                              simplified.vertices[:-2, :])


# Only consider angles in 0 <= angle <= pi/2, otherwise
# using min/max will get the expected results out of order:
# min/max for simplification code depends on original vector,
# and if angle is outside above range then simplification
# min/max will be opposite from actual min/max.
@pytest.mark.parametrize('angle', [0, np.pi/4, np.pi/3, np.pi/2])
@pytest.mark.parametrize('offset', [0, .5])
def test_angled_antiparallel(angle, offset):
    scale = 5
    np.random.seed(19680801)
    # get 15 random offsets
    # TODO: guarantee offset > 0 results in some offsets < 0
    vert_offsets = (np.random.rand(15) - offset) * scale
    # always start at 0 so rotation makes sense
    vert_offsets[0] = 0
    # always take the first step the same direction
    vert_offsets[1] = 1
    # compute points along a diagonal line
    x = np.sin(angle) * vert_offsets
    y = np.cos(angle) * vert_offsets

    # will check these later
    x_max = x[1:].max()
    x_min = x[1:].min()

    y_max = y[1:].max()
    y_min = y[1:].min()

    if offset > 0:
        p_expected = Path([[0, 0],
                           [x_max, y_max],
                           [x_min, y_min],
                           [x[-1], y[-1]],
                           [0, 0]],
                          codes=[1, 2, 2, 2, 0])

    else:
        p_expected = Path([[0, 0],
                           [x_max, y_max],
                           [x[-1], y[-1]],
                           [0, 0]],
                          codes=[1, 2, 2, 0])

    p = Path(np.vstack([x, y]).T)
    p2 = p.cleaned(simplify=True)

    assert_array_almost_equal(p_expected.vertices,
                              p2.vertices)
    assert_array_equal(p_expected.codes, p2.codes)


def test_sine_plus_noise():
    np.random.seed(0)
    x = (np.sin(np.linspace(0, np.pi * 2.0, 50000)) +
         np.random.uniform(size=50000) * 0.01)

    fig, ax = plt.subplots()
    p1 = ax.plot(x, solid_joinstyle='round', linewidth=2.0)

    # Ensure that the path's transform takes the new axes limits into account.
    fig.canvas.draw()
    path = p1[0].get_path()
    transform = p1[0].get_transform()
    path = transform.transform_path(path)
    simplified = path.cleaned(simplify=True)

    assert simplified.vertices.size == 25240


@image_comparison(['simplify_curve'], remove_text=True, tol=0.017)
def test_simplify_curve():
    pp1 = patches.PathPatch(
        Path([(0, 0), (1, 0), (1, 1), (np.nan, 1), (0, 0), (2, 0), (2, 2),
              (0, 0)],
             [Path.MOVETO, Path.CURVE3, Path.CURVE3, Path.CURVE3, Path.CURVE3,
              Path.CURVE3, Path.CURVE3, Path.CLOSEPOLY]),
        fc="none")

    fig, ax = plt.subplots()
    ax.add_patch(pp1)
    ax.set_xlim((0, 2))
    ax.set_ylim((0, 2))


@check_figures_equal()
def test_closed_path_nan_removal(fig_test, fig_ref):
    ax_test = fig_test.subplots(2, 2).flatten()
    ax_ref = fig_ref.subplots(2, 2).flatten()

    # NaN on the first point also removes the last point, because it's closed.
    path = Path(
        [[-3, np.nan], [3, -3], [3, 3], [-3, 3], [-3, -3]],
        [Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.CLOSEPOLY])
    ax_test[0].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-3, np.nan], [3, -3], [3, 3], [-3, 3], [-3, np.nan]],
        [Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.LINETO])
    ax_ref[0].add_patch(patches.PathPatch(path, facecolor='none'))

    # NaN on second-last point should not re-close.
    path = Path(
        [[-2, -2], [2, -2], [2, 2], [-2, np.nan], [-2, -2]],
        [Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.CLOSEPOLY])
    ax_test[0].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-2, -2], [2, -2], [2, 2], [-2, np.nan], [-2, -2]],
        [Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.LINETO])
    ax_ref[0].add_patch(patches.PathPatch(path, facecolor='none'))

    # Test multiple loops in a single path (with same paths as above).
    path = Path(
        [[-3, np.nan], [3, -3], [3, 3], [-3, 3], [-3, -3],
         [-2, -2], [2, -2], [2, 2], [-2, np.nan], [-2, -2]],
        [Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.CLOSEPOLY,
         Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.CLOSEPOLY])
    ax_test[1].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-3, np.nan], [3, -3], [3, 3], [-3, 3], [-3, np.nan],
         [-2, -2], [2, -2], [2, 2], [-2, np.nan], [-2, -2]],
        [Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.LINETO,
         Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.LINETO])
    ax_ref[1].add_patch(patches.PathPatch(path, facecolor='none'))

    # NaN in first point of CURVE3 should not re-close, and hide entire curve.
    path = Path(
        [[-1, -1], [1, -1], [1, np.nan], [0, 1], [-1, 1], [-1, -1]],
        [Path.MOVETO, Path.LINETO, Path.CURVE3, Path.CURVE3, Path.LINETO,
         Path.CLOSEPOLY])
    ax_test[2].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-1, -1], [1, -1], [1, np.nan], [0, 1], [-1, 1], [-1, -1]],
        [Path.MOVETO, Path.LINETO, Path.CURVE3, Path.CURVE3, Path.LINETO,
         Path.CLOSEPOLY])
    ax_ref[2].add_patch(patches.PathPatch(path, facecolor='none'))

    # NaN in second point of CURVE3 should not re-close, and hide entire curve
    # plus next line segment.
    path = Path(
        [[-3, -3], [3, -3], [3, 0], [0, np.nan], [-3, 3], [-3, -3]],
        [Path.MOVETO, Path.LINETO, Path.CURVE3, Path.CURVE3, Path.LINETO,
         Path.LINETO])
    ax_test[2].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-3, -3], [3, -3], [3, 0], [0, np.nan], [-3, 3], [-3, -3]],
        [Path.MOVETO, Path.LINETO, Path.CURVE3, Path.CURVE3, Path.LINETO,
         Path.LINETO])
    ax_ref[2].add_patch(patches.PathPatch(path, facecolor='none'))

    # NaN in first point of CURVE4 should not re-close, and hide entire curve.
    path = Path(
        [[-1, -1], [1, -1], [1, np.nan], [0, 0], [0, 1], [-1, 1], [-1, -1]],
        [Path.MOVETO, Path.LINETO, Path.CURVE4, Path.CURVE4, Path.CURVE4,
         Path.LINETO, Path.CLOSEPOLY])
    ax_test[3].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-1, -1], [1, -1], [1, np.nan], [0, 0], [0, 1], [-1, 1], [-1, -1]],
        [Path.MOVETO, Path.LINETO, Path.CURVE4, Path.CURVE4, Path.CURVE4,
         Path.LINETO, Path.CLOSEPOLY])
    ax_ref[3].add_patch(patches.PathPatch(path, facecolor='none'))

    # NaN in second point of CURVE4 should not re-close, and hide entire curve.
    path = Path(
        [[-2, -2], [2, -2], [2, 0], [0, np.nan], [0, 2], [-2, 2], [-2, -2]],
        [Path.MOVETO, Path.LINETO, Path.CURVE4, Path.CURVE4, Path.CURVE4,
         Path.LINETO, Path.LINETO])
    ax_test[3].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-2, -2], [2, -2], [2, 0], [0, np.nan], [0, 2], [-2, 2], [-2, -2]],
        [Path.MOVETO, Path.LINETO, Path.CURVE4, Path.CURVE4, Path.CURVE4,
         Path.LINETO, Path.LINETO])
    ax_ref[3].add_patch(patches.PathPatch(path, facecolor='none'))

    # NaN in third point of CURVE4 should not re-close, and hide entire curve
    # plus next line segment.
    path = Path(
        [[-3, -3], [3, -3], [3, 0], [0, 0], [0, np.nan], [-3, 3], [-3, -3]],
        [Path.MOVETO, Path.LINETO, Path.CURVE4, Path.CURVE4, Path.CURVE4,
         Path.LINETO, Path.LINETO])
    ax_test[3].add_patch(patches.PathPatch(path, facecolor='none'))
    path = Path(
        [[-3, -3], [3, -3], [3, 0], [0, 0], [0, np.nan], [-3, 3], [-3, -3]],
        [Path.MOVETO, Path.LINETO, Path.CURVE4, Path.CURVE4, Path.CURVE4,
         Path.LINETO, Path.LINETO])
    ax_ref[3].add_patch(patches.PathPatch(path, facecolor='none'))

    # Keep everything clean.
    for ax in [*ax_test.flat, *ax_ref.flat]:
        ax.set(xlim=(-3.5, 3.5), ylim=(-3.5, 3.5))
    remove_ticks_and_titles(fig_test)
    remove_ticks_and_titles(fig_ref)


@check_figures_equal()
def test_closed_path_clipping(fig_test, fig_ref):
    vertices = []
    for roll in range(8):
        offset = 0.1 * roll + 0.1

        # A U-like pattern.
        pattern = [
            [-0.5, 1.5], [-0.5, -0.5], [1.5, -0.5], [1.5, 1.5],  # Outer square
            # With a notch in the top.
            [1 - offset / 2, 1.5], [1 - offset / 2, offset],
            [offset / 2, offset], [offset / 2, 1.5],
        ]

        # Place the initial/final point anywhere in/out of the clipping area.
        pattern = np.roll(pattern, roll, axis=0)
        pattern = np.concatenate((pattern, pattern[:1, :]))

        vertices.append(pattern)

    # Multiple subpaths are used here to ensure they aren't broken by closed
    # loop clipping.
    codes = np.full(len(vertices[0]), Path.LINETO)
    codes[0] = Path.MOVETO
    codes[-1] = Path.CLOSEPOLY
    codes = np.tile(codes, len(vertices))
    vertices = np.concatenate(vertices)

    fig_test.set_size_inches((5, 5))
    path = Path(vertices, codes)
    fig_test.add_artist(patches.PathPatch(path, facecolor='none'))

    # For reference, we draw the same thing, but unclosed by using a line to
    # the last point only.
    fig_ref.set_size_inches((5, 5))
    codes = codes.copy()
    codes[codes == Path.CLOSEPOLY] = Path.LINETO
    path = Path(vertices, codes)
    fig_ref.add_artist(patches.PathPatch(path, facecolor='none'))


@image_comparison(['hatch_simplify'], remove_text=True)
def test_hatch():
    fig, ax = plt.subplots()
    ax.add_patch(plt.Rectangle((0, 0), 1, 1, fill=False, hatch="/"))
    ax.set_xlim((0.45, 0.55))
    ax.set_ylim((0.45, 0.55))


@image_comparison(['fft_peaks'], remove_text=True)
def test_fft_peaks():
    fig, ax = plt.subplots()
    t = np.arange(65536)
    p1 = ax.plot(abs(np.fft.fft(np.sin(2*np.pi*.01*t)*np.blackman(len(t)))))

    # Ensure that the path's transform takes the new axes limits into account.
    fig.canvas.draw()
    path = p1[0].get_path()
    transform = p1[0].get_transform()
    path = transform.transform_path(path)
    simplified = path.cleaned(simplify=True)

    assert simplified.vertices.size == 36


def test_start_with_moveto():
    # Should be entirely clipped away to a single MOVETO
    data = b"""
ZwAAAAku+v9UAQAA+Tj6/z8CAADpQ/r/KAMAANlO+v8QBAAAyVn6//UEAAC6ZPr/2gUAAKpv+v+8
BgAAm3r6/50HAACLhfr/ewgAAHyQ+v9ZCQAAbZv6/zQKAABepvr/DgsAAE+x+v/lCwAAQLz6/7wM
AAAxx/r/kA0AACPS+v9jDgAAFN36/zQPAAAF6Pr/AxAAAPfy+v/QEAAA6f36/5wRAADbCPv/ZhIA
AMwT+/8uEwAAvh77//UTAACwKfv/uRQAAKM0+/98FQAAlT/7/z0WAACHSvv//RYAAHlV+/+7FwAA
bGD7/3cYAABea/v/MRkAAFF2+//pGQAARIH7/6AaAAA3jPv/VRsAACmX+/8JHAAAHKL7/7ocAAAP
rfv/ah0AAAO4+/8YHgAA9sL7/8QeAADpzfv/bx8AANzY+/8YIAAA0OP7/78gAADD7vv/ZCEAALf5
+/8IIgAAqwT8/6kiAACeD/z/SiMAAJIa/P/oIwAAhiX8/4QkAAB6MPz/HyUAAG47/P+4JQAAYkb8
/1AmAABWUfz/5SYAAEpc/P95JwAAPmf8/wsoAAAzcvz/nCgAACd9/P8qKQAAHIj8/7cpAAAQk/z/
QyoAAAWe/P/MKgAA+aj8/1QrAADus/z/2isAAOO+/P9eLAAA2Mn8/+AsAADM1Pz/YS0AAMHf/P/g
LQAAtur8/10uAACr9fz/2C4AAKEA/f9SLwAAlgv9/8ovAACLFv3/QDAAAIAh/f+1MAAAdSz9/ycx
AABrN/3/mDEAAGBC/f8IMgAAVk39/3UyAABLWP3/4TIAAEFj/f9LMwAANm79/7MzAAAsef3/GjQA
ACKE/f9+NAAAF4/9/+E0AAANmv3/QzUAAAOl/f+iNQAA+a/9/wA2AADvuv3/XDYAAOXF/f+2NgAA
29D9/w83AADR2/3/ZjcAAMfm/f+7NwAAvfH9/w44AACz/P3/XzgAAKkH/v+vOAAAnxL+//04AACW
Hf7/SjkAAIwo/v+UOQAAgjP+/905AAB5Pv7/JDoAAG9J/v9pOgAAZVT+/606AABcX/7/7zoAAFJq
/v8vOwAASXX+/207AAA/gP7/qjsAADaL/v/lOwAALZb+/x48AAAjof7/VTwAABqs/v+LPAAAELf+
/788AAAHwv7/8TwAAP7M/v8hPQAA9df+/1A9AADr4v7/fT0AAOLt/v+oPQAA2fj+/9E9AADQA///
+T0AAMYO//8fPgAAvRn//0M+AAC0JP//ZT4AAKsv//+GPgAAojr//6U+AACZRf//wj4AAJBQ///d
PgAAh1v///c+AAB+Zv//Dz8AAHRx//8lPwAAa3z//zk/AABih///TD8AAFmS//9dPwAAUJ3//2w/
AABHqP//ej8AAD6z//+FPwAANb7//48/AAAsyf//lz8AACPU//+ePwAAGt///6M/AAAR6v//pj8A
AAj1//+nPwAA/////w=="""

    verts = np.frombuffer(base64.decodebytes(data), dtype='<i4')
    verts = verts.reshape((len(verts) // 2, 2))
    path = Path(verts)
    segs = path.iter_segments(transforms.IdentityTransform(),
                              clip=(0.0, 0.0, 100.0, 100.0))
    segs = list(segs)
    assert len(segs) == 1
    assert segs[0][1] == Path.MOVETO


def test_throw_rendering_complexity_exceeded():
    plt.rcParams['path.simplify'] = False
    xx = np.arange(2_000_000)
    yy = np.random.rand(2_000_000)
    yy[1000] = np.nan

    fig, ax = plt.subplots()
    ax.plot(xx, yy)
    with pytest.raises(OverflowError):
        fig.savefig(io.BytesIO())


@image_comparison(['clipper_edge'], remove_text=True)
def test_clipper():
    dat = (0, 1, 0, 2, 0, 3, 0, 4, 0, 5)
    fig = plt.figure(figsize=(2, 1))
    fig.subplots_adjust(left=0, bottom=0, wspace=0, hspace=0)

    ax = fig.add_axes((0, 0, 1.0, 1.0), ylim=(0, 5), autoscale_on=False)
    ax.plot(dat)
    ax.xaxis.set_major_locator(plt.MultipleLocator(1))
    ax.yaxis.set_major_locator(plt.MultipleLocator(1))
    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_ticks_position('left')

    ax.set_xlim(5, 9)


@image_comparison(['para_equal_perp'], remove_text=True)
def test_para_equal_perp():
    x = np.array([0, 1, 2, 1, 0, -1, 0, 1] + [1] * 128)
    y = np.array([1, 1, 2, 1, 0, -1, 0, 0] + [0] * 128)

    fig, ax = plt.subplots()
    ax.plot(x + 1, y + 1)
    ax.plot(x + 1, y + 1, 'ro')


@image_comparison(['clipping_with_nans'])
def test_clipping_with_nans():
    x = np.linspace(0, 3.14 * 2, 3000)
    y = np.sin(x)
    x[::100] = np.nan

    fig, ax = plt.subplots()
    ax.plot(x, y)
    ax.set_ylim(-0.25, 0.25)


def test_clipping_full():
    p = Path([[1e30, 1e30]] * 5)
    simplified = list(p.iter_segments(clip=[0, 0, 100, 100]))
    assert simplified == []

    p = Path([[50, 40], [75, 65]], [1, 2])
    simplified = list(p.iter_segments(clip=[0, 0, 100, 100]))
    assert ([(list(x), y) for x, y in simplified] ==
            [([50, 40], 1), ([75, 65], 2)])

    p = Path([[50, 40]], [1])
    simplified = list(p.iter_segments(clip=[0, 0, 100, 100]))
    assert ([(list(x), y) for x, y in simplified] ==
            [([50, 40], 1)])

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_simplification.py
NikolaBlagojevic/pyrecodes
import numpy as np
from numpy.testing import assert_array_almost_equal
import pytest
import matplotlib.pyplot as plt
from matplotlib.testing.decorators import image_comparison
import matplotlib.transforms as mtransforms


def velocity_field():
    Y, X = np.mgrid[-3:3:100j, -3:3:200j]
    U = -1 - X**2 + Y
    V = 1 + X - Y**2
    return X, Y, U, V


def swirl_velocity_field():
    x = np.linspace(-3., 3., 200)
    y = np.linspace(-3., 3., 100)
    X, Y = np.meshgrid(x, y)
    a = 0.1
    U = np.cos(a) * (-Y) - np.sin(a) * X
    V = np.sin(a) * (-Y) + np.cos(a) * X
    return x, y, U, V


@image_comparison(['streamplot_startpoints'], remove_text=True, style='mpl20',
                  extensions=['png'])
def test_startpoints():
    X, Y, U, V = velocity_field()
    start_x, start_y = np.meshgrid(np.linspace(X.min(), X.max(), 5),
                                   np.linspace(Y.min(), Y.max(), 5))
    start_points = np.column_stack([start_x.ravel(), start_y.ravel()])
    plt.streamplot(X, Y, U, V, start_points=start_points)
    plt.plot(start_x, start_y, 'ok')


@image_comparison(['streamplot_colormap'], remove_text=True, style='mpl20',
                  tol=0.022)
def test_colormap():
    X, Y, U, V = velocity_field()
    plt.streamplot(X, Y, U, V, color=U, density=0.6, linewidth=2,
                   cmap=plt.cm.autumn)
    plt.colorbar()


@image_comparison(['streamplot_linewidth'], remove_text=True, style='mpl20',
                  tol=0.002)
def test_linewidth():
    X, Y, U, V = velocity_field()
    speed = np.hypot(U, V)
    lw = 5 * speed / speed.max()
    ax = plt.figure().subplots()
    ax.streamplot(X, Y, U, V, density=[0.5, 1], color='k', linewidth=lw)


@image_comparison(['streamplot_masks_and_nans'],
                  remove_text=True, style='mpl20')
def test_masks_and_nans():
    X, Y, U, V = velocity_field()
    mask = np.zeros(U.shape, dtype=bool)
    mask[40:60, 80:120] = 1
    U[:20, :40] = np.nan
    U = np.ma.array(U, mask=mask)
    ax = plt.figure().subplots()
    with np.errstate(invalid='ignore'):
        ax.streamplot(X, Y, U, V, color=U, cmap=plt.cm.Blues)


@image_comparison(['streamplot_maxlength.png'],
                  remove_text=True, style='mpl20', tol=0.302)
def test_maxlength():
    x, y, U, V = swirl_velocity_field()
    ax = plt.figure().subplots()
    ax.streamplot(x, y, U, V, maxlength=10., start_points=[[0., 1.5]],
                  linewidth=2, density=2)
    assert ax.get_xlim()[-1] == ax.get_ylim()[-1] == 3
    # Compatibility for old test image
    ax.set(xlim=(None, 3.2555988021882305), ylim=(None, 3.078326760195413))


@image_comparison(['streamplot_maxlength_no_broken.png'],
                  remove_text=True, style='mpl20', tol=0.302)
def test_maxlength_no_broken():
    x, y, U, V = swirl_velocity_field()
    ax = plt.figure().subplots()
    ax.streamplot(x, y, U, V, maxlength=10., start_points=[[0., 1.5]],
                  linewidth=2, density=2, broken_streamlines=False)
    assert ax.get_xlim()[-1] == ax.get_ylim()[-1] == 3
    # Compatibility for old test image
    ax.set(xlim=(None, 3.2555988021882305), ylim=(None, 3.078326760195413))


@image_comparison(['streamplot_direction.png'],
                  remove_text=True, style='mpl20', tol=0.073)
def test_direction():
    x, y, U, V = swirl_velocity_field()
    plt.streamplot(x, y, U, V, integration_direction='backward',
                   maxlength=1.5, start_points=[[1.5, 0.]],
                   linewidth=2, density=2)


def test_streamplot_limits():
    ax = plt.axes()
    x = np.linspace(-5, 10, 20)
    y = np.linspace(-2, 4, 10)
    y, x = np.meshgrid(y, x)
    trans = mtransforms.Affine2D().translate(25, 32) + ax.transData
    plt.barbs(x, y, np.sin(x), np.cos(y), transform=trans)
    # The calculated bounds are approximately the bounds of the original data,
    # this is because the entire path is taken into account when updating the
    # datalim.
    assert_array_almost_equal(ax.dataLim.bounds, (20, 30, 15, 6),
                              decimal=1)


def test_streamplot_grid():
    u = np.ones((2, 2))
    v = np.zeros((2, 2))

    # Test for same rows and columns
    x = np.array([[10, 20], [10, 30]])
    y = np.array([[10, 10], [20, 20]])

    with pytest.raises(ValueError, match="The rows of 'x' must be equal"):
        plt.streamplot(x, y, u, v)

    x = np.array([[10, 20], [10, 20]])
    y = np.array([[10, 10], [20, 30]])

    with pytest.raises(ValueError, match="The columns of 'y' must be equal"):
        plt.streamplot(x, y, u, v)

    x = np.array([[10, 20], [10, 20]])
    y = np.array([[10, 10], [20, 20]])
    plt.streamplot(x, y, u, v)

    # Test for maximum dimensions
    x = np.array([0, 10])
    y = np.array([[[0, 10]]])

    with pytest.raises(ValueError, match="'y' can have at maximum "
                                         "2 dimensions"):
        plt.streamplot(x, y, u, v)

    # Test for equal spacing
    u = np.ones((3, 3))
    v = np.zeros((3, 3))
    x = np.array([0, 10, 20])
    y = np.array([0, 10, 30])

    with pytest.raises(ValueError, match="'y' values must be equally spaced"):
        plt.streamplot(x, y, u, v)

    # Test for strictly increasing
    x = np.array([0, 20, 40])
    y = np.array([0, 20, 10])

    with pytest.raises(ValueError, match="'y' must be strictly increasing"):
        plt.streamplot(x, y, u, v)


def test_streamplot_inputs():  # test no exception occurs.
    # fully-masked
    plt.streamplot(np.arange(3), np.arange(3),
                   np.full((3, 3), np.nan), np.full((3, 3), np.nan),
                   color=np.random.rand(3, 3))
    # array-likes
    plt.streamplot(range(3), range(3),
                   np.random.rand(3, 3), np.random.rand(3, 3))

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_streamplot.py
NikolaBlagojevic/pyrecodes
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.testing.decorators import image_comparison, check_figures_equal

from matplotlib.table import CustomCell, Table
from matplotlib.path import Path
from matplotlib.transforms import Bbox


def test_non_square():
    # Check that creating a non-square table works
    cellcolors = ['b', 'r']
    plt.table(cellColours=cellcolors)


@image_comparison(['table_zorder.png'], remove_text=True)
def test_zorder():
    data = [[66386, 174296],
            [58230, 381139]]

    colLabels = ('Freeze', 'Wind')
    rowLabels = ['%d year' % x for x in (100, 50)]

    cellText = []
    yoff = np.zeros(len(colLabels))
    for row in reversed(data):
        yoff += row
        cellText.append(['%1.1f' % (x/1000.0) for x in yoff])

    t = np.linspace(0, 2*np.pi, 100)
    plt.plot(t, np.cos(t), lw=4, zorder=2)

    plt.table(cellText=cellText,
              rowLabels=rowLabels,
              colLabels=colLabels,
              loc='center',
              zorder=-2,
              )

    plt.table(cellText=cellText,
              rowLabels=rowLabels,
              colLabels=colLabels,
              loc='upper center',
              zorder=4,
              )
    plt.yticks([])


@image_comparison(['table_labels.png'])
def test_label_colours():
    dim = 3

    c = np.linspace(0, 1, dim)
    colours = plt.cm.RdYlGn(c)
    cellText = [['1'] * dim] * dim

    fig = plt.figure()

    ax1 = fig.add_subplot(4, 1, 1)
    ax1.axis('off')
    ax1.table(cellText=cellText,
              rowColours=colours,
              loc='best')

    ax2 = fig.add_subplot(4, 1, 2)
    ax2.axis('off')
    ax2.table(cellText=cellText,
              rowColours=colours,
              rowLabels=['Header'] * dim,
              loc='best')

    ax3 = fig.add_subplot(4, 1, 3)
    ax3.axis('off')
    ax3.table(cellText=cellText,
              colColours=colours,
              loc='best')

    ax4 = fig.add_subplot(4, 1, 4)
    ax4.axis('off')
    ax4.table(cellText=cellText,
              colColours=colours,
              colLabels=['Header'] * dim,
              loc='best')


@image_comparison(['table_cell_manipulation.png'], remove_text=True)
def test_diff_cell_table():
    cells = ('horizontal', 'vertical', 'open', 'closed', 'T', 'R', 'B', 'L')
    cellText = [['1'] * len(cells)] * 2
    colWidths = [0.1] * len(cells)

    _, axs = plt.subplots(nrows=len(cells), figsize=(4, len(cells)+1))
    for ax, cell in zip(axs, cells):
        ax.table(
                colWidths=colWidths,
                cellText=cellText,
                loc='center',
                edges=cell,
                )
        ax.axis('off')
    plt.tight_layout()


def test_customcell():
    types = ('horizontal', 'vertical', 'open', 'closed', 'T', 'R', 'B', 'L')
    codes = (
        (Path.MOVETO, Path.LINETO, Path.MOVETO, Path.LINETO, Path.MOVETO),
        (Path.MOVETO, Path.MOVETO, Path.LINETO, Path.MOVETO, Path.LINETO),
        (Path.MOVETO, Path.MOVETO, Path.MOVETO, Path.MOVETO, Path.MOVETO),
        (Path.MOVETO, Path.LINETO, Path.LINETO, Path.LINETO, Path.CLOSEPOLY),
        (Path.MOVETO, Path.MOVETO, Path.MOVETO, Path.LINETO, Path.MOVETO),
        (Path.MOVETO, Path.MOVETO, Path.LINETO, Path.MOVETO, Path.MOVETO),
        (Path.MOVETO, Path.LINETO, Path.MOVETO, Path.MOVETO, Path.MOVETO),
        (Path.MOVETO, Path.MOVETO, Path.MOVETO, Path.MOVETO, Path.LINETO),
        )

    for t, c in zip(types, codes):
        cell = CustomCell((0, 0), visible_edges=t, width=1, height=1)
        code = tuple(s for _, s in cell.get_path().iter_segments())
        assert c == code


@image_comparison(['table_auto_column.png'])
def test_auto_column():
    fig = plt.figure()

    # iterable list input
    ax1 = fig.add_subplot(4, 1, 1)
    ax1.axis('off')
    tb1 = ax1.table(
        cellText=[['Fit Text', 2],
                  ['very long long text, Longer text than default', 1]],
        rowLabels=["A", "B"],
        colLabels=["Col1", "Col2"],
        loc="center")
    tb1.auto_set_font_size(False)
    tb1.set_fontsize(12)
    tb1.auto_set_column_width([-1, 0, 1])

    # iterable tuple input
    ax2 = fig.add_subplot(4, 1, 2)
    ax2.axis('off')
    tb2 = ax2.table(
        cellText=[['Fit Text', 2],
                  ['very long long text, Longer text than default', 1]],
        rowLabels=["A", "B"],
        colLabels=["Col1", "Col2"],
        loc="center")
    tb2.auto_set_font_size(False)
    tb2.set_fontsize(12)
    tb2.auto_set_column_width((-1, 0, 1))

    # 3 single inputs
    ax3 = fig.add_subplot(4, 1, 3)
    ax3.axis('off')
    tb3 = ax3.table(
        cellText=[['Fit Text', 2],
                  ['very long long text, Longer text than default', 1]],
        rowLabels=["A", "B"],
        colLabels=["Col1", "Col2"],
        loc="center")
    tb3.auto_set_font_size(False)
    tb3.set_fontsize(12)
    tb3.auto_set_column_width(-1)
    tb3.auto_set_column_width(0)
    tb3.auto_set_column_width(1)

    # 4 non integer iterable input
    ax4 = fig.add_subplot(4, 1, 4)
    ax4.axis('off')
    tb4 = ax4.table(
        cellText=[['Fit Text', 2],
                  ['very long long text, Longer text than default', 1]],
        rowLabels=["A", "B"],
        colLabels=["Col1", "Col2"],
        loc="center")
    tb4.auto_set_font_size(False)
    tb4.set_fontsize(12)
    tb4.auto_set_column_width("-101")


def test_table_cells():
    fig, ax = plt.subplots()
    table = Table(ax)

    cell = table.add_cell(1, 2, 1, 1)
    assert isinstance(cell, CustomCell)
    assert cell is table[1, 2]

    cell2 = CustomCell((0, 0), 1, 2, visible_edges=None)
    table[2, 1] = cell2
    assert table[2, 1] is cell2

    # make sure getitem support has not broken
    # properties and setp
    table.properties()
    plt.setp(table)


@check_figures_equal(extensions=["png"])
def test_table_bbox(fig_test, fig_ref):
    data = [[2, 3],
            [4, 5]]

    col_labels = ('Foo', 'Bar')
    row_labels = ('Ada', 'Bob')

    cell_text = [[f"{x}" for x in row] for row in data]

    ax_list = fig_test.subplots()
    ax_list.table(cellText=cell_text,
                  rowLabels=row_labels,
                  colLabels=col_labels,
                  loc='center',
                  bbox=[0.1, 0.2, 0.8, 0.6]
                  )

    ax_bbox = fig_ref.subplots()
    ax_bbox.table(cellText=cell_text,
                  rowLabels=row_labels,
                  colLabels=col_labels,
                  loc='center',
                  bbox=Bbox.from_extents(0.1, 0.2, 0.9, 0.8)
                  )

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_table.py
NikolaBlagojevic/pyrecodes
import os
from pathlib import Path
import re
import subprocess
import sys

import matplotlib.pyplot as plt
from matplotlib.texmanager import TexManager
from matplotlib.testing._markers import needs_usetex
import pytest


def test_fontconfig_preamble():
    """Test that the preamble is included in the source."""
    plt.rcParams['text.usetex'] = True

    src1 = TexManager()._get_tex_source("", fontsize=12)
    plt.rcParams['text.latex.preamble'] = '\\usepackage{txfonts}'
    src2 = TexManager()._get_tex_source("", fontsize=12)

    assert src1 != src2


@pytest.mark.parametrize(
    "rc, preamble, family", [
        ({"font.family": "sans-serif", "font.sans-serif": "helvetica"},
         r"\usepackage{helvet}", r"\sffamily"),
        ({"font.family": "serif", "font.serif": "palatino"},
         r"\usepackage{mathpazo}", r"\rmfamily"),
        ({"font.family": "cursive", "font.cursive": "zapf chancery"},
         r"\usepackage{chancery}", r"\rmfamily"),
        ({"font.family": "monospace", "font.monospace": "courier"},
         r"\usepackage{courier}", r"\ttfamily"),
        ({"font.family": "helvetica"}, r"\usepackage{helvet}", r"\sffamily"),
        ({"font.family": "palatino"}, r"\usepackage{mathpazo}", r"\rmfamily"),
        ({"font.family": "zapf chancery"},
         r"\usepackage{chancery}", r"\rmfamily"),
        ({"font.family": "courier"}, r"\usepackage{courier}", r"\ttfamily")
    ])
def test_font_selection(rc, preamble, family):
    plt.rcParams.update(rc)
    tm = TexManager()
    src = Path(tm.make_tex("hello, world", fontsize=12)).read_text()
    assert preamble in src
    assert [*re.findall(r"\\\w+family", src)] == [family]


@needs_usetex
def test_unicode_characters():
    # Smoke test to see that Unicode characters does not cause issues
    # See #23019
    plt.rcParams['text.usetex'] = True
    fig, ax = plt.subplots()
    ax.set_ylabel('\\textit{Velocity (\N{DEGREE SIGN}/sec)}')
    ax.set_xlabel('\N{VULGAR FRACTION ONE QUARTER}Öøæ')
    fig.canvas.draw()

    # But not all characters.
    # Should raise RuntimeError, not UnicodeDecodeError
    with pytest.raises(RuntimeError):
        ax.set_title('\N{SNOWMAN}')
        fig.canvas.draw()


@needs_usetex
def test_openin_any_paranoid():
    completed = subprocess.run(
        [sys.executable, "-c",
         'import matplotlib.pyplot as plt;'
         'plt.rcParams.update({"text.usetex": True});'
         'plt.title("paranoid");'
         'plt.show(block=False);'],
        env={**os.environ, 'openin_any': 'p'}, check=True, capture_output=True)
    assert completed.stderr == b""

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_texmanager.py
NikolaBlagojevic/pyrecodes
import copy

from matplotlib.textpath import TextPath


def test_copy():
    tp = TextPath((0, 0), ".")
    assert copy.deepcopy(tp).vertices is not tp.vertices
    assert (copy.deepcopy(tp).vertices == tp.vertices).all()
    assert copy.copy(tp).vertices is tp.vertices

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_textpath.py
NikolaBlagojevic/pyrecodes
import warnings

import numpy as np
from numpy.testing import assert_array_equal
import pytest

import matplotlib as mpl
from matplotlib.testing.decorators import image_comparison
import matplotlib.pyplot as plt
from matplotlib.offsetbox import AnchoredOffsetbox, DrawingArea
from matplotlib.patches import Rectangle


def example_plot(ax, fontsize=12):
    ax.plot([1, 2])
    ax.locator_params(nbins=3)
    ax.set_xlabel('x-label', fontsize=fontsize)
    ax.set_ylabel('y-label', fontsize=fontsize)
    ax.set_title('Title', fontsize=fontsize)


@image_comparison(['tight_layout1'], tol=1.9)
def test_tight_layout1():
    """Test tight_layout for a single subplot."""
    fig, ax = plt.subplots()
    example_plot(ax, fontsize=24)
    plt.tight_layout()


@image_comparison(['tight_layout2'])
def test_tight_layout2():
    """Test tight_layout for multiple subplots."""
    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)
    example_plot(ax1)
    example_plot(ax2)
    example_plot(ax3)
    example_plot(ax4)
    plt.tight_layout()


@image_comparison(['tight_layout3'])
def test_tight_layout3():
    """Test tight_layout for multiple subplots."""
    ax1 = plt.subplot(221)
    ax2 = plt.subplot(223)
    ax3 = plt.subplot(122)
    example_plot(ax1)
    example_plot(ax2)
    example_plot(ax3)
    plt.tight_layout()


@image_comparison(['tight_layout4'], freetype_version=('2.5.5', '2.6.1'),
                  tol=0.015)
def test_tight_layout4():
    """Test tight_layout for subplot2grid."""
    ax1 = plt.subplot2grid((3, 3), (0, 0))
    ax2 = plt.subplot2grid((3, 3), (0, 1), colspan=2)
    ax3 = plt.subplot2grid((3, 3), (1, 0), colspan=2, rowspan=2)
    ax4 = plt.subplot2grid((3, 3), (1, 2), rowspan=2)
    example_plot(ax1)
    example_plot(ax2)
    example_plot(ax3)
    example_plot(ax4)
    plt.tight_layout()


@image_comparison(['tight_layout5'])
def test_tight_layout5():
    """Test tight_layout for image."""
    ax = plt.subplot()
    arr = np.arange(100).reshape((10, 10))
    ax.imshow(arr, interpolation="none")
    plt.tight_layout()


@image_comparison(['tight_layout6'])
def test_tight_layout6():
    """Test tight_layout for gridspec."""

    # This raises warnings since tight layout cannot
    # do this fully automatically. But the test is
    # correct since the layout is manually edited
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", UserWarning)
        fig = plt.figure()

        gs1 = mpl.gridspec.GridSpec(2, 1)
        ax1 = fig.add_subplot(gs1[0])
        ax2 = fig.add_subplot(gs1[1])

        example_plot(ax1)
        example_plot(ax2)

        gs1.tight_layout(fig, rect=[0, 0, 0.5, 1])

        gs2 = mpl.gridspec.GridSpec(3, 1)

        for ss in gs2:
            ax = fig.add_subplot(ss)
            example_plot(ax)
            ax.set_title("")
            ax.set_xlabel("")

        ax.set_xlabel("x-label", fontsize=12)

        gs2.tight_layout(fig, rect=[0.5, 0, 1, 1], h_pad=0.45)

        top = min(gs1.top, gs2.top)
        bottom = max(gs1.bottom, gs2.bottom)

        gs1.tight_layout(fig, rect=[None, 0 + (bottom-gs1.bottom),
                                    0.5, 1 - (gs1.top-top)])
        gs2.tight_layout(fig, rect=[0.5, 0 + (bottom-gs2.bottom),
                                    None, 1 - (gs2.top-top)],
                         h_pad=0.45)


@image_comparison(['tight_layout7'], tol=1.9)
def test_tight_layout7():
    # tight layout with left and right titles
    fontsize = 24
    fig, ax = plt.subplots()
    ax.plot([1, 2])
    ax.locator_params(nbins=3)
    ax.set_xlabel('x-label', fontsize=fontsize)
    ax.set_ylabel('y-label', fontsize=fontsize)
    ax.set_title('Left Title', loc='left', fontsize=fontsize)
    ax.set_title('Right Title', loc='right', fontsize=fontsize)
    plt.tight_layout()


@image_comparison(['tight_layout8'])
def test_tight_layout8():
    """Test automatic use of tight_layout."""
    fig = plt.figure()
    fig.set_layout_engine(layout='tight', pad=0.1)
    ax = fig.add_subplot()
    example_plot(ax, fontsize=24)
    fig.draw_without_rendering()


@image_comparison(['tight_layout9'])
def test_tight_layout9():
    # Test tight_layout for non-visible subplots
    # GH 8244
    f, axarr = plt.subplots(2, 2)
    axarr[1][1].set_visible(False)
    plt.tight_layout()


def test_outward_ticks():
    """Test automatic use of tight_layout."""
    fig = plt.figure()
    ax = fig.add_subplot(221)
    ax.xaxis.set_tick_params(tickdir='out', length=16, width=3)
    ax.yaxis.set_tick_params(tickdir='out', length=16, width=3)
    ax.xaxis.set_tick_params(
        tickdir='out', length=32, width=3, tick1On=True, which='minor')
    ax.yaxis.set_tick_params(
        tickdir='out', length=32, width=3, tick1On=True, which='minor')
    ax.xaxis.set_ticks([0], minor=True)
    ax.yaxis.set_ticks([0], minor=True)
    ax = fig.add_subplot(222)
    ax.xaxis.set_tick_params(tickdir='in', length=32, width=3)
    ax.yaxis.set_tick_params(tickdir='in', length=32, width=3)
    ax = fig.add_subplot(223)
    ax.xaxis.set_tick_params(tickdir='inout', length=32, width=3)
    ax.yaxis.set_tick_params(tickdir='inout', length=32, width=3)
    ax = fig.add_subplot(224)
    ax.xaxis.set_tick_params(tickdir='out', length=32, width=3)
    ax.yaxis.set_tick_params(tickdir='out', length=32, width=3)
    plt.tight_layout()
    # These values were obtained after visual checking that they correspond
    # to a tight layouting that did take the ticks into account.
    ans = [[[0.091, 0.607], [0.433, 0.933]],
           [[0.579, 0.607], [0.922, 0.933]],
           [[0.091, 0.140], [0.433, 0.466]],
           [[0.579, 0.140], [0.922, 0.466]]]
    for nn, ax in enumerate(fig.axes):
        assert_array_equal(np.round(ax.get_position().get_points(), 3),
                           ans[nn])


def add_offsetboxes(ax, size=10, margin=.1, color='black'):
    """
    Surround ax with OffsetBoxes
    """
    m, mp = margin, 1+margin
    anchor_points = [(-m, -m), (-m, .5), (-m, mp),
                     (mp, .5), (.5, mp), (mp, mp),
                     (.5, -m), (mp, -m), (.5, -m)]
    for point in anchor_points:
        da = DrawingArea(size, size)
        background = Rectangle((0, 0), width=size,
                               height=size,
                               facecolor=color,
                               edgecolor='None',
                               linewidth=0,
                               antialiased=False)
        da.add_artist(background)

        anchored_box = AnchoredOffsetbox(
            loc='center',
            child=da,
            pad=0.,
            frameon=False,
            bbox_to_anchor=point,
            bbox_transform=ax.transAxes,
            borderpad=0.)
        ax.add_artist(anchored_box)
    return anchored_box


@image_comparison(['tight_layout_offsetboxes1', 'tight_layout_offsetboxes2'])
def test_tight_layout_offsetboxes():
    # 1.
    # - Create 4 subplots
    # - Plot a diagonal line on them
    # - Surround each plot with 7 boxes
    # - Use tight_layout
    # - See that the squares are included in the tight_layout
    #   and that the squares in the middle do not overlap
    #
    # 2.
    # - Make the squares around the right side axes invisible
    # - See that the invisible squares do not affect the
    #   tight_layout
    rows = cols = 2
    colors = ['red', 'blue', 'green', 'yellow']
    x = y = [0, 1]

    def _subplots():
        _, axs = plt.subplots(rows, cols)
        axs = axs.flat
        for ax, color in zip(axs, colors):
            ax.plot(x, y, color=color)
            add_offsetboxes(ax, 20, color=color)
        return axs

    # 1.
    axs = _subplots()
    plt.tight_layout()

    # 2.
    axs = _subplots()
    for ax in (axs[cols-1::rows]):
        for child in ax.get_children():
            if isinstance(child, AnchoredOffsetbox):
                child.set_visible(False)

    plt.tight_layout()


def test_empty_layout():
    """Test that tight layout doesn't cause an error when there are no axes."""
    fig = plt.gcf()
    fig.tight_layout()


@pytest.mark.parametrize("label", ["xlabel", "ylabel"])
def test_verybig_decorators(label):
    """Test that no warning emitted when xlabel/ylabel too big."""
    fig, ax = plt.subplots(figsize=(3, 2))
    ax.set(**{label: 'a' * 100})


def test_big_decorators_horizontal():
    """Test that doesn't warn when xlabel too big."""
    fig, axs = plt.subplots(1, 2, figsize=(3, 2))
    axs[0].set_xlabel('a' * 30)
    axs[1].set_xlabel('b' * 30)


def test_big_decorators_vertical():
    """Test that doesn't warn when ylabel too big."""
    fig, axs = plt.subplots(2, 1, figsize=(3, 2))
    axs[0].set_ylabel('a' * 20)
    axs[1].set_ylabel('b' * 20)


def test_badsubplotgrid():
    # test that we get warning for mismatched subplot grids, not than an error
    plt.subplot2grid((4, 5), (0, 0))
    # this is the bad entry:
    plt.subplot2grid((5, 5), (0, 3), colspan=3, rowspan=5)
    with pytest.warns(UserWarning):
        plt.tight_layout()


def test_collapsed():
    # test that if the amount of space required to make all the axes
    # decorations fit would mean that the actual Axes would end up with size
    # zero (i.e. margins add up to more than the available width) that a call
    # to tight_layout will not get applied:
    fig, ax = plt.subplots(tight_layout=True)
    ax.set_xlim([0, 1])
    ax.set_ylim([0, 1])

    ax.annotate('BIG LONG STRING', xy=(1.25, 2), xytext=(10.5, 1.75),
                annotation_clip=False)
    p1 = ax.get_position()
    with pytest.warns(UserWarning):
        plt.tight_layout()
        p2 = ax.get_position()
        assert p1.width == p2.width
    # test that passing a rect doesn't crash...
    with pytest.warns(UserWarning):
        plt.tight_layout(rect=[0, 0, 0.8, 0.8])


def test_suptitle():
    fig, ax = plt.subplots(tight_layout=True)
    st = fig.suptitle("foo")
    t = ax.set_title("bar")
    fig.canvas.draw()
    assert st.get_window_extent().y0 > t.get_window_extent().y1


@pytest.mark.backend("pdf")
def test_non_agg_renderer(monkeypatch, recwarn):
    unpatched_init = mpl.backend_bases.RendererBase.__init__

    def __init__(self, *args, **kwargs):
        # Check that we don't instantiate any other renderer than a pdf
        # renderer to perform pdf tight layout.
        assert isinstance(self, mpl.backends.backend_pdf.RendererPdf)
        unpatched_init(self, *args, **kwargs)

    monkeypatch.setattr(mpl.backend_bases.RendererBase, "__init__", __init__)
    fig, ax = plt.subplots()
    fig.tight_layout()


def test_manual_colorbar():
    # This should warn, but not raise
    fig, axes = plt.subplots(1, 2)
    pts = axes[1].scatter([0, 1], [0, 1], c=[1, 5])
    ax_rect = axes[1].get_position()
    cax = fig.add_axes(
        [ax_rect.x1 + 0.005, ax_rect.y0, 0.015, ax_rect.height]
    )
    fig.colorbar(pts, cax=cax)
    with pytest.warns(UserWarning, match="This figure includes Axes"):
        fig.tight_layout()


def test_clipped_to_axes():
    # Ensure that _fully_clipped_to_axes() returns True under default
    # conditions for all projection types. Axes.get_tightbbox()
    # uses this to skip artists in layout calculations.
    arr = np.arange(100).reshape((10, 10))
    fig = plt.figure(figsize=(6, 2))
    ax1 = fig.add_subplot(131, projection='rectilinear')
    ax2 = fig.add_subplot(132, projection='mollweide')
    ax3 = fig.add_subplot(133, projection='polar')
    for ax in (ax1, ax2, ax3):
        # Default conditions (clipped by ax.bbox or ax.patch)
        ax.grid(False)
        h, = ax.plot(arr[:, 0])
        m = ax.pcolor(arr)
        assert h._fully_clipped_to_axes()
        assert m._fully_clipped_to_axes()
        # Non-default conditions (not clipped by ax.patch)
        rect = Rectangle((0, 0), 0.5, 0.5, transform=ax.transAxes)
        h.set_clip_path(rect)
        m.set_clip_path(rect.get_path(), rect.get_transform())
        assert not h._fully_clipped_to_axes()
        assert not m._fully_clipped_to_axes()


def test_tight_pads():
    fig, ax = plt.subplots()
    with pytest.warns(PendingDeprecationWarning,
                      match='will be deprecated'):
        fig.set_tight_layout({'pad': 0.15})
    fig.draw_without_rendering()


def test_tight_kwargs():
    fig, ax = plt.subplots(tight_layout={'pad': 0.15})
    fig.draw_without_rendering()


def test_tight_toggle():
    fig, ax = plt.subplots()
    with pytest.warns(PendingDeprecationWarning):
        fig.set_tight_layout(True)
        assert fig.get_tight_layout()
        fig.set_tight_layout(False)
        assert not fig.get_tight_layout()
        fig.set_tight_layout(True)
        assert fig.get_tight_layout()

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_tightlayout.py
NikolaBlagojevic/pyrecodes
from datetime import datetime, timezone, timedelta
import platform
from unittest.mock import MagicMock

import matplotlib.pyplot as plt
from matplotlib.testing.decorators import check_figures_equal, image_comparison
import matplotlib.units as munits
from matplotlib.category import UnitData
import numpy as np
import pytest


# Basic class that wraps numpy array and has units
class Quantity:
    def __init__(self, data, units):
        self.magnitude = data
        self.units = units

    def to(self, new_units):
        factors = {('hours', 'seconds'): 3600, ('minutes', 'hours'): 1 / 60,
                   ('minutes', 'seconds'): 60, ('feet', 'miles'): 1 / 5280.,
                   ('feet', 'inches'): 12, ('miles', 'inches'): 12 * 5280}
        if self.units != new_units:
            mult = factors[self.units, new_units]
            return Quantity(mult * self.magnitude, new_units)
        else:
            return Quantity(self.magnitude, self.units)

    def __copy__(self):
        return Quantity(self.magnitude, self.units)

    def __getattr__(self, attr):
        return getattr(self.magnitude, attr)

    def __getitem__(self, item):
        if np.iterable(self.magnitude):
            return Quantity(self.magnitude[item], self.units)
        else:
            return Quantity(self.magnitude, self.units)

    def __array__(self):
        return np.asarray(self.magnitude)


@pytest.fixture
def quantity_converter():
    # Create an instance of the conversion interface and
    # mock so we can check methods called
    qc = munits.ConversionInterface()

    def convert(value, unit, axis):
        if hasattr(value, 'units'):
            return value.to(unit).magnitude
        elif np.iterable(value):
            try:
                return [v.to(unit).magnitude for v in value]
            except AttributeError:
                return [Quantity(v, axis.get_units()).to(unit).magnitude
                        for v in value]
        else:
            return Quantity(value, axis.get_units()).to(unit).magnitude

    def default_units(value, axis):
        if hasattr(value, 'units'):
            return value.units
        elif np.iterable(value):
            for v in value:
                if hasattr(v, 'units'):
                    return v.units
            return None

    qc.convert = MagicMock(side_effect=convert)
    qc.axisinfo = MagicMock(side_effect=lambda u, a:
                            munits.AxisInfo(label=u, default_limits=(0, 100)))
    qc.default_units = MagicMock(side_effect=default_units)
    return qc


# Tests that the conversion machinery works properly for classes that
# work as a facade over numpy arrays (like pint)
@image_comparison(['plot_pint.png'], style='mpl20',
                  tol=0 if platform.machine() == 'x86_64' else 0.01)
def test_numpy_facade(quantity_converter):
    # use former defaults to match existing baseline image
    plt.rcParams['axes.formatter.limits'] = -7, 7

    # Register the class
    munits.registry[Quantity] = quantity_converter

    # Simple test
    y = Quantity(np.linspace(0, 30), 'miles')
    x = Quantity(np.linspace(0, 5), 'hours')

    fig, ax = plt.subplots()
    fig.subplots_adjust(left=0.15)  # Make space for label
    ax.plot(x, y, 'tab:blue')
    ax.axhline(Quantity(26400, 'feet'), color='tab:red')
    ax.axvline(Quantity(120, 'minutes'), color='tab:green')
    ax.yaxis.set_units('inches')
    ax.xaxis.set_units('seconds')

    assert quantity_converter.convert.called
    assert quantity_converter.axisinfo.called
    assert quantity_converter.default_units.called


# Tests gh-8908
@image_comparison(['plot_masked_units.png'], remove_text=True, style='mpl20',
                  tol=0 if platform.machine() == 'x86_64' else 0.01)
def test_plot_masked_units():
    data = np.linspace(-5, 5)
    data_masked = np.ma.array(data, mask=(data > -2) & (data < 2))
    data_masked_units = Quantity(data_masked, 'meters')

    fig, ax = plt.subplots()
    ax.plot(data_masked_units)


def test_empty_set_limits_with_units(quantity_converter):
    # Register the class
    munits.registry[Quantity] = quantity_converter

    fig, ax = plt.subplots()
    ax.set_xlim(Quantity(-1, 'meters'), Quantity(6, 'meters'))
    ax.set_ylim(Quantity(-1, 'hours'), Quantity(16, 'hours'))


@image_comparison(['jpl_bar_units.png'],
                  savefig_kwarg={'dpi': 120}, style='mpl20')
def test_jpl_bar_units():
    import matplotlib.testing.jpl_units as units
    units.register()

    day = units.Duration("ET", 24.0 * 60.0 * 60.0)
    x = [0 * units.km, 1 * units.km, 2 * units.km]
    w = [1 * day, 2 * day, 3 * day]
    b = units.Epoch("ET", dt=datetime(2009, 4, 25))
    fig, ax = plt.subplots()
    ax.bar(x, w, bottom=b)
    ax.set_ylim([b - 1 * day, b + w[-1] + (1.001) * day])


@image_comparison(['jpl_barh_units.png'],
                  savefig_kwarg={'dpi': 120}, style='mpl20')
def test_jpl_barh_units():
    import matplotlib.testing.jpl_units as units
    units.register()

    day = units.Duration("ET", 24.0 * 60.0 * 60.0)
    x = [0 * units.km, 1 * units.km, 2 * units.km]
    w = [1 * day, 2 * day, 3 * day]
    b = units.Epoch("ET", dt=datetime(2009, 4, 25))

    fig, ax = plt.subplots()
    ax.barh(x, w, left=b)
    ax.set_xlim([b - 1 * day, b + w[-1] + (1.001) * day])


def test_empty_arrays():
    # Check that plotting an empty array with a dtype works
    plt.scatter(np.array([], dtype='datetime64[ns]'), np.array([]))


def test_scatter_element0_masked():
    times = np.arange('2005-02', '2005-03', dtype='datetime64[D]')
    y = np.arange(len(times), dtype=float)
    y[0] = np.nan
    fig, ax = plt.subplots()
    ax.scatter(times, y)
    fig.canvas.draw()


def test_errorbar_mixed_units():
    x = np.arange(10)
    y = [datetime(2020, 5, i * 2 + 1) for i in x]
    fig, ax = plt.subplots()
    ax.errorbar(x, y, timedelta(days=0.5))
    fig.canvas.draw()


@check_figures_equal(extensions=["png"])
def test_subclass(fig_test, fig_ref):
    class subdate(datetime):
        pass

    fig_test.subplots().plot(subdate(2000, 1, 1), 0, "o")
    fig_ref.subplots().plot(datetime(2000, 1, 1), 0, "o")


def test_shared_axis_quantity(quantity_converter):
    munits.registry[Quantity] = quantity_converter
    x = Quantity(np.linspace(0, 1, 10), "hours")
    y1 = Quantity(np.linspace(1, 2, 10), "feet")
    y2 = Quantity(np.linspace(3, 4, 10), "feet")
    fig, (ax1, ax2) = plt.subplots(2, 1, sharex='all', sharey='all')
    ax1.plot(x, y1)
    ax2.plot(x, y2)
    assert ax1.xaxis.get_units() == ax2.xaxis.get_units() == "hours"
    assert ax2.yaxis.get_units() == ax2.yaxis.get_units() == "feet"
    ax1.xaxis.set_units("seconds")
    ax2.yaxis.set_units("inches")
    assert ax1.xaxis.get_units() == ax2.xaxis.get_units() == "seconds"
    assert ax1.yaxis.get_units() == ax2.yaxis.get_units() == "inches"


def test_shared_axis_datetime():
    # datetime uses dates.DateConverter
    y1 = [datetime(2020, i, 1, tzinfo=timezone.utc) for i in range(1, 13)]
    y2 = [datetime(2021, i, 1, tzinfo=timezone.utc) for i in range(1, 13)]
    fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True)
    ax1.plot(y1)
    ax2.plot(y2)
    ax1.yaxis.set_units(timezone(timedelta(hours=5)))
    assert ax2.yaxis.units == timezone(timedelta(hours=5))


def test_shared_axis_categorical():
    # str uses category.StrCategoryConverter
    d1 = {"a": 1, "b": 2}
    d2 = {"a": 3, "b": 4}
    fig, (ax1, ax2) = plt.subplots(1, 2, sharex=True, sharey=True)
    ax1.plot(d1.keys(), d1.values())
    ax2.plot(d2.keys(), d2.values())
    ax1.xaxis.set_units(UnitData(["c", "d"]))
    assert "c" in ax2.xaxis.get_units()._mapping.keys()


def test_empty_default_limits(quantity_converter):
    munits.registry[Quantity] = quantity_converter
    fig, ax1 = plt.subplots()
    ax1.xaxis.update_units(Quantity([10], "miles"))
    fig.draw_without_rendering()
    assert ax1.get_xlim() == (0, 100)
    ax1.yaxis.update_units(Quantity([10], "miles"))
    fig.draw_without_rendering()
    assert ax1.get_ylim() == (0, 100)

    fig, ax = plt.subplots()
    ax.axhline(30)
    ax.plot(Quantity(np.arange(0, 3), "miles"),
            Quantity(np.arange(0, 6, 2), "feet"))
    fig.draw_without_rendering()
    assert ax.get_xlim() == (0, 2)
    assert ax.get_ylim() == (0, 30)

    fig, ax = plt.subplots()
    ax.axvline(30)
    ax.plot(Quantity(np.arange(0, 3), "miles"),
            Quantity(np.arange(0, 6, 2), "feet"))
    fig.draw_without_rendering()
    assert ax.get_xlim() == (0, 30)
    assert ax.get_ylim() == (0, 4)

    fig, ax = plt.subplots()
    ax.xaxis.update_units(Quantity([10], "miles"))
    ax.axhline(30)
    fig.draw_without_rendering()
    assert ax.get_xlim() == (0, 100)
    assert ax.get_ylim() == (28.5, 31.5)

    fig, ax = plt.subplots()
    ax.yaxis.update_units(Quantity([10], "miles"))
    ax.axvline(30)
    fig.draw_without_rendering()
    assert ax.get_ylim() == (0, 100)
    assert ax.get_xlim() == (28.5, 31.5)


# test array-like objects...
class Kernel:
    def __init__(self, array):
        self._array = np.asanyarray(array)

    def __array__(self):
        return self._array

    @property
    def shape(self):
        return self._array.shape


def test_plot_kernel():
    # just a smoketest that fail
    kernel = Kernel([1, 2, 3, 4, 5])
    plt.plot(kernel)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_units.py
NikolaBlagojevic/pyrecodes
from tempfile import TemporaryFile

import numpy as np
import pytest

import matplotlib as mpl
from matplotlib import dviread
from matplotlib.testing import _has_tex_package
from matplotlib.testing.decorators import check_figures_equal, image_comparison
from matplotlib.testing._markers import needs_usetex
import matplotlib.pyplot as plt


pytestmark = needs_usetex


@image_comparison(
    baseline_images=['test_usetex'],
    extensions=['pdf', 'png'],
    style="mpl20")
def test_usetex():
    mpl.rcParams['text.usetex'] = True
    fig, ax = plt.subplots()
    kwargs = {"verticalalignment": "baseline", "size": 24,
              "bbox": dict(pad=0, edgecolor="k", facecolor="none")}
    ax.text(0.2, 0.7,
            # the \LaTeX macro exercises character sizing and placement,
            # \left[ ... \right\} draw some variable-height characters,
            # \sqrt and \frac draw horizontal rules, \mathrm changes the font
            r'\LaTeX\ $\left[\int\limits_e^{2e}'
            r'\sqrt\frac{\log^3 x}{x}\,\mathrm{d}x \right\}$',
            **kwargs)
    ax.text(0.2, 0.3, "lg", **kwargs)
    ax.text(0.4, 0.3, r"$\frac{1}{2}\pi$", **kwargs)
    ax.text(0.6, 0.3, "$p^{3^A}$", **kwargs)
    ax.text(0.8, 0.3, "$p_{3_2}$", **kwargs)
    for x in {t.get_position()[0] for t in ax.texts}:
        ax.axvline(x)
    for y in {t.get_position()[1] for t in ax.texts}:
        ax.axhline(y)
    ax.set_axis_off()


@check_figures_equal()
def test_empty(fig_test, fig_ref):
    mpl.rcParams['text.usetex'] = True
    fig_test.text(.5, .5, "% a comment")


@check_figures_equal()
def test_unicode_minus(fig_test, fig_ref):
    mpl.rcParams['text.usetex'] = True
    fig_test.text(.5, .5, "$-$")
    fig_ref.text(.5, .5, "\N{MINUS SIGN}")


def test_mathdefault():
    plt.rcParams["axes.formatter.use_mathtext"] = True
    fig = plt.figure()
    fig.add_subplot().set_xlim(-1, 1)
    # Check that \mathdefault commands generated by tickers don't cause
    # problems when later switching usetex on.
    mpl.rcParams['text.usetex'] = True
    fig.canvas.draw()


@image_comparison(['eqnarray.png'])
def test_multiline_eqnarray():
    text = (
        r'\begin{eqnarray*}'
        r'foo\\'
        r'bar\\'
        r'baz\\'
        r'\end{eqnarray*}'
    )

    fig = plt.figure(figsize=(1, 1))
    fig.text(0.5, 0.5, text, usetex=True,
             horizontalalignment='center', verticalalignment='center')


@pytest.mark.parametrize("fontsize", [8, 10, 12])
def test_minus_no_descent(fontsize):
    # Test special-casing of minus descent in DviFont._height_depth_of, by
    # checking that overdrawing a 1 and a -1 results in an overall height
    # equivalent to drawing either of them separately.
    mpl.style.use("mpl20")
    mpl.rcParams['font.size'] = fontsize
    heights = {}
    fig = plt.figure()
    for vals in [(1,), (-1,), (-1, 1)]:
        fig.clear()
        for x in vals:
            fig.text(.5, .5, f"${x}$", usetex=True)
        fig.canvas.draw()
        # The following counts the number of non-fully-blank pixel rows.
        heights[vals] = ((np.array(fig.canvas.buffer_rgba())[..., 0] != 255)
                         .any(axis=1).sum())
    assert len({*heights.values()}) == 1


@pytest.mark.parametrize('pkg', ['xcolor', 'chemformula'])
def test_usetex_packages(pkg):
    if not _has_tex_package(pkg):
        pytest.skip(f'{pkg} is not available')
    mpl.rcParams['text.usetex'] = True

    fig = plt.figure()
    text = fig.text(0.5, 0.5, "Some text 0123456789")
    fig.canvas.draw()

    mpl.rcParams['text.latex.preamble'] = (
        r'\PassOptionsToPackage{dvipsnames}{xcolor}\usepackage{%s}' % pkg)
    fig = plt.figure()
    text2 = fig.text(0.5, 0.5, "Some text 0123456789")
    fig.canvas.draw()
    np.testing.assert_array_equal(text2.get_window_extent(),
                                  text.get_window_extent())


@pytest.mark.parametrize(
    "preamble",
    [r"\usepackage[full]{textcomp}", r"\usepackage{underscore}"],
)
def test_latex_pkg_already_loaded(preamble):
    plt.rcParams["text.latex.preamble"] = preamble
    fig = plt.figure()
    fig.text(.5, .5, "hello, world", usetex=True)
    fig.canvas.draw()


def test_usetex_with_underscore():
    plt.rcParams["text.usetex"] = True
    df = {"a_b": range(5)[::-1], "c": range(5)}
    fig, ax = plt.subplots()
    ax.plot("c", "a_b", data=df)
    ax.legend()
    ax.text(0, 0, "foo_bar", usetex=True)
    plt.draw()


@pytest.mark.flaky(reruns=3)  # Tends to hit a TeX cache lock on AppVeyor.
@pytest.mark.parametrize("fmt", ["pdf", "svg"])
def test_missing_psfont(fmt, monkeypatch):
    """An error is raised if a TeX font lacks a Type-1 equivalent"""
    monkeypatch.setattr(
        dviread.PsfontsMap, '__getitem__',
        lambda self, k: dviread.PsFont(
            texname=b'texfont', psname=b'Some Font',
            effects=None, encoding=None, filename=None))
    mpl.rcParams['text.usetex'] = True
    fig, ax = plt.subplots()
    ax.text(0.5, 0.5, 'hello')
    with TemporaryFile() as tmpfile, pytest.raises(ValueError):
        fig.savefig(tmpfile, format=fmt)

env_pyrecodes/lib/python3.9/site-packages/matplotlib/tests/test_usetex.py
NikolaBlagojevic/pyrecodes
from matplotlib.testing.conftest import (mpl_test_settings,  # noqa
                                         pytest_configure, pytest_unconfigure)

env_pyrecodes/lib/python3.9/site-packages/mpl_toolkits/axes_grid1/tests/conftest.py
NikolaBlagojevic/pyrecodes
from matplotlib.testing.conftest import (mpl_test_settings,  # noqa
                                         pytest_configure, pytest_unconfigure)

env_pyrecodes/lib/python3.9/site-packages/mpl_toolkits/axisartist/tests/conftest.py
NikolaBlagojevic/pyrecodes
import matplotlib.pyplot as plt
from matplotlib.testing.decorators import image_comparison

from mpl_toolkits.axisartist import AxisArtistHelperRectlinear
from mpl_toolkits.axisartist.axis_artist import (AxisArtist, AxisLabel,
                                                 LabelBase, Ticks, TickLabels)


@image_comparison(['axis_artist_ticks.png'], style='default')
def test_ticks():
    fig, ax = plt.subplots()

    ax.xaxis.set_visible(False)
    ax.yaxis.set_visible(False)

    locs_angles = [((i / 10, 0.0), i * 30) for i in range(-1, 12)]

    ticks_in = Ticks(ticksize=10, axis=ax.xaxis)
    ticks_in.set_locs_angles(locs_angles)
    ax.add_artist(ticks_in)

    ticks_out = Ticks(ticksize=10, tick_out=True, color='C3', axis=ax.xaxis)
    ticks_out.set_locs_angles(locs_angles)
    ax.add_artist(ticks_out)


@image_comparison(['axis_artist_labelbase.png'], style='default')
def test_labelbase():
    # Remove this line when this test image is regenerated.
    plt.rcParams['text.kerning_factor'] = 6

    fig, ax = plt.subplots()

    ax.plot([0.5], [0.5], "o")

    label = LabelBase(0.5, 0.5, "Test")
    label._ref_angle = -90
    label._offset_radius = 50
    label.set_rotation(-90)
    label.set(ha="center", va="top")
    ax.add_artist(label)


@image_comparison(['axis_artist_ticklabels.png'], style='default')
def test_ticklabels():
    # Remove this line when this test image is regenerated.
    plt.rcParams['text.kerning_factor'] = 6

    fig, ax = plt.subplots()

    ax.xaxis.set_visible(False)
    ax.yaxis.set_visible(False)

    ax.plot([0.2, 0.4], [0.5, 0.5], "o")

    ticks = Ticks(ticksize=10, axis=ax.xaxis)
    ax.add_artist(ticks)
    locs_angles_labels = [((0.2, 0.5), -90, "0.2"),
                          ((0.4, 0.5), -120, "0.4")]
    tick_locs_angles = [(xy, a + 180) for xy, a, l in locs_angles_labels]
    ticks.set_locs_angles(tick_locs_angles)

    ticklabels = TickLabels(axis_direction="left")
    ticklabels._locs_angles_labels = locs_angles_labels
    ticklabels.set_pad(10)
    ax.add_artist(ticklabels)

    ax.plot([0.5], [0.5], "s")
    axislabel = AxisLabel(0.5, 0.5, "Test")
    axislabel._offset_radius = 20
    axislabel._ref_angle = 0
    axislabel.set_axis_direction("bottom")
    ax.add_artist(axislabel)

    ax.set_xlim(0, 1)
    ax.set_ylim(0, 1)


@image_comparison(['axis_artist.png'], style='default')
def test_axis_artist():
    # Remove this line when this test image is regenerated.
    plt.rcParams['text.kerning_factor'] = 6

    fig, ax = plt.subplots()

    ax.xaxis.set_visible(False)
    ax.yaxis.set_visible(False)

    for loc in ('left', 'right', 'bottom'):
        _helper = AxisArtistHelperRectlinear.Fixed(ax, loc=loc)
        axisline = AxisArtist(ax, _helper, offset=None, axis_direction=loc)
        ax.add_artist(axisline)

    # Settings for bottom AxisArtist.
    axisline.set_label("TTT")
    axisline.major_ticks.set_tick_out(False)
    axisline.label.set_pad(5)

    ax.set_ylabel("Test")

env_pyrecodes/lib/python3.9/site-packages/mpl_toolkits/axisartist/tests/test_axis_artist.py
NikolaBlagojevic/pyrecodes
import numpy as np

import matplotlib.pyplot as plt
from matplotlib.path import Path
from matplotlib.projections import PolarAxes
from matplotlib.transforms import Affine2D, Transform
from matplotlib.testing.decorators import image_comparison

from mpl_toolkits.axisartist import SubplotHost
from mpl_toolkits.axes_grid1.parasite_axes import host_axes_class_factory
from mpl_toolkits.axisartist import angle_helper
from mpl_toolkits.axisartist.axislines import Axes
from mpl_toolkits.axisartist.grid_helper_curvelinear import \
    GridHelperCurveLinear


@image_comparison(['custom_transform.png'], style='default', tol=0.2)
def test_custom_transform():
    class MyTransform(Transform):
        input_dims = output_dims = 2

        def __init__(self, resolution):
            """
            Resolution is the number of steps to interpolate between each input
            line segment to approximate its path in transformed space.
            """
            Transform.__init__(self)
            self._resolution = resolution

        def transform(self, ll):
            x, y = ll.T
            return np.column_stack([x, y - x])

        transform_non_affine = transform

        def transform_path(self, path):
            ipath = path.interpolated(self._resolution)
            return Path(self.transform(ipath.vertices), ipath.codes)

        transform_path_non_affine = transform_path

        def inverted(self):
            return MyTransformInv(self._resolution)

    class MyTransformInv(Transform):
        input_dims = output_dims = 2

        def __init__(self, resolution):
            Transform.__init__(self)
            self._resolution = resolution

        def transform(self, ll):
            x, y = ll.T
            return np.column_stack([x, y + x])

        def inverted(self):
            return MyTransform(self._resolution)

    fig = plt.figure()

    SubplotHost = host_axes_class_factory(Axes)

    tr = MyTransform(1)
    grid_helper = GridHelperCurveLinear(tr)
    ax1 = SubplotHost(fig, 1, 1, 1, grid_helper=grid_helper)
    fig.add_subplot(ax1)

    ax2 = ax1.get_aux_axes(tr, viewlim_mode="equal")
    ax2.plot([3, 6], [5.0, 10.])

    ax1.set_aspect(1.)
    ax1.set_xlim(0, 10)
    ax1.set_ylim(0, 10)

    ax1.grid(True)


# Remove tol & kerning_factor when this test image is regenerated.
@image_comparison(['polar_box.png'], style='default', tol=0.27)
def test_polar_box():
    plt.rcParams['text.kerning_factor'] = 6

    fig = plt.figure(figsize=(5, 5))

    # PolarAxes.PolarTransform takes radian. However, we want our coordinate
    # system in degree
    tr = Affine2D().scale(np.pi / 180., 1.) + PolarAxes.PolarTransform()

    # polar projection, which involves cycle, and also has limits in
    # its coordinates, needs a special method to find the extremes
    # (min, max of the coordinate within the view).
    extreme_finder = angle_helper.ExtremeFinderCycle(20, 20,
                                                     lon_cycle=360,
                                                     lat_cycle=None,
                                                     lon_minmax=None,
                                                     lat_minmax=(0, np.inf))

    grid_locator1 = angle_helper.LocatorDMS(12)
    tick_formatter1 = angle_helper.FormatterDMS()

    grid_helper = GridHelperCurveLinear(tr,
                                        extreme_finder=extreme_finder,
                                        grid_locator1=grid_locator1,
                                        tick_formatter1=tick_formatter1)

    ax1 = SubplotHost(fig, 1, 1, 1, grid_helper=grid_helper)

    ax1.axis["right"].major_ticklabels.set_visible(True)
    ax1.axis["top"].major_ticklabels.set_visible(True)

    # let right axis shows ticklabels for 1st coordinate (angle)
    ax1.axis["right"].get_helper().nth_coord_ticks = 0
    # let bottom axis shows ticklabels for 2nd coordinate (radius)
    ax1.axis["bottom"].get_helper().nth_coord_ticks = 1

    fig.add_subplot(ax1)

    ax1.axis["lat"] = axis = grid_helper.new_floating_axis(0, 45, axes=ax1)
    axis.label.set_text("Test")
    axis.label.set_visible(True)
    axis.get_helper().set_extremes(2, 12)

    ax1.axis["lon"] = axis = grid_helper.new_floating_axis(1, 6, axes=ax1)
    axis.label.set_text("Test 2")
    axis.get_helper().set_extremes(-180, 90)

    # A parasite axes with given transform
    ax2 = ax1.get_aux_axes(tr, viewlim_mode="equal")
    assert ax2.transData == tr + ax1.transData
    # Anything you draw in ax2 will match the ticks and grids of ax1.
    ax2.plot(np.linspace(0, 30, 50), np.linspace(10, 10, 50))

    ax1.set_aspect(1.)
    ax1.set_xlim(-5, 12)
    ax1.set_ylim(-5, 10)

    ax1.grid(True)


# Remove tol & kerning_factor when this test image is regenerated.
@image_comparison(['axis_direction.png'], style='default', tol=0.12)
def test_axis_direction():
    plt.rcParams['text.kerning_factor'] = 6

    fig = plt.figure(figsize=(5, 5))

    # PolarAxes.PolarTransform takes radian. However, we want our coordinate
    # system in degree
    tr = Affine2D().scale(np.pi / 180., 1.) + PolarAxes.PolarTransform()

    # polar projection, which involves cycle, and also has limits in
    # its coordinates, needs a special method to find the extremes
    # (min, max of the coordinate within the view).

    # 20, 20 : number of sampling points along x, y direction
    extreme_finder = angle_helper.ExtremeFinderCycle(20, 20,
                                                     lon_cycle=360,
                                                     lat_cycle=None,
                                                     lon_minmax=None,
                                                     lat_minmax=(0, np.inf),
                                                     )

    grid_locator1 = angle_helper.LocatorDMS(12)
    tick_formatter1 = angle_helper.FormatterDMS()

    grid_helper = GridHelperCurveLinear(tr,
                                        extreme_finder=extreme_finder,
                                        grid_locator1=grid_locator1,
                                        tick_formatter1=tick_formatter1)

    ax1 = SubplotHost(fig, 1, 1, 1, grid_helper=grid_helper)

    for axis in ax1.axis.values():
        axis.set_visible(False)

    fig.add_subplot(ax1)

    ax1.axis["lat1"] = axis = grid_helper.new_floating_axis(
        0, 130,
        axes=ax1, axis_direction="left")
    axis.label.set_text("Test")
    axis.label.set_visible(True)
    axis.get_helper().set_extremes(0.001, 10)

    ax1.axis["lat2"] = axis = grid_helper.new_floating_axis(
        0, 50,
        axes=ax1, axis_direction="right")
    axis.label.set_text("Test")
    axis.label.set_visible(True)
    axis.get_helper().set_extremes(0.001, 10)

    ax1.axis["lon"] = axis = grid_helper.new_floating_axis(
        1, 10,
        axes=ax1, axis_direction="bottom")
    axis.label.set_text("Test 2")
    axis.get_helper().set_extremes(50, 130)
    axis.major_ticklabels.set_axis_direction("top")
    axis.label.set_axis_direction("top")

    grid_helper.grid_finder.grid_locator1.set_params(nbins=5)
    grid_helper.grid_finder.grid_locator2.set_params(nbins=5)

    ax1.set_aspect(1.)
    ax1.set_xlim(-8, 8)
    ax1.set_ylim(-4, 12)

    ax1.grid(True)

env_pyrecodes/lib/python3.9/site-packages/mpl_toolkits/axisartist/tests/test_grid_helper_curvelinear.py
NikolaBlagojevic/pyrecodes
from matplotlib.testing.conftest import (mpl_test_settings,  # noqa
                                         pytest_configure, pytest_unconfigure)

env_pyrecodes/lib/python3.9/site-packages/mpl_toolkits/mplot3d/tests/conftest.py
NikolaBlagojevic/pyrecodes
import matplotlib.pyplot as plt

from matplotlib.backend_bases import MouseEvent


def test_scatter_3d_projection_conservation():
    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')
    # fix axes3d projection
    ax.roll = 0
    ax.elev = 0
    ax.azim = -45
    ax.stale = True

    x = [0, 1, 2, 3, 4]
    scatter_collection = ax.scatter(x, x, x)
    fig.canvas.draw_idle()

    # Get scatter location on canvas and freeze the data
    scatter_offset = scatter_collection.get_offsets()
    scatter_location = ax.transData.transform(scatter_offset)

    # Yaw -44 and -46 are enough to produce two set of scatter
    # with opposite z-order without moving points too far
    for azim in (-44, -46):
        ax.azim = azim
        ax.stale = True
        fig.canvas.draw_idle()

        for i in range(5):
            # Create a mouse event used to locate and to get index
            # from each dots
            event = MouseEvent("button_press_event", fig.canvas,
                               *scatter_location[i, :])
            contains, ind = scatter_collection.contains(event)
            assert contains is True
            assert len(ind["ind"]) == 1
            assert ind["ind"][0] == i

env_pyrecodes/lib/python3.9/site-packages/mpl_toolkits/mplot3d/tests/test_art3d.py
NikolaBlagojevic/pyrecodes
"""Notebook format validators."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.


import json
import os
import pprint
import warnings
from copy import deepcopy
from textwrap import dedent
from typing import Any, Optional, Tuple

from ._imports import import_item
from .corpus.words import generate_corpus_id
from .json_compat import ValidationError, _validator_for_name, get_current_validator
from .reader import get_version
from .warnings import DuplicateCellId, MissingIDFieldWarning

validators = {}
_deprecated = object()


def _relax_additional_properties(obj):
    """relax any `additionalProperties`"""
    if isinstance(obj, dict):
        for key, value in obj.items():
            value = (  # noqa
                True if key == "additionalProperties" else _relax_additional_properties(value)
            )
            obj[key] = value
    elif isinstance(obj, list):
        for i, value in enumerate(obj):
            obj[i] = _relax_additional_properties(value)
    return obj


def _allow_undefined(schema):
    schema["definitions"]["cell"]["oneOf"].append({"$ref": "#/definitions/unrecognized_cell"})
    schema["definitions"]["output"]["oneOf"].append({"$ref": "#/definitions/unrecognized_output"})
    return schema


def get_validator(version=None, version_minor=None, relax_add_props=False, name=None):
    """Load the JSON schema into a Validator"""
    if version is None:
        from . import current_nbformat

        version = current_nbformat

    v = import_item("nbformat.v%s" % version)
    current_minor = getattr(v, "nbformat_minor", 0)
    if version_minor is None:
        version_minor = current_minor

    current_validator = _validator_for_name(name) if name else get_current_validator()

    version_tuple = (current_validator.name, version, version_minor)

    if version_tuple not in validators:
        try:
            schema_json = _get_schema_json(v, version=version, version_minor=version_minor)
        except AttributeError:
            return None

        if current_minor < version_minor:
            # notebook from the future, relax all `additionalProperties: False` requirements
            schema_json = _relax_additional_properties(schema_json)
            # and allow undefined cell types and outputs
            schema_json = _allow_undefined(schema_json)

        validators[version_tuple] = current_validator(schema_json)

    if relax_add_props:
        try:
            schema_json = _get_schema_json(v, version=version, version_minor=version_minor)
        except AttributeError:
            return None

        # this allows properties to be added for intermediate
        # representations while validating for all other kinds of errors
        schema_json = _relax_additional_properties(schema_json)
        validators[version_tuple] = current_validator(schema_json)

    return validators[version_tuple]


def _get_schema_json(v, version=None, version_minor=None):
    """
    Gets the json schema from a given imported library and nbformat version.
    """
    if (version, version_minor) in v.nbformat_schema:
        schema_path = os.path.join(
            os.path.dirname(v.__file__), v.nbformat_schema[(version, version_minor)]
        )
    elif version_minor > v.nbformat_minor:
        # load the latest schema
        schema_path = os.path.join(os.path.dirname(v.__file__), v.nbformat_schema[(None, None)])
    else:
        msg = "Cannot find appropriate nbformat schema file."
        raise AttributeError(msg)
    with open(schema_path, encoding='utf-8') as f:
        schema_json = json.load(f)
    return schema_json


def isvalid(nbjson, ref=None, version=None, version_minor=None):
    """Checks whether the given notebook JSON conforms to the current
    notebook format schema. Returns True if the JSON is valid, and
    False otherwise.

    To see the individual errors that were encountered, please use the
    `validate` function instead.
    """
    orig = deepcopy(nbjson)
    try:
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", category=DeprecationWarning)
            warnings.filterwarnings("ignore", category=MissingIDFieldWarning)
            validate(nbjson, ref, version, version_minor, repair_duplicate_cell_ids=False)
    except ValidationError:
        return False
    else:
        return True
    finally:
        if nbjson != orig:
            raise AssertionError


def _format_as_index(indices):
    """
    (from jsonschema._utils.format_as_index, copied to avoid relying on private API)

    Construct a single string containing indexing operations for the indices.

    For example, [1, 2, "foo"] -> [1][2]["foo"]
    """

    if not indices:
        return ""
    return "[%s]" % "][".join(repr(index) for index in indices)


_ITEM_LIMIT = 16
_STR_LIMIT = 64


def _truncate_obj(obj):
    """Truncate objects for use in validation tracebacks

    Cell and output lists are squashed, as are long strings, lists, and dicts.
    """
    if isinstance(obj, dict):
        truncated_dict = {k: _truncate_obj(v) for k, v in list(obj.items())[:_ITEM_LIMIT]}
        if isinstance(truncated_dict.get("cells"), list):
            truncated_dict["cells"] = ["...%i cells..." % len(obj["cells"])]
        if isinstance(truncated_dict.get("outputs"), list):
            truncated_dict["outputs"] = ["...%i outputs..." % len(obj["outputs"])]

        if len(obj) > _ITEM_LIMIT:
            truncated_dict["..."] = "%i keys truncated" % (len(obj) - _ITEM_LIMIT)
        return truncated_dict
    elif isinstance(obj, list):
        truncated_list = [_truncate_obj(item) for item in obj[:_ITEM_LIMIT]]
        if len(obj) > _ITEM_LIMIT:
            truncated_list.append("...%i items truncated..." % (len(obj) - _ITEM_LIMIT))
        return truncated_list
    elif isinstance(obj, str):
        truncated_str = obj[:_STR_LIMIT]
        if len(obj) > _STR_LIMIT:
            truncated_str += "..."
        return truncated_str
    else:
        return obj


class NotebookValidationError(ValidationError):
    """Schema ValidationError with truncated representation

    to avoid massive verbose tracebacks.
    """

    def __init__(self, original, ref=None):
        """Initialize the error class."""
        self.original = original
        self.ref = getattr(self.original, "ref", ref)
        self.message = self.original.message

    def __getattr__(self, key):
        """Get an attribute from the error."""
        return getattr(self.original, key)

    def __unicode__(self):
        """Custom str for validation errors

        avoids dumping full schema and notebook to logs
        """
        error = self.original
        instance = _truncate_obj(error.instance)

        return "\n".join(
            [
                error.message,
                "",
                "Failed validating {!r} in {}{}:".format(
                    error.validator,
                    self.ref or "notebook",
                    _format_as_index(list(error.relative_schema_path)[:-1]),
                ),
                "",
                "On instance%s:" % _format_as_index(error.relative_path),
                pprint.pformat(instance, width=78),
            ]
        )

    __str__ = __unicode__


def better_validation_error(error, version, version_minor):
    """Get better ValidationError on oneOf failures

    oneOf errors aren't informative.
    if it's a cell type or output_type error,
    try validating directly based on the type for a better error message
    """
    if not len(error.schema_path):
        return error
    key = error.schema_path[-1]
    ref = None
    if key.endswith("Of"):
        if isinstance(error.instance, dict):
            if "cell_type" in error.instance:
                ref = error.instance["cell_type"] + "_cell"
            elif "output_type" in error.instance:
                ref = error.instance["output_type"]

        if ref:
            try:
                validate(
                    error.instance,
                    ref,
                    version=version,
                    version_minor=version_minor,
                )
            except ValidationError as sub_error:
                # keep extending relative path
                error.relative_path.extend(sub_error.relative_path)
                sub_error.relative_path = error.relative_path
                better = better_validation_error(sub_error, version, version_minor)
                if better.ref is None:
                    better.ref = ref
                return better
            except Exception:  # noqa
                # if it fails for some reason,
                # let the original error through
                pass
    return NotebookValidationError(error, ref)


def normalize(
    nbdict: Any,
    version: Optional[int] = None,
    version_minor: Optional[int] = None,
    *,
    relax_add_props: bool = False,
    strip_invalid_metadata: bool = False,
) -> Tuple[int, Any]:
    """
    Normalise a notebook prior to validation.

    This tries to implement a couple of normalisation steps to standardise
    notebooks and make validation easier.

    You should in general not rely on this function and make sure the notebooks
    that reach nbformat are already in a normal form. If not you likely have a bug,
    and may have security issues.

    Parameters
    ----------
    nbdict : dict
        notebook document
    version : int
    version_minor : int
    relax_add_props : bool
        Whether to allow extra property in the Json schema validating the
        notebook.
    strip_invalid_metadata : bool
        Whether to strip metadata that does not exist in the Json schema when
        validating the notebook.

    Returns
    -------
    changes : int
        number of changes in the notebooks
    notebook : dict
        deep-copy of the original object with relevant changes.

    """
    nbdict = deepcopy(nbdict)
    nbdict_version, nbdict_version_minor = get_version(nbdict)
    if version is None:
        version = nbdict_version
    if version_minor is None:
        version_minor = nbdict_version_minor
    return _normalize(
        nbdict,
        version,
        version_minor,
        True,
        relax_add_props=relax_add_props,
        strip_invalid_metadata=strip_invalid_metadata,
    )


def _normalize(
    nbdict: Any,
    version: int,
    version_minor: int,
    repair_duplicate_cell_ids: bool,
    relax_add_props: bool,
    strip_invalid_metadata: bool,
) -> Tuple[int, Any]:
    """
    Private normalisation routine.

    This function attempts to normalize the `nbdict` passed to it.

    As `_normalize()` is currently used both in `validate()` (for
    historical reasons), and in the `normalize()` public function,
    `_normalize()` does currently mutate `nbdict`.
    Ideally, once `validate()` stops calling `_normalize()`, `_normalize()`
    may stop mutating `nbdict`.

    """
    changes = 0

    if (version, version_minor) >= (4, 5):
        # if we support cell ids ensure default ids are provided
        for cell in nbdict["cells"]:
            if "id" not in cell:
                warnings.warn(
                    "Code cell is missing an id field, this will become"
                    " a hard error in future nbformat versions. You may want"
                    " to use `normalize()` on your notebooks before validations"
                    " (available since nbformat 5.1.4). Previous versions of nbformat"
                    " are fixing this issue transparently, and will stop doing so"
                    " in the future.",
                    MissingIDFieldWarning,
                    stacklevel=3,
                )
                # Generate cell ids if any are missing
                if repair_duplicate_cell_ids:
                    cell["id"] = generate_corpus_id()
                    changes += 1

        # if we support cell ids check for uniqueness when validating the whole notebook
        seen_ids = set()
        for cell in nbdict["cells"]:
            if "id" not in cell:
                continue
            cell_id = cell["id"]
            if cell_id in seen_ids:
                # Best effort to repair if we find a duplicate id
                if repair_duplicate_cell_ids:
                    new_id = generate_corpus_id()
                    cell["id"] = new_id
                    changes += 1
                    warnings.warn(
                        f"Non-unique cell id {cell_id!r} detected. Corrected to {new_id!r}.",
                        DuplicateCellId,
                        stacklevel=3,
                    )
                else:
                    msg = f"Non-unique cell id '{cell_id}' detected."
                    raise ValidationError(msg)
            seen_ids.add(cell_id)
    if strip_invalid_metadata:
        changes += _strip_invalida_metadata(
            nbdict, version, version_minor, relax_add_props=relax_add_props
        )
    return changes, nbdict


def _dep_warn(field):
    warnings.warn(
        dedent(
            f"""`{field}` kwargs of validate has been deprecated for security
        reasons, and will be removed soon.

        Please explicitly use the `n_changes, new_notebook = nbformat.validator.normalize(old_notebook, ...)` if you wish to
        normalise your notebook. `normalize` is available since nbformat 5.5.0

        """
        ),
        DeprecationWarning,
        stacklevel=3,
    )


def validate(  # noqa
    nbdict: Any = None,
    ref: Optional[str] = None,
    version: Optional[int] = None,
    version_minor: Optional[int] = None,
    relax_add_props: bool = False,
    nbjson: Any = None,
    repair_duplicate_cell_ids: bool = _deprecated,  # type: ignore
    strip_invalid_metadata: bool = _deprecated,  # type: ignore
) -> None:
    """Checks whether the given notebook dict-like object
    conforms to the relevant notebook format schema.

    Parameters
    ----------
    nbdict : dict
        notebook document
    ref : optional, str
        reference to the subset of the schema we want to validate against.
        for example ``"markdown_cell"``, `"code_cell"` ....
    version : int
    version_minor : int
    relax_add_props : bool
        Wether to allow extra properties in the JSON schema validating the notebook.
        When True, all known fields are validated, but unknown fields are ignored.
    nbjson
    repair_duplicate_cell_ids : bool
        Deprecated since 5.5.0 - will be removed in the future.
    strip_invalid_metadata : bool
        Deprecated since 5.5.0 - will be removed in the future.

    Returns
    -------
    None

    Raises
    ------
    ValidationError if not valid.

    Notes
    -----
    Prior to Nbformat 5.5.0 the `validate` and `isvalid` method would silently
    try to fix invalid notebook and mutate arguments. This behavior is deprecated
    and will be removed in a near future.

    Please explicitly call `normalize` if you need to normalize notebooks.
    """
    assert isinstance(ref, str) or ref is None  # noqa

    if strip_invalid_metadata is _deprecated:
        strip_invalid_metadata = False
    else:
        _dep_warn("strip_invalid_metadata")
        pass

    if repair_duplicate_cell_ids is _deprecated:
        repair_duplicate_cell_ids = True
    else:
        _dep_warn("repair_duplicate_cell_ids")
        pass

    # backwards compatibility for nbjson argument
    if nbdict is not None:
        pass
    elif nbjson is not None:
        nbdict = nbjson
    else:
        msg = "validate() missing 1 required argument: 'nbdict'"
        raise TypeError(msg)

    if ref is None:
        # if ref is not specified, we have a whole notebook, so we can get the version
        nbdict_version, nbdict_version_minor = get_version(nbdict)
        if version is None:
            version = nbdict_version
        if version_minor is None:
            version_minor = nbdict_version_minor
    else:
        # if ref is specified, and we don't have a version number, assume we're validating against 1.0
        if version is None:  # noqa
            version, version_minor = 1, 0

    if ref is None:
        assert isinstance(version, int)  # noqa
        assert isinstance(version_minor, int)  # noqa
        _normalize(
            nbdict,
            version,
            version_minor,
            repair_duplicate_cell_ids,
            relax_add_props=relax_add_props,
            strip_invalid_metadata=strip_invalid_metadata,
        )

    for error in iter_validate(
        nbdict,
        ref=ref,
        version=version,
        version_minor=version_minor,
        relax_add_props=relax_add_props,
        strip_invalid_metadata=strip_invalid_metadata,
    ):
        raise error


def _get_errors(
    nbdict: Any, version: int, version_minor: int, relax_add_props: bool, *args: Any
) -> Any:
    validator = get_validator(version, version_minor, relax_add_props=relax_add_props)
    if not validator:
        msg = f"No schema for validating v{version}.{version_minor} notebooks"
        raise ValidationError(msg)
    iter_errors = validator.iter_errors(nbdict, *args)
    errors = list(iter_errors)
    # jsonschema gives the best error messages.
    if len(errors) and validator.name != "jsonschema":
        validator = get_validator(
            version=version,
            version_minor=version_minor,
            relax_add_props=relax_add_props,
            name="jsonschema",
        )
        return validator.iter_errors(nbdict, *args)
    return iter(errors)


def _strip_invalida_metadata(  # noqa
    nbdict: Any, version: int, version_minor: int, relax_add_props: bool
) -> int:
    """
    This function tries to extract metadata errors from the validator and fix
    them if necessary. This mostly mean stripping unknown keys from metadata
    fields, or removing metadata fields altogether.

    Parameters
    ----------
    nbdict : dict
        notebook document
    version : int
    version_minor : int
    relax_add_props : bool
        Wether to allow extra property in the Json schema validating the
        notebook.

    Returns
    -------
    int
        number of modifications

    """
    errors = _get_errors(nbdict, version, version_minor, relax_add_props)
    changes = 0
    if len(list(errors)) > 0:
        # jsonschema gives a better error tree.
        validator = get_validator(
            version=version,
            version_minor=version_minor,
            relax_add_props=relax_add_props,
            name="jsonschema",
        )
        if not validator:
            msg = f"No jsonschema for validating v{version}.{version_minor} notebooks"
            raise ValidationError(msg)
        errors = validator.iter_errors(nbdict)
        error_tree = validator.error_tree(errors)
        if "metadata" in error_tree:
            for key in error_tree["metadata"]:
                nbdict["metadata"].pop(key, None)
                changes += 1

        if "cells" in error_tree:
            number_of_cells = len(nbdict.get("cells", 0))
            for cell_idx in range(number_of_cells):
                # Cells don't report individual metadata keys as having failed validation
                # Instead it reports that it failed to validate against each cell-type definition.
                # We have to delve into why those definitions failed to uncover which metadata
                # keys are misbehaving.
                if "oneOf" in error_tree["cells"][cell_idx].errors:
                    intended_cell_type = nbdict["cells"][cell_idx]["cell_type"]
                    schemas_by_index = [
                        ref["$ref"]
                        for ref in error_tree["cells"][cell_idx].errors["oneOf"].schema["oneOf"]
                    ]
                    cell_type_definition_name = f"#/definitions/{intended_cell_type}_cell"
                    if cell_type_definition_name in schemas_by_index:
                        schema_index = schemas_by_index.index(cell_type_definition_name)
                        for error in error_tree["cells"][cell_idx].errors["oneOf"].context:
                            rel_path = error.relative_path
                            error_for_intended_schema = error.schema_path[0] == schema_index
                            is_top_level_metadata_key = (
                                len(rel_path) == 2 and rel_path[0] == "metadata"  # noqa
                            )
                            if error_for_intended_schema and is_top_level_metadata_key:
                                nbdict["cells"][cell_idx]["metadata"].pop(rel_path[1], None)
                                changes += 1

    return changes


def iter_validate(
    nbdict=None,
    ref=None,
    version=None,
    version_minor=None,
    relax_add_props=False,
    nbjson=None,
    strip_invalid_metadata=False,
):
    """Checks whether the given notebook dict-like object conforms to the
    relevant notebook format schema.

    Returns a generator of all ValidationErrors if not valid.

    Notes
    -----
    To fix: For security reasons, this function should *never* mutate its `nbdict` argument, and
    should *never* try to validate a mutated or modified version of its notebook.

    """
    # backwards compatibility for nbjson argument
    if nbdict is not None:
        pass
    elif nbjson is not None:
        nbdict = nbjson
    else:
        msg = "iter_validate() missing 1 required argument: 'nbdict'"
        raise TypeError(msg)

    if version is None:
        version, version_minor = get_version(nbdict)

    if ref:
        try:
            errors = _get_errors(
                nbdict,
                version,
                version_minor,
                relax_add_props,
                {"$ref": "#/definitions/%s" % ref},
            )
        except ValidationError as e:
            yield e
            return

    else:
        if strip_invalid_metadata:
            _strip_invalida_metadata(nbdict, version, version_minor, relax_add_props)

        # Validate one more time to ensure that us removing metadata
        # didn't cause another complex validation issue in the schema.
        # Also to ensure that higher-level errors produced by individual metadata validation
        # failures are removed.
        try:
            errors = _get_errors(nbdict, version, version_minor, relax_add_props)
        except ValidationError as e:
            yield e
            return

    for error in errors:
        yield better_validation_error(error, version, version_minor)

env_pyrecodes/lib/python3.9/site-packages/nbformat/validator.py
NikolaBlagojevic/pyrecodes
from .nh3 import *

__doc__ = nh3.__doc__
if hasattr(nh3, "__all__"):
    __all__ = nh3.__all__
env_pyrecodes/lib/python3.9/site-packages/nh3/__init__.py
NikolaBlagojevic/pyrecodes
# This file is generated by numpy's setup.py
# It contains system_info results at the time of building this package.
__all__ = ["get_info","show"]


import os
import sys

extra_dll_dir = os.path.join(os.path.dirname(__file__), '.libs')

if sys.platform == 'win32' and os.path.isdir(extra_dll_dir):
    os.add_dll_directory(extra_dll_dir)

openblas64__info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None)], 'runtime_library_dirs': ['/usr/local/lib']}
blas_ilp64_opt_info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None)], 'runtime_library_dirs': ['/usr/local/lib']}
openblas64__lapack_info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None), ('HAVE_LAPACKE', None)], 'runtime_library_dirs': ['/usr/local/lib']}
lapack_ilp64_opt_info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None), ('HAVE_LAPACKE', None)], 'runtime_library_dirs': ['/usr/local/lib']}

def get_info(name):
    g = globals()
    return g.get(name, g.get(name + "_info", {}))

def show():
    """
    Show libraries in the system on which NumPy was built.

    Print information about various resources (libraries, library
    directories, include directories, etc.) in the system on which
    NumPy was built.

    See Also
    --------
    get_include : Returns the directory containing NumPy C
                  header files.

    Notes
    -----
    1. Classes specifying the information to be printed are defined
       in the `numpy.distutils.system_info` module.

       Information may include:

       * ``language``: language used to write the libraries (mostly
         C or f77)
       * ``libraries``: names of libraries found in the system
       * ``library_dirs``: directories containing the libraries
       * ``include_dirs``: directories containing library header files
       * ``src_dirs``: directories containing library source files
       * ``define_macros``: preprocessor macros used by
         ``distutils.setup``
       * ``baseline``: minimum CPU features required
       * ``found``: dispatched features supported in the system
       * ``not found``: dispatched features that are not supported
         in the system

    2. NumPy BLAS/LAPACK Installation Notes

       Installing a numpy wheel (``pip install numpy`` or force it
       via ``pip install numpy --only-binary :numpy: numpy``) includes
       an OpenBLAS implementation of the BLAS and LAPACK linear algebra
       APIs. In this case, ``library_dirs`` reports the original build
       time configuration as compiled with gcc/gfortran; at run time
       the OpenBLAS library is in
       ``site-packages/numpy.libs/`` (linux), or
       ``site-packages/numpy/.dylibs/`` (macOS), or
       ``site-packages/numpy/.libs/`` (windows).

       Installing numpy from source
       (``pip install numpy --no-binary numpy``) searches for BLAS and
       LAPACK dynamic link libraries at build time as influenced by
       environment variables NPY_BLAS_LIBS, NPY_CBLAS_LIBS, and
       NPY_LAPACK_LIBS; or NPY_BLAS_ORDER and NPY_LAPACK_ORDER;
       or the optional file ``~/.numpy-site.cfg``.
       NumPy remembers those locations and expects to load the same
       libraries at run-time.
       In NumPy 1.21+ on macOS, 'accelerate' (Apple's Accelerate BLAS
       library) is in the default build-time search order after
       'openblas'.

    Examples
    --------
    >>> import numpy as np
    >>> np.show_config()
    blas_opt_info:
        language = c
        define_macros = [('HAVE_CBLAS', None)]
        libraries = ['openblas', 'openblas']
        library_dirs = ['/usr/local/lib']
    """
    from numpy.core._multiarray_umath import (
        __cpu_features__, __cpu_baseline__, __cpu_dispatch__
    )
    for name,info_dict in globals().items():
        if name[0] == "_" or type(info_dict) is not type({}): continue
        print(name + ":")
        if not info_dict:
            print("  NOT AVAILABLE")
        for k,v in info_dict.items():
            v = str(v)
            if k == "sources" and len(v) > 200:
                v = v[:60] + " ...\n... " + v[-60:]
            print("    %s = %s" % (k,v))

    features_found, features_not_found = [], []
    for feature in __cpu_dispatch__:
        if __cpu_features__[feature]:
            features_found.append(feature)
        else:
            features_not_found.append(feature)

    print("Supported SIMD extensions in this NumPy install:")
    print("    baseline = %s" % (','.join(__cpu_baseline__)))
    print("    found = %s" % (','.join(features_found)))
    print("    not found = %s" % (','.join(features_not_found)))


env_pyrecodes/lib/python3.9/site-packages/numpy/__config__.py
NikolaBlagojevic/pyrecodes

# This file was generated by 'versioneer.py' (0.26) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.

import json

version_json = '''
{
 "date": "2023-02-05T11:25:52-0500",
 "dirty": false,
 "error": null,
 "full-revisionid": "85f38ab180ece5290f64e8ddbd9cf06ad8fa4a5e",
 "version": "1.24.2"
}
'''  # END VERSION_JSON


def get_versions():
    return json.loads(version_json)

env_pyrecodes/lib/python3.9/site-packages/numpy/_version.py
NikolaBlagojevic/pyrecodes
import collections.abc
import tempfile
import sys
import warnings
import operator
import io
import itertools
import functools
import ctypes
import os
import gc
import re
import weakref
import pytest
from contextlib import contextmanager

from numpy.compat import pickle

import pathlib
import builtins
from decimal import Decimal
import mmap

import numpy as np
import numpy.core._multiarray_tests as _multiarray_tests
from numpy.core._rational_tests import rational
from numpy.testing import (
    assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal,
    assert_array_equal, assert_raises_regex, assert_array_almost_equal,
    assert_allclose, IS_PYPY, IS_PYSTON, HAS_REFCOUNT, assert_array_less,
    runstring, temppath, suppress_warnings, break_cycles,
    )
from numpy.testing._private.utils import requires_memory, _no_tracing
from numpy.core.tests._locales import CommaDecimalPointLocale
from numpy.lib.recfunctions import repack_fields

# Need to test an object that does not fully implement math interface
from datetime import timedelta, datetime


def _aligned_zeros(shape, dtype=float, order="C", align=None):
    """
    Allocate a new ndarray with aligned memory.

    The ndarray is guaranteed *not* aligned to twice the requested alignment.
    Eg, if align=4, guarantees it is not aligned to 8. If align=None uses
    dtype.alignment."""
    dtype = np.dtype(dtype)
    if dtype == np.dtype(object):
        # Can't do this, fall back to standard allocation (which
        # should always be sufficiently aligned)
        if align is not None:
            raise ValueError("object array alignment not supported")
        return np.zeros(shape, dtype=dtype, order=order)
    if align is None:
        align = dtype.alignment
    if not hasattr(shape, '__len__'):
        shape = (shape,)
    size = functools.reduce(operator.mul, shape) * dtype.itemsize
    buf = np.empty(size + 2*align + 1, np.uint8)

    ptr = buf.__array_interface__['data'][0]
    offset = ptr % align
    if offset != 0:
        offset = align - offset
    if (ptr % (2*align)) == 0:
        offset += align

    # Note: slices producing 0-size arrays do not necessarily change
    # data pointer --- so we use and allocate size+1
    buf = buf[offset:offset+size+1][:-1]
    buf.fill(0)
    data = np.ndarray(shape, dtype, buf, order=order)
    return data


class TestFlags:
    def setup_method(self):
        self.a = np.arange(10)

    def test_writeable(self):
        mydict = locals()
        self.a.flags.writeable = False
        assert_raises(ValueError, runstring, 'self.a[0] = 3', mydict)
        assert_raises(ValueError, runstring, 'self.a[0:1].itemset(3)', mydict)
        self.a.flags.writeable = True
        self.a[0] = 5
        self.a[0] = 0

    def test_writeable_any_base(self):
        # Ensure that any base being writeable is sufficient to change flag;
        # this is especially interesting for arrays from an array interface.
        arr = np.arange(10)

        class subclass(np.ndarray):
            pass

        # Create subclass so base will not be collapsed, this is OK to change
        view1 = arr.view(subclass)
        view2 = view1[...]
        arr.flags.writeable = False
        view2.flags.writeable = False
        view2.flags.writeable = True  # Can be set to True again.

        arr = np.arange(10)

        class frominterface:
            def __init__(self, arr):
                self.arr = arr
                self.__array_interface__ = arr.__array_interface__

        view1 = np.asarray(frominterface)
        view2 = view1[...]
        view2.flags.writeable = False
        view2.flags.writeable = True

        view1.flags.writeable = False
        view2.flags.writeable = False
        with assert_raises(ValueError):
            # Must assume not writeable, since only base is not:
            view2.flags.writeable = True

    def test_writeable_from_readonly(self):
        # gh-9440 - make sure fromstring, from buffer on readonly buffers
        # set writeable False
        data = b'\x00' * 100
        vals = np.frombuffer(data, 'B')
        assert_raises(ValueError, vals.setflags, write=True)
        types = np.dtype( [('vals', 'u1'), ('res3', 'S4')] )
        values = np.core.records.fromstring(data, types)
        vals = values['vals']
        assert_raises(ValueError, vals.setflags, write=True)

    def test_writeable_from_buffer(self):
        data = bytearray(b'\x00' * 100)
        vals = np.frombuffer(data, 'B')
        assert_(vals.flags.writeable)
        vals.setflags(write=False)
        assert_(vals.flags.writeable is False)
        vals.setflags(write=True)
        assert_(vals.flags.writeable)
        types = np.dtype( [('vals', 'u1'), ('res3', 'S4')] )
        values = np.core.records.fromstring(data, types)
        vals = values['vals']
        assert_(vals.flags.writeable)
        vals.setflags(write=False)
        assert_(vals.flags.writeable is False)
        vals.setflags(write=True)
        assert_(vals.flags.writeable)

    @pytest.mark.skipif(IS_PYPY, reason="PyPy always copies")
    def test_writeable_pickle(self):
        import pickle
        # Small arrays will be copied without setting base.
        # See condition for using PyArray_SetBaseObject in
        # array_setstate.
        a = np.arange(1000)
        for v in range(pickle.HIGHEST_PROTOCOL):
            vals = pickle.loads(pickle.dumps(a, v))
            assert_(vals.flags.writeable)
            assert_(isinstance(vals.base, bytes))

    def test_writeable_from_c_data(self):
        # Test that the writeable flag can be changed for an array wrapping
        # low level C-data, but not owning its data.
        # Also see that this is deprecated to change from python.
        from numpy.core._multiarray_tests import get_c_wrapping_array

        arr_writeable = get_c_wrapping_array(True)
        assert not arr_writeable.flags.owndata
        assert arr_writeable.flags.writeable
        view = arr_writeable[...]

        # Toggling the writeable flag works on the view:
        view.flags.writeable = False
        assert not view.flags.writeable
        view.flags.writeable = True
        assert view.flags.writeable
        # Flag can be unset on the arr_writeable:
        arr_writeable.flags.writeable = False

        arr_readonly = get_c_wrapping_array(False)
        assert not arr_readonly.flags.owndata
        assert not arr_readonly.flags.writeable

        for arr in [arr_writeable, arr_readonly]:
            view = arr[...]
            view.flags.writeable = False  # make sure it is readonly
            arr.flags.writeable = False
            assert not arr.flags.writeable

            with assert_raises(ValueError):
                view.flags.writeable = True

            with warnings.catch_warnings():
                warnings.simplefilter("error", DeprecationWarning)
                with assert_raises(DeprecationWarning):
                    arr.flags.writeable = True

            with assert_warns(DeprecationWarning):
                arr.flags.writeable = True

    def test_warnonwrite(self):
        a = np.arange(10)
        a.flags._warn_on_write = True
        with warnings.catch_warnings(record=True) as w:
            warnings.filterwarnings('always')
            a[1] = 10
            a[2] = 10
            # only warn once
            assert_(len(w) == 1)

    @pytest.mark.parametrize(["flag", "flag_value", "writeable"],
            [("writeable", True, True),
             # Delete _warn_on_write after deprecation and simplify
             # the parameterization:
             ("_warn_on_write", True, False),
             ("writeable", False, False)])
    def test_readonly_flag_protocols(self, flag, flag_value, writeable):
        a = np.arange(10)
        setattr(a.flags, flag, flag_value)

        class MyArr():
            __array_struct__ = a.__array_struct__

        assert memoryview(a).readonly is not writeable
        assert a.__array_interface__['data'][1] is not writeable
        assert np.asarray(MyArr()).flags.writeable is writeable

    def test_otherflags(self):
        assert_equal(self.a.flags.carray, True)
        assert_equal(self.a.flags['C'], True)
        assert_equal(self.a.flags.farray, False)
        assert_equal(self.a.flags.behaved, True)
        assert_equal(self.a.flags.fnc, False)
        assert_equal(self.a.flags.forc, True)
        assert_equal(self.a.flags.owndata, True)
        assert_equal(self.a.flags.writeable, True)
        assert_equal(self.a.flags.aligned, True)
        assert_equal(self.a.flags.writebackifcopy, False)
        assert_equal(self.a.flags['X'], False)
        assert_equal(self.a.flags['WRITEBACKIFCOPY'], False)

    def test_string_align(self):
        a = np.zeros(4, dtype=np.dtype('|S4'))
        assert_(a.flags.aligned)
        # not power of two are accessed byte-wise and thus considered aligned
        a = np.zeros(5, dtype=np.dtype('|S4'))
        assert_(a.flags.aligned)

    def test_void_align(self):
        a = np.zeros(4, dtype=np.dtype([("a", "i4"), ("b", "i4")]))
        assert_(a.flags.aligned)


class TestHash:
    # see #3793
    def test_int(self):
        for st, ut, s in [(np.int8, np.uint8, 8),
                          (np.int16, np.uint16, 16),
                          (np.int32, np.uint32, 32),
                          (np.int64, np.uint64, 64)]:
            for i in range(1, s):
                assert_equal(hash(st(-2**i)), hash(-2**i),
                             err_msg="%r: -2**%d" % (st, i))
                assert_equal(hash(st(2**(i - 1))), hash(2**(i - 1)),
                             err_msg="%r: 2**%d" % (st, i - 1))
                assert_equal(hash(st(2**i - 1)), hash(2**i - 1),
                             err_msg="%r: 2**%d - 1" % (st, i))

                i = max(i - 1, 1)
                assert_equal(hash(ut(2**(i - 1))), hash(2**(i - 1)),
                             err_msg="%r: 2**%d" % (ut, i - 1))
                assert_equal(hash(ut(2**i - 1)), hash(2**i - 1),
                             err_msg="%r: 2**%d - 1" % (ut, i))


class TestAttributes:
    def setup_method(self):
        self.one = np.arange(10)
        self.two = np.arange(20).reshape(4, 5)
        self.three = np.arange(60, dtype=np.float64).reshape(2, 5, 6)

    def test_attributes(self):
        assert_equal(self.one.shape, (10,))
        assert_equal(self.two.shape, (4, 5))
        assert_equal(self.three.shape, (2, 5, 6))
        self.three.shape = (10, 3, 2)
        assert_equal(self.three.shape, (10, 3, 2))
        self.three.shape = (2, 5, 6)
        assert_equal(self.one.strides, (self.one.itemsize,))
        num = self.two.itemsize
        assert_equal(self.two.strides, (5*num, num))
        num = self.three.itemsize
        assert_equal(self.three.strides, (30*num, 6*num, num))
        assert_equal(self.one.ndim, 1)
        assert_equal(self.two.ndim, 2)
        assert_equal(self.three.ndim, 3)
        num = self.two.itemsize
        assert_equal(self.two.size, 20)
        assert_equal(self.two.nbytes, 20*num)
        assert_equal(self.two.itemsize, self.two.dtype.itemsize)
        assert_equal(self.two.base, np.arange(20))

    def test_dtypeattr(self):
        assert_equal(self.one.dtype, np.dtype(np.int_))
        assert_equal(self.three.dtype, np.dtype(np.float_))
        assert_equal(self.one.dtype.char, 'l')
        assert_equal(self.three.dtype.char, 'd')
        assert_(self.three.dtype.str[0] in '<>')
        assert_equal(self.one.dtype.str[1], 'i')
        assert_equal(self.three.dtype.str[1], 'f')

    def test_int_subclassing(self):
        # Regression test for https://github.com/numpy/numpy/pull/3526

        numpy_int = np.int_(0)

        # int_ doesn't inherit from Python int, because it's not fixed-width
        assert_(not isinstance(numpy_int, int))

    def test_stridesattr(self):
        x = self.one

        def make_array(size, offset, strides):
            return np.ndarray(size, buffer=x, dtype=int,
                              offset=offset*x.itemsize,
                              strides=strides*x.itemsize)

        assert_equal(make_array(4, 4, -1), np.array([4, 3, 2, 1]))
        assert_raises(ValueError, make_array, 4, 4, -2)
        assert_raises(ValueError, make_array, 4, 2, -1)
        assert_raises(ValueError, make_array, 8, 3, 1)
        assert_equal(make_array(8, 3, 0), np.array([3]*8))
        # Check behavior reported in gh-2503:
        assert_raises(ValueError, make_array, (2, 3), 5, np.array([-2, -3]))
        make_array(0, 0, 10)

    def test_set_stridesattr(self):
        x = self.one

        def make_array(size, offset, strides):
            try:
                r = np.ndarray([size], dtype=int, buffer=x,
                               offset=offset*x.itemsize)
            except Exception as e:
                raise RuntimeError(e)
            r.strides = strides = strides*x.itemsize
            return r

        assert_equal(make_array(4, 4, -1), np.array([4, 3, 2, 1]))
        assert_equal(make_array(7, 3, 1), np.array([3, 4, 5, 6, 7, 8, 9]))
        assert_raises(ValueError, make_array, 4, 4, -2)
        assert_raises(ValueError, make_array, 4, 2, -1)
        assert_raises(RuntimeError, make_array, 8, 3, 1)
        # Check that the true extent of the array is used.
        # Test relies on as_strided base not exposing a buffer.
        x = np.lib.stride_tricks.as_strided(np.arange(1), (10, 10), (0, 0))

        def set_strides(arr, strides):
            arr.strides = strides

        assert_raises(ValueError, set_strides, x, (10*x.itemsize, x.itemsize))

        # Test for offset calculations:
        x = np.lib.stride_tricks.as_strided(np.arange(10, dtype=np.int8)[-1],
                                                    shape=(10,), strides=(-1,))
        assert_raises(ValueError, set_strides, x[::-1], -1)
        a = x[::-1]
        a.strides = 1
        a[::2].strides = 2

        # test 0d
        arr_0d = np.array(0)
        arr_0d.strides = ()
        assert_raises(TypeError, set_strides, arr_0d, None)

    def test_fill(self):
        for t in "?bhilqpBHILQPfdgFDGO":
            x = np.empty((3, 2, 1), t)
            y = np.empty((3, 2, 1), t)
            x.fill(1)
            y[...] = 1
            assert_equal(x, y)

    def test_fill_max_uint64(self):
        x = np.empty((3, 2, 1), dtype=np.uint64)
        y = np.empty((3, 2, 1), dtype=np.uint64)
        value = 2**64 - 1
        y[...] = value
        x.fill(value)
        assert_array_equal(x, y)

    def test_fill_struct_array(self):
        # Filling from a scalar
        x = np.array([(0, 0.0), (1, 1.0)], dtype='i4,f8')
        x.fill(x[0])
        assert_equal(x['f1'][1], x['f1'][0])
        # Filling from a tuple that can be converted
        # to a scalar
        x = np.zeros(2, dtype=[('a', 'f8'), ('b', 'i4')])
        x.fill((3.5, -2))
        assert_array_equal(x['a'], [3.5, 3.5])
        assert_array_equal(x['b'], [-2, -2])

    def test_fill_readonly(self):
        # gh-22922
        a = np.zeros(11)
        a.setflags(write=False)
        with pytest.raises(ValueError, match=".*read-only"):
            a.fill(0)


class TestArrayConstruction:
    def test_array(self):
        d = np.ones(6)
        r = np.array([d, d])
        assert_equal(r, np.ones((2, 6)))

        d = np.ones(6)
        tgt = np.ones((2, 6))
        r = np.array([d, d])
        assert_equal(r, tgt)
        tgt[1] = 2
        r = np.array([d, d + 1])
        assert_equal(r, tgt)

        d = np.ones(6)
        r = np.array([[d, d]])
        assert_equal(r, np.ones((1, 2, 6)))

        d = np.ones(6)
        r = np.array([[d, d], [d, d]])
        assert_equal(r, np.ones((2, 2, 6)))

        d = np.ones((6, 6))
        r = np.array([d, d])
        assert_equal(r, np.ones((2, 6, 6)))

        d = np.ones((6, ))
        r = np.array([[d, d + 1], d + 2], dtype=object)
        assert_equal(len(r), 2)
        assert_equal(r[0], [d, d + 1])
        assert_equal(r[1], d + 2)

        tgt = np.ones((2, 3), dtype=bool)
        tgt[0, 2] = False
        tgt[1, 0:2] = False
        r = np.array([[True, True, False], [False, False, True]])
        assert_equal(r, tgt)
        r = np.array([[True, False], [True, False], [False, True]])
        assert_equal(r, tgt.T)

    def test_array_empty(self):
        assert_raises(TypeError, np.array)

    def test_0d_array_shape(self):
        assert np.ones(np.array(3)).shape == (3,)

    def test_array_copy_false(self):
        d = np.array([1, 2, 3])
        e = np.array(d, copy=False)
        d[1] = 3
        assert_array_equal(e, [1, 3, 3])
        e = np.array(d, copy=False, order='F')
        d[1] = 4
        assert_array_equal(e, [1, 4, 3])
        e[2] = 7
        assert_array_equal(d, [1, 4, 7])

    def test_array_copy_true(self):
        d = np.array([[1,2,3], [1, 2, 3]])
        e = np.array(d, copy=True)
        d[0, 1] = 3
        e[0, 2] = -7
        assert_array_equal(e, [[1, 2, -7], [1, 2, 3]])
        assert_array_equal(d, [[1, 3, 3], [1, 2, 3]])
        e = np.array(d, copy=True, order='F')
        d[0, 1] = 5
        e[0, 2] = 7
        assert_array_equal(e, [[1, 3, 7], [1, 2, 3]])
        assert_array_equal(d, [[1, 5, 3], [1,2,3]])

    def test_array_cont(self):
        d = np.ones(10)[::2]
        assert_(np.ascontiguousarray(d).flags.c_contiguous)
        assert_(np.ascontiguousarray(d).flags.f_contiguous)
        assert_(np.asfortranarray(d).flags.c_contiguous)
        assert_(np.asfortranarray(d).flags.f_contiguous)
        d = np.ones((10, 10))[::2,::2]
        assert_(np.ascontiguousarray(d).flags.c_contiguous)
        assert_(np.asfortranarray(d).flags.f_contiguous)

    @pytest.mark.parametrize("func",
            [np.array,
             np.asarray,
             np.asanyarray,
             np.ascontiguousarray,
             np.asfortranarray])
    def test_bad_arguments_error(self, func):
        with pytest.raises(TypeError):
            func(3, dtype="bad dtype")
        with pytest.raises(TypeError):
            func()  # missing arguments
        with pytest.raises(TypeError):
            func(1, 2, 3, 4, 5, 6, 7, 8)  # too many arguments

    @pytest.mark.parametrize("func",
            [np.array,
             np.asarray,
             np.asanyarray,
             np.ascontiguousarray,
             np.asfortranarray])
    def test_array_as_keyword(self, func):
        # This should likely be made positional only, but do not change
        # the name accidentally.
        if func is np.array:
            func(object=3)
        else:
            func(a=3)


class TestAssignment:
    def test_assignment_broadcasting(self):
        a = np.arange(6).reshape(2, 3)

        # Broadcasting the input to the output
        a[...] = np.arange(3)
        assert_equal(a, [[0, 1, 2], [0, 1, 2]])
        a[...] = np.arange(2).reshape(2, 1)
        assert_equal(a, [[0, 0, 0], [1, 1, 1]])

        # For compatibility with <= 1.5, a limited version of broadcasting
        # the output to the input.
        #
        # This behavior is inconsistent with NumPy broadcasting
        # in general, because it only uses one of the two broadcasting
        # rules (adding a new "1" dimension to the left of the shape),
        # applied to the output instead of an input. In NumPy 2.0, this kind
        # of broadcasting assignment will likely be disallowed.
        a[...] = np.arange(6)[::-1].reshape(1, 2, 3)
        assert_equal(a, [[5, 4, 3], [2, 1, 0]])
        # The other type of broadcasting would require a reduction operation.

        def assign(a, b):
            a[...] = b

        assert_raises(ValueError, assign, a, np.arange(12).reshape(2, 2, 3))

    def test_assignment_errors(self):
        # Address issue #2276
        class C:
            pass
        a = np.zeros(1)

        def assign(v):
            a[0] = v

        assert_raises((AttributeError, TypeError), assign, C())
        assert_raises(ValueError, assign, [1])

    def test_unicode_assignment(self):
        # gh-5049
        from numpy.core.numeric import set_string_function

        @contextmanager
        def inject_str(s):
            """ replace ndarray.__str__ temporarily """
            set_string_function(lambda x: s, repr=False)
            try:
                yield
            finally:
                set_string_function(None, repr=False)

        a1d = np.array(['test'])
        a0d = np.array('done')
        with inject_str('bad'):
            a1d[0] = a0d  # previously this would invoke __str__
        assert_equal(a1d[0], 'done')

        # this would crash for the same reason
        np.array([np.array('\xe5\xe4\xf6')])

    def test_stringlike_empty_list(self):
        # gh-8902
        u = np.array(['done'])
        b = np.array([b'done'])

        class bad_sequence:
            def __getitem__(self): pass
            def __len__(self): raise RuntimeError

        assert_raises(ValueError, operator.setitem, u, 0, [])
        assert_raises(ValueError, operator.setitem, b, 0, [])

        assert_raises(ValueError, operator.setitem, u, 0, bad_sequence())
        assert_raises(ValueError, operator.setitem, b, 0, bad_sequence())

    def test_longdouble_assignment(self):
        # only relevant if longdouble is larger than float
        # we're looking for loss of precision

        for dtype in (np.longdouble, np.longcomplex):
            # gh-8902
            tinyb = np.nextafter(np.longdouble(0), 1).astype(dtype)
            tinya = np.nextafter(np.longdouble(0), -1).astype(dtype)

            # construction
            tiny1d = np.array([tinya])
            assert_equal(tiny1d[0], tinya)

            # scalar = scalar
            tiny1d[0] = tinyb
            assert_equal(tiny1d[0], tinyb)

            # 0d = scalar
            tiny1d[0, ...] = tinya
            assert_equal(tiny1d[0], tinya)

            # 0d = 0d
            tiny1d[0, ...] = tinyb[...]
            assert_equal(tiny1d[0], tinyb)

            # scalar = 0d
            tiny1d[0] = tinyb[...]
            assert_equal(tiny1d[0], tinyb)

            arr = np.array([np.array(tinya)])
            assert_equal(arr[0], tinya)

    def test_cast_to_string(self):
        # cast to str should do "str(scalar)", not "str(scalar.item())"
        # Example: In python2, str(float) is truncated, so we want to avoid
        # str(np.float64(...).item()) as this would incorrectly truncate.
        a = np.zeros(1, dtype='S20')
        a[:] = np.array(['1.12345678901234567890'], dtype='f8')
        assert_equal(a[0], b"1.1234567890123457")


class TestDtypedescr:
    def test_construction(self):
        d1 = np.dtype('i4')
        assert_equal(d1, np.dtype(np.int32))
        d2 = np.dtype('f8')
        assert_equal(d2, np.dtype(np.float64))

    def test_byteorders(self):
        assert_(np.dtype('<i4') != np.dtype('>i4'))
        assert_(np.dtype([('a', '<i4')]) != np.dtype([('a', '>i4')]))

    def test_structured_non_void(self):
        fields = [('a', '<i2'), ('b', '<i2')]
        dt_int = np.dtype(('i4', fields))
        assert_equal(str(dt_int), "(numpy.int32, [('a', '<i2'), ('b', '<i2')])")

        # gh-9821
        arr_int = np.zeros(4, dt_int)
        assert_equal(repr(arr_int),
            "array([0, 0, 0, 0], dtype=(numpy.int32, [('a', '<i2'), ('b', '<i2')]))")


class TestZeroRank:
    def setup_method(self):
        self.d = np.array(0), np.array('x', object)

    def test_ellipsis_subscript(self):
        a, b = self.d
        assert_equal(a[...], 0)
        assert_equal(b[...], 'x')
        assert_(a[...].base is a)  # `a[...] is a` in numpy <1.9.
        assert_(b[...].base is b)  # `b[...] is b` in numpy <1.9.

    def test_empty_subscript(self):
        a, b = self.d
        assert_equal(a[()], 0)
        assert_equal(b[()], 'x')
        assert_(type(a[()]) is a.dtype.type)
        assert_(type(b[()]) is str)

    def test_invalid_subscript(self):
        a, b = self.d
        assert_raises(IndexError, lambda x: x[0], a)
        assert_raises(IndexError, lambda x: x[0], b)
        assert_raises(IndexError, lambda x: x[np.array([], int)], a)
        assert_raises(IndexError, lambda x: x[np.array([], int)], b)

    def test_ellipsis_subscript_assignment(self):
        a, b = self.d
        a[...] = 42
        assert_equal(a, 42)
        b[...] = ''
        assert_equal(b.item(), '')

    def test_empty_subscript_assignment(self):
        a, b = self.d
        a[()] = 42
        assert_equal(a, 42)
        b[()] = ''
        assert_equal(b.item(), '')

    def test_invalid_subscript_assignment(self):
        a, b = self.d

        def assign(x, i, v):
            x[i] = v

        assert_raises(IndexError, assign, a, 0, 42)
        assert_raises(IndexError, assign, b, 0, '')
        assert_raises(ValueError, assign, a, (), '')

    def test_newaxis(self):
        a, b = self.d
        assert_equal(a[np.newaxis].shape, (1,))
        assert_equal(a[..., np.newaxis].shape, (1,))
        assert_equal(a[np.newaxis, ...].shape, (1,))
        assert_equal(a[..., np.newaxis].shape, (1,))
        assert_equal(a[np.newaxis, ..., np.newaxis].shape, (1, 1))
        assert_equal(a[..., np.newaxis, np.newaxis].shape, (1, 1))
        assert_equal(a[np.newaxis, np.newaxis, ...].shape, (1, 1))
        assert_equal(a[(np.newaxis,)*10].shape, (1,)*10)

    def test_invalid_newaxis(self):
        a, b = self.d

        def subscript(x, i):
            x[i]

        assert_raises(IndexError, subscript, a, (np.newaxis, 0))
        assert_raises(IndexError, subscript, a, (np.newaxis,)*50)

    def test_constructor(self):
        x = np.ndarray(())
        x[()] = 5
        assert_equal(x[()], 5)
        y = np.ndarray((), buffer=x)
        y[()] = 6
        assert_equal(x[()], 6)

        # strides and shape must be the same length
        with pytest.raises(ValueError):
            np.ndarray((2,), strides=())
        with pytest.raises(ValueError):
            np.ndarray((), strides=(2,))

    def test_output(self):
        x = np.array(2)
        assert_raises(ValueError, np.add, x, [1], x)

    def test_real_imag(self):
        # contiguity checks are for gh-11245
        x = np.array(1j)
        xr = x.real
        xi = x.imag

        assert_equal(xr, np.array(0))
        assert_(type(xr) is np.ndarray)
        assert_equal(xr.flags.contiguous, True)
        assert_equal(xr.flags.f_contiguous, True)

        assert_equal(xi, np.array(1))
        assert_(type(xi) is np.ndarray)
        assert_equal(xi.flags.contiguous, True)
        assert_equal(xi.flags.f_contiguous, True)


class TestScalarIndexing:
    def setup_method(self):
        self.d = np.array([0, 1])[0]

    def test_ellipsis_subscript(self):
        a = self.d
        assert_equal(a[...], 0)
        assert_equal(a[...].shape, ())

    def test_empty_subscript(self):
        a = self.d
        assert_equal(a[()], 0)
        assert_equal(a[()].shape, ())

    def test_invalid_subscript(self):
        a = self.d
        assert_raises(IndexError, lambda x: x[0], a)
        assert_raises(IndexError, lambda x: x[np.array([], int)], a)

    def test_invalid_subscript_assignment(self):
        a = self.d

        def assign(x, i, v):
            x[i] = v

        assert_raises(TypeError, assign, a, 0, 42)

    def test_newaxis(self):
        a = self.d
        assert_equal(a[np.newaxis].shape, (1,))
        assert_equal(a[..., np.newaxis].shape, (1,))
        assert_equal(a[np.newaxis, ...].shape, (1,))
        assert_equal(a[..., np.newaxis].shape, (1,))
        assert_equal(a[np.newaxis, ..., np.newaxis].shape, (1, 1))
        assert_equal(a[..., np.newaxis, np.newaxis].shape, (1, 1))
        assert_equal(a[np.newaxis, np.newaxis, ...].shape, (1, 1))
        assert_equal(a[(np.newaxis,)*10].shape, (1,)*10)

    def test_invalid_newaxis(self):
        a = self.d

        def subscript(x, i):
            x[i]

        assert_raises(IndexError, subscript, a, (np.newaxis, 0))
        assert_raises(IndexError, subscript, a, (np.newaxis,)*50)

    def test_overlapping_assignment(self):
        # With positive strides
        a = np.arange(4)
        a[:-1] = a[1:]
        assert_equal(a, [1, 2, 3, 3])

        a = np.arange(4)
        a[1:] = a[:-1]
        assert_equal(a, [0, 0, 1, 2])

        # With positive and negative strides
        a = np.arange(4)
        a[:] = a[::-1]
        assert_equal(a, [3, 2, 1, 0])

        a = np.arange(6).reshape(2, 3)
        a[::-1,:] = a[:, ::-1]
        assert_equal(a, [[5, 4, 3], [2, 1, 0]])

        a = np.arange(6).reshape(2, 3)
        a[::-1, ::-1] = a[:, ::-1]
        assert_equal(a, [[3, 4, 5], [0, 1, 2]])

        # With just one element overlapping
        a = np.arange(5)
        a[:3] = a[2:]
        assert_equal(a, [2, 3, 4, 3, 4])

        a = np.arange(5)
        a[2:] = a[:3]
        assert_equal(a, [0, 1, 0, 1, 2])

        a = np.arange(5)
        a[2::-1] = a[2:]
        assert_equal(a, [4, 3, 2, 3, 4])

        a = np.arange(5)
        a[2:] = a[2::-1]
        assert_equal(a, [0, 1, 2, 1, 0])

        a = np.arange(5)
        a[2::-1] = a[:1:-1]
        assert_equal(a, [2, 3, 4, 3, 4])

        a = np.arange(5)
        a[:1:-1] = a[2::-1]
        assert_equal(a, [0, 1, 0, 1, 2])


class TestCreation:
    """
    Test the np.array constructor
    """
    def test_from_attribute(self):
        class x:
            def __array__(self, dtype=None):
                pass

        assert_raises(ValueError, np.array, x())

    def test_from_string(self):
        types = np.typecodes['AllInteger'] + np.typecodes['Float']
        nstr = ['123', '123']
        result = np.array([123, 123], dtype=int)
        for type in types:
            msg = 'String conversion for %s' % type
            assert_equal(np.array(nstr, dtype=type), result, err_msg=msg)

    def test_void(self):
        arr = np.array([], dtype='V')
        assert arr.dtype == 'V8'  # current default
        # Same length scalars (those that go to the same void) work:
        arr = np.array([b"1234", b"1234"], dtype="V")
        assert arr.dtype == "V4"

        # Promoting different lengths will fail (pre 1.20 this worked)
        # by going via S5 and casting to V5.
        with pytest.raises(TypeError):
            np.array([b"1234", b"12345"], dtype="V")
        with pytest.raises(TypeError):
            np.array([b"12345", b"1234"], dtype="V")

        # Check the same for the casting path:
        arr = np.array([b"1234", b"1234"], dtype="O").astype("V")
        assert arr.dtype == "V4"
        with pytest.raises(TypeError):
            np.array([b"1234", b"12345"], dtype="O").astype("V")

    @pytest.mark.parametrize("idx",
            [pytest.param(Ellipsis, id="arr"), pytest.param((), id="scalar")])
    def test_structured_void_promotion(self, idx):
        arr = np.array(
            [np.array(1, dtype="i,i")[idx], np.array(2, dtype='i,i')[idx]],
            dtype="V")
        assert_array_equal(arr, np.array([(1, 1), (2, 2)], dtype="i,i"))
        # The following fails to promote the two dtypes, resulting in an error
        with pytest.raises(TypeError):
            np.array(
                [np.array(1, dtype="i,i")[idx], np.array(2, dtype='i,i,i')[idx]],
                dtype="V")


    def test_too_big_error(self):
        # 45341 is the smallest integer greater than sqrt(2**31 - 1).
        # 3037000500 is the smallest integer greater than sqrt(2**63 - 1).
        # We want to make sure that the square byte array with those dimensions
        # is too big on 32 or 64 bit systems respectively.
        if np.iinfo('intp').max == 2**31 - 1:
            shape = (46341, 46341)
        elif np.iinfo('intp').max == 2**63 - 1:
            shape = (3037000500, 3037000500)
        else:
            return
        assert_raises(ValueError, np.empty, shape, dtype=np.int8)
        assert_raises(ValueError, np.zeros, shape, dtype=np.int8)
        assert_raises(ValueError, np.ones, shape, dtype=np.int8)

    @pytest.mark.skipif(np.dtype(np.intp).itemsize != 8,
                        reason="malloc may not fail on 32 bit systems")
    def test_malloc_fails(self):
        # This test is guaranteed to fail due to a too large allocation
        with assert_raises(np.core._exceptions._ArrayMemoryError):
            np.empty(np.iinfo(np.intp).max, dtype=np.uint8)

    def test_zeros(self):
        types = np.typecodes['AllInteger'] + np.typecodes['AllFloat']
        for dt in types:
            d = np.zeros((13,), dtype=dt)
            assert_equal(np.count_nonzero(d), 0)
            # true for ieee floats
            assert_equal(d.sum(), 0)
            assert_(not d.any())

            d = np.zeros(2, dtype='(2,4)i4')
            assert_equal(np.count_nonzero(d), 0)
            assert_equal(d.sum(), 0)
            assert_(not d.any())

            d = np.zeros(2, dtype='4i4')
            assert_equal(np.count_nonzero(d), 0)
            assert_equal(d.sum(), 0)
            assert_(not d.any())

            d = np.zeros(2, dtype='(2,4)i4, (2,4)i4')
            assert_equal(np.count_nonzero(d), 0)

    @pytest.mark.slow
    def test_zeros_big(self):
        # test big array as they might be allocated different by the system
        types = np.typecodes['AllInteger'] + np.typecodes['AllFloat']
        for dt in types:
            d = np.zeros((30 * 1024**2,), dtype=dt)
            assert_(not d.any())
            # This test can fail on 32-bit systems due to insufficient
            # contiguous memory. Deallocating the previous array increases the
            # chance of success.
            del(d)

    def test_zeros_obj(self):
        # test initialization from PyLong(0)
        d = np.zeros((13,), dtype=object)
        assert_array_equal(d, [0] * 13)
        assert_equal(np.count_nonzero(d), 0)

    def test_zeros_obj_obj(self):
        d = np.zeros(10, dtype=[('k', object, 2)])
        assert_array_equal(d['k'], 0)

    def test_zeros_like_like_zeros(self):
        # test zeros_like returns the same as zeros
        for c in np.typecodes['All']:
            if c == 'V':
                continue
            d = np.zeros((3,3), dtype=c)
            assert_array_equal(np.zeros_like(d), d)
            assert_equal(np.zeros_like(d).dtype, d.dtype)
        # explicitly check some special cases
        d = np.zeros((3,3), dtype='S5')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)
        d = np.zeros((3,3), dtype='U5')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)

        d = np.zeros((3,3), dtype='<i4')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)
        d = np.zeros((3,3), dtype='>i4')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)

        d = np.zeros((3,3), dtype='<M8[s]')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)
        d = np.zeros((3,3), dtype='>M8[s]')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)

        d = np.zeros((3,3), dtype='f4,f4')
        assert_array_equal(np.zeros_like(d), d)
        assert_equal(np.zeros_like(d).dtype, d.dtype)

    def test_empty_unicode(self):
        # don't throw decode errors on garbage memory
        for i in range(5, 100, 5):
            d = np.empty(i, dtype='U')
            str(d)

    def test_sequence_non_homogeneous(self):
        assert_equal(np.array([4, 2**80]).dtype, object)
        assert_equal(np.array([4, 2**80, 4]).dtype, object)
        assert_equal(np.array([2**80, 4]).dtype, object)
        assert_equal(np.array([2**80] * 3).dtype, object)
        assert_equal(np.array([[1, 1],[1j, 1j]]).dtype, complex)
        assert_equal(np.array([[1j, 1j],[1, 1]]).dtype, complex)
        assert_equal(np.array([[1, 1, 1],[1, 1j, 1.], [1, 1, 1]]).dtype, complex)

    def test_non_sequence_sequence(self):
        """Should not segfault.

        Class Fail breaks the sequence protocol for new style classes, i.e.,
        those derived from object. Class Map is a mapping type indicated by
        raising a ValueError. At some point we may raise a warning instead
        of an error in the Fail case.

        """
        class Fail:
            def __len__(self):
                return 1

            def __getitem__(self, index):
                raise ValueError()

        class Map:
            def __len__(self):
                return 1

            def __getitem__(self, index):
                raise KeyError()

        a = np.array([Map()])
        assert_(a.shape == (1,))
        assert_(a.dtype == np.dtype(object))
        assert_raises(ValueError, np.array, [Fail()])

    def test_no_len_object_type(self):
        # gh-5100, want object array from iterable object without len()
        class Point2:
            def __init__(self):
                pass

            def __getitem__(self, ind):
                if ind in [0, 1]:
                    return ind
                else:
                    raise IndexError()
        d = np.array([Point2(), Point2(), Point2()])
        assert_equal(d.dtype, np.dtype(object))

    def test_false_len_sequence(self):
        # gh-7264, segfault for this example
        class C:
            def __getitem__(self, i):
                raise IndexError
            def __len__(self):
                return 42

        a = np.array(C()) # segfault?
        assert_equal(len(a), 0)

    def test_false_len_iterable(self):
        # Special case where a bad __getitem__ makes us fall back on __iter__:
        class C:
            def __getitem__(self, x):
                raise Exception
            def __iter__(self):
                return iter(())
            def __len__(self):
                return 2

        a = np.empty(2)
        with assert_raises(ValueError):
            a[:] = C()  # Segfault!

        np.array(C()) == list(C())

    def test_failed_len_sequence(self):
        # gh-7393
        class A:
            def __init__(self, data):
                self._data = data
            def __getitem__(self, item):
                return type(self)(self._data[item])
            def __len__(self):
                return len(self._data)

        # len(d) should give 3, but len(d[0]) will fail
        d = A([1,2,3])
        assert_equal(len(np.array(d)), 3)

    def test_array_too_big(self):
        # Test that array creation succeeds for arrays addressable by intp
        # on the byte level and fails for too large arrays.
        buf = np.zeros(100)

        max_bytes = np.iinfo(np.intp).max
        for dtype in ["intp", "S20", "b"]:
            dtype = np.dtype(dtype)
            itemsize = dtype.itemsize

            np.ndarray(buffer=buf, strides=(0,),
                       shape=(max_bytes//itemsize,), dtype=dtype)
            assert_raises(ValueError, np.ndarray, buffer=buf, strides=(0,),
                          shape=(max_bytes//itemsize + 1,), dtype=dtype)

    def _ragged_creation(self, seq):
        # without dtype=object, the ragged object raises
        with pytest.raises(ValueError, match=".*detected shape was"):
            a = np.array(seq)

        return np.array(seq, dtype=object)

    def test_ragged_ndim_object(self):
        # Lists of mismatching depths are treated as object arrays
        a = self._ragged_creation([[1], 2, 3])
        assert_equal(a.shape, (3,))
        assert_equal(a.dtype, object)

        a = self._ragged_creation([1, [2], 3])
        assert_equal(a.shape, (3,))
        assert_equal(a.dtype, object)

        a = self._ragged_creation([1, 2, [3]])
        assert_equal(a.shape, (3,))
        assert_equal(a.dtype, object)

    def test_ragged_shape_object(self):
        # The ragged dimension of a list is turned into an object array
        a = self._ragged_creation([[1, 1], [2], [3]])
        assert_equal(a.shape, (3,))
        assert_equal(a.dtype, object)

        a = self._ragged_creation([[1], [2, 2], [3]])
        assert_equal(a.shape, (3,))
        assert_equal(a.dtype, object)

        a = self._ragged_creation([[1], [2], [3, 3]])
        assert a.shape == (3,)
        assert a.dtype == object

    def test_array_of_ragged_array(self):
        outer = np.array([None, None])
        outer[0] = outer[1] = np.array([1, 2, 3])
        assert np.array(outer).shape == (2,)
        assert np.array([outer]).shape == (1, 2)

        outer_ragged = np.array([None, None])
        outer_ragged[0] = np.array([1, 2, 3])
        outer_ragged[1] = np.array([1, 2, 3, 4])
        # should both of these emit deprecation warnings?
        assert np.array(outer_ragged).shape == (2,)
        assert np.array([outer_ragged]).shape == (1, 2,)

    def test_deep_nonragged_object(self):
        # None of these should raise, even though they are missing dtype=object
        a = np.array([[[Decimal(1)]]])
        a = np.array([1, Decimal(1)])
        a = np.array([[1], [Decimal(1)]])

    @pytest.mark.parametrize("dtype", [object, "O,O", "O,(3)O", "(2,3)O"])
    @pytest.mark.parametrize("function", [
            np.ndarray, np.empty,
            lambda shape, dtype: np.empty_like(np.empty(shape, dtype=dtype))])
    def test_object_initialized_to_None(self, function, dtype):
        # NumPy has support for object fields to be NULL (meaning None)
        # but generally, we should always fill with the proper None, and
        # downstream may rely on that.  (For fully initialized arrays!)
        arr = function(3, dtype=dtype)
        # We expect a fill value of None, which is not NULL:
        expected = np.array(None).tobytes()
        expected = expected * (arr.nbytes // len(expected))
        assert arr.tobytes() == expected

class TestStructured:
    def test_subarray_field_access(self):
        a = np.zeros((3, 5), dtype=[('a', ('i4', (2, 2)))])
        a['a'] = np.arange(60).reshape(3, 5, 2, 2)

        # Since the subarray is always in C-order, a transpose
        # does not swap the subarray:
        assert_array_equal(a.T['a'], a['a'].transpose(1, 0, 2, 3))

        # In Fortran order, the subarray gets appended
        # like in all other cases, not prepended as a special case
        b = a.copy(order='F')
        assert_equal(a['a'].shape, b['a'].shape)
        assert_equal(a.T['a'].shape, a.T.copy()['a'].shape)

    def test_subarray_comparison(self):
        # Check that comparisons between record arrays with
        # multi-dimensional field types work properly
        a = np.rec.fromrecords(
            [([1, 2, 3], 'a', [[1, 2], [3, 4]]), ([3, 3, 3], 'b', [[0, 0], [0, 0]])],
            dtype=[('a', ('f4', 3)), ('b', object), ('c', ('i4', (2, 2)))])
        b = a.copy()
        assert_equal(a == b, [True, True])
        assert_equal(a != b, [False, False])
        b[1].b = 'c'
        assert_equal(a == b, [True, False])
        assert_equal(a != b, [False, True])
        for i in range(3):
            b[0].a = a[0].a
            b[0].a[i] = 5
            assert_equal(a == b, [False, False])
            assert_equal(a != b, [True, True])
        for i in range(2):
            for j in range(2):
                b = a.copy()
                b[0].c[i, j] = 10
                assert_equal(a == b, [False, True])
                assert_equal(a != b, [True, False])

        # Check that broadcasting with a subarray works, including cases that
        # require promotion to work:
        a = np.array([[(0,)], [(1,)]], dtype=[('a', 'f8')])
        b = np.array([(0,), (0,), (1,)], dtype=[('a', 'f8')])
        assert_equal(a == b, [[True, True, False], [False, False, True]])
        assert_equal(b == a, [[True, True, False], [False, False, True]])
        a = np.array([[(0,)], [(1,)]], dtype=[('a', 'f8', (1,))])
        b = np.array([(0,), (0,), (1,)], dtype=[('a', 'f8', (1,))])
        assert_equal(a == b, [[True, True, False], [False, False, True]])
        assert_equal(b == a, [[True, True, False], [False, False, True]])
        a = np.array([[([0, 0],)], [([1, 1],)]], dtype=[('a', 'f8', (2,))])
        b = np.array([([0, 0],), ([0, 1],), ([1, 1],)], dtype=[('a', 'f8', (2,))])
        assert_equal(a == b, [[True, False, False], [False, False, True]])
        assert_equal(b == a, [[True, False, False], [False, False, True]])

        # Check that broadcasting Fortran-style arrays with a subarray work
        a = np.array([[([0, 0],)], [([1, 1],)]], dtype=[('a', 'f8', (2,))], order='F')
        b = np.array([([0, 0],), ([0, 1],), ([1, 1],)], dtype=[('a', 'f8', (2,))])
        assert_equal(a == b, [[True, False, False], [False, False, True]])
        assert_equal(b == a, [[True, False, False], [False, False, True]])

        # Check that incompatible sub-array shapes don't result to broadcasting
        x = np.zeros((1,), dtype=[('a', ('f4', (1, 2))), ('b', 'i1')])
        y = np.zeros((1,), dtype=[('a', ('f4', (2,))), ('b', 'i1')])
        # The main importance is that it does not return True:
        with pytest.raises(TypeError):
            x == y

        x = np.zeros((1,), dtype=[('a', ('f4', (2, 1))), ('b', 'i1')])
        y = np.zeros((1,), dtype=[('a', ('f4', (2,))), ('b', 'i1')])
        # The main importance is that it does not return True:
        with pytest.raises(TypeError):
            x == y

    def test_empty_structured_array_comparison(self):
        # Check that comparison works on empty arrays with nontrivially
        # shaped fields
        a = np.zeros(0, [('a', '<f8', (1, 1))])
        assert_equal(a, a)
        a = np.zeros(0, [('a', '<f8', (1,))])
        assert_equal(a, a)
        a = np.zeros((0, 0), [('a', '<f8', (1, 1))])
        assert_equal(a, a)
        a = np.zeros((1, 0, 1), [('a', '<f8', (1, 1))])
        assert_equal(a, a)

    def test_structured_comparisons_with_promotion(self):
        # Check that structured arrays can be compared so long as their
        # dtypes promote fine:
        a = np.array([(5, 42), (10, 1)], dtype=[('a', '>i8'), ('b', '<f8')])
        b = np.array([(5, 43), (10, 1)], dtype=[('a', '<i8'), ('b', '>f8')])
        assert_equal(a == b, [False, True])
        assert_equal(a != b, [True, False])

        a = np.array([(5, 42), (10, 1)], dtype=[('a', '>f8'), ('b', '<f8')])
        b = np.array([(5, 43), (10, 1)], dtype=[('a', '<i8'), ('b', '>i8')])
        assert_equal(a == b, [False, True])
        assert_equal(a != b, [True, False])

        # Including with embedded subarray dtype (although subarray comparison
        # itself may still be a bit weird and compare the raw data)
        a = np.array([(5, 42), (10, 1)], dtype=[('a', '10>f8'), ('b', '5<f8')])
        b = np.array([(5, 43), (10, 1)], dtype=[('a', '10<i8'), ('b', '5>i8')])
        assert_equal(a == b, [False, True])
        assert_equal(a != b, [True, False])

    def test_void_comparison_failures(self):
        # In principle, one could decide to return an array of False for some
        # if comparisons are impossible.  But right now we return TypeError
        # when "void" dtype are involved.
        x = np.zeros(3, dtype=[('a', 'i1')])
        y = np.zeros(3)
        # Cannot compare non-structured to structured:
        with pytest.raises(TypeError):
            x == y

        # Added title prevents promotion, but casts are OK:
        y = np.zeros(3, dtype=[(('title', 'a'), 'i1')])
        assert np.can_cast(y.dtype, x.dtype)
        with pytest.raises(TypeError):
            x == y

        x = np.zeros(3, dtype="V7")
        y = np.zeros(3, dtype="V8")
        with pytest.raises(TypeError):
            x == y

    def test_casting(self):
        # Check that casting a structured array to change its byte order
        # works
        a = np.array([(1,)], dtype=[('a', '<i4')])
        assert_(np.can_cast(a.dtype, [('a', '>i4')], casting='unsafe'))
        b = a.astype([('a', '>i4')])
        assert_equal(b, a.byteswap().newbyteorder())
        assert_equal(a['a'][0], b['a'][0])

        # Check that equality comparison works on structured arrays if
        # they are 'equiv'-castable
        a = np.array([(5, 42), (10, 1)], dtype=[('a', '>i4'), ('b', '<f8')])
        b = np.array([(5, 42), (10, 1)], dtype=[('a', '<i4'), ('b', '>f8')])
        assert_(np.can_cast(a.dtype, b.dtype, casting='equiv'))
        assert_equal(a == b, [True, True])

        # Check that 'equiv' casting can change byte order
        assert_(np.can_cast(a.dtype, b.dtype, casting='equiv'))
        c = a.astype(b.dtype, casting='equiv')
        assert_equal(a == c, [True, True])

        # Check that 'safe' casting can change byte order and up-cast
        # fields
        t = [('a', '<i8'), ('b', '>f8')]
        assert_(np.can_cast(a.dtype, t, casting='safe'))
        c = a.astype(t, casting='safe')
        assert_equal((c == np.array([(5, 42), (10, 1)], dtype=t)),
                     [True, True])

        # Check that 'same_kind' casting can change byte order and
        # change field widths within a "kind"
        t = [('a', '<i4'), ('b', '>f4')]
        assert_(np.can_cast(a.dtype, t, casting='same_kind'))
        c = a.astype(t, casting='same_kind')
        assert_equal((c == np.array([(5, 42), (10, 1)], dtype=t)),
                     [True, True])

        # Check that casting fails if the casting rule should fail on
        # any of the fields
        t = [('a', '>i8'), ('b', '<f4')]
        assert_(not np.can_cast(a.dtype, t, casting='safe'))
        assert_raises(TypeError, a.astype, t, casting='safe')
        t = [('a', '>i2'), ('b', '<f8')]
        assert_(not np.can_cast(a.dtype, t, casting='equiv'))
        assert_raises(TypeError, a.astype, t, casting='equiv')
        t = [('a', '>i8'), ('b', '<i2')]
        assert_(not np.can_cast(a.dtype, t, casting='same_kind'))
        assert_raises(TypeError, a.astype, t, casting='same_kind')
        assert_(not np.can_cast(a.dtype, b.dtype, casting='no'))
        assert_raises(TypeError, a.astype, b.dtype, casting='no')

        # Check that non-'unsafe' casting can't change the set of field names
        for casting in ['no', 'safe', 'equiv', 'same_kind']:
            t = [('a', '>i4')]
            assert_(not np.can_cast(a.dtype, t, casting=casting))
            t = [('a', '>i4'), ('b', '<f8'), ('c', 'i4')]
            assert_(not np.can_cast(a.dtype, t, casting=casting))

    def test_objview(self):
        # https://github.com/numpy/numpy/issues/3286
        a = np.array([], dtype=[('a', 'f'), ('b', 'f'), ('c', 'O')])
        a[['a', 'b']]  # TypeError?

        # https://github.com/numpy/numpy/issues/3253
        dat2 = np.zeros(3, [('A', 'i'), ('B', '|O')])
        dat2[['B', 'A']]  # TypeError?

    def test_setfield(self):
        # https://github.com/numpy/numpy/issues/3126
        struct_dt = np.dtype([('elem', 'i4', 5),])
        dt = np.dtype([('field', 'i4', 10),('struct', struct_dt)])
        x = np.zeros(1, dt)
        x[0]['field'] = np.ones(10, dtype='i4')
        x[0]['struct'] = np.ones(1, dtype=struct_dt)
        assert_equal(x[0]['field'], np.ones(10, dtype='i4'))

    def test_setfield_object(self):
        # make sure object field assignment with ndarray value
        # on void scalar mimics setitem behavior
        b = np.zeros(1, dtype=[('x', 'O')])
        # next line should work identically to b['x'][0] = np.arange(3)
        b[0]['x'] = np.arange(3)
        assert_equal(b[0]['x'], np.arange(3))

        # check that broadcasting check still works
        c = np.zeros(1, dtype=[('x', 'O', 5)])

        def testassign():
            c[0]['x'] = np.arange(3)

        assert_raises(ValueError, testassign)

    def test_zero_width_string(self):
        # Test for PR #6430 / issues #473, #4955, #2585

        dt = np.dtype([('I', int), ('S', 'S0')])

        x = np.zeros(4, dtype=dt)

        assert_equal(x['S'], [b'', b'', b'', b''])
        assert_equal(x['S'].itemsize, 0)

        x['S'] = ['a', 'b', 'c', 'd']
        assert_equal(x['S'], [b'', b'', b'', b''])
        assert_equal(x['I'], [0, 0, 0, 0])

        # Variation on test case from #4955
        x['S'][x['I'] == 0] = 'hello'
        assert_equal(x['S'], [b'', b'', b'', b''])
        assert_equal(x['I'], [0, 0, 0, 0])

        # Variation on test case from #2585
        x['S'] = 'A'
        assert_equal(x['S'], [b'', b'', b'', b''])
        assert_equal(x['I'], [0, 0, 0, 0])

        # Allow zero-width dtypes in ndarray constructor
        y = np.ndarray(4, dtype=x['S'].dtype)
        assert_equal(y.itemsize, 0)
        assert_equal(x['S'], y)

        # More tests for indexing an array with zero-width fields
        assert_equal(np.zeros(4, dtype=[('a', 'S0,S0'),
                                        ('b', 'u1')])['a'].itemsize, 0)
        assert_equal(np.empty(3, dtype='S0,S0').itemsize, 0)
        assert_equal(np.zeros(4, dtype='S0,u1')['f0'].itemsize, 0)

        xx = x['S'].reshape((2, 2))
        assert_equal(xx.itemsize, 0)
        assert_equal(xx, [[b'', b''], [b'', b'']])
        # check for no uninitialized memory due to viewing S0 array
        assert_equal(xx[:].dtype, xx.dtype)
        assert_array_equal(eval(repr(xx), dict(array=np.array)), xx)

        b = io.BytesIO()
        np.save(b, xx)

        b.seek(0)
        yy = np.load(b)
        assert_equal(yy.itemsize, 0)
        assert_equal(xx, yy)

        with temppath(suffix='.npy') as tmp:
            np.save(tmp, xx)
            yy = np.load(tmp)
            assert_equal(yy.itemsize, 0)
            assert_equal(xx, yy)

    def test_base_attr(self):
        a = np.zeros(3, dtype='i4,f4')
        b = a[0]
        assert_(b.base is a)

    def test_assignment(self):
        def testassign(arr, v):
            c = arr.copy()
            c[0] = v  # assign using setitem
            c[1:] = v # assign using "dtype_transfer" code paths
            return c

        dt = np.dtype([('foo', 'i8'), ('bar', 'i8')])
        arr = np.ones(2, dt)
        v1 = np.array([(2,3)], dtype=[('foo', 'i8'), ('bar', 'i8')])
        v2 = np.array([(2,3)], dtype=[('bar', 'i8'), ('foo', 'i8')])
        v3 = np.array([(2,3)], dtype=[('bar', 'i8'), ('baz', 'i8')])
        v4 = np.array([(2,)],  dtype=[('bar', 'i8')])
        v5 = np.array([(2,3)], dtype=[('foo', 'f8'), ('bar', 'f8')])
        w = arr.view({'names': ['bar'], 'formats': ['i8'], 'offsets': [8]})

        ans = np.array([(2,3),(2,3)], dtype=dt)
        assert_equal(testassign(arr, v1), ans)
        assert_equal(testassign(arr, v2), ans)
        assert_equal(testassign(arr, v3), ans)
        assert_raises(TypeError, lambda: testassign(arr, v4))
        assert_equal(testassign(arr, v5), ans)
        w[:] = 4
        assert_equal(arr, np.array([(1,4),(1,4)], dtype=dt))

        # test field-reordering, assignment by position, and self-assignment
        a = np.array([(1,2,3)],
                     dtype=[('foo', 'i8'), ('bar', 'i8'), ('baz', 'f4')])
        a[['foo', 'bar']] = a[['bar', 'foo']]
        assert_equal(a[0].item(), (2,1,3))

        # test that this works even for 'simple_unaligned' structs
        # (ie, that PyArray_EquivTypes cares about field order too)
        a = np.array([(1,2)], dtype=[('a', 'i4'), ('b', 'i4')])
        a[['a', 'b']] = a[['b', 'a']]
        assert_equal(a[0].item(), (2,1))

    def test_scalar_assignment(self):
        with assert_raises(ValueError):
            arr = np.arange(25).reshape(5, 5)
            arr.itemset(3)

    def test_structuredscalar_indexing(self):
        # test gh-7262
        x = np.empty(shape=1, dtype="(2)3S,(2)3U")
        assert_equal(x[["f0","f1"]][0], x[0][["f0","f1"]])
        assert_equal(x[0], x[0][()])

    def test_multiindex_titles(self):
        a = np.zeros(4, dtype=[(('a', 'b'), 'i'), ('c', 'i'), ('d', 'i')])
        assert_raises(KeyError, lambda : a[['a','c']])
        assert_raises(KeyError, lambda : a[['a','a']])
        assert_raises(ValueError, lambda : a[['b','b']])  # field exists, but repeated
        a[['b','c']]  # no exception

    def test_structured_cast_promotion_fieldorder(self):
        # gh-15494
        # dtypes with different field names are not promotable
        A = ("a", "<i8")
        B = ("b", ">i8")
        ab = np.array([(1, 2)], dtype=[A, B])
        ba = np.array([(1, 2)], dtype=[B, A])
        assert_raises(TypeError, np.concatenate, ab, ba)
        assert_raises(TypeError, np.result_type, ab.dtype, ba.dtype)
        assert_raises(TypeError, np.promote_types, ab.dtype, ba.dtype)

        # dtypes with same field names/order but different memory offsets
        # and byte-order are promotable to packed nbo.
        assert_equal(np.promote_types(ab.dtype, ba[['a', 'b']].dtype),
                     repack_fields(ab.dtype.newbyteorder('N')))

        # gh-13667
        # dtypes with different fieldnames but castable field types are castable
        assert_equal(np.can_cast(ab.dtype, ba.dtype), True)
        assert_equal(ab.astype(ba.dtype).dtype, ba.dtype)
        assert_equal(np.can_cast('f8,i8', [('f0', 'f8'), ('f1', 'i8')]), True)
        assert_equal(np.can_cast('f8,i8', [('f1', 'f8'), ('f0', 'i8')]), True)
        assert_equal(np.can_cast('f8,i8', [('f1', 'i8'), ('f0', 'f8')]), False)
        assert_equal(np.can_cast('f8,i8', [('f1', 'i8'), ('f0', 'f8')],
                                 casting='unsafe'), True)

        ab[:] = ba  # make sure assignment still works

        # tests of type-promotion of corresponding fields
        dt1 = np.dtype([("", "i4")])
        dt2 = np.dtype([("", "i8")])
        assert_equal(np.promote_types(dt1, dt2), np.dtype([('f0', 'i8')]))
        assert_equal(np.promote_types(dt2, dt1), np.dtype([('f0', 'i8')]))
        assert_raises(TypeError, np.promote_types, dt1, np.dtype([("", "V3")]))
        assert_equal(np.promote_types('i4,f8', 'i8,f4'),
                     np.dtype([('f0', 'i8'), ('f1', 'f8')]))
        # test nested case
        dt1nest = np.dtype([("", dt1)])
        dt2nest = np.dtype([("", dt2)])
        assert_equal(np.promote_types(dt1nest, dt2nest),
                     np.dtype([('f0', np.dtype([('f0', 'i8')]))]))

        # note that offsets are lost when promoting:
        dt = np.dtype({'names': ['x'], 'formats': ['i4'], 'offsets': [8]})
        a = np.ones(3, dtype=dt)
        assert_equal(np.concatenate([a, a]).dtype, np.dtype([('x', 'i4')]))

    @pytest.mark.parametrize("dtype_dict", [
            dict(names=["a", "b"], formats=["i4", "f"], itemsize=100),
            dict(names=["a", "b"], formats=["i4", "f"],
                 offsets=[0, 12])])
    @pytest.mark.parametrize("align", [True, False])
    def test_structured_promotion_packs(self, dtype_dict, align):
        # Structured dtypes are packed when promoted (we consider the packed
        # form to be "canonical"), so tere is no extra padding.
        dtype = np.dtype(dtype_dict, align=align)
        # Remove non "canonical" dtype options:
        dtype_dict.pop("itemsize", None)
        dtype_dict.pop("offsets", None)
        expected = np.dtype(dtype_dict, align=align)

        res = np.promote_types(dtype, dtype)
        assert res.itemsize == expected.itemsize
        assert res.fields == expected.fields

        # But the "expected" one, should just be returned unchanged:
        res = np.promote_types(expected, expected)
        assert res is expected

    def test_structured_asarray_is_view(self):
        # A scalar viewing an array preserves its view even when creating a
        # new array. This test documents behaviour, it may not be the best
        # desired behaviour.
        arr = np.array([1], dtype="i,i")
        scalar = arr[0]
        assert not scalar.flags.owndata  # view into the array
        assert np.asarray(scalar).base is scalar
        # But never when a dtype is passed in:
        assert np.asarray(scalar, dtype=scalar.dtype).base is None
        # A scalar which owns its data does not have this property.
        # It is not easy to create one, one method is to use pickle:
        scalar = pickle.loads(pickle.dumps(scalar))
        assert scalar.flags.owndata
        assert np.asarray(scalar).base is None

class TestBool:
    def test_test_interning(self):
        a0 = np.bool_(0)
        b0 = np.bool_(False)
        assert_(a0 is b0)
        a1 = np.bool_(1)
        b1 = np.bool_(True)
        assert_(a1 is b1)
        assert_(np.array([True])[0] is a1)
        assert_(np.array(True)[()] is a1)

    def test_sum(self):
        d = np.ones(101, dtype=bool)
        assert_equal(d.sum(), d.size)
        assert_equal(d[::2].sum(), d[::2].size)
        assert_equal(d[::-2].sum(), d[::-2].size)

        d = np.frombuffer(b'\xff\xff' * 100, dtype=bool)
        assert_equal(d.sum(), d.size)
        assert_equal(d[::2].sum(), d[::2].size)
        assert_equal(d[::-2].sum(), d[::-2].size)

    def check_count_nonzero(self, power, length):
        powers = [2 ** i for i in range(length)]
        for i in range(2**power):
            l = [(i & x) != 0 for x in powers]
            a = np.array(l, dtype=bool)
            c = builtins.sum(l)
            assert_equal(np.count_nonzero(a), c)
            av = a.view(np.uint8)
            av *= 3
            assert_equal(np.count_nonzero(a), c)
            av *= 4
            assert_equal(np.count_nonzero(a), c)
            av[av != 0] = 0xFF
            assert_equal(np.count_nonzero(a), c)

    def test_count_nonzero(self):
        # check all 12 bit combinations in a length 17 array
        # covers most cases of the 16 byte unrolled code
        self.check_count_nonzero(12, 17)

    @pytest.mark.slow
    def test_count_nonzero_all(self):
        # check all combinations in a length 17 array
        # covers all cases of the 16 byte unrolled code
        self.check_count_nonzero(17, 17)

    def test_count_nonzero_unaligned(self):
        # prevent mistakes as e.g. gh-4060
        for o in range(7):
            a = np.zeros((18,), dtype=bool)[o+1:]
            a[:o] = True
            assert_equal(np.count_nonzero(a), builtins.sum(a.tolist()))
            a = np.ones((18,), dtype=bool)[o+1:]
            a[:o] = False
            assert_equal(np.count_nonzero(a), builtins.sum(a.tolist()))

    def _test_cast_from_flexible(self, dtype):
        # empty string -> false
        for n in range(3):
            v = np.array(b'', (dtype, n))
            assert_equal(bool(v), False)
            assert_equal(bool(v[()]), False)
            assert_equal(v.astype(bool), False)
            assert_(isinstance(v.astype(bool), np.ndarray))
            assert_(v[()].astype(bool) is np.False_)

        # anything else -> true
        for n in range(1, 4):
            for val in [b'a', b'0', b' ']:
                v = np.array(val, (dtype, n))
                assert_equal(bool(v), True)
                assert_equal(bool(v[()]), True)
                assert_equal(v.astype(bool), True)
                assert_(isinstance(v.astype(bool), np.ndarray))
                assert_(v[()].astype(bool) is np.True_)

    def test_cast_from_void(self):
        self._test_cast_from_flexible(np.void)

    @pytest.mark.xfail(reason="See gh-9847")
    def test_cast_from_unicode(self):
        self._test_cast_from_flexible(np.unicode_)

    @pytest.mark.xfail(reason="See gh-9847")
    def test_cast_from_bytes(self):
        self._test_cast_from_flexible(np.bytes_)


class TestZeroSizeFlexible:
    @staticmethod
    def _zeros(shape, dtype=str):
        dtype = np.dtype(dtype)
        if dtype == np.void:
            return np.zeros(shape, dtype=(dtype, 0))

        # not constructable directly
        dtype = np.dtype([('x', dtype, 0)])
        return np.zeros(shape, dtype=dtype)['x']

    def test_create(self):
        zs = self._zeros(10, bytes)
        assert_equal(zs.itemsize, 0)
        zs = self._zeros(10, np.void)
        assert_equal(zs.itemsize, 0)
        zs = self._zeros(10, str)
        assert_equal(zs.itemsize, 0)

    def _test_sort_partition(self, name, kinds, **kwargs):
        # Previously, these would all hang
        for dt in [bytes, np.void, str]:
            zs = self._zeros(10, dt)
            sort_method = getattr(zs, name)
            sort_func = getattr(np, name)
            for kind in kinds:
                sort_method(kind=kind, **kwargs)
                sort_func(zs, kind=kind, **kwargs)

    def test_sort(self):
        self._test_sort_partition('sort', kinds='qhs')

    def test_argsort(self):
        self._test_sort_partition('argsort', kinds='qhs')

    def test_partition(self):
        self._test_sort_partition('partition', kinds=['introselect'], kth=2)

    def test_argpartition(self):
        self._test_sort_partition('argpartition', kinds=['introselect'], kth=2)

    def test_resize(self):
        # previously an error
        for dt in [bytes, np.void, str]:
            zs = self._zeros(10, dt)
            zs.resize(25)
            zs.resize((10, 10))

    def test_view(self):
        for dt in [bytes, np.void, str]:
            zs = self._zeros(10, dt)

            # viewing as itself should be allowed
            assert_equal(zs.view(dt).dtype, np.dtype(dt))

            # viewing as any non-empty type gives an empty result
            assert_equal(zs.view((dt, 1)).shape, (0,))

    def test_dumps(self):
        zs = self._zeros(10, int)
        assert_equal(zs, pickle.loads(zs.dumps()))

    def test_pickle(self):
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            for dt in [bytes, np.void, str]:
                zs = self._zeros(10, dt)
                p = pickle.dumps(zs, protocol=proto)
                zs2 = pickle.loads(p)

                assert_equal(zs.dtype, zs2.dtype)

    def test_pickle_empty(self):
        """Checking if an empty array pickled and un-pickled will not cause a
        segmentation fault"""
        arr = np.array([]).reshape(999999, 0)
        pk_dmp = pickle.dumps(arr)
        pk_load = pickle.loads(pk_dmp)

        assert pk_load.size == 0

    @pytest.mark.skipif(pickle.HIGHEST_PROTOCOL < 5,
                        reason="requires pickle protocol 5")
    def test_pickle_with_buffercallback(self):
        array = np.arange(10)
        buffers = []
        bytes_string = pickle.dumps(array, buffer_callback=buffers.append,
                                    protocol=5)
        array_from_buffer = pickle.loads(bytes_string, buffers=buffers)
        # when using pickle protocol 5 with buffer callbacks,
        # array_from_buffer is reconstructed from a buffer holding a view
        # to the initial array's data, so modifying an element in array
        # should modify it in array_from_buffer too.
        array[0] = -1
        assert array_from_buffer[0] == -1, array_from_buffer[0]


class TestMethods:

    sort_kinds = ['quicksort', 'heapsort', 'stable']

    def test_all_where(self):
        a = np.array([[True, False, True],
                      [False, False, False],
                      [True, True, True]])
        wh_full = np.array([[True, False, True],
                            [False, False, False],
                            [True, False, True]])
        wh_lower = np.array([[False],
                             [False],
                             [True]])
        for _ax in [0, None]:
            assert_equal(a.all(axis=_ax, where=wh_lower),
                        np.all(a[wh_lower[:,0],:], axis=_ax))
            assert_equal(np.all(a, axis=_ax, where=wh_lower),
                         a[wh_lower[:,0],:].all(axis=_ax))

        assert_equal(a.all(where=wh_full), True)
        assert_equal(np.all(a, where=wh_full), True)
        assert_equal(a.all(where=False), True)
        assert_equal(np.all(a, where=False), True)

    def test_any_where(self):
        a = np.array([[True, False, True],
                      [False, False, False],
                      [True, True, True]])
        wh_full = np.array([[False, True, False],
                            [True, True, True],
                            [False, False, False]])
        wh_middle = np.array([[False],
                              [True],
                              [False]])
        for _ax in [0, None]:
            assert_equal(a.any(axis=_ax, where=wh_middle),
                         np.any(a[wh_middle[:,0],:], axis=_ax))
            assert_equal(np.any(a, axis=_ax, where=wh_middle),
                         a[wh_middle[:,0],:].any(axis=_ax))
        assert_equal(a.any(where=wh_full), False)
        assert_equal(np.any(a, where=wh_full), False)
        assert_equal(a.any(where=False), False)
        assert_equal(np.any(a, where=False), False)

    def test_compress(self):
        tgt = [[5, 6, 7, 8, 9]]
        arr = np.arange(10).reshape(2, 5)
        out = arr.compress([0, 1], axis=0)
        assert_equal(out, tgt)

        tgt = [[1, 3], [6, 8]]
        out = arr.compress([0, 1, 0, 1, 0], axis=1)
        assert_equal(out, tgt)

        tgt = [[1], [6]]
        arr = np.arange(10).reshape(2, 5)
        out = arr.compress([0, 1], axis=1)
        assert_equal(out, tgt)

        arr = np.arange(10).reshape(2, 5)
        out = arr.compress([0, 1])
        assert_equal(out, 1)

    def test_choose(self):
        x = 2*np.ones((3,), dtype=int)
        y = 3*np.ones((3,), dtype=int)
        x2 = 2*np.ones((2, 3), dtype=int)
        y2 = 3*np.ones((2, 3), dtype=int)
        ind = np.array([0, 0, 1])

        A = ind.choose((x, y))
        assert_equal(A, [2, 2, 3])

        A = ind.choose((x2, y2))
        assert_equal(A, [[2, 2, 3], [2, 2, 3]])

        A = ind.choose((x, y2))
        assert_equal(A, [[2, 2, 3], [2, 2, 3]])

        oned = np.ones(1)
        # gh-12031, caused SEGFAULT
        assert_raises(TypeError, oned.choose,np.void(0), [oned])

        out = np.array(0)
        ret = np.choose(np.array(1), [10, 20, 30], out=out)
        assert out is ret
        assert_equal(out[()], 20)

        # gh-6272 check overlap on out
        x = np.arange(5)
        y = np.choose([0,0,0], [x[:3], x[:3], x[:3]], out=x[1:4], mode='wrap')
        assert_equal(y, np.array([0, 1, 2]))

    def test_prod(self):
        ba = [1, 2, 10, 11, 6, 5, 4]
        ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]

        for ctype in [np.int16, np.uint16, np.int32, np.uint32,
                      np.float32, np.float64, np.complex64, np.complex128]:
            a = np.array(ba, ctype)
            a2 = np.array(ba2, ctype)
            if ctype in ['1', 'b']:
                assert_raises(ArithmeticError, a.prod)
                assert_raises(ArithmeticError, a2.prod, axis=1)
            else:
                assert_equal(a.prod(axis=0), 26400)
                assert_array_equal(a2.prod(axis=0),
                                   np.array([50, 36, 84, 180], ctype))
                assert_array_equal(a2.prod(axis=-1),
                                   np.array([24, 1890, 600], ctype))

    def test_repeat(self):
        m = np.array([1, 2, 3, 4, 5, 6])
        m_rect = m.reshape((2, 3))

        A = m.repeat([1, 3, 2, 1, 1, 2])
        assert_equal(A, [1, 2, 2, 2, 3,
                         3, 4, 5, 6, 6])

        A = m.repeat(2)
        assert_equal(A, [1, 1, 2, 2, 3, 3,
                         4, 4, 5, 5, 6, 6])

        A = m_rect.repeat([2, 1], axis=0)
        assert_equal(A, [[1, 2, 3],
                         [1, 2, 3],
                         [4, 5, 6]])

        A = m_rect.repeat([1, 3, 2], axis=1)
        assert_equal(A, [[1, 2, 2, 2, 3, 3],
                         [4, 5, 5, 5, 6, 6]])

        A = m_rect.repeat(2, axis=0)
        assert_equal(A, [[1, 2, 3],
                         [1, 2, 3],
                         [4, 5, 6],
                         [4, 5, 6]])

        A = m_rect.repeat(2, axis=1)
        assert_equal(A, [[1, 1, 2, 2, 3, 3],
                         [4, 4, 5, 5, 6, 6]])

    def test_reshape(self):
        arr = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]])

        tgt = [[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]]
        assert_equal(arr.reshape(2, 6), tgt)

        tgt = [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]
        assert_equal(arr.reshape(3, 4), tgt)

        tgt = [[1, 10, 8, 6], [4, 2, 11, 9], [7, 5, 3, 12]]
        assert_equal(arr.reshape((3, 4), order='F'), tgt)

        tgt = [[1, 4, 7, 10], [2, 5, 8, 11], [3, 6, 9, 12]]
        assert_equal(arr.T.reshape((3, 4), order='C'), tgt)

    def test_round(self):
        def check_round(arr, expected, *round_args):
            assert_equal(arr.round(*round_args), expected)
            # With output array
            out = np.zeros_like(arr)
            res = arr.round(*round_args, out=out)
            assert_equal(out, expected)
            assert out is res

        check_round(np.array([1.2, 1.5]), [1, 2])
        check_round(np.array(1.5), 2)
        check_round(np.array([12.2, 15.5]), [10, 20], -1)
        check_round(np.array([12.15, 15.51]), [12.2, 15.5], 1)
        # Complex rounding
        check_round(np.array([4.5 + 1.5j]), [4 + 2j])
        check_round(np.array([12.5 + 15.5j]), [10 + 20j], -1)

    def test_squeeze(self):
        a = np.array([[[1], [2], [3]]])
        assert_equal(a.squeeze(), [1, 2, 3])
        assert_equal(a.squeeze(axis=(0,)), [[1], [2], [3]])
        assert_raises(ValueError, a.squeeze, axis=(1,))
        assert_equal(a.squeeze(axis=(2,)), [[1, 2, 3]])

    def test_transpose(self):
        a = np.array([[1, 2], [3, 4]])
        assert_equal(a.transpose(), [[1, 3], [2, 4]])
        assert_raises(ValueError, lambda: a.transpose(0))
        assert_raises(ValueError, lambda: a.transpose(0, 0))
        assert_raises(ValueError, lambda: a.transpose(0, 1, 2))

    def test_sort(self):
        # test ordering for floats and complex containing nans. It is only
        # necessary to check the less-than comparison, so sorts that
        # only follow the insertion sort path are sufficient. We only
        # test doubles and complex doubles as the logic is the same.

        # check doubles
        msg = "Test real sort order with nans"
        a = np.array([np.nan, 1, 0])
        b = np.sort(a)
        assert_equal(b, a[::-1], msg)
        # check complex
        msg = "Test complex sort order with nans"
        a = np.zeros(9, dtype=np.complex128)
        a.real += [np.nan, np.nan, np.nan, 1, 0, 1, 1, 0, 0]
        a.imag += [np.nan, 1, 0, np.nan, np.nan, 1, 0, 1, 0]
        b = np.sort(a)
        assert_equal(b, a[::-1], msg)

    # all c scalar sorts use the same code with different types
    # so it suffices to run a quick check with one type. The number
    # of sorted items must be greater than ~50 to check the actual
    # algorithm because quick and merge sort fall over to insertion
    # sort for small arrays.

    @pytest.mark.parametrize('dtype', [np.uint8, np.uint16, np.uint32, np.uint64,
                                       np.float16, np.float32, np.float64,
                                       np.longdouble])
    def test_sort_unsigned(self, dtype):
        a = np.arange(101, dtype=dtype)
        b = a[::-1].copy()
        for kind in self.sort_kinds:
            msg = "scalar sort, kind=%s" % kind
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)
            c = b.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    @pytest.mark.parametrize('dtype',
                             [np.int8, np.int16, np.int32, np.int64, np.float16,
                              np.float32, np.float64, np.longdouble])
    def test_sort_signed(self, dtype):
        a = np.arange(-50, 51, dtype=dtype)
        b = a[::-1].copy()
        for kind in self.sort_kinds:
            msg = "scalar sort, kind=%s" % (kind)
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)
            c = b.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    @pytest.mark.parametrize('dtype', [np.float32, np.float64, np.longdouble])
    @pytest.mark.parametrize('part', ['real', 'imag'])
    def test_sort_complex(self, part, dtype):
        # test complex sorts. These use the same code as the scalars
        # but the compare function differs.
        cdtype = {
            np.single: np.csingle,
            np.double: np.cdouble,
            np.longdouble: np.clongdouble,
        }[dtype]
        a = np.arange(-50, 51, dtype=dtype)
        b = a[::-1].copy()
        ai = (a * (1+1j)).astype(cdtype)
        bi = (b * (1+1j)).astype(cdtype)
        setattr(ai, part, 1)
        setattr(bi, part, 1)
        for kind in self.sort_kinds:
            msg = "complex sort, %s part == 1, kind=%s" % (part, kind)
            c = ai.copy()
            c.sort(kind=kind)
            assert_equal(c, ai, msg)
            c = bi.copy()
            c.sort(kind=kind)
            assert_equal(c, ai, msg)

    def test_sort_complex_byte_swapping(self):
        # test sorting of complex arrays requiring byte-swapping, gh-5441
        for endianness in '<>':
            for dt in np.typecodes['Complex']:
                arr = np.array([1+3.j, 2+2.j, 3+1.j], dtype=endianness + dt)
                c = arr.copy()
                c.sort()
                msg = 'byte-swapped complex sort, dtype={0}'.format(dt)
                assert_equal(c, arr, msg)

    @pytest.mark.parametrize('dtype', [np.bytes_, np.unicode_])
    def test_sort_string(self, dtype):
        # np.array will perform the encoding to bytes for us in the bytes test
        a = np.array(['aaaaaaaa' + chr(i) for i in range(101)], dtype=dtype)
        b = a[::-1].copy()
        for kind in self.sort_kinds:
            msg = "kind=%s" % kind
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)
            c = b.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    def test_sort_object(self):
        # test object array sorts.
        a = np.empty((101,), dtype=object)
        a[:] = list(range(101))
        b = a[::-1]
        for kind in ['q', 'h', 'm']:
            msg = "kind=%s" % kind
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)
            c = b.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    def test_sort_structured(self):
        # test record array sorts.
        dt = np.dtype([('f', float), ('i', int)])
        a = np.array([(i, i) for i in range(101)], dtype=dt)
        b = a[::-1]
        for kind in ['q', 'h', 'm']:
            msg = "kind=%s" % kind
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)
            c = b.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    @pytest.mark.parametrize('dtype', ['datetime64[D]', 'timedelta64[D]'])
    def test_sort_time(self, dtype):
        # test datetime64 and timedelta64 sorts.
        a = np.arange(0, 101, dtype=dtype)
        b = a[::-1]
        for kind in ['q', 'h', 'm']:
            msg = "kind=%s" % kind
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)
            c = b.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    def test_sort_axis(self):
        # check axis handling. This should be the same for all type
        # specific sorts, so we only check it for one type and one kind
        a = np.array([[3, 2], [1, 0]])
        b = np.array([[1, 0], [3, 2]])
        c = np.array([[2, 3], [0, 1]])
        d = a.copy()
        d.sort(axis=0)
        assert_equal(d, b, "test sort with axis=0")
        d = a.copy()
        d.sort(axis=1)
        assert_equal(d, c, "test sort with axis=1")
        d = a.copy()
        d.sort()
        assert_equal(d, c, "test sort with default axis")

    def test_sort_size_0(self):
        # check axis handling for multidimensional empty arrays
        a = np.array([])
        a.shape = (3, 2, 1, 0)
        for axis in range(-a.ndim, a.ndim):
            msg = 'test empty array sort with axis={0}'.format(axis)
            assert_equal(np.sort(a, axis=axis), a, msg)
        msg = 'test empty array sort with axis=None'
        assert_equal(np.sort(a, axis=None), a.ravel(), msg)

    def test_sort_bad_ordering(self):
        # test generic class with bogus ordering,
        # should not segfault.
        class Boom:
            def __lt__(self, other):
                return True

        a = np.array([Boom()] * 100, dtype=object)
        for kind in self.sort_kinds:
            msg = "kind=%s" % kind
            c = a.copy()
            c.sort(kind=kind)
            assert_equal(c, a, msg)

    def test_void_sort(self):
        # gh-8210 - previously segfaulted
        for i in range(4):
            rand = np.random.randint(256, size=4000, dtype=np.uint8)
            arr = rand.view('V4')
            arr[::-1].sort()

        dt = np.dtype([('val', 'i4', (1,))])
        for i in range(4):
            rand = np.random.randint(256, size=4000, dtype=np.uint8)
            arr = rand.view(dt)
            arr[::-1].sort()

    def test_sort_raises(self):
        #gh-9404
        arr = np.array([0, datetime.now(), 1], dtype=object)
        for kind in self.sort_kinds:
            assert_raises(TypeError, arr.sort, kind=kind)
        #gh-3879
        class Raiser:
            def raises_anything(*args, **kwargs):
                raise TypeError("SOMETHING ERRORED")
            __eq__ = __ne__ = __lt__ = __gt__ = __ge__ = __le__ = raises_anything
        arr = np.array([[Raiser(), n] for n in range(10)]).reshape(-1)
        np.random.shuffle(arr)
        for kind in self.sort_kinds:
            assert_raises(TypeError, arr.sort, kind=kind)

    def test_sort_degraded(self):
        # test degraded dataset would take minutes to run with normal qsort
        d = np.arange(1000000)
        do = d.copy()
        x = d
        # create a median of 3 killer where each median is the sorted second
        # last element of the quicksort partition
        while x.size > 3:
            mid = x.size // 2
            x[mid], x[-2] = x[-2], x[mid]
            x = x[:-2]

        assert_equal(np.sort(d), do)
        assert_equal(d[np.argsort(d)], do)

    def test_copy(self):
        def assert_fortran(arr):
            assert_(arr.flags.fortran)
            assert_(arr.flags.f_contiguous)
            assert_(not arr.flags.c_contiguous)

        def assert_c(arr):
            assert_(not arr.flags.fortran)
            assert_(not arr.flags.f_contiguous)
            assert_(arr.flags.c_contiguous)

        a = np.empty((2, 2), order='F')
        # Test copying a Fortran array
        assert_c(a.copy())
        assert_c(a.copy('C'))
        assert_fortran(a.copy('F'))
        assert_fortran(a.copy('A'))

        # Now test starting with a C array.
        a = np.empty((2, 2), order='C')
        assert_c(a.copy())
        assert_c(a.copy('C'))
        assert_fortran(a.copy('F'))
        assert_c(a.copy('A'))

    @pytest.mark.parametrize("dtype", ['O', np.int32, 'i,O'])
    def test__deepcopy__(self, dtype):
        # Force the entry of NULLs into array
        a = np.empty(4, dtype=dtype)
        ctypes.memset(a.ctypes.data, 0, a.nbytes)

        # Ensure no error is raised, see gh-21833
        b = a.__deepcopy__({})

        a[0] = 42
        with pytest.raises(AssertionError):
            assert_array_equal(a, b)

    def test__deepcopy__catches_failure(self):
        class MyObj:
            def __deepcopy__(self, *args, **kwargs):
                raise RuntimeError

        arr = np.array([1, MyObj(), 3], dtype='O')
        with pytest.raises(RuntimeError):
            arr.__deepcopy__({})

    def test_sort_order(self):
        # Test sorting an array with fields
        x1 = np.array([21, 32, 14])
        x2 = np.array(['my', 'first', 'name'])
        x3 = np.array([3.1, 4.5, 6.2])
        r = np.rec.fromarrays([x1, x2, x3], names='id,word,number')

        r.sort(order=['id'])
        assert_equal(r.id, np.array([14, 21, 32]))
        assert_equal(r.word, np.array(['name', 'my', 'first']))
        assert_equal(r.number, np.array([6.2, 3.1, 4.5]))

        r.sort(order=['word'])
        assert_equal(r.id, np.array([32, 21, 14]))
        assert_equal(r.word, np.array(['first', 'my', 'name']))
        assert_equal(r.number, np.array([4.5, 3.1, 6.2]))

        r.sort(order=['number'])
        assert_equal(r.id, np.array([21, 32, 14]))
        assert_equal(r.word, np.array(['my', 'first', 'name']))
        assert_equal(r.number, np.array([3.1, 4.5, 6.2]))

        assert_raises_regex(ValueError, 'duplicate',
            lambda: r.sort(order=['id', 'id']))

        if sys.byteorder == 'little':
            strtype = '>i2'
        else:
            strtype = '<i2'
        mydtype = [('name', 'U5'), ('col2', strtype)]
        r = np.array([('a', 1), ('b', 255), ('c', 3), ('d', 258)],
                     dtype=mydtype)
        r.sort(order='col2')
        assert_equal(r['col2'], [1, 3, 255, 258])
        assert_equal(r, np.array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
                                 dtype=mydtype))

    def test_argsort(self):
        # all c scalar argsorts use the same code with different types
        # so it suffices to run a quick check with one type. The number
        # of sorted items must be greater than ~50 to check the actual
        # algorithm because quick and merge sort fall over to insertion
        # sort for small arrays.

        for dtype in [np.int32, np.uint32, np.float32]:
            a = np.arange(101, dtype=dtype)
            b = a[::-1].copy()
            for kind in self.sort_kinds:
                msg = "scalar argsort, kind=%s, dtype=%s" % (kind, dtype)
                assert_equal(a.copy().argsort(kind=kind), a, msg)
                assert_equal(b.copy().argsort(kind=kind), b, msg)

        # test complex argsorts. These use the same code as the scalars
        # but the compare function differs.
        ai = a*1j + 1
        bi = b*1j + 1
        for kind in self.sort_kinds:
            msg = "complex argsort, kind=%s" % kind
            assert_equal(ai.copy().argsort(kind=kind), a, msg)
            assert_equal(bi.copy().argsort(kind=kind), b, msg)
        ai = a + 1j
        bi = b + 1j
        for kind in self.sort_kinds:
            msg = "complex argsort, kind=%s" % kind
            assert_equal(ai.copy().argsort(kind=kind), a, msg)
            assert_equal(bi.copy().argsort(kind=kind), b, msg)

        # test argsort of complex arrays requiring byte-swapping, gh-5441
        for endianness in '<>':
            for dt in np.typecodes['Complex']:
                arr = np.array([1+3.j, 2+2.j, 3+1.j], dtype=endianness + dt)
                msg = 'byte-swapped complex argsort, dtype={0}'.format(dt)
                assert_equal(arr.argsort(),
                             np.arange(len(arr), dtype=np.intp), msg)

        # test string argsorts.
        s = 'aaaaaaaa'
        a = np.array([s + chr(i) for i in range(101)])
        b = a[::-1].copy()
        r = np.arange(101)
        rr = r[::-1]
        for kind in self.sort_kinds:
            msg = "string argsort, kind=%s" % kind
            assert_equal(a.copy().argsort(kind=kind), r, msg)
            assert_equal(b.copy().argsort(kind=kind), rr, msg)

        # test unicode argsorts.
        s = 'aaaaaaaa'
        a = np.array([s + chr(i) for i in range(101)], dtype=np.unicode_)
        b = a[::-1]
        r = np.arange(101)
        rr = r[::-1]
        for kind in self.sort_kinds:
            msg = "unicode argsort, kind=%s" % kind
            assert_equal(a.copy().argsort(kind=kind), r, msg)
            assert_equal(b.copy().argsort(kind=kind), rr, msg)

        # test object array argsorts.
        a = np.empty((101,), dtype=object)
        a[:] = list(range(101))
        b = a[::-1]
        r = np.arange(101)
        rr = r[::-1]
        for kind in self.sort_kinds:
            msg = "object argsort, kind=%s" % kind
            assert_equal(a.copy().argsort(kind=kind), r, msg)
            assert_equal(b.copy().argsort(kind=kind), rr, msg)

        # test structured array argsorts.
        dt = np.dtype([('f', float), ('i', int)])
        a = np.array([(i, i) for i in range(101)], dtype=dt)
        b = a[::-1]
        r = np.arange(101)
        rr = r[::-1]
        for kind in self.sort_kinds:
            msg = "structured array argsort, kind=%s" % kind
            assert_equal(a.copy().argsort(kind=kind), r, msg)
            assert_equal(b.copy().argsort(kind=kind), rr, msg)

        # test datetime64 argsorts.
        a = np.arange(0, 101, dtype='datetime64[D]')
        b = a[::-1]
        r = np.arange(101)
        rr = r[::-1]
        for kind in ['q', 'h', 'm']:
            msg = "datetime64 argsort, kind=%s" % kind
            assert_equal(a.copy().argsort(kind=kind), r, msg)
            assert_equal(b.copy().argsort(kind=kind), rr, msg)

        # test timedelta64 argsorts.
        a = np.arange(0, 101, dtype='timedelta64[D]')
        b = a[::-1]
        r = np.arange(101)
        rr = r[::-1]
        for kind in ['q', 'h', 'm']:
            msg = "timedelta64 argsort, kind=%s" % kind
            assert_equal(a.copy().argsort(kind=kind), r, msg)
            assert_equal(b.copy().argsort(kind=kind), rr, msg)

        # check axis handling. This should be the same for all type
        # specific argsorts, so we only check it for one type and one kind
        a = np.array([[3, 2], [1, 0]])
        b = np.array([[1, 1], [0, 0]])
        c = np.array([[1, 0], [1, 0]])
        assert_equal(a.copy().argsort(axis=0), b)
        assert_equal(a.copy().argsort(axis=1), c)
        assert_equal(a.copy().argsort(), c)

        # check axis handling for multidimensional empty arrays
        a = np.array([])
        a.shape = (3, 2, 1, 0)
        for axis in range(-a.ndim, a.ndim):
            msg = 'test empty array argsort with axis={0}'.format(axis)
            assert_equal(np.argsort(a, axis=axis),
                         np.zeros_like(a, dtype=np.intp), msg)
        msg = 'test empty array argsort with axis=None'
        assert_equal(np.argsort(a, axis=None),
                     np.zeros_like(a.ravel(), dtype=np.intp), msg)

        # check that stable argsorts are stable
        r = np.arange(100)
        # scalars
        a = np.zeros(100)
        assert_equal(a.argsort(kind='m'), r)
        # complex
        a = np.zeros(100, dtype=complex)
        assert_equal(a.argsort(kind='m'), r)
        # string
        a = np.array(['aaaaaaaaa' for i in range(100)])
        assert_equal(a.argsort(kind='m'), r)
        # unicode
        a = np.array(['aaaaaaaaa' for i in range(100)], dtype=np.unicode_)
        assert_equal(a.argsort(kind='m'), r)

    def test_sort_unicode_kind(self):
        d = np.arange(10)
        k = b'\xc3\xa4'.decode("UTF8")
        assert_raises(ValueError, d.sort, kind=k)
        assert_raises(ValueError, d.argsort, kind=k)

    @pytest.mark.parametrize('a', [
        np.array([0, 1, np.nan], dtype=np.float16),
        np.array([0, 1, np.nan], dtype=np.float32),
        np.array([0, 1, np.nan]),
    ])
    def test_searchsorted_floats(self, a):
        # test for floats arrays containing nans. Explicitly test
        # half, single, and double precision floats to verify that
        # the NaN-handling is correct.
        msg = "Test real (%s) searchsorted with nans, side='l'" % a.dtype
        b = a.searchsorted(a, side='left')
        assert_equal(b, np.arange(3), msg)
        msg = "Test real (%s) searchsorted with nans, side='r'" % a.dtype
        b = a.searchsorted(a, side='right')
        assert_equal(b, np.arange(1, 4), msg)
        # check keyword arguments
        a.searchsorted(v=1)
        x = np.array([0, 1, np.nan], dtype='float32')
        y = np.searchsorted(x, x[-1])
        assert_equal(y, 2)

    def test_searchsorted_complex(self):
        # test for complex arrays containing nans.
        # The search sorted routines use the compare functions for the
        # array type, so this checks if that is consistent with the sort
        # order.
        # check double complex
        a = np.zeros(9, dtype=np.complex128)
        a.real += [0, 0, 1, 1, 0, 1, np.nan, np.nan, np.nan]
        a.imag += [0, 1, 0, 1, np.nan, np.nan, 0, 1, np.nan]
        msg = "Test complex searchsorted with nans, side='l'"
        b = a.searchsorted(a, side='left')
        assert_equal(b, np.arange(9), msg)
        msg = "Test complex searchsorted with nans, side='r'"
        b = a.searchsorted(a, side='right')
        assert_equal(b, np.arange(1, 10), msg)
        msg = "Test searchsorted with little endian, side='l'"
        a = np.array([0, 128], dtype='<i4')
        b = a.searchsorted(np.array(128, dtype='<i4'))
        assert_equal(b, 1, msg)
        msg = "Test searchsorted with big endian, side='l'"
        a = np.array([0, 128], dtype='>i4')
        b = a.searchsorted(np.array(128, dtype='>i4'))
        assert_equal(b, 1, msg)

    def test_searchsorted_n_elements(self):
        # Check 0 elements
        a = np.ones(0)
        b = a.searchsorted([0, 1, 2], 'left')
        assert_equal(b, [0, 0, 0])
        b = a.searchsorted([0, 1, 2], 'right')
        assert_equal(b, [0, 0, 0])
        a = np.ones(1)
        # Check 1 element
        b = a.searchsorted([0, 1, 2], 'left')
        assert_equal(b, [0, 0, 1])
        b = a.searchsorted([0, 1, 2], 'right')
        assert_equal(b, [0, 1, 1])
        # Check all elements equal
        a = np.ones(2)
        b = a.searchsorted([0, 1, 2], 'left')
        assert_equal(b, [0, 0, 2])
        b = a.searchsorted([0, 1, 2], 'right')
        assert_equal(b, [0, 2, 2])

    def test_searchsorted_unaligned_array(self):
        # Test searching unaligned array
        a = np.arange(10)
        aligned = np.empty(a.itemsize * a.size + 1, 'uint8')
        unaligned = aligned[1:].view(a.dtype)
        unaligned[:] = a
        # Test searching unaligned array
        b = unaligned.searchsorted(a, 'left')
        assert_equal(b, a)
        b = unaligned.searchsorted(a, 'right')
        assert_equal(b, a + 1)
        # Test searching for unaligned keys
        b = a.searchsorted(unaligned, 'left')
        assert_equal(b, a)
        b = a.searchsorted(unaligned, 'right')
        assert_equal(b, a + 1)

    def test_searchsorted_resetting(self):
        # Test smart resetting of binsearch indices
        a = np.arange(5)
        b = a.searchsorted([6, 5, 4], 'left')
        assert_equal(b, [5, 5, 4])
        b = a.searchsorted([6, 5, 4], 'right')
        assert_equal(b, [5, 5, 5])

    def test_searchsorted_type_specific(self):
        # Test all type specific binary search functions
        types = ''.join((np.typecodes['AllInteger'], np.typecodes['AllFloat'],
                         np.typecodes['Datetime'], '?O'))
        for dt in types:
            if dt == 'M':
                dt = 'M8[D]'
            if dt == '?':
                a = np.arange(2, dtype=dt)
                out = np.arange(2)
            else:
                a = np.arange(0, 5, dtype=dt)
                out = np.arange(5)
            b = a.searchsorted(a, 'left')
            assert_equal(b, out)
            b = a.searchsorted(a, 'right')
            assert_equal(b, out + 1)
            # Test empty array, use a fresh array to get warnings in
            # valgrind if access happens.
            e = np.ndarray(shape=0, buffer=b'', dtype=dt)
            b = e.searchsorted(a, 'left')
            assert_array_equal(b, np.zeros(len(a), dtype=np.intp))
            b = a.searchsorted(e, 'left')
            assert_array_equal(b, np.zeros(0, dtype=np.intp))

    def test_searchsorted_unicode(self):
        # Test searchsorted on unicode strings.

        # 1.6.1 contained a string length miscalculation in
        # arraytypes.c.src:UNICODE_compare() which manifested as
        # incorrect/inconsistent results from searchsorted.
        a = np.array(['P:\\20x_dapi_cy3\\20x_dapi_cy3_20100185_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100186_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100187_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100189_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100190_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100191_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100192_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100193_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100194_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100195_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100196_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100197_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100198_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100199_1'],
                     dtype=np.unicode_)
        ind = np.arange(len(a))
        assert_equal([a.searchsorted(v, 'left') for v in a], ind)
        assert_equal([a.searchsorted(v, 'right') for v in a], ind + 1)
        assert_equal([a.searchsorted(a[i], 'left') for i in ind], ind)
        assert_equal([a.searchsorted(a[i], 'right') for i in ind], ind + 1)

    def test_searchsorted_with_invalid_sorter(self):
        a = np.array([5, 2, 1, 3, 4])
        s = np.argsort(a)
        assert_raises(TypeError, np.searchsorted, a, 0,
                      sorter=np.array((1, (2, 3)), dtype=object))
        assert_raises(TypeError, np.searchsorted, a, 0, sorter=[1.1])
        assert_raises(ValueError, np.searchsorted, a, 0, sorter=[1, 2, 3, 4])
        assert_raises(ValueError, np.searchsorted, a, 0, sorter=[1, 2, 3, 4, 5, 6])

        # bounds check
        assert_raises(ValueError, np.searchsorted, a, 4, sorter=[0, 1, 2, 3, 5])
        assert_raises(ValueError, np.searchsorted, a, 0, sorter=[-1, 0, 1, 2, 3])
        assert_raises(ValueError, np.searchsorted, a, 0, sorter=[4, 0, -1, 2, 3])

    def test_searchsorted_with_sorter(self):
        a = np.random.rand(300)
        s = a.argsort()
        b = np.sort(a)
        k = np.linspace(0, 1, 20)
        assert_equal(b.searchsorted(k), a.searchsorted(k, sorter=s))

        a = np.array([0, 1, 2, 3, 5]*20)
        s = a.argsort()
        k = [0, 1, 2, 3, 5]
        expected = [0, 20, 40, 60, 80]
        assert_equal(a.searchsorted(k, side='left', sorter=s), expected)
        expected = [20, 40, 60, 80, 100]
        assert_equal(a.searchsorted(k, side='right', sorter=s), expected)

        # Test searching unaligned array
        keys = np.arange(10)
        a = keys.copy()
        np.random.shuffle(s)
        s = a.argsort()
        aligned = np.empty(a.itemsize * a.size + 1, 'uint8')
        unaligned = aligned[1:].view(a.dtype)
        # Test searching unaligned array
        unaligned[:] = a
        b = unaligned.searchsorted(keys, 'left', s)
        assert_equal(b, keys)
        b = unaligned.searchsorted(keys, 'right', s)
        assert_equal(b, keys + 1)
        # Test searching for unaligned keys
        unaligned[:] = keys
        b = a.searchsorted(unaligned, 'left', s)
        assert_equal(b, keys)
        b = a.searchsorted(unaligned, 'right', s)
        assert_equal(b, keys + 1)

        # Test all type specific indirect binary search functions
        types = ''.join((np.typecodes['AllInteger'], np.typecodes['AllFloat'],
                         np.typecodes['Datetime'], '?O'))
        for dt in types:
            if dt == 'M':
                dt = 'M8[D]'
            if dt == '?':
                a = np.array([1, 0], dtype=dt)
                # We want the sorter array to be of a type that is different
                # from np.intp in all platforms, to check for #4698
                s = np.array([1, 0], dtype=np.int16)
                out = np.array([1, 0])
            else:
                a = np.array([3, 4, 1, 2, 0], dtype=dt)
                # We want the sorter array to be of a type that is different
                # from np.intp in all platforms, to check for #4698
                s = np.array([4, 2, 3, 0, 1], dtype=np.int16)
                out = np.array([3, 4, 1, 2, 0], dtype=np.intp)
            b = a.searchsorted(a, 'left', s)
            assert_equal(b, out)
            b = a.searchsorted(a, 'right', s)
            assert_equal(b, out + 1)
            # Test empty array, use a fresh array to get warnings in
            # valgrind if access happens.
            e = np.ndarray(shape=0, buffer=b'', dtype=dt)
            b = e.searchsorted(a, 'left', s[:0])
            assert_array_equal(b, np.zeros(len(a), dtype=np.intp))
            b = a.searchsorted(e, 'left', s)
            assert_array_equal(b, np.zeros(0, dtype=np.intp))

        # Test non-contiguous sorter array
        a = np.array([3, 4, 1, 2, 0])
        srt = np.empty((10,), dtype=np.intp)
        srt[1::2] = -1
        srt[::2] = [4, 2, 3, 0, 1]
        s = srt[::2]
        out = np.array([3, 4, 1, 2, 0], dtype=np.intp)
        b = a.searchsorted(a, 'left', s)
        assert_equal(b, out)
        b = a.searchsorted(a, 'right', s)
        assert_equal(b, out + 1)

    def test_searchsorted_return_type(self):
        # Functions returning indices should always return base ndarrays
        class A(np.ndarray):
            pass
        a = np.arange(5).view(A)
        b = np.arange(1, 3).view(A)
        s = np.arange(5).view(A)
        assert_(not isinstance(a.searchsorted(b, 'left'), A))
        assert_(not isinstance(a.searchsorted(b, 'right'), A))
        assert_(not isinstance(a.searchsorted(b, 'left', s), A))
        assert_(not isinstance(a.searchsorted(b, 'right', s), A))

    @pytest.mark.parametrize("dtype", np.typecodes["All"])
    def test_argpartition_out_of_range(self, dtype):
        # Test out of range values in kth raise an error, gh-5469
        d = np.arange(10).astype(dtype=dtype)
        assert_raises(ValueError, d.argpartition, 10)
        assert_raises(ValueError, d.argpartition, -11)

    @pytest.mark.parametrize("dtype", np.typecodes["All"])
    def test_partition_out_of_range(self, dtype):
        # Test out of range values in kth raise an error, gh-5469
        d = np.arange(10).astype(dtype=dtype)
        assert_raises(ValueError, d.partition, 10)
        assert_raises(ValueError, d.partition, -11)

    def test_argpartition_integer(self):
        # Test non-integer values in kth raise an error/
        d = np.arange(10)
        assert_raises(TypeError, d.argpartition, 9.)
        # Test also for generic type argpartition, which uses sorting
        # and used to not bound check kth
        d_obj = np.arange(10, dtype=object)
        assert_raises(TypeError, d_obj.argpartition, 9.)

    def test_partition_integer(self):
        # Test out of range values in kth raise an error, gh-5469
        d = np.arange(10)
        assert_raises(TypeError, d.partition, 9.)
        # Test also for generic type partition, which uses sorting
        # and used to not bound check kth
        d_obj = np.arange(10, dtype=object)
        assert_raises(TypeError, d_obj.partition, 9.)

    @pytest.mark.parametrize("kth_dtype", np.typecodes["AllInteger"])
    def test_partition_empty_array(self, kth_dtype):
        # check axis handling for multidimensional empty arrays
        kth = np.array(0, dtype=kth_dtype)[()]
        a = np.array([])
        a.shape = (3, 2, 1, 0)
        for axis in range(-a.ndim, a.ndim):
            msg = 'test empty array partition with axis={0}'.format(axis)
            assert_equal(np.partition(a, kth, axis=axis), a, msg)
        msg = 'test empty array partition with axis=None'
        assert_equal(np.partition(a, kth, axis=None), a.ravel(), msg)

    @pytest.mark.parametrize("kth_dtype", np.typecodes["AllInteger"])
    def test_argpartition_empty_array(self, kth_dtype):
        # check axis handling for multidimensional empty arrays
        kth = np.array(0, dtype=kth_dtype)[()]
        a = np.array([])
        a.shape = (3, 2, 1, 0)
        for axis in range(-a.ndim, a.ndim):
            msg = 'test empty array argpartition with axis={0}'.format(axis)
            assert_equal(np.partition(a, kth, axis=axis),
                         np.zeros_like(a, dtype=np.intp), msg)
        msg = 'test empty array argpartition with axis=None'
        assert_equal(np.partition(a, kth, axis=None),
                     np.zeros_like(a.ravel(), dtype=np.intp), msg)

    def test_partition(self):
        d = np.arange(10)
        assert_raises(TypeError, np.partition, d, 2, kind=1)
        assert_raises(ValueError, np.partition, d, 2, kind="nonsense")
        assert_raises(ValueError, np.argpartition, d, 2, kind="nonsense")
        assert_raises(ValueError, d.partition, 2, axis=0, kind="nonsense")
        assert_raises(ValueError, d.argpartition, 2, axis=0, kind="nonsense")
        for k in ("introselect",):
            d = np.array([])
            assert_array_equal(np.partition(d, 0, kind=k), d)
            assert_array_equal(np.argpartition(d, 0, kind=k), d)
            d = np.ones(1)
            assert_array_equal(np.partition(d, 0, kind=k)[0], d)
            assert_array_equal(d[np.argpartition(d, 0, kind=k)],
                               np.partition(d, 0, kind=k))

            # kth not modified
            kth = np.array([30, 15, 5])
            okth = kth.copy()
            np.partition(np.arange(40), kth)
            assert_array_equal(kth, okth)

            for r in ([2, 1], [1, 2], [1, 1]):
                d = np.array(r)
                tgt = np.sort(d)
                assert_array_equal(np.partition(d, 0, kind=k)[0], tgt[0])
                assert_array_equal(np.partition(d, 1, kind=k)[1], tgt[1])
                assert_array_equal(d[np.argpartition(d, 0, kind=k)],
                                   np.partition(d, 0, kind=k))
                assert_array_equal(d[np.argpartition(d, 1, kind=k)],
                                   np.partition(d, 1, kind=k))
                for i in range(d.size):
                    d[i:].partition(0, kind=k)
                assert_array_equal(d, tgt)

            for r in ([3, 2, 1], [1, 2, 3], [2, 1, 3], [2, 3, 1],
                      [1, 1, 1], [1, 2, 2], [2, 2, 1], [1, 2, 1]):
                d = np.array(r)
                tgt = np.sort(d)
                assert_array_equal(np.partition(d, 0, kind=k)[0], tgt[0])
                assert_array_equal(np.partition(d, 1, kind=k)[1], tgt[1])
                assert_array_equal(np.partition(d, 2, kind=k)[2], tgt[2])
                assert_array_equal(d[np.argpartition(d, 0, kind=k)],
                                   np.partition(d, 0, kind=k))
                assert_array_equal(d[np.argpartition(d, 1, kind=k)],
                                   np.partition(d, 1, kind=k))
                assert_array_equal(d[np.argpartition(d, 2, kind=k)],
                                   np.partition(d, 2, kind=k))
                for i in range(d.size):
                    d[i:].partition(0, kind=k)
                assert_array_equal(d, tgt)

            d = np.ones(50)
            assert_array_equal(np.partition(d, 0, kind=k), d)
            assert_array_equal(d[np.argpartition(d, 0, kind=k)],
                               np.partition(d, 0, kind=k))

            # sorted
            d = np.arange(49)
            assert_equal(np.partition(d, 5, kind=k)[5], 5)
            assert_equal(np.partition(d, 15, kind=k)[15], 15)
            assert_array_equal(d[np.argpartition(d, 5, kind=k)],
                               np.partition(d, 5, kind=k))
            assert_array_equal(d[np.argpartition(d, 15, kind=k)],
                               np.partition(d, 15, kind=k))

            # rsorted
            d = np.arange(47)[::-1]
            assert_equal(np.partition(d, 6, kind=k)[6], 6)
            assert_equal(np.partition(d, 16, kind=k)[16], 16)
            assert_array_equal(d[np.argpartition(d, 6, kind=k)],
                               np.partition(d, 6, kind=k))
            assert_array_equal(d[np.argpartition(d, 16, kind=k)],
                               np.partition(d, 16, kind=k))

            assert_array_equal(np.partition(d, -6, kind=k),
                               np.partition(d, 41, kind=k))
            assert_array_equal(np.partition(d, -16, kind=k),
                               np.partition(d, 31, kind=k))
            assert_array_equal(d[np.argpartition(d, -6, kind=k)],
                               np.partition(d, 41, kind=k))

            # median of 3 killer, O(n^2) on pure median 3 pivot quickselect
            # exercises the median of median of 5 code used to keep O(n)
            d = np.arange(1000000)
            x = np.roll(d, d.size // 2)
            mid = x.size // 2 + 1
            assert_equal(np.partition(x, mid)[mid], mid)
            d = np.arange(1000001)
            x = np.roll(d, d.size // 2 + 1)
            mid = x.size // 2 + 1
            assert_equal(np.partition(x, mid)[mid], mid)

            # max
            d = np.ones(10)
            d[1] = 4
            assert_equal(np.partition(d, (2, -1))[-1], 4)
            assert_equal(np.partition(d, (2, -1))[2], 1)
            assert_equal(d[np.argpartition(d, (2, -1))][-1], 4)
            assert_equal(d[np.argpartition(d, (2, -1))][2], 1)
            d[1] = np.nan
            assert_(np.isnan(d[np.argpartition(d, (2, -1))][-1]))
            assert_(np.isnan(np.partition(d, (2, -1))[-1]))

            # equal elements
            d = np.arange(47) % 7
            tgt = np.sort(np.arange(47) % 7)
            np.random.shuffle(d)
            for i in range(d.size):
                assert_equal(np.partition(d, i, kind=k)[i], tgt[i])
            assert_array_equal(d[np.argpartition(d, 6, kind=k)],
                               np.partition(d, 6, kind=k))
            assert_array_equal(d[np.argpartition(d, 16, kind=k)],
                               np.partition(d, 16, kind=k))
            for i in range(d.size):
                d[i:].partition(0, kind=k)
            assert_array_equal(d, tgt)

            d = np.array([0, 1, 2, 3, 4, 5, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
                          7, 7, 7, 7, 7, 9])
            kth = [0, 3, 19, 20]
            assert_equal(np.partition(d, kth, kind=k)[kth], (0, 3, 7, 7))
            assert_equal(d[np.argpartition(d, kth, kind=k)][kth], (0, 3, 7, 7))

            d = np.array([2, 1])
            d.partition(0, kind=k)
            assert_raises(ValueError, d.partition, 2)
            assert_raises(np.AxisError, d.partition, 3, axis=1)
            assert_raises(ValueError, np.partition, d, 2)
            assert_raises(np.AxisError, np.partition, d, 2, axis=1)
            assert_raises(ValueError, d.argpartition, 2)
            assert_raises(np.AxisError, d.argpartition, 3, axis=1)
            assert_raises(ValueError, np.argpartition, d, 2)
            assert_raises(np.AxisError, np.argpartition, d, 2, axis=1)
            d = np.arange(10).reshape((2, 5))
            d.partition(1, axis=0, kind=k)
            d.partition(4, axis=1, kind=k)
            np.partition(d, 1, axis=0, kind=k)
            np.partition(d, 4, axis=1, kind=k)
            np.partition(d, 1, axis=None, kind=k)
            np.partition(d, 9, axis=None, kind=k)
            d.argpartition(1, axis=0, kind=k)
            d.argpartition(4, axis=1, kind=k)
            np.argpartition(d, 1, axis=0, kind=k)
            np.argpartition(d, 4, axis=1, kind=k)
            np.argpartition(d, 1, axis=None, kind=k)
            np.argpartition(d, 9, axis=None, kind=k)
            assert_raises(ValueError, d.partition, 2, axis=0)
            assert_raises(ValueError, d.partition, 11, axis=1)
            assert_raises(TypeError, d.partition, 2, axis=None)
            assert_raises(ValueError, np.partition, d, 9, axis=1)
            assert_raises(ValueError, np.partition, d, 11, axis=None)
            assert_raises(ValueError, d.argpartition, 2, axis=0)
            assert_raises(ValueError, d.argpartition, 11, axis=1)
            assert_raises(ValueError, np.argpartition, d, 9, axis=1)
            assert_raises(ValueError, np.argpartition, d, 11, axis=None)

            td = [(dt, s) for dt in [np.int32, np.float32, np.complex64]
                  for s in (9, 16)]
            for dt, s in td:
                aae = assert_array_equal
                at = assert_

                d = np.arange(s, dtype=dt)
                np.random.shuffle(d)
                d1 = np.tile(np.arange(s, dtype=dt), (4, 1))
                map(np.random.shuffle, d1)
                d0 = np.transpose(d1)
                for i in range(d.size):
                    p = np.partition(d, i, kind=k)
                    assert_equal(p[i], i)
                    # all before are smaller
                    assert_array_less(p[:i], p[i])
                    # all after are larger
                    assert_array_less(p[i], p[i + 1:])
                    aae(p, d[np.argpartition(d, i, kind=k)])

                    p = np.partition(d1, i, axis=1, kind=k)
                    aae(p[:, i], np.array([i] * d1.shape[0], dtype=dt))
                    # array_less does not seem to work right
                    at((p[:, :i].T <= p[:, i]).all(),
                       msg="%d: %r <= %r" % (i, p[:, i], p[:, :i].T))
                    at((p[:, i + 1:].T > p[:, i]).all(),
                       msg="%d: %r < %r" % (i, p[:, i], p[:, i + 1:].T))
                    aae(p, d1[np.arange(d1.shape[0])[:, None],
                        np.argpartition(d1, i, axis=1, kind=k)])

                    p = np.partition(d0, i, axis=0, kind=k)
                    aae(p[i, :], np.array([i] * d1.shape[0], dtype=dt))
                    # array_less does not seem to work right
                    at((p[:i, :] <= p[i, :]).all(),
                       msg="%d: %r <= %r" % (i, p[i, :], p[:i, :]))
                    at((p[i + 1:, :] > p[i, :]).all(),
                       msg="%d: %r < %r" % (i, p[i, :], p[:, i + 1:]))
                    aae(p, d0[np.argpartition(d0, i, axis=0, kind=k),
                        np.arange(d0.shape[1])[None, :]])

                    # check inplace
                    dc = d.copy()
                    dc.partition(i, kind=k)
                    assert_equal(dc, np.partition(d, i, kind=k))
                    dc = d0.copy()
                    dc.partition(i, axis=0, kind=k)
                    assert_equal(dc, np.partition(d0, i, axis=0, kind=k))
                    dc = d1.copy()
                    dc.partition(i, axis=1, kind=k)
                    assert_equal(dc, np.partition(d1, i, axis=1, kind=k))

    def assert_partitioned(self, d, kth):
        prev = 0
        for k in np.sort(kth):
            assert_array_less(d[prev:k], d[k], err_msg='kth %d' % k)
            assert_((d[k:] >= d[k]).all(),
                    msg="kth %d, %r not greater equal %d" % (k, d[k:], d[k]))
            prev = k + 1

    def test_partition_iterative(self):
            d = np.arange(17)
            kth = (0, 1, 2, 429, 231)
            assert_raises(ValueError, d.partition, kth)
            assert_raises(ValueError, d.argpartition, kth)
            d = np.arange(10).reshape((2, 5))
            assert_raises(ValueError, d.partition, kth, axis=0)
            assert_raises(ValueError, d.partition, kth, axis=1)
            assert_raises(ValueError, np.partition, d, kth, axis=1)
            assert_raises(ValueError, np.partition, d, kth, axis=None)

            d = np.array([3, 4, 2, 1])
            p = np.partition(d, (0, 3))
            self.assert_partitioned(p, (0, 3))
            self.assert_partitioned(d[np.argpartition(d, (0, 3))], (0, 3))

            assert_array_equal(p, np.partition(d, (-3, -1)))
            assert_array_equal(p, d[np.argpartition(d, (-3, -1))])

            d = np.arange(17)
            np.random.shuffle(d)
            d.partition(range(d.size))
            assert_array_equal(np.arange(17), d)
            np.random.shuffle(d)
            assert_array_equal(np.arange(17), d[d.argpartition(range(d.size))])

            # test unsorted kth
            d = np.arange(17)
            np.random.shuffle(d)
            keys = np.array([1, 3, 8, -2])
            np.random.shuffle(d)
            p = np.partition(d, keys)
            self.assert_partitioned(p, keys)
            p = d[np.argpartition(d, keys)]
            self.assert_partitioned(p, keys)
            np.random.shuffle(keys)
            assert_array_equal(np.partition(d, keys), p)
            assert_array_equal(d[np.argpartition(d, keys)], p)

            # equal kth
            d = np.arange(20)[::-1]
            self.assert_partitioned(np.partition(d, [5]*4), [5])
            self.assert_partitioned(np.partition(d, [5]*4 + [6, 13]),
                                    [5]*4 + [6, 13])
            self.assert_partitioned(d[np.argpartition(d, [5]*4)], [5])
            self.assert_partitioned(d[np.argpartition(d, [5]*4 + [6, 13])],
                                    [5]*4 + [6, 13])

            d = np.arange(12)
            np.random.shuffle(d)
            d1 = np.tile(np.arange(12), (4, 1))
            map(np.random.shuffle, d1)
            d0 = np.transpose(d1)

            kth = (1, 6, 7, -1)
            p = np.partition(d1, kth, axis=1)
            pa = d1[np.arange(d1.shape[0])[:, None],
                    d1.argpartition(kth, axis=1)]
            assert_array_equal(p, pa)
            for i in range(d1.shape[0]):
                self.assert_partitioned(p[i,:], kth)
            p = np.partition(d0, kth, axis=0)
            pa = d0[np.argpartition(d0, kth, axis=0),
                    np.arange(d0.shape[1])[None,:]]
            assert_array_equal(p, pa)
            for i in range(d0.shape[1]):
                self.assert_partitioned(p[:, i], kth)

    def test_partition_cdtype(self):
        d = np.array([('Galahad', 1.7, 38), ('Arthur', 1.8, 41),
                   ('Lancelot', 1.9, 38)],
                  dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])

        tgt = np.sort(d, order=['age', 'height'])
        assert_array_equal(np.partition(d, range(d.size),
                                        order=['age', 'height']),
                           tgt)
        assert_array_equal(d[np.argpartition(d, range(d.size),
                                             order=['age', 'height'])],
                           tgt)
        for k in range(d.size):
            assert_equal(np.partition(d, k, order=['age', 'height'])[k],
                        tgt[k])
            assert_equal(d[np.argpartition(d, k, order=['age', 'height'])][k],
                         tgt[k])

        d = np.array(['Galahad', 'Arthur', 'zebra', 'Lancelot'])
        tgt = np.sort(d)
        assert_array_equal(np.partition(d, range(d.size)), tgt)
        for k in range(d.size):
            assert_equal(np.partition(d, k)[k], tgt[k])
            assert_equal(d[np.argpartition(d, k)][k], tgt[k])

    def test_partition_unicode_kind(self):
        d = np.arange(10)
        k = b'\xc3\xa4'.decode("UTF8")
        assert_raises(ValueError, d.partition, 2, kind=k)
        assert_raises(ValueError, d.argpartition, 2, kind=k)

    def test_partition_fuzz(self):
        # a few rounds of random data testing
        for j in range(10, 30):
            for i in range(1, j - 2):
                d = np.arange(j)
                np.random.shuffle(d)
                d = d % np.random.randint(2, 30)
                idx = np.random.randint(d.size)
                kth = [0, idx, i, i + 1]
                tgt = np.sort(d)[kth]
                assert_array_equal(np.partition(d, kth)[kth], tgt,
                                   err_msg="data: %r\n kth: %r" % (d, kth))

    @pytest.mark.parametrize("kth_dtype", np.typecodes["AllInteger"])
    def test_argpartition_gh5524(self, kth_dtype):
        #  A test for functionality of argpartition on lists.
        kth = np.array(1, dtype=kth_dtype)[()]
        d = [6, 7, 3, 2, 9, 0]
        p = np.argpartition(d, kth)
        self.assert_partitioned(np.array(d)[p],[1])

    def test_flatten(self):
        x0 = np.array([[1, 2, 3], [4, 5, 6]], np.int32)
        x1 = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], np.int32)
        y0 = np.array([1, 2, 3, 4, 5, 6], np.int32)
        y0f = np.array([1, 4, 2, 5, 3, 6], np.int32)
        y1 = np.array([1, 2, 3, 4, 5, 6, 7, 8], np.int32)
        y1f = np.array([1, 5, 3, 7, 2, 6, 4, 8], np.int32)
        assert_equal(x0.flatten(), y0)
        assert_equal(x0.flatten('F'), y0f)
        assert_equal(x0.flatten('F'), x0.T.flatten())
        assert_equal(x1.flatten(), y1)
        assert_equal(x1.flatten('F'), y1f)
        assert_equal(x1.flatten('F'), x1.T.flatten())


    @pytest.mark.parametrize('func', (np.dot, np.matmul))
    def test_arr_mult(self, func):
        a = np.array([[1, 0], [0, 1]])
        b = np.array([[0, 1], [1, 0]])
        c = np.array([[9, 1], [1, -9]])
        d = np.arange(24).reshape(4, 6)
        ddt = np.array(
            [[  55,  145,  235,  325],
             [ 145,  451,  757, 1063],
             [ 235,  757, 1279, 1801],
             [ 325, 1063, 1801, 2539]]
        )
        dtd = np.array(
            [[504, 540, 576, 612, 648, 684],
             [540, 580, 620, 660, 700, 740],
             [576, 620, 664, 708, 752, 796],
             [612, 660, 708, 756, 804, 852],
             [648, 700, 752, 804, 856, 908],
             [684, 740, 796, 852, 908, 964]]
        )


        # gemm vs syrk optimizations
        for et in [np.float32, np.float64, np.complex64, np.complex128]:
            eaf = a.astype(et)
            assert_equal(func(eaf, eaf), eaf)
            assert_equal(func(eaf.T, eaf), eaf)
            assert_equal(func(eaf, eaf.T), eaf)
            assert_equal(func(eaf.T, eaf.T), eaf)
            assert_equal(func(eaf.T.copy(), eaf), eaf)
            assert_equal(func(eaf, eaf.T.copy()), eaf)
            assert_equal(func(eaf.T.copy(), eaf.T.copy()), eaf)

        # syrk validations
        for et in [np.float32, np.float64, np.complex64, np.complex128]:
            eaf = a.astype(et)
            ebf = b.astype(et)
            assert_equal(func(ebf, ebf), eaf)
            assert_equal(func(ebf.T, ebf), eaf)
            assert_equal(func(ebf, ebf.T), eaf)
            assert_equal(func(ebf.T, ebf.T), eaf)

        # syrk - different shape, stride, and view validations
        for et in [np.float32, np.float64, np.complex64, np.complex128]:
            edf = d.astype(et)
            assert_equal(
                func(edf[::-1, :], edf.T),
                func(edf[::-1, :].copy(), edf.T.copy())
            )
            assert_equal(
                func(edf[:, ::-1], edf.T),
                func(edf[:, ::-1].copy(), edf.T.copy())
            )
            assert_equal(
                func(edf, edf[::-1, :].T),
                func(edf, edf[::-1, :].T.copy())
            )
            assert_equal(
                func(edf, edf[:, ::-1].T),
                func(edf, edf[:, ::-1].T.copy())
            )
            assert_equal(
                func(edf[:edf.shape[0] // 2, :], edf[::2, :].T),
                func(edf[:edf.shape[0] // 2, :].copy(), edf[::2, :].T.copy())
            )
            assert_equal(
                func(edf[::2, :], edf[:edf.shape[0] // 2, :].T),
                func(edf[::2, :].copy(), edf[:edf.shape[0] // 2, :].T.copy())
            )

        # syrk - different shape
        for et in [np.float32, np.float64, np.complex64, np.complex128]:
            edf = d.astype(et)
            eddtf = ddt.astype(et)
            edtdf = dtd.astype(et)
            assert_equal(func(edf, edf.T), eddtf)
            assert_equal(func(edf.T, edf), edtdf)

    @pytest.mark.parametrize('func', (np.dot, np.matmul))
    @pytest.mark.parametrize('dtype', 'ifdFD')
    def test_no_dgemv(self, func, dtype):
        # check vector arg for contiguous before gemv
        # gh-12156
        a = np.arange(8.0, dtype=dtype).reshape(2, 4)
        b = np.broadcast_to(1., (4, 1))
        ret1 = func(a, b)
        ret2 = func(a, b.copy())
        assert_equal(ret1, ret2)

        ret1 = func(b.T, a.T)
        ret2 = func(b.T.copy(), a.T)
        assert_equal(ret1, ret2)

        # check for unaligned data
        dt = np.dtype(dtype)
        a = np.zeros(8 * dt.itemsize // 2 + 1, dtype='int16')[1:].view(dtype)
        a = a.reshape(2, 4)
        b = a[0]
        # make sure it is not aligned
        assert_(a.__array_interface__['data'][0] % dt.itemsize != 0)
        ret1 = func(a, b)
        ret2 = func(a.copy(), b.copy())
        assert_equal(ret1, ret2)

        ret1 = func(b.T, a.T)
        ret2 = func(b.T.copy(), a.T.copy())
        assert_equal(ret1, ret2)

    def test_dot(self):
        a = np.array([[1, 0], [0, 1]])
        b = np.array([[0, 1], [1, 0]])
        c = np.array([[9, 1], [1, -9]])
        # function versus methods
        assert_equal(np.dot(a, b), a.dot(b))
        assert_equal(np.dot(np.dot(a, b), c), a.dot(b).dot(c))

        # test passing in an output array
        c = np.zeros_like(a)
        a.dot(b, c)
        assert_equal(c, np.dot(a, b))

        # test keyword args
        c = np.zeros_like(a)
        a.dot(b=b, out=c)
        assert_equal(c, np.dot(a, b))

    def test_dot_type_mismatch(self):
        c = 1.
        A = np.array((1,1), dtype='i,i')

        assert_raises(TypeError, np.dot, c, A)
        assert_raises(TypeError, np.dot, A, c)

    def test_dot_out_mem_overlap(self):
        np.random.seed(1)

        # Test BLAS and non-BLAS code paths, including all dtypes
        # that dot() supports
        dtypes = [np.dtype(code) for code in np.typecodes['All']
                  if code not in 'USVM']
        for dtype in dtypes:
            a = np.random.rand(3, 3).astype(dtype)

            # Valid dot() output arrays must be aligned
            b = _aligned_zeros((3, 3), dtype=dtype)
            b[...] = np.random.rand(3, 3)

            y = np.dot(a, b)
            x = np.dot(a, b, out=b)
            assert_equal(x, y, err_msg=repr(dtype))

            # Check invalid output array
            assert_raises(ValueError, np.dot, a, b, out=b[::2])
            assert_raises(ValueError, np.dot, a, b, out=b.T)

    def test_dot_matmul_out(self):
        # gh-9641
        class Sub(np.ndarray):
            pass
        a = np.ones((2, 2)).view(Sub)
        b = np.ones((2, 2)).view(Sub)
        out = np.ones((2, 2))

        # make sure out can be any ndarray (not only subclass of inputs)
        np.dot(a, b, out=out)
        np.matmul(a, b, out=out)

    def test_dot_matmul_inner_array_casting_fails(self):

        class A:
            def __array__(self, *args, **kwargs):
                raise NotImplementedError

        # Don't override the error from calling __array__()
        assert_raises(NotImplementedError, np.dot, A(), A())
        assert_raises(NotImplementedError, np.matmul, A(), A())
        assert_raises(NotImplementedError, np.inner, A(), A())

    def test_matmul_out(self):
        # overlapping memory
        a = np.arange(18).reshape(2, 3, 3)
        b = np.matmul(a, a)
        c = np.matmul(a, a, out=a)
        assert_(c is a)
        assert_equal(c, b)
        a = np.arange(18).reshape(2, 3, 3)
        c = np.matmul(a, a, out=a[::-1, ...])
        assert_(c.base is a.base)
        assert_equal(c, b)

    def test_diagonal(self):
        a = np.arange(12).reshape((3, 4))
        assert_equal(a.diagonal(), [0, 5, 10])
        assert_equal(a.diagonal(0), [0, 5, 10])
        assert_equal(a.diagonal(1), [1, 6, 11])
        assert_equal(a.diagonal(-1), [4, 9])
        assert_raises(np.AxisError, a.diagonal, axis1=0, axis2=5)
        assert_raises(np.AxisError, a.diagonal, axis1=5, axis2=0)
        assert_raises(np.AxisError, a.diagonal, axis1=5, axis2=5)
        assert_raises(ValueError, a.diagonal, axis1=1, axis2=1)

        b = np.arange(8).reshape((2, 2, 2))
        assert_equal(b.diagonal(), [[0, 6], [1, 7]])
        assert_equal(b.diagonal(0), [[0, 6], [1, 7]])
        assert_equal(b.diagonal(1), [[2], [3]])
        assert_equal(b.diagonal(-1), [[4], [5]])
        assert_raises(ValueError, b.diagonal, axis1=0, axis2=0)
        assert_equal(b.diagonal(0, 1, 2), [[0, 3], [4, 7]])
        assert_equal(b.diagonal(0, 0, 1), [[0, 6], [1, 7]])
        assert_equal(b.diagonal(offset=1, axis1=0, axis2=2), [[1], [3]])
        # Order of axis argument doesn't matter:
        assert_equal(b.diagonal(0, 2, 1), [[0, 3], [4, 7]])

    def test_diagonal_view_notwriteable(self):
        a = np.eye(3).diagonal()
        assert_(not a.flags.writeable)
        assert_(not a.flags.owndata)

        a = np.diagonal(np.eye(3))
        assert_(not a.flags.writeable)
        assert_(not a.flags.owndata)

        a = np.diag(np.eye(3))
        assert_(not a.flags.writeable)
        assert_(not a.flags.owndata)

    def test_diagonal_memleak(self):
        # Regression test for a bug that crept in at one point
        a = np.zeros((100, 100))
        if HAS_REFCOUNT:
            assert_(sys.getrefcount(a) < 50)
        for i in range(100):
            a.diagonal()
        if HAS_REFCOUNT:
            assert_(sys.getrefcount(a) < 50)

    def test_size_zero_memleak(self):
        # Regression test for issue 9615
        # Exercises a special-case code path for dot products of length
        # zero in cblasfuncs (making it is specific to floating dtypes).
        a = np.array([], dtype=np.float64)
        x = np.array(2.0)
        for _ in range(100):
            np.dot(a, a, out=x)
        if HAS_REFCOUNT:
            assert_(sys.getrefcount(x) < 50)

    def test_trace(self):
        a = np.arange(12).reshape((3, 4))
        assert_equal(a.trace(), 15)
        assert_equal(a.trace(0), 15)
        assert_equal(a.trace(1), 18)
        assert_equal(a.trace(-1), 13)

        b = np.arange(8).reshape((2, 2, 2))
        assert_equal(b.trace(), [6, 8])
        assert_equal(b.trace(0), [6, 8])
        assert_equal(b.trace(1), [2, 3])
        assert_equal(b.trace(-1), [4, 5])
        assert_equal(b.trace(0, 0, 1), [6, 8])
        assert_equal(b.trace(0, 0, 2), [5, 9])
        assert_equal(b.trace(0, 1, 2), [3, 11])
        assert_equal(b.trace(offset=1, axis1=0, axis2=2), [1, 3])

        out = np.array(1)
        ret = a.trace(out=out)
        assert ret is out

    def test_trace_subclass(self):
        # The class would need to overwrite trace to ensure single-element
        # output also has the right subclass.
        class MyArray(np.ndarray):
            pass

        b = np.arange(8).reshape((2, 2, 2)).view(MyArray)
        t = b.trace()
        assert_(isinstance(t, MyArray))

    def test_put(self):
        icodes = np.typecodes['AllInteger']
        fcodes = np.typecodes['AllFloat']
        for dt in icodes + fcodes + 'O':
            tgt = np.array([0, 1, 0, 3, 0, 5], dtype=dt)

            # test 1-d
            a = np.zeros(6, dtype=dt)
            a.put([1, 3, 5], [1, 3, 5])
            assert_equal(a, tgt)

            # test 2-d
            a = np.zeros((2, 3), dtype=dt)
            a.put([1, 3, 5], [1, 3, 5])
            assert_equal(a, tgt.reshape(2, 3))

        for dt in '?':
            tgt = np.array([False, True, False, True, False, True], dtype=dt)

            # test 1-d
            a = np.zeros(6, dtype=dt)
            a.put([1, 3, 5], [True]*3)
            assert_equal(a, tgt)

            # test 2-d
            a = np.zeros((2, 3), dtype=dt)
            a.put([1, 3, 5], [True]*3)
            assert_equal(a, tgt.reshape(2, 3))

        # check must be writeable
        a = np.zeros(6)
        a.flags.writeable = False
        assert_raises(ValueError, a.put, [1, 3, 5], [1, 3, 5])

        # when calling np.put, make sure a
        # TypeError is raised if the object
        # isn't an ndarray
        bad_array = [1, 2, 3]
        assert_raises(TypeError, np.put, bad_array, [0, 2], 5)

    def test_ravel(self):
        a = np.array([[0, 1], [2, 3]])
        assert_equal(a.ravel(), [0, 1, 2, 3])
        assert_(not a.ravel().flags.owndata)
        assert_equal(a.ravel('F'), [0, 2, 1, 3])
        assert_equal(a.ravel(order='C'), [0, 1, 2, 3])
        assert_equal(a.ravel(order='F'), [0, 2, 1, 3])
        assert_equal(a.ravel(order='A'), [0, 1, 2, 3])
        assert_(not a.ravel(order='A').flags.owndata)
        assert_equal(a.ravel(order='K'), [0, 1, 2, 3])
        assert_(not a.ravel(order='K').flags.owndata)
        assert_equal(a.ravel(), a.reshape(-1))

        a = np.array([[0, 1], [2, 3]], order='F')
        assert_equal(a.ravel(), [0, 1, 2, 3])
        assert_equal(a.ravel(order='A'), [0, 2, 1, 3])
        assert_equal(a.ravel(order='K'), [0, 2, 1, 3])
        assert_(not a.ravel(order='A').flags.owndata)
        assert_(not a.ravel(order='K').flags.owndata)
        assert_equal(a.ravel(), a.reshape(-1))
        assert_equal(a.ravel(order='A'), a.reshape(-1, order='A'))

        a = np.array([[0, 1], [2, 3]])[::-1, :]
        assert_equal(a.ravel(), [2, 3, 0, 1])
        assert_equal(a.ravel(order='C'), [2, 3, 0, 1])
        assert_equal(a.ravel(order='F'), [2, 0, 3, 1])
        assert_equal(a.ravel(order='A'), [2, 3, 0, 1])
        # 'K' doesn't reverse the axes of negative strides
        assert_equal(a.ravel(order='K'), [2, 3, 0, 1])
        assert_(a.ravel(order='K').flags.owndata)

        # Test simple 1-d copy behaviour:
        a = np.arange(10)[::2]
        assert_(a.ravel('K').flags.owndata)
        assert_(a.ravel('C').flags.owndata)
        assert_(a.ravel('F').flags.owndata)

        # Not contiguous and 1-sized axis with non matching stride
        a = np.arange(2**3 * 2)[::2]
        a = a.reshape(2, 1, 2, 2).swapaxes(-1, -2)
        strides = list(a.strides)
        strides[1] = 123
        a.strides = strides
        assert_(a.ravel(order='K').flags.owndata)
        assert_equal(a.ravel('K'), np.arange(0, 15, 2))

        # contiguous and 1-sized axis with non matching stride works:
        a = np.arange(2**3)
        a = a.reshape(2, 1, 2, 2).swapaxes(-1, -2)
        strides = list(a.strides)
        strides[1] = 123
        a.strides = strides
        assert_(np.may_share_memory(a.ravel(order='K'), a))
        assert_equal(a.ravel(order='K'), np.arange(2**3))

        # Test negative strides (not very interesting since non-contiguous):
        a = np.arange(4)[::-1].reshape(2, 2)
        assert_(a.ravel(order='C').flags.owndata)
        assert_(a.ravel(order='K').flags.owndata)
        assert_equal(a.ravel('C'), [3, 2, 1, 0])
        assert_equal(a.ravel('K'), [3, 2, 1, 0])

        # 1-element tidy strides test:
        a = np.array([[1]])
        a.strides = (123, 432)
        # If the following stride is not 8, NPY_RELAXED_STRIDES_DEBUG is
        # messing them up on purpose:
        if np.ones(1).strides == (8,):
            assert_(np.may_share_memory(a.ravel('K'), a))
            assert_equal(a.ravel('K').strides, (a.dtype.itemsize,))

        for order in ('C', 'F', 'A', 'K'):
            # 0-d corner case:
            a = np.array(0)
            assert_equal(a.ravel(order), [0])
            assert_(np.may_share_memory(a.ravel(order), a))

        # Test that certain non-inplace ravels work right (mostly) for 'K':
        b = np.arange(2**4 * 2)[::2].reshape(2, 2, 2, 2)
        a = b[..., ::2]
        assert_equal(a.ravel('K'), [0, 4, 8, 12, 16, 20, 24, 28])
        assert_equal(a.ravel('C'), [0, 4, 8, 12, 16, 20, 24, 28])
        assert_equal(a.ravel('A'), [0, 4, 8, 12, 16, 20, 24, 28])
        assert_equal(a.ravel('F'), [0, 16, 8, 24, 4, 20, 12, 28])

        a = b[::2, ...]
        assert_equal(a.ravel('K'), [0, 2, 4, 6, 8, 10, 12, 14])
        assert_equal(a.ravel('C'), [0, 2, 4, 6, 8, 10, 12, 14])
        assert_equal(a.ravel('A'), [0, 2, 4, 6, 8, 10, 12, 14])
        assert_equal(a.ravel('F'), [0, 8, 4, 12, 2, 10, 6, 14])

    def test_ravel_subclass(self):
        class ArraySubclass(np.ndarray):
            pass

        a = np.arange(10).view(ArraySubclass)
        assert_(isinstance(a.ravel('C'), ArraySubclass))
        assert_(isinstance(a.ravel('F'), ArraySubclass))
        assert_(isinstance(a.ravel('A'), ArraySubclass))
        assert_(isinstance(a.ravel('K'), ArraySubclass))

        a = np.arange(10)[::2].view(ArraySubclass)
        assert_(isinstance(a.ravel('C'), ArraySubclass))
        assert_(isinstance(a.ravel('F'), ArraySubclass))
        assert_(isinstance(a.ravel('A'), ArraySubclass))
        assert_(isinstance(a.ravel('K'), ArraySubclass))

    def test_swapaxes(self):
        a = np.arange(1*2*3*4).reshape(1, 2, 3, 4).copy()
        idx = np.indices(a.shape)
        assert_(a.flags['OWNDATA'])
        b = a.copy()
        # check exceptions
        assert_raises(np.AxisError, a.swapaxes, -5, 0)
        assert_raises(np.AxisError, a.swapaxes, 4, 0)
        assert_raises(np.AxisError, a.swapaxes, 0, -5)
        assert_raises(np.AxisError, a.swapaxes, 0, 4)

        for i in range(-4, 4):
            for j in range(-4, 4):
                for k, src in enumerate((a, b)):
                    c = src.swapaxes(i, j)
                    # check shape
                    shape = list(src.shape)
                    shape[i] = src.shape[j]
                    shape[j] = src.shape[i]
                    assert_equal(c.shape, shape, str((i, j, k)))
                    # check array contents
                    i0, i1, i2, i3 = [dim-1 for dim in c.shape]
                    j0, j1, j2, j3 = [dim-1 for dim in src.shape]
                    assert_equal(src[idx[j0], idx[j1], idx[j2], idx[j3]],
                                 c[idx[i0], idx[i1], idx[i2], idx[i3]],
                                 str((i, j, k)))
                    # check a view is always returned, gh-5260
                    assert_(not c.flags['OWNDATA'], str((i, j, k)))
                    # check on non-contiguous input array
                    if k == 1:
                        b = c

    def test_conjugate(self):
        a = np.array([1-1j, 1+1j, 23+23.0j])
        ac = a.conj()
        assert_equal(a.real, ac.real)
        assert_equal(a.imag, -ac.imag)
        assert_equal(ac, a.conjugate())
        assert_equal(ac, np.conjugate(a))

        a = np.array([1-1j, 1+1j, 23+23.0j], 'F')
        ac = a.conj()
        assert_equal(a.real, ac.real)
        assert_equal(a.imag, -ac.imag)
        assert_equal(ac, a.conjugate())
        assert_equal(ac, np.conjugate(a))

        a = np.array([1, 2, 3])
        ac = a.conj()
        assert_equal(a, ac)
        assert_equal(ac, a.conjugate())
        assert_equal(ac, np.conjugate(a))

        a = np.array([1.0, 2.0, 3.0])
        ac = a.conj()
        assert_equal(a, ac)
        assert_equal(ac, a.conjugate())
        assert_equal(ac, np.conjugate(a))

        a = np.array([1-1j, 1+1j, 1, 2.0], object)
        ac = a.conj()
        assert_equal(ac, [k.conjugate() for k in a])
        assert_equal(ac, a.conjugate())
        assert_equal(ac, np.conjugate(a))

        a = np.array([1-1j, 1, 2.0, 'f'], object)
        assert_raises(TypeError, lambda: a.conj())
        assert_raises(TypeError, lambda: a.conjugate())

    def test_conjugate_out(self):
        # Minimal test for the out argument being passed on correctly
        # NOTE: The ability to pass `out` is currently undocumented!
        a = np.array([1-1j, 1+1j, 23+23.0j])
        out = np.empty_like(a)
        res = a.conjugate(out)
        assert res is out
        assert_array_equal(out, a.conjugate())

    def test__complex__(self):
        dtypes = ['i1', 'i2', 'i4', 'i8',
                  'u1', 'u2', 'u4', 'u8',
                  'f', 'd', 'g', 'F', 'D', 'G',
                  '?', 'O']
        for dt in dtypes:
            a = np.array(7, dtype=dt)
            b = np.array([7], dtype=dt)
            c = np.array([[[[[7]]]]], dtype=dt)

            msg = 'dtype: {0}'.format(dt)
            ap = complex(a)
            assert_equal(ap, a, msg)
            bp = complex(b)
            assert_equal(bp, b, msg)
            cp = complex(c)
            assert_equal(cp, c, msg)

    def test__complex__should_not_work(self):
        dtypes = ['i1', 'i2', 'i4', 'i8',
                  'u1', 'u2', 'u4', 'u8',
                  'f', 'd', 'g', 'F', 'D', 'G',
                  '?', 'O']
        for dt in dtypes:
            a = np.array([1, 2, 3], dtype=dt)
            assert_raises(TypeError, complex, a)

        dt = np.dtype([('a', 'f8'), ('b', 'i1')])
        b = np.array((1.0, 3), dtype=dt)
        assert_raises(TypeError, complex, b)

        c = np.array([(1.0, 3), (2e-3, 7)], dtype=dt)
        assert_raises(TypeError, complex, c)

        d = np.array('1+1j')
        assert_raises(TypeError, complex, d)

        e = np.array(['1+1j'], 'U')
        assert_raises(TypeError, complex, e)

class TestCequenceMethods:
    def test_array_contains(self):
        assert_(4.0 in np.arange(16.).reshape(4,4))
        assert_(20.0 not in np.arange(16.).reshape(4,4))

class TestBinop:
    def test_inplace(self):
        # test refcount 1 inplace conversion
        assert_array_almost_equal(np.array([0.5]) * np.array([1.0, 2.0]),
                                  [0.5, 1.0])

        d = np.array([0.5, 0.5])[::2]
        assert_array_almost_equal(d * (d * np.array([1.0, 2.0])),
                                  [0.25, 0.5])

        a = np.array([0.5])
        b = np.array([0.5])
        c = a + b
        c = a - b
        c = a * b
        c = a / b
        assert_equal(a, b)
        assert_almost_equal(c, 1.)

        c = a + b * 2. / b * a - a / b
        assert_equal(a, b)
        assert_equal(c, 0.5)

        # true divide
        a = np.array([5])
        b = np.array([3])
        c = (a * a) / b

        assert_almost_equal(c, 25 / 3)
        assert_equal(a, 5)
        assert_equal(b, 3)

    # ndarray.__rop__ always calls ufunc
    # ndarray.__iop__ always calls ufunc
    # ndarray.__op__, __rop__:
    #   - defer if other has __array_ufunc__ and it is None
    #           or other is not a subclass and has higher array priority
    #   - else, call ufunc
    def test_ufunc_binop_interaction(self):
        # Python method name (without underscores)
        #   -> (numpy ufunc, has_in_place_version, preferred_dtype)
        ops = {
            'add':      (np.add, True, float),
            'sub':      (np.subtract, True, float),
            'mul':      (np.multiply, True, float),
            'truediv':  (np.true_divide, True, float),
            'floordiv': (np.floor_divide, True, float),
            'mod':      (np.remainder, True, float),
            'divmod':   (np.divmod, False, float),
            'pow':      (np.power, True, int),
            'lshift':   (np.left_shift, True, int),
            'rshift':   (np.right_shift, True, int),
            'and':      (np.bitwise_and, True, int),
            'xor':      (np.bitwise_xor, True, int),
            'or':       (np.bitwise_or, True, int),
            'matmul':   (np.matmul, False, float),
            # 'ge':       (np.less_equal, False),
            # 'gt':       (np.less, False),
            # 'le':       (np.greater_equal, False),
            # 'lt':       (np.greater, False),
            # 'eq':       (np.equal, False),
            # 'ne':       (np.not_equal, False),
        }

        class Coerced(Exception):
            pass

        def array_impl(self):
            raise Coerced

        def op_impl(self, other):
            return "forward"

        def rop_impl(self, other):
            return "reverse"

        def iop_impl(self, other):
            return "in-place"

        def array_ufunc_impl(self, ufunc, method, *args, **kwargs):
            return ("__array_ufunc__", ufunc, method, args, kwargs)

        # Create an object with the given base, in the given module, with a
        # bunch of placeholder __op__ methods, and optionally a
        # __array_ufunc__ and __array_priority__.
        def make_obj(base, array_priority=False, array_ufunc=False,
                     alleged_module="__main__"):
            class_namespace = {"__array__": array_impl}
            if array_priority is not False:
                class_namespace["__array_priority__"] = array_priority
            for op in ops:
                class_namespace["__{0}__".format(op)] = op_impl
                class_namespace["__r{0}__".format(op)] = rop_impl
                class_namespace["__i{0}__".format(op)] = iop_impl
            if array_ufunc is not False:
                class_namespace["__array_ufunc__"] = array_ufunc
            eval_namespace = {"base": base,
                              "class_namespace": class_namespace,
                              "__name__": alleged_module,
                              }
            MyType = eval("type('MyType', (base,), class_namespace)",
                          eval_namespace)
            if issubclass(MyType, np.ndarray):
                # Use this range to avoid special case weirdnesses around
                # divide-by-0, pow(x, 2), overflow due to pow(big, big), etc.
                return np.arange(3, 7).reshape(2, 2).view(MyType)
            else:
                return MyType()

        def check(obj, binop_override_expected, ufunc_override_expected,
                  inplace_override_expected, check_scalar=True):
            for op, (ufunc, has_inplace, dtype) in ops.items():
                err_msg = ('op: %s, ufunc: %s, has_inplace: %s, dtype: %s'
                           % (op, ufunc, has_inplace, dtype))
                check_objs = [np.arange(3, 7, dtype=dtype).reshape(2, 2)]
                if check_scalar:
                    check_objs.append(check_objs[0][0])
                for arr in check_objs:
                    arr_method = getattr(arr, "__{0}__".format(op))

                    def first_out_arg(result):
                        if op == "divmod":
                            assert_(isinstance(result, tuple))
                            return result[0]
                        else:
                            return result

                    # arr __op__ obj
                    if binop_override_expected:
                        assert_equal(arr_method(obj), NotImplemented, err_msg)
                    elif ufunc_override_expected:
                        assert_equal(arr_method(obj)[0], "__array_ufunc__",
                                     err_msg)
                    else:
                        if (isinstance(obj, np.ndarray) and
                            (type(obj).__array_ufunc__ is
                             np.ndarray.__array_ufunc__)):
                            # __array__ gets ignored
                            res = first_out_arg(arr_method(obj))
                            assert_(res.__class__ is obj.__class__, err_msg)
                        else:
                            assert_raises((TypeError, Coerced),
                                          arr_method, obj, err_msg=err_msg)
                    # obj __op__ arr
                    arr_rmethod = getattr(arr, "__r{0}__".format(op))
                    if ufunc_override_expected:
                        res = arr_rmethod(obj)
                        assert_equal(res[0], "__array_ufunc__",
                                     err_msg=err_msg)
                        assert_equal(res[1], ufunc, err_msg=err_msg)
                    else:
                        if (isinstance(obj, np.ndarray) and
                                (type(obj).__array_ufunc__ is
                                 np.ndarray.__array_ufunc__)):
                            # __array__ gets ignored
                            res = first_out_arg(arr_rmethod(obj))
                            assert_(res.__class__ is obj.__class__, err_msg)
                        else:
                            # __array_ufunc__ = "asdf" creates a TypeError
                            assert_raises((TypeError, Coerced),
                                          arr_rmethod, obj, err_msg=err_msg)

                    # arr __iop__ obj
                    # array scalars don't have in-place operators
                    if has_inplace and isinstance(arr, np.ndarray):
                        arr_imethod = getattr(arr, "__i{0}__".format(op))
                        if inplace_override_expected:
                            assert_equal(arr_method(obj), NotImplemented,
                                         err_msg=err_msg)
                        elif ufunc_override_expected:
                            res = arr_imethod(obj)
                            assert_equal(res[0], "__array_ufunc__", err_msg)
                            assert_equal(res[1], ufunc, err_msg)
                            assert_(type(res[-1]["out"]) is tuple, err_msg)
                            assert_(res[-1]["out"][0] is arr, err_msg)
                        else:
                            if (isinstance(obj, np.ndarray) and
                                    (type(obj).__array_ufunc__ is
                                    np.ndarray.__array_ufunc__)):
                                # __array__ gets ignored
                                assert_(arr_imethod(obj) is arr, err_msg)
                            else:
                                assert_raises((TypeError, Coerced),
                                              arr_imethod, obj,
                                              err_msg=err_msg)

                    op_fn = getattr(operator, op, None)
                    if op_fn is None:
                        op_fn = getattr(operator, op + "_", None)
                    if op_fn is None:
                        op_fn = getattr(builtins, op)
                    assert_equal(op_fn(obj, arr), "forward", err_msg)
                    if not isinstance(obj, np.ndarray):
                        if binop_override_expected:
                            assert_equal(op_fn(arr, obj), "reverse", err_msg)
                        elif ufunc_override_expected:
                            assert_equal(op_fn(arr, obj)[0], "__array_ufunc__",
                                         err_msg)
                    if ufunc_override_expected:
                        assert_equal(ufunc(obj, arr)[0], "__array_ufunc__",
                                     err_msg)

        # No array priority, no array_ufunc -> nothing called
        check(make_obj(object), False, False, False)
        # Negative array priority, no array_ufunc -> nothing called
        # (has to be very negative, because scalar priority is -1000000.0)
        check(make_obj(object, array_priority=-2**30), False, False, False)
        # Positive array priority, no array_ufunc -> binops and iops only
        check(make_obj(object, array_priority=1), True, False, True)
        # ndarray ignores array_priority for ndarray subclasses
        check(make_obj(np.ndarray, array_priority=1), False, False, False,
              check_scalar=False)
        # Positive array_priority and array_ufunc -> array_ufunc only
        check(make_obj(object, array_priority=1,
                       array_ufunc=array_ufunc_impl), False, True, False)
        check(make_obj(np.ndarray, array_priority=1,
                       array_ufunc=array_ufunc_impl), False, True, False)
        # array_ufunc set to None -> defer binops only
        check(make_obj(object, array_ufunc=None), True, False, False)
        check(make_obj(np.ndarray, array_ufunc=None), True, False, False,
              check_scalar=False)

    @pytest.mark.parametrize("priority", [None, "runtime error"])
    def test_ufunc_binop_bad_array_priority(self, priority):
        # Mainly checks that this does not crash.  The second array has a lower
        # priority than -1 ("error value").  If the __radd__ actually exists,
        # bad things can happen (I think via the scalar paths).
        # In principle both of these can probably just be errors in the future.
        class BadPriority:
            @property
            def __array_priority__(self):
                if priority == "runtime error":
                    raise RuntimeError("RuntimeError in __array_priority__!")
                return priority

            def __radd__(self, other):
                return "result"

        class LowPriority(np.ndarray):
            __array_priority__ = -1000

        # Priority failure uses the same as scalars (smaller -1000).  So the
        # LowPriority wins with 'result' for each element (inner operation).
        res = np.arange(3).view(LowPriority) + BadPriority()
        assert res.shape == (3,)
        assert res[0] == 'result'


    def test_ufunc_override_normalize_signature(self):
        # gh-5674
        class SomeClass:
            def __array_ufunc__(self, ufunc, method, *inputs, **kw):
                return kw

        a = SomeClass()
        kw = np.add(a, [1])
        assert_('sig' not in kw and 'signature' not in kw)
        kw = np.add(a, [1], sig='ii->i')
        assert_('sig' not in kw and 'signature' in kw)
        assert_equal(kw['signature'], 'ii->i')
        kw = np.add(a, [1], signature='ii->i')
        assert_('sig' not in kw and 'signature' in kw)
        assert_equal(kw['signature'], 'ii->i')

    def test_array_ufunc_index(self):
        # Check that index is set appropriately, also if only an output
        # is passed on (latter is another regression tests for github bug 4753)
        # This also checks implicitly that 'out' is always a tuple.
        class CheckIndex:
            def __array_ufunc__(self, ufunc, method, *inputs, **kw):
                for i, a in enumerate(inputs):
                    if a is self:
                        return i
                # calls below mean we must be in an output.
                for j, a in enumerate(kw['out']):
                    if a is self:
                        return (j,)

        a = CheckIndex()
        dummy = np.arange(2.)
        # 1 input, 1 output
        assert_equal(np.sin(a), 0)
        assert_equal(np.sin(dummy, a), (0,))
        assert_equal(np.sin(dummy, out=a), (0,))
        assert_equal(np.sin(dummy, out=(a,)), (0,))
        assert_equal(np.sin(a, a), 0)
        assert_equal(np.sin(a, out=a), 0)
        assert_equal(np.sin(a, out=(a,)), 0)
        # 1 input, 2 outputs
        assert_equal(np.modf(dummy, a), (0,))
        assert_equal(np.modf(dummy, None, a), (1,))
        assert_equal(np.modf(dummy, dummy, a), (1,))
        assert_equal(np.modf(dummy, out=(a, None)), (0,))
        assert_equal(np.modf(dummy, out=(a, dummy)), (0,))
        assert_equal(np.modf(dummy, out=(None, a)), (1,))
        assert_equal(np.modf(dummy, out=(dummy, a)), (1,))
        assert_equal(np.modf(a, out=(dummy, a)), 0)
        with assert_raises(TypeError):
            # Out argument must be tuple, since there are multiple outputs
            np.modf(dummy, out=a)

        assert_raises(ValueError, np.modf, dummy, out=(a,))

        # 2 inputs, 1 output
        assert_equal(np.add(a, dummy), 0)
        assert_equal(np.add(dummy, a), 1)
        assert_equal(np.add(dummy, dummy, a), (0,))
        assert_equal(np.add(dummy, a, a), 1)
        assert_equal(np.add(dummy, dummy, out=a), (0,))
        assert_equal(np.add(dummy, dummy, out=(a,)), (0,))
        assert_equal(np.add(a, dummy, out=a), 0)

    def test_out_override(self):
        # regression test for github bug 4753
        class OutClass(np.ndarray):
            def __array_ufunc__(self, ufunc, method, *inputs, **kw):
                if 'out' in kw:
                    tmp_kw = kw.copy()
                    tmp_kw.pop('out')
                    func = getattr(ufunc, method)
                    kw['out'][0][...] = func(*inputs, **tmp_kw)

        A = np.array([0]).view(OutClass)
        B = np.array([5])
        C = np.array([6])
        np.multiply(C, B, A)
        assert_equal(A[0], 30)
        assert_(isinstance(A, OutClass))
        A[0] = 0
        np.multiply(C, B, out=A)
        assert_equal(A[0], 30)
        assert_(isinstance(A, OutClass))

    def test_pow_override_with_errors(self):
        # regression test for gh-9112
        class PowerOnly(np.ndarray):
            def __array_ufunc__(self, ufunc, method, *inputs, **kw):
                if ufunc is not np.power:
                    raise NotImplementedError
                return "POWER!"
        # explicit cast to float, to ensure the fast power path is taken.
        a = np.array(5., dtype=np.float64).view(PowerOnly)
        assert_equal(a ** 2.5, "POWER!")
        with assert_raises(NotImplementedError):
            a ** 0.5
        with assert_raises(NotImplementedError):
            a ** 0
        with assert_raises(NotImplementedError):
            a ** 1
        with assert_raises(NotImplementedError):
            a ** -1
        with assert_raises(NotImplementedError):
            a ** 2

    def test_pow_array_object_dtype(self):
        # test pow on arrays of object dtype
        class SomeClass:
            def __init__(self, num=None):
                self.num = num

            # want to ensure a fast pow path is not taken
            def __mul__(self, other):
                raise AssertionError('__mul__ should not be called')

            def __div__(self, other):
                raise AssertionError('__div__ should not be called')

            def __pow__(self, exp):
                return SomeClass(num=self.num ** exp)

            def __eq__(self, other):
                if isinstance(other, SomeClass):
                    return self.num == other.num

            __rpow__ = __pow__

        def pow_for(exp, arr):
            return np.array([x ** exp for x in arr])

        obj_arr = np.array([SomeClass(1), SomeClass(2), SomeClass(3)])

        assert_equal(obj_arr ** 0.5, pow_for(0.5, obj_arr))
        assert_equal(obj_arr ** 0, pow_for(0, obj_arr))
        assert_equal(obj_arr ** 1, pow_for(1, obj_arr))
        assert_equal(obj_arr ** -1, pow_for(-1, obj_arr))
        assert_equal(obj_arr ** 2, pow_for(2, obj_arr))

    def test_pos_array_ufunc_override(self):
        class A(np.ndarray):
            def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
                return getattr(ufunc, method)(*[i.view(np.ndarray) for
                                                i in inputs], **kwargs)
        tst = np.array('foo').view(A)
        with assert_raises(TypeError):
            +tst


class TestTemporaryElide:
    # elision is only triggered on relatively large arrays

    def test_extension_incref_elide(self):
        # test extension (e.g. cython) calling PyNumber_* slots without
        # increasing the reference counts
        #
        # def incref_elide(a):
        #    d = input.copy() # refcount 1
        #    return d, d + d # PyNumber_Add without increasing refcount
        from numpy.core._multiarray_tests import incref_elide
        d = np.ones(100000)
        orig, res = incref_elide(d)
        d + d
        # the return original should not be changed to an inplace operation
        assert_array_equal(orig, d)
        assert_array_equal(res, d + d)

    def test_extension_incref_elide_stack(self):
        # scanning if the refcount == 1 object is on the python stack to check
        # that we are called directly from python is flawed as object may still
        # be above the stack pointer and we have no access to the top of it
        #
        # def incref_elide_l(d):
        #    return l[4] + l[4] # PyNumber_Add without increasing refcount
        from numpy.core._multiarray_tests import incref_elide_l
        # padding with 1 makes sure the object on the stack is not overwritten
        l = [1, 1, 1, 1, np.ones(100000)]
        res = incref_elide_l(l)
        # the return original should not be changed to an inplace operation
        assert_array_equal(l[4], np.ones(100000))
        assert_array_equal(res, l[4] + l[4])

    def test_temporary_with_cast(self):
        # check that we don't elide into a temporary which would need casting
        d = np.ones(200000, dtype=np.int64)
        assert_equal(((d + d) + 2**222).dtype, np.dtype('O'))

        r = ((d + d) / 2)
        assert_equal(r.dtype, np.dtype('f8'))

        r = np.true_divide((d + d), 2)
        assert_equal(r.dtype, np.dtype('f8'))

        r = ((d + d) / 2.)
        assert_equal(r.dtype, np.dtype('f8'))

        r = ((d + d) // 2)
        assert_equal(r.dtype, np.dtype(np.int64))

        # commutative elision into the astype result
        f = np.ones(100000, dtype=np.float32)
        assert_equal(((f + f) + f.astype(np.float64)).dtype, np.dtype('f8'))

        # no elision into lower type
        d = f.astype(np.float64)
        assert_equal(((f + f) + d).dtype, d.dtype)
        l = np.ones(100000, dtype=np.longdouble)
        assert_equal(((d + d) + l).dtype, l.dtype)

        # test unary abs with different output dtype
        for dt in (np.complex64, np.complex128, np.clongdouble):
            c = np.ones(100000, dtype=dt)
            r = abs(c * 2.0)
            assert_equal(r.dtype, np.dtype('f%d' % (c.itemsize // 2)))

    def test_elide_broadcast(self):
        # test no elision on broadcast to higher dimension
        # only triggers elision code path in debug mode as triggering it in
        # normal mode needs 256kb large matching dimension, so a lot of memory
        d = np.ones((2000, 1), dtype=int)
        b = np.ones((2000), dtype=bool)
        r = (1 - d) + b
        assert_equal(r, 1)
        assert_equal(r.shape, (2000, 2000))

    def test_elide_scalar(self):
        # check inplace op does not create ndarray from scalars
        a = np.bool_()
        assert_(type(~(a & a)) is np.bool_)

    def test_elide_scalar_readonly(self):
        # The imaginary part of a real array is readonly. This needs to go
        # through fast_scalar_power which is only called for powers of
        # +1, -1, 0, 0.5, and 2, so use 2. Also need valid refcount for
        # elision which can be gotten for the imaginary part of a real
        # array. Should not error.
        a = np.empty(100000, dtype=np.float64)
        a.imag ** 2

    def test_elide_readonly(self):
        # don't try to elide readonly temporaries
        r = np.asarray(np.broadcast_to(np.zeros(1), 100000).flat) * 0.0
        assert_equal(r, 0)

    def test_elide_updateifcopy(self):
        a = np.ones(2**20)[::2]
        b = a.flat.__array__() + 1
        del b
        assert_equal(a, 1)


class TestCAPI:
    def test_IsPythonScalar(self):
        from numpy.core._multiarray_tests import IsPythonScalar
        assert_(IsPythonScalar(b'foobar'))
        assert_(IsPythonScalar(1))
        assert_(IsPythonScalar(2**80))
        assert_(IsPythonScalar(2.))
        assert_(IsPythonScalar("a"))

    @pytest.mark.parametrize("converter",
             [_multiarray_tests.run_scalar_intp_converter,
              _multiarray_tests.run_scalar_intp_from_sequence])
    def test_intp_sequence_converters(self, converter):
        # Test simple values (-1 is special for error return paths)
        assert converter(10) == (10,)
        assert converter(-1) == (-1,)
        # A 0-D array looks a bit like a sequence but must take the integer
        # path:
        assert converter(np.array(123)) == (123,)
        # Test simple sequences (intp_from_sequence only supports length 1):
        assert converter((10,)) == (10,)
        assert converter(np.array([11])) == (11,)

    @pytest.mark.parametrize("converter",
             [_multiarray_tests.run_scalar_intp_converter,
              _multiarray_tests.run_scalar_intp_from_sequence])
    @pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8),
            reason="PyPy bug in error formatting")
    def test_intp_sequence_converters_errors(self, converter):
        with pytest.raises(TypeError,
                match="expected a sequence of integers or a single integer, "):
            converter(object())
        with pytest.raises(TypeError,
                match="expected a sequence of integers or a single integer, "
                      "got '32.0'"):
            converter(32.)
        with pytest.raises(TypeError,
                match="'float' object cannot be interpreted as an integer"):
            converter([32.])
        with pytest.raises(ValueError,
                match="Maximum allowed dimension"):
            # These converters currently convert overflows to a ValueError
            converter(2**64)


class TestSubscripting:
    def test_test_zero_rank(self):
        x = np.array([1, 2, 3])
        assert_(isinstance(x[0], np.int_))
        assert_(type(x[0, ...]) is np.ndarray)


class TestPickling:
    @pytest.mark.skipif(pickle.HIGHEST_PROTOCOL >= 5,
                        reason=('this tests the error messages when trying to'
                                'protocol 5 although it is not available'))
    def test_correct_protocol5_error_message(self):
        array = np.arange(10)

    def test_record_array_with_object_dtype(self):
        my_object = object()

        arr_with_object = np.array(
                [(my_object, 1, 2.0)],
                dtype=[('a', object), ('b', int), ('c', float)])
        arr_without_object = np.array(
                [('xxx', 1, 2.0)],
                dtype=[('a', str), ('b', int), ('c', float)])

        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            depickled_arr_with_object = pickle.loads(
                    pickle.dumps(arr_with_object, protocol=proto))
            depickled_arr_without_object = pickle.loads(
                    pickle.dumps(arr_without_object, protocol=proto))

            assert_equal(arr_with_object.dtype,
                         depickled_arr_with_object.dtype)
            assert_equal(arr_without_object.dtype,
                         depickled_arr_without_object.dtype)

    @pytest.mark.skipif(pickle.HIGHEST_PROTOCOL < 5,
                        reason="requires pickle protocol 5")
    def test_f_contiguous_array(self):
        f_contiguous_array = np.array([[1, 2, 3], [4, 5, 6]], order='F')
        buffers = []

        # When using pickle protocol 5, Fortran-contiguous arrays can be
        # serialized using out-of-band buffers
        bytes_string = pickle.dumps(f_contiguous_array, protocol=5,
                                    buffer_callback=buffers.append)

        assert len(buffers) > 0

        depickled_f_contiguous_array = pickle.loads(bytes_string,
                                                    buffers=buffers)

        assert_equal(f_contiguous_array, depickled_f_contiguous_array)

    def test_non_contiguous_array(self):
        non_contiguous_array = np.arange(12).reshape(3, 4)[:, :2]
        assert not non_contiguous_array.flags.c_contiguous
        assert not non_contiguous_array.flags.f_contiguous

        # make sure non-contiguous arrays can be pickled-depickled
        # using any protocol
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            depickled_non_contiguous_array = pickle.loads(
                    pickle.dumps(non_contiguous_array, protocol=proto))

            assert_equal(non_contiguous_array, depickled_non_contiguous_array)

    def test_roundtrip(self):
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            carray = np.array([[2, 9], [7, 0], [3, 8]])
            DATA = [
                carray,
                np.transpose(carray),
                np.array([('xxx', 1, 2.0)], dtype=[('a', (str, 3)), ('b', int),
                                                   ('c', float)])
            ]

            refs = [weakref.ref(a) for a in DATA]
            for a in DATA:
                assert_equal(
                        a, pickle.loads(pickle.dumps(a, protocol=proto)),
                        err_msg="%r" % a)
            del a, DATA, carray
            break_cycles()
            # check for reference leaks (gh-12793)
            for ref in refs:
                assert ref() is None

    def _loads(self, obj):
        return pickle.loads(obj, encoding='latin1')

    # version 0 pickles, using protocol=2 to pickle
    # version 0 doesn't have a version field
    def test_version0_int8(self):
        s = b'\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x04\x85cnumpy\ndtype\nq\x04U\x02i1K\x00K\x01\x87Rq\x05(U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x04\x01\x02\x03\x04tb.'
        a = np.array([1, 2, 3, 4], dtype=np.int8)
        p = self._loads(s)
        assert_equal(a, p)

    def test_version0_float32(self):
        s = b'\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x04\x85cnumpy\ndtype\nq\x04U\x02f4K\x00K\x01\x87Rq\x05(U\x01<NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x10\x00\x00\x80?\x00\x00\x00@\x00\x00@@\x00\x00\x80@tb.'
        a = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
        p = self._loads(s)
        assert_equal(a, p)

    def test_version0_object(self):
        s = b'\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x02\x85cnumpy\ndtype\nq\x04U\x02O8K\x00K\x01\x87Rq\x05(U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89]q\x06(}q\x07U\x01aK\x01s}q\x08U\x01bK\x02setb.'
        a = np.array([{'a': 1}, {'b': 2}])
        p = self._loads(s)
        assert_equal(a, p)

    # version 1 pickles, using protocol=2 to pickle
    def test_version1_int8(self):
        s = b'\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x01K\x04\x85cnumpy\ndtype\nq\x04U\x02i1K\x00K\x01\x87Rq\x05(K\x01U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x04\x01\x02\x03\x04tb.'
        a = np.array([1, 2, 3, 4], dtype=np.int8)
        p = self._loads(s)
        assert_equal(a, p)

    def test_version1_float32(self):
        s = b'\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x01K\x04\x85cnumpy\ndtype\nq\x04U\x02f4K\x00K\x01\x87Rq\x05(K\x01U\x01<NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x10\x00\x00\x80?\x00\x00\x00@\x00\x00@@\x00\x00\x80@tb.'
        a = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
        p = self._loads(s)
        assert_equal(a, p)

    def test_version1_object(self):
        s = b'\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x01K\x02\x85cnumpy\ndtype\nq\x04U\x02O8K\x00K\x01\x87Rq\x05(K\x01U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89]q\x06(}q\x07U\x01aK\x01s}q\x08U\x01bK\x02setb.'
        a = np.array([{'a': 1}, {'b': 2}])
        p = self._loads(s)
        assert_equal(a, p)

    def test_subarray_int_shape(self):
        s = b"cnumpy.core.multiarray\n_reconstruct\np0\n(cnumpy\nndarray\np1\n(I0\ntp2\nS'b'\np3\ntp4\nRp5\n(I1\n(I1\ntp6\ncnumpy\ndtype\np7\n(S'V6'\np8\nI0\nI1\ntp9\nRp10\n(I3\nS'|'\np11\nN(S'a'\np12\ng3\ntp13\n(dp14\ng12\n(g7\n(S'V4'\np15\nI0\nI1\ntp16\nRp17\n(I3\nS'|'\np18\n(g7\n(S'i1'\np19\nI0\nI1\ntp20\nRp21\n(I3\nS'|'\np22\nNNNI-1\nI-1\nI0\ntp23\nb(I2\nI2\ntp24\ntp25\nNNI4\nI1\nI0\ntp26\nbI0\ntp27\nsg3\n(g7\n(S'V2'\np28\nI0\nI1\ntp29\nRp30\n(I3\nS'|'\np31\n(g21\nI2\ntp32\nNNI2\nI1\nI0\ntp33\nbI4\ntp34\nsI6\nI1\nI0\ntp35\nbI00\nS'\\x01\\x01\\x01\\x01\\x01\\x02'\np36\ntp37\nb."
        a = np.array([(1, (1, 2))], dtype=[('a', 'i1', (2, 2)), ('b', 'i1', 2)])
        p = self._loads(s)
        assert_equal(a, p)

    def test_datetime64_byteorder(self):
        original = np.array([['2015-02-24T00:00:00.000000000']], dtype='datetime64[ns]')

        original_byte_reversed = original.copy(order='K')
        original_byte_reversed.dtype = original_byte_reversed.dtype.newbyteorder('S')
        original_byte_reversed.byteswap(inplace=True)

        new = pickle.loads(pickle.dumps(original_byte_reversed))

        assert_equal(original.dtype, new.dtype)


class TestFancyIndexing:
    def test_list(self):
        x = np.ones((1, 1))
        x[:, [0]] = 2.0
        assert_array_equal(x, np.array([[2.0]]))

        x = np.ones((1, 1, 1))
        x[:, :, [0]] = 2.0
        assert_array_equal(x, np.array([[[2.0]]]))

    def test_tuple(self):
        x = np.ones((1, 1))
        x[:, (0,)] = 2.0
        assert_array_equal(x, np.array([[2.0]]))
        x = np.ones((1, 1, 1))
        x[:, :, (0,)] = 2.0
        assert_array_equal(x, np.array([[[2.0]]]))

    def test_mask(self):
        x = np.array([1, 2, 3, 4])
        m = np.array([0, 1, 0, 0], bool)
        assert_array_equal(x[m], np.array([2]))

    def test_mask2(self):
        x = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
        m = np.array([0, 1], bool)
        m2 = np.array([[0, 1, 0, 0], [1, 0, 0, 0]], bool)
        m3 = np.array([[0, 1, 0, 0], [0, 0, 0, 0]], bool)
        assert_array_equal(x[m], np.array([[5, 6, 7, 8]]))
        assert_array_equal(x[m2], np.array([2, 5]))
        assert_array_equal(x[m3], np.array([2]))

    def test_assign_mask(self):
        x = np.array([1, 2, 3, 4])
        m = np.array([0, 1, 0, 0], bool)
        x[m] = 5
        assert_array_equal(x, np.array([1, 5, 3, 4]))

    def test_assign_mask2(self):
        xorig = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
        m = np.array([0, 1], bool)
        m2 = np.array([[0, 1, 0, 0], [1, 0, 0, 0]], bool)
        m3 = np.array([[0, 1, 0, 0], [0, 0, 0, 0]], bool)
        x = xorig.copy()
        x[m] = 10
        assert_array_equal(x, np.array([[1, 2, 3, 4], [10, 10, 10, 10]]))
        x = xorig.copy()
        x[m2] = 10
        assert_array_equal(x, np.array([[1, 10, 3, 4], [10, 6, 7, 8]]))
        x = xorig.copy()
        x[m3] = 10
        assert_array_equal(x, np.array([[1, 10, 3, 4], [5, 6, 7, 8]]))


class TestStringCompare:
    def test_string(self):
        g1 = np.array(["This", "is", "example"])
        g2 = np.array(["This", "was", "example"])
        assert_array_equal(g1 == g2, [g1[i] == g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 != g2, [g1[i] != g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 <= g2, [g1[i] <= g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 >= g2, [g1[i] >= g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 < g2, [g1[i] < g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 > g2, [g1[i] > g2[i] for i in [0, 1, 2]])

    def test_mixed(self):
        g1 = np.array(["spam", "spa", "spammer", "and eggs"])
        g2 = "spam"
        assert_array_equal(g1 == g2, [x == g2 for x in g1])
        assert_array_equal(g1 != g2, [x != g2 for x in g1])
        assert_array_equal(g1 < g2, [x < g2 for x in g1])
        assert_array_equal(g1 > g2, [x > g2 for x in g1])
        assert_array_equal(g1 <= g2, [x <= g2 for x in g1])
        assert_array_equal(g1 >= g2, [x >= g2 for x in g1])

    def test_unicode(self):
        g1 = np.array(["This", "is", "example"])
        g2 = np.array(["This", "was", "example"])
        assert_array_equal(g1 == g2, [g1[i] == g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 != g2, [g1[i] != g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 <= g2, [g1[i] <= g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 >= g2, [g1[i] >= g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 < g2,  [g1[i] < g2[i] for i in [0, 1, 2]])
        assert_array_equal(g1 > g2,  [g1[i] > g2[i] for i in [0, 1, 2]])

class TestArgmaxArgminCommon:

    sizes = [(), (3,), (3, 2), (2, 3),
             (3, 3), (2, 3, 4), (4, 3, 2),
             (1, 2, 3, 4), (2, 3, 4, 1),
             (3, 4, 1, 2), (4, 1, 2, 3),
             (64,), (128,), (256,)]

    @pytest.mark.parametrize("size, axis", itertools.chain(*[[(size, axis)
        for axis in list(range(-len(size), len(size))) + [None]]
        for size in sizes]))
    @pytest.mark.parametrize('method', [np.argmax, np.argmin])
    def test_np_argmin_argmax_keepdims(self, size, axis, method):

        arr = np.random.normal(size=size)

        # contiguous arrays
        if axis is None:
            new_shape = [1 for _ in range(len(size))]
        else:
            new_shape = list(size)
            new_shape[axis] = 1
        new_shape = tuple(new_shape)

        _res_orig = method(arr, axis=axis)
        res_orig = _res_orig.reshape(new_shape)
        res = method(arr, axis=axis, keepdims=True)
        assert_equal(res, res_orig)
        assert_(res.shape == new_shape)
        outarray = np.empty(res.shape, dtype=res.dtype)
        res1 = method(arr, axis=axis, out=outarray,
                            keepdims=True)
        assert_(res1 is outarray)
        assert_equal(res, outarray)

        if len(size) > 0:
            wrong_shape = list(new_shape)
            if axis is not None:
                wrong_shape[axis] = 2
            else:
                wrong_shape[0] = 2
            wrong_outarray = np.empty(wrong_shape, dtype=res.dtype)
            with pytest.raises(ValueError):
                method(arr.T, axis=axis,
                        out=wrong_outarray, keepdims=True)

        # non-contiguous arrays
        if axis is None:
            new_shape = [1 for _ in range(len(size))]
        else:
            new_shape = list(size)[::-1]
            new_shape[axis] = 1
        new_shape = tuple(new_shape)

        _res_orig = method(arr.T, axis=axis)
        res_orig = _res_orig.reshape(new_shape)
        res = method(arr.T, axis=axis, keepdims=True)
        assert_equal(res, res_orig)
        assert_(res.shape == new_shape)
        outarray = np.empty(new_shape[::-1], dtype=res.dtype)
        outarray = outarray.T
        res1 = method(arr.T, axis=axis, out=outarray,
                            keepdims=True)
        assert_(res1 is outarray)
        assert_equal(res, outarray)

        if len(size) > 0:
            # one dimension lesser for non-zero sized
            # array should raise an error
            with pytest.raises(ValueError):
                method(arr[0], axis=axis,
                        out=outarray, keepdims=True)

        if len(size) > 0:
            wrong_shape = list(new_shape)
            if axis is not None:
                wrong_shape[axis] = 2
            else:
                wrong_shape[0] = 2
            wrong_outarray = np.empty(wrong_shape, dtype=res.dtype)
            with pytest.raises(ValueError):
                method(arr.T, axis=axis,
                        out=wrong_outarray, keepdims=True)

    @pytest.mark.parametrize('method', ['max', 'min'])
    def test_all(self, method):
        a = np.random.normal(0, 1, (4, 5, 6, 7, 8))
        arg_method = getattr(a, 'arg' + method)
        val_method = getattr(a, method)
        for i in range(a.ndim):
            a_maxmin = val_method(i)
            aarg_maxmin = arg_method(i)
            axes = list(range(a.ndim))
            axes.remove(i)
            assert_(np.all(a_maxmin == aarg_maxmin.choose(
                                        *a.transpose(i, *axes))))

    @pytest.mark.parametrize('method', ['argmax', 'argmin'])
    def test_output_shape(self, method):
        # see also gh-616
        a = np.ones((10, 5))
        arg_method = getattr(a, method)
        # Check some simple shape mismatches
        out = np.ones(11, dtype=np.int_)
        assert_raises(ValueError, arg_method, -1, out)

        out = np.ones((2, 5), dtype=np.int_)
        assert_raises(ValueError, arg_method, -1, out)

        # these could be relaxed possibly (used to allow even the previous)
        out = np.ones((1, 10), dtype=np.int_)
        assert_raises(ValueError, arg_method, -1, out)

        out = np.ones(10, dtype=np.int_)
        arg_method(-1, out=out)
        assert_equal(out, arg_method(-1))

    @pytest.mark.parametrize('ndim', [0, 1])
    @pytest.mark.parametrize('method', ['argmax', 'argmin'])
    def test_ret_is_out(self, ndim, method):
        a = np.ones((4,) + (256,)*ndim)
        arg_method = getattr(a, method)
        out = np.empty((256,)*ndim, dtype=np.intp)
        ret = arg_method(axis=0, out=out)
        assert ret is out

    @pytest.mark.parametrize('np_array, method, idx, val',
        [(np.zeros, 'argmax', 5942, "as"),
         (np.ones, 'argmin', 6001, "0")])
    def test_unicode(self, np_array, method, idx, val):
        d = np_array(6031, dtype='<U9')
        arg_method = getattr(d, method)
        d[idx] = val
        assert_equal(arg_method(), idx)

    @pytest.mark.parametrize('arr_method, np_method',
        [('argmax', np.argmax),
         ('argmin', np.argmin)])
    def test_np_vs_ndarray(self, arr_method, np_method):
        # make sure both ndarray.argmax/argmin and
        # numpy.argmax/argmin support out/axis args
        a = np.random.normal(size=(2, 3))
        arg_method = getattr(a, arr_method)

        # check positional args
        out1 = np.zeros(2, dtype=int)
        out2 = np.zeros(2, dtype=int)
        assert_equal(arg_method(1, out1), np_method(a, 1, out2))
        assert_equal(out1, out2)

        # check keyword args
        out1 = np.zeros(3, dtype=int)
        out2 = np.zeros(3, dtype=int)
        assert_equal(arg_method(out=out1, axis=0),
                     np_method(a, out=out2, axis=0))
        assert_equal(out1, out2)

    @pytest.mark.leaks_references(reason="replaces None with NULL.")
    @pytest.mark.parametrize('method, vals',
        [('argmax', (10, 30)),
         ('argmin', (30, 10))])
    def test_object_with_NULLs(self, method, vals):
        # See gh-6032
        a = np.empty(4, dtype='O')
        arg_method = getattr(a, method)
        ctypes.memset(a.ctypes.data, 0, a.nbytes)
        assert_equal(arg_method(), 0)
        a[3] = vals[0]
        assert_equal(arg_method(), 3)
        a[1] = vals[1]
        assert_equal(arg_method(), 1)

class TestArgmax:
    usg_data = [
        ([1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0], 0),
        ([3, 3, 3, 3,  2,  2,  2,  2], 0),
        ([0, 1, 2, 3,  4,  5,  6,  7], 7),
        ([7, 6, 5, 4,  3,  2,  1,  0], 0)
    ]
    sg_data = usg_data + [
        ([1, 2, 3, 4, -4, -3, -2, -1], 3),
        ([1, 2, 3, 4, -1, -2, -3, -4], 3)
    ]
    darr = [(np.array(d[0], dtype=t), d[1]) for d, t in (
        itertools.product(usg_data, (
            np.uint8, np.uint16, np.uint32, np.uint64
        ))
    )]
    darr = darr + [(np.array(d[0], dtype=t), d[1]) for d, t in (
        itertools.product(sg_data, (
            np.int8, np.int16, np.int32, np.int64, np.float32, np.float64
        ))
    )]
    darr = darr + [(np.array(d[0], dtype=t), d[1]) for d, t in (
        itertools.product((
            ([0, 1, 2, 3, np.nan], 4),
            ([0, 1, 2, np.nan, 3], 3),
            ([np.nan, 0, 1, 2, 3], 0),
            ([np.nan, 0, np.nan, 2, 3], 0),
            # To hit the tail of SIMD multi-level(x4, x1) inner loops
            # on variant SIMD widthes
            ([1] * (2*5-1) + [np.nan], 2*5-1),
            ([1] * (4*5-1) + [np.nan], 4*5-1),
            ([1] * (8*5-1) + [np.nan], 8*5-1),
            ([1] * (16*5-1) + [np.nan], 16*5-1),
            ([1] * (32*5-1) + [np.nan], 32*5-1)
        ), (
            np.float32, np.float64
        ))
    )]
    nan_arr = darr + [
        ([0, 1, 2, 3, complex(0, np.nan)], 4),
        ([0, 1, 2, 3, complex(np.nan, 0)], 4),
        ([0, 1, 2, complex(np.nan, 0), 3], 3),
        ([0, 1, 2, complex(0, np.nan), 3], 3),
        ([complex(0, np.nan), 0, 1, 2, 3], 0),
        ([complex(np.nan, np.nan), 0, 1, 2, 3], 0),
        ([complex(np.nan, 0), complex(np.nan, 2), complex(np.nan, 1)], 0),
        ([complex(np.nan, np.nan), complex(np.nan, 2), complex(np.nan, 1)], 0),
        ([complex(np.nan, 0), complex(np.nan, 2), complex(np.nan, np.nan)], 0),

        ([complex(0, 0), complex(0, 2), complex(0, 1)], 1),
        ([complex(1, 0), complex(0, 2), complex(0, 1)], 0),
        ([complex(1, 0), complex(0, 2), complex(1, 1)], 2),

        ([np.datetime64('1923-04-14T12:43:12'),
          np.datetime64('1994-06-21T14:43:15'),
          np.datetime64('2001-10-15T04:10:32'),
          np.datetime64('1995-11-25T16:02:16'),
          np.datetime64('2005-01-04T03:14:12'),
          np.datetime64('2041-12-03T14:05:03')], 5),
        ([np.datetime64('1935-09-14T04:40:11'),
          np.datetime64('1949-10-12T12:32:11'),
          np.datetime64('2010-01-03T05:14:12'),
          np.datetime64('2015-11-20T12:20:59'),
          np.datetime64('1932-09-23T10:10:13'),
          np.datetime64('2014-10-10T03:50:30')], 3),
        # Assorted tests with NaTs
        ([np.datetime64('NaT'),
          np.datetime64('NaT'),
          np.datetime64('2010-01-03T05:14:12'),
          np.datetime64('NaT'),
          np.datetime64('2015-09-23T10:10:13'),
          np.datetime64('1932-10-10T03:50:30')], 0),
        ([np.datetime64('2059-03-14T12:43:12'),
          np.datetime64('1996-09-21T14:43:15'),
          np.datetime64('NaT'),
          np.datetime64('2022-12-25T16:02:16'),
          np.datetime64('1963-10-04T03:14:12'),
          np.datetime64('2013-05-08T18:15:23')], 2),
        ([np.timedelta64(2, 's'),
          np.timedelta64(1, 's'),
          np.timedelta64('NaT', 's'),
          np.timedelta64(3, 's')], 2),
        ([np.timedelta64('NaT', 's')] * 3, 0),

        ([timedelta(days=5, seconds=14), timedelta(days=2, seconds=35),
          timedelta(days=-1, seconds=23)], 0),
        ([timedelta(days=1, seconds=43), timedelta(days=10, seconds=5),
          timedelta(days=5, seconds=14)], 1),
        ([timedelta(days=10, seconds=24), timedelta(days=10, seconds=5),
          timedelta(days=10, seconds=43)], 2),

        ([False, False, False, False, True], 4),
        ([False, False, False, True, False], 3),
        ([True, False, False, False, False], 0),
        ([True, False, True, False, False], 0),
    ]

    @pytest.mark.parametrize('data', nan_arr)
    def test_combinations(self, data):
        arr, pos = data
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning,
                        "invalid value encountered in reduce")
            val = np.max(arr)

        assert_equal(np.argmax(arr), pos, err_msg="%r" % arr)
        assert_equal(arr[np.argmax(arr)], val, err_msg="%r" % arr)

        # add padding to test SIMD loops
        rarr = np.repeat(arr, 129)
        rpos = pos * 129
        assert_equal(np.argmax(rarr), rpos, err_msg="%r" % rarr)
        assert_equal(rarr[np.argmax(rarr)], val, err_msg="%r" % rarr)

        padd = np.repeat(np.min(arr), 513)
        rarr = np.concatenate((arr, padd))
        rpos = pos
        assert_equal(np.argmax(rarr), rpos, err_msg="%r" % rarr)
        assert_equal(rarr[np.argmax(rarr)], val, err_msg="%r" % rarr)


    def test_maximum_signed_integers(self):

        a = np.array([1, 2**7 - 1, -2**7], dtype=np.int8)
        assert_equal(np.argmax(a), 1)
        a.repeat(129)
        assert_equal(np.argmax(a), 1)

        a = np.array([1, 2**15 - 1, -2**15], dtype=np.int16)
        assert_equal(np.argmax(a), 1)
        a.repeat(129)
        assert_equal(np.argmax(a), 1)

        a = np.array([1, 2**31 - 1, -2**31], dtype=np.int32)
        assert_equal(np.argmax(a), 1)
        a.repeat(129)
        assert_equal(np.argmax(a), 1)

        a = np.array([1, 2**63 - 1, -2**63], dtype=np.int64)
        assert_equal(np.argmax(a), 1)
        a.repeat(129)
        assert_equal(np.argmax(a), 1)

class TestArgmin:
    usg_data = [
        ([1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0], 8),
        ([3, 3, 3, 3,  2,  2,  2,  2], 4),
        ([0, 1, 2, 3,  4,  5,  6,  7], 0),
        ([7, 6, 5, 4,  3,  2,  1,  0], 7)
    ]
    sg_data = usg_data + [
        ([1, 2, 3, 4, -4, -3, -2, -1], 4),
        ([1, 2, 3, 4, -1, -2, -3, -4], 7)
    ]
    darr = [(np.array(d[0], dtype=t), d[1]) for d, t in (
        itertools.product(usg_data, (
            np.uint8, np.uint16, np.uint32, np.uint64
        ))
    )]
    darr = darr + [(np.array(d[0], dtype=t), d[1]) for d, t in (
        itertools.product(sg_data, (
            np.int8, np.int16, np.int32, np.int64, np.float32, np.float64
        ))
    )]
    darr = darr + [(np.array(d[0], dtype=t), d[1]) for d, t in (
        itertools.product((
            ([0, 1, 2, 3, np.nan], 4),
            ([0, 1, 2, np.nan, 3], 3),
            ([np.nan, 0, 1, 2, 3], 0),
            ([np.nan, 0, np.nan, 2, 3], 0),
            # To hit the tail of SIMD multi-level(x4, x1) inner loops
            # on variant SIMD widthes
            ([1] * (2*5-1) + [np.nan], 2*5-1),
            ([1] * (4*5-1) + [np.nan], 4*5-1),
            ([1] * (8*5-1) + [np.nan], 8*5-1),
            ([1] * (16*5-1) + [np.nan], 16*5-1),
            ([1] * (32*5-1) + [np.nan], 32*5-1)
        ), (
            np.float32, np.float64
        ))
    )]
    nan_arr = darr + [
        ([0, 1, 2, 3, complex(0, np.nan)], 4),
        ([0, 1, 2, 3, complex(np.nan, 0)], 4),
        ([0, 1, 2, complex(np.nan, 0), 3], 3),
        ([0, 1, 2, complex(0, np.nan), 3], 3),
        ([complex(0, np.nan), 0, 1, 2, 3], 0),
        ([complex(np.nan, np.nan), 0, 1, 2, 3], 0),
        ([complex(np.nan, 0), complex(np.nan, 2), complex(np.nan, 1)], 0),
        ([complex(np.nan, np.nan), complex(np.nan, 2), complex(np.nan, 1)], 0),
        ([complex(np.nan, 0), complex(np.nan, 2), complex(np.nan, np.nan)], 0),

        ([complex(0, 0), complex(0, 2), complex(0, 1)], 0),
        ([complex(1, 0), complex(0, 2), complex(0, 1)], 2),
        ([complex(1, 0), complex(0, 2), complex(1, 1)], 1),

        ([np.datetime64('1923-04-14T12:43:12'),
          np.datetime64('1994-06-21T14:43:15'),
          np.datetime64('2001-10-15T04:10:32'),
          np.datetime64('1995-11-25T16:02:16'),
          np.datetime64('2005-01-04T03:14:12'),
          np.datetime64('2041-12-03T14:05:03')], 0),
        ([np.datetime64('1935-09-14T04:40:11'),
          np.datetime64('1949-10-12T12:32:11'),
          np.datetime64('2010-01-03T05:14:12'),
          np.datetime64('2014-11-20T12:20:59'),
          np.datetime64('2015-09-23T10:10:13'),
          np.datetime64('1932-10-10T03:50:30')], 5),
        # Assorted tests with NaTs
        ([np.datetime64('NaT'),
          np.datetime64('NaT'),
          np.datetime64('2010-01-03T05:14:12'),
          np.datetime64('NaT'),
          np.datetime64('2015-09-23T10:10:13'),
          np.datetime64('1932-10-10T03:50:30')], 0),
        ([np.datetime64('2059-03-14T12:43:12'),
          np.datetime64('1996-09-21T14:43:15'),
          np.datetime64('NaT'),
          np.datetime64('2022-12-25T16:02:16'),
          np.datetime64('1963-10-04T03:14:12'),
          np.datetime64('2013-05-08T18:15:23')], 2),
        ([np.timedelta64(2, 's'),
          np.timedelta64(1, 's'),
          np.timedelta64('NaT', 's'),
          np.timedelta64(3, 's')], 2),
        ([np.timedelta64('NaT', 's')] * 3, 0),

        ([timedelta(days=5, seconds=14), timedelta(days=2, seconds=35),
          timedelta(days=-1, seconds=23)], 2),
        ([timedelta(days=1, seconds=43), timedelta(days=10, seconds=5),
          timedelta(days=5, seconds=14)], 0),
        ([timedelta(days=10, seconds=24), timedelta(days=10, seconds=5),
          timedelta(days=10, seconds=43)], 1),

        ([True, True, True, True, False], 4),
        ([True, True, True, False, True], 3),
        ([False, True, True, True, True], 0),
        ([False, True, False, True, True], 0),
    ]

    @pytest.mark.parametrize('data', nan_arr)
    def test_combinations(self, data):
        arr, pos = data
        with suppress_warnings() as sup:
            sup.filter(RuntimeWarning,
                       "invalid value encountered in reduce")
            min_val = np.min(arr)

        assert_equal(np.argmin(arr), pos, err_msg="%r" % arr)
        assert_equal(arr[np.argmin(arr)], min_val, err_msg="%r" % arr)

        # add padding to test SIMD loops
        rarr = np.repeat(arr, 129)
        rpos = pos * 129
        assert_equal(np.argmin(rarr), rpos, err_msg="%r" % rarr)
        assert_equal(rarr[np.argmin(rarr)], min_val, err_msg="%r" % rarr)

        padd = np.repeat(np.max(arr), 513)
        rarr = np.concatenate((arr, padd))
        rpos = pos
        assert_equal(np.argmin(rarr), rpos, err_msg="%r" % rarr)
        assert_equal(rarr[np.argmin(rarr)], min_val, err_msg="%r" % rarr)

    def test_minimum_signed_integers(self):

        a = np.array([1, -2**7, -2**7 + 1, 2**7 - 1], dtype=np.int8)
        assert_equal(np.argmin(a), 1)
        a.repeat(129)
        assert_equal(np.argmin(a), 1)

        a = np.array([1, -2**15, -2**15 + 1, 2**15 - 1], dtype=np.int16)
        assert_equal(np.argmin(a), 1)
        a.repeat(129)
        assert_equal(np.argmin(a), 1)

        a = np.array([1, -2**31, -2**31 + 1, 2**31 - 1], dtype=np.int32)
        assert_equal(np.argmin(a), 1)
        a.repeat(129)
        assert_equal(np.argmin(a), 1)

        a = np.array([1, -2**63, -2**63 + 1, 2**63 - 1], dtype=np.int64)
        assert_equal(np.argmin(a), 1)
        a.repeat(129)
        assert_equal(np.argmin(a), 1)

class TestMinMax:

    def test_scalar(self):
        assert_raises(np.AxisError, np.amax, 1, 1)
        assert_raises(np.AxisError, np.amin, 1, 1)

        assert_equal(np.amax(1, axis=0), 1)
        assert_equal(np.amin(1, axis=0), 1)
        assert_equal(np.amax(1, axis=None), 1)
        assert_equal(np.amin(1, axis=None), 1)

    def test_axis(self):
        assert_raises(np.AxisError, np.amax, [1, 2, 3], 1000)
        assert_equal(np.amax([[1, 2, 3]], axis=1), 3)

    def test_datetime(self):
        # Do not ignore NaT
        for dtype in ('m8[s]', 'm8[Y]'):
            a = np.arange(10).astype(dtype)
            assert_equal(np.amin(a), a[0])
            assert_equal(np.amax(a), a[9])
            a[3] = 'NaT'
            assert_equal(np.amin(a), a[3])
            assert_equal(np.amax(a), a[3])


class TestNewaxis:
    def test_basic(self):
        sk = np.array([0, -0.1, 0.1])
        res = 250*sk[:, np.newaxis]
        assert_almost_equal(res.ravel(), 250*sk)


class TestClip:
    def _check_range(self, x, cmin, cmax):
        assert_(np.all(x >= cmin))
        assert_(np.all(x <= cmax))

    def _clip_type(self, type_group, array_max,
                   clip_min, clip_max, inplace=False,
                   expected_min=None, expected_max=None):
        if expected_min is None:
            expected_min = clip_min
        if expected_max is None:
            expected_max = clip_max

        for T in np.sctypes[type_group]:
            if sys.byteorder == 'little':
                byte_orders = ['=', '>']
            else:
                byte_orders = ['<', '=']

            for byteorder in byte_orders:
                dtype = np.dtype(T).newbyteorder(byteorder)

                x = (np.random.random(1000) * array_max).astype(dtype)
                if inplace:
                    # The tests that call us pass clip_min and clip_max that
                    # might not fit in the destination dtype. They were written
                    # assuming the previous unsafe casting, which now must be
                    # passed explicitly to avoid a warning.
                    x.clip(clip_min, clip_max, x, casting='unsafe')
                else:
                    x = x.clip(clip_min, clip_max)
                    byteorder = '='

                if x.dtype.byteorder == '|':
                    byteorder = '|'
                assert_equal(x.dtype.byteorder, byteorder)
                self._check_range(x, expected_min, expected_max)
        return x

    def test_basic(self):
        for inplace in [False, True]:
            self._clip_type(
                'float', 1024, -12.8, 100.2, inplace=inplace)
            self._clip_type(
                'float', 1024, 0, 0, inplace=inplace)

            self._clip_type(
                'int', 1024, -120, 100, inplace=inplace)
            self._clip_type(
                'int', 1024, 0, 0, inplace=inplace)

            self._clip_type(
                'uint', 1024, 0, 0, inplace=inplace)
            self._clip_type(
                'uint', 1024, -120, 100, inplace=inplace, expected_min=0)

    def test_record_array(self):
        rec = np.array([(-5, 2.0, 3.0), (5.0, 4.0, 3.0)],
                       dtype=[('x', '<f8'), ('y', '<f8'), ('z', '<f8')])
        y = rec['x'].clip(-0.3, 0.5)
        self._check_range(y, -0.3, 0.5)

    def test_max_or_min(self):
        val = np.array([0, 1, 2, 3, 4, 5, 6, 7])
        x = val.clip(3)
        assert_(np.all(x >= 3))
        x = val.clip(min=3)
        assert_(np.all(x >= 3))
        x = val.clip(max=4)
        assert_(np.all(x <= 4))

    def test_nan(self):
        input_arr = np.array([-2., np.nan, 0.5, 3., 0.25, np.nan])
        result = input_arr.clip(-1, 1)
        expected = np.array([-1., np.nan, 0.5, 1., 0.25, np.nan])
        assert_array_equal(result, expected)


class TestCompress:
    def test_axis(self):
        tgt = [[5, 6, 7, 8, 9]]
        arr = np.arange(10).reshape(2, 5)
        out = np.compress([0, 1], arr, axis=0)
        assert_equal(out, tgt)

        tgt = [[1, 3], [6, 8]]
        out = np.compress([0, 1, 0, 1, 0], arr, axis=1)
        assert_equal(out, tgt)

    def test_truncate(self):
        tgt = [[1], [6]]
        arr = np.arange(10).reshape(2, 5)
        out = np.compress([0, 1], arr, axis=1)
        assert_equal(out, tgt)

    def test_flatten(self):
        arr = np.arange(10).reshape(2, 5)
        out = np.compress([0, 1], arr)
        assert_equal(out, 1)


class TestPutmask:
    def tst_basic(self, x, T, mask, val):
        np.putmask(x, mask, val)
        assert_equal(x[mask], np.array(val, T))

    def test_ip_types(self):
        unchecked_types = [bytes, str, np.void]

        x = np.random.random(1000)*100
        mask = x < 40

        for val in [-100, 0, 15]:
            for types in np.sctypes.values():
                for T in types:
                    if T not in unchecked_types:
                        if val < 0 and np.dtype(T).kind == "u":
                            val = np.iinfo(T).max - 99
                        self.tst_basic(x.copy().astype(T), T, mask, val)

            # Also test string of a length which uses an untypical length
            dt = np.dtype("S3")
            self.tst_basic(x.astype(dt), dt.type, mask, dt.type(val)[:3])

    def test_mask_size(self):
        assert_raises(ValueError, np.putmask, np.array([1, 2, 3]), [True], 5)

    @pytest.mark.parametrize('dtype', ('>i4', '<i4'))
    def test_byteorder(self, dtype):
        x = np.array([1, 2, 3], dtype)
        np.putmask(x, [True, False, True], -1)
        assert_array_equal(x, [-1, 2, -1])

    def test_record_array(self):
        # Note mixed byteorder.
        rec = np.array([(-5, 2.0, 3.0), (5.0, 4.0, 3.0)],
                      dtype=[('x', '<f8'), ('y', '>f8'), ('z', '<f8')])
        np.putmask(rec['x'], [True, False], 10)
        assert_array_equal(rec['x'], [10, 5])
        assert_array_equal(rec['y'], [2, 4])
        assert_array_equal(rec['z'], [3, 3])
        np.putmask(rec['y'], [True, False], 11)
        assert_array_equal(rec['x'], [10, 5])
        assert_array_equal(rec['y'], [11, 4])
        assert_array_equal(rec['z'], [3, 3])

    def test_overlaps(self):
        # gh-6272 check overlap
        x = np.array([True, False, True, False])
        np.putmask(x[1:4], [True, True, True], x[:3])
        assert_equal(x, np.array([True, True, False, True]))

        x = np.array([True, False, True, False])
        np.putmask(x[1:4], x[:3], [True, False, True])
        assert_equal(x, np.array([True, True, True, True]))

    def test_writeable(self):
        a = np.arange(5)
        a.flags.writeable = False

        with pytest.raises(ValueError):
            np.putmask(a, a >= 2, 3)


class TestTake:
    def tst_basic(self, x):
        ind = list(range(x.shape[0]))
        assert_array_equal(x.take(ind, axis=0), x)

    def test_ip_types(self):
        unchecked_types = [bytes, str, np.void]

        x = np.random.random(24)*100
        x.shape = 2, 3, 4
        for types in np.sctypes.values():
            for T in types:
                if T not in unchecked_types:
                    self.tst_basic(x.copy().astype(T))

            # Also test string of a length which uses an untypical length
            self.tst_basic(x.astype("S3"))

    def test_raise(self):
        x = np.random.random(24)*100
        x.shape = 2, 3, 4
        assert_raises(IndexError, x.take, [0, 1, 2], axis=0)
        assert_raises(IndexError, x.take, [-3], axis=0)
        assert_array_equal(x.take([-1], axis=0)[0], x[1])

    def test_clip(self):
        x = np.random.random(24)*100
        x.shape = 2, 3, 4
        assert_array_equal(x.take([-1], axis=0, mode='clip')[0], x[0])
        assert_array_equal(x.take([2], axis=0, mode='clip')[0], x[1])

    def test_wrap(self):
        x = np.random.random(24)*100
        x.shape = 2, 3, 4
        assert_array_equal(x.take([-1], axis=0, mode='wrap')[0], x[1])
        assert_array_equal(x.take([2], axis=0, mode='wrap')[0], x[0])
        assert_array_equal(x.take([3], axis=0, mode='wrap')[0], x[1])

    @pytest.mark.parametrize('dtype', ('>i4', '<i4'))
    def test_byteorder(self, dtype):
        x = np.array([1, 2, 3], dtype)
        assert_array_equal(x.take([0, 2, 1]), [1, 3, 2])

    def test_record_array(self):
        # Note mixed byteorder.
        rec = np.array([(-5, 2.0, 3.0), (5.0, 4.0, 3.0)],
                      dtype=[('x', '<f8'), ('y', '>f8'), ('z', '<f8')])
        rec1 = rec.take([1])
        assert_(rec1['x'] == 5.0 and rec1['y'] == 4.0)

    def test_out_overlap(self):
        # gh-6272 check overlap on out
        x = np.arange(5)
        y = np.take(x, [1, 2, 3], out=x[2:5], mode='wrap')
        assert_equal(y, np.array([1, 2, 3]))

    @pytest.mark.parametrize('shape', [(1, 2), (1,), ()])
    def test_ret_is_out(self, shape):
        # 0d arrays should not be an exception to this rule
        x = np.arange(5)
        inds = np.zeros(shape, dtype=np.intp)
        out = np.zeros(shape, dtype=x.dtype)
        ret = np.take(x, inds, out=out)
        assert ret is out


class TestLexsort:
    @pytest.mark.parametrize('dtype',[
        np.uint8, np.uint16, np.uint32, np.uint64,
        np.int8, np.int16, np.int32, np.int64,
        np.float16, np.float32, np.float64
    ])
    def test_basic(self, dtype):
        a = np.array([1, 2, 1, 3, 1, 5], dtype=dtype)
        b = np.array([0, 4, 5, 6, 2, 3], dtype=dtype)
        idx = np.lexsort((b, a))
        expected_idx = np.array([0, 4, 2, 1, 3, 5])
        assert_array_equal(idx, expected_idx)
        assert_array_equal(a[idx], np.sort(a))

    def test_mixed(self):
        a = np.array([1, 2, 1, 3, 1, 5])
        b = np.array([0, 4, 5, 6, 2, 3], dtype='datetime64[D]')

        idx = np.lexsort((b, a))
        expected_idx = np.array([0, 4, 2, 1, 3, 5])
        assert_array_equal(idx, expected_idx)

    def test_datetime(self):
        a = np.array([0,0,0], dtype='datetime64[D]')
        b = np.array([2,1,0], dtype='datetime64[D]')
        idx = np.lexsort((b, a))
        expected_idx = np.array([2, 1, 0])
        assert_array_equal(idx, expected_idx)

        a = np.array([0,0,0], dtype='timedelta64[D]')
        b = np.array([2,1,0], dtype='timedelta64[D]')
        idx = np.lexsort((b, a))
        expected_idx = np.array([2, 1, 0])
        assert_array_equal(idx, expected_idx)

    def test_object(self):  # gh-6312
        a = np.random.choice(10, 1000)
        b = np.random.choice(['abc', 'xy', 'wz', 'efghi', 'qwst', 'x'], 1000)

        for u in a, b:
            left = np.lexsort((u.astype('O'),))
            right = np.argsort(u, kind='mergesort')
            assert_array_equal(left, right)

        for u, v in (a, b), (b, a):
            idx = np.lexsort((u, v))
            assert_array_equal(idx, np.lexsort((u.astype('O'), v)))
            assert_array_equal(idx, np.lexsort((u, v.astype('O'))))
            u, v = np.array(u, dtype='object'), np.array(v, dtype='object')
            assert_array_equal(idx, np.lexsort((u, v)))

    def test_invalid_axis(self): # gh-7528
        x = np.linspace(0., 1., 42*3).reshape(42, 3)
        assert_raises(np.AxisError, np.lexsort, x, axis=2)

class TestIO:
    """Test tofile, fromfile, tobytes, and fromstring"""

    @pytest.fixture()
    def x(self):
        shape = (2, 4, 3)
        rand = np.random.random
        x = rand(shape) + rand(shape).astype(complex) * 1j
        x[0, :, 1] = [np.nan, np.inf, -np.inf, np.nan]
        return x

    @pytest.fixture(params=["string", "path_obj"])
    def tmp_filename(self, tmp_path, request):
        # This fixture covers two cases:
        # one where the filename is a string and
        # another where it is a pathlib object
        filename = tmp_path / "file"
        if request.param == "string":
            filename = str(filename)
        yield filename

    def test_nofile(self):
        # this should probably be supported as a file
        # but for now test for proper errors
        b = io.BytesIO()
        assert_raises(OSError, np.fromfile, b, np.uint8, 80)
        d = np.ones(7)
        assert_raises(OSError, lambda x: x.tofile(b), d)

    def test_bool_fromstring(self):
        v = np.array([True, False, True, False], dtype=np.bool_)
        y = np.fromstring('1 0 -2.3 0.0', sep=' ', dtype=np.bool_)
        assert_array_equal(v, y)

    def test_uint64_fromstring(self):
        d = np.fromstring("9923372036854775807 104783749223640",
                          dtype=np.uint64, sep=' ')
        e = np.array([9923372036854775807, 104783749223640], dtype=np.uint64)
        assert_array_equal(d, e)

    def test_int64_fromstring(self):
        d = np.fromstring("-25041670086757 104783749223640",
                          dtype=np.int64, sep=' ')
        e = np.array([-25041670086757, 104783749223640], dtype=np.int64)
        assert_array_equal(d, e)

    def test_fromstring_count0(self):
        d = np.fromstring("1,2", sep=",", dtype=np.int64, count=0)
        assert d.shape == (0,)

    def test_empty_files_text(self, tmp_filename):
        with open(tmp_filename, 'w') as f:
            pass
        y = np.fromfile(tmp_filename)
        assert_(y.size == 0, "Array not empty")

    def test_empty_files_binary(self, tmp_filename):
        with open(tmp_filename, 'wb') as f:
            pass
        y = np.fromfile(tmp_filename, sep=" ")
        assert_(y.size == 0, "Array not empty")

    def test_roundtrip_file(self, x, tmp_filename):
        with open(tmp_filename, 'wb') as f:
            x.tofile(f)
        # NB. doesn't work with flush+seek, due to use of C stdio
        with open(tmp_filename, 'rb') as f:
            y = np.fromfile(f, dtype=x.dtype)
        assert_array_equal(y, x.flat)

    def test_roundtrip(self, x, tmp_filename):
        x.tofile(tmp_filename)
        y = np.fromfile(tmp_filename, dtype=x.dtype)
        assert_array_equal(y, x.flat)

    def test_roundtrip_dump_pathlib(self, x, tmp_filename):
        p = pathlib.Path(tmp_filename)
        x.dump(p)
        y = np.load(p, allow_pickle=True)
        assert_array_equal(y, x)

    def test_roundtrip_binary_str(self, x):
        s = x.tobytes()
        y = np.frombuffer(s, dtype=x.dtype)
        assert_array_equal(y, x.flat)

        s = x.tobytes('F')
        y = np.frombuffer(s, dtype=x.dtype)
        assert_array_equal(y, x.flatten('F'))

    def test_roundtrip_str(self, x):
        x = x.real.ravel()
        s = "@".join(map(str, x))
        y = np.fromstring(s, sep="@")
        # NB. str imbues less precision
        nan_mask = ~np.isfinite(x)
        assert_array_equal(x[nan_mask], y[nan_mask])
        assert_array_almost_equal(x[~nan_mask], y[~nan_mask], decimal=5)

    def test_roundtrip_repr(self, x):
        x = x.real.ravel()
        s = "@".join(map(repr, x))
        y = np.fromstring(s, sep="@")
        assert_array_equal(x, y)

    def test_unseekable_fromfile(self, x, tmp_filename):
        # gh-6246
        x.tofile(tmp_filename)

        def fail(*args, **kwargs):
            raise OSError('Can not tell or seek')

        with io.open(tmp_filename, 'rb', buffering=0) as f:
            f.seek = fail
            f.tell = fail
            assert_raises(OSError, np.fromfile, f, dtype=x.dtype)

    def test_io_open_unbuffered_fromfile(self, x, tmp_filename):
        # gh-6632
        x.tofile(tmp_filename)
        with io.open(tmp_filename, 'rb', buffering=0) as f:
            y = np.fromfile(f, dtype=x.dtype)
            assert_array_equal(y, x.flat)

    def test_largish_file(self, tmp_filename):
        # check the fallocate path on files > 16MB
        d = np.zeros(4 * 1024 ** 2)
        d.tofile(tmp_filename)
        assert_equal(os.path.getsize(tmp_filename), d.nbytes)
        assert_array_equal(d, np.fromfile(tmp_filename))
        # check offset
        with open(tmp_filename, "r+b") as f:
            f.seek(d.nbytes)
            d.tofile(f)
            assert_equal(os.path.getsize(tmp_filename), d.nbytes * 2)
        # check append mode (gh-8329)
        open(tmp_filename, "w").close()  # delete file contents
        with open(tmp_filename, "ab") as f:
            d.tofile(f)
        assert_array_equal(d, np.fromfile(tmp_filename))
        with open(tmp_filename, "ab") as f:
            d.tofile(f)
        assert_equal(os.path.getsize(tmp_filename), d.nbytes * 2)

    def test_io_open_buffered_fromfile(self, x, tmp_filename):
        # gh-6632
        x.tofile(tmp_filename)
        with io.open(tmp_filename, 'rb', buffering=-1) as f:
            y = np.fromfile(f, dtype=x.dtype)
        assert_array_equal(y, x.flat)

    def test_file_position_after_fromfile(self, tmp_filename):
        # gh-4118
        sizes = [io.DEFAULT_BUFFER_SIZE//8,
                 io.DEFAULT_BUFFER_SIZE,
                 io.DEFAULT_BUFFER_SIZE*8]

        for size in sizes:
            with open(tmp_filename, 'wb') as f:
                f.seek(size-1)
                f.write(b'\0')

            for mode in ['rb', 'r+b']:
                err_msg = "%d %s" % (size, mode)

                with open(tmp_filename, mode) as f:
                    f.read(2)
                    np.fromfile(f, dtype=np.float64, count=1)
                    pos = f.tell()
                assert_equal(pos, 10, err_msg=err_msg)

    def test_file_position_after_tofile(self, tmp_filename):
        # gh-4118
        sizes = [io.DEFAULT_BUFFER_SIZE//8,
                 io.DEFAULT_BUFFER_SIZE,
                 io.DEFAULT_BUFFER_SIZE*8]

        for size in sizes:
            err_msg = "%d" % (size,)

            with open(tmp_filename, 'wb') as f:
                f.seek(size-1)
                f.write(b'\0')
                f.seek(10)
                f.write(b'12')
                np.array([0], dtype=np.float64).tofile(f)
                pos = f.tell()
            assert_equal(pos, 10 + 2 + 8, err_msg=err_msg)

            with open(tmp_filename, 'r+b') as f:
                f.read(2)
                f.seek(0, 1)  # seek between read&write required by ANSI C
                np.array([0], dtype=np.float64).tofile(f)
                pos = f.tell()
            assert_equal(pos, 10, err_msg=err_msg)

    def test_load_object_array_fromfile(self, tmp_filename):
        # gh-12300
        with open(tmp_filename, 'w') as f:
            # Ensure we have a file with consistent contents
            pass

        with open(tmp_filename, 'rb') as f:
            assert_raises_regex(ValueError, "Cannot read into object array",
                                np.fromfile, f, dtype=object)

        assert_raises_regex(ValueError, "Cannot read into object array",
                            np.fromfile, tmp_filename, dtype=object)

    def test_fromfile_offset(self, x, tmp_filename):
        with open(tmp_filename, 'wb') as f:
            x.tofile(f)

        with open(tmp_filename, 'rb') as f:
            y = np.fromfile(f, dtype=x.dtype, offset=0)
            assert_array_equal(y, x.flat)

        with open(tmp_filename, 'rb') as f:
            count_items = len(x.flat) // 8
            offset_items = len(x.flat) // 4
            offset_bytes = x.dtype.itemsize * offset_items
            y = np.fromfile(
                f, dtype=x.dtype, count=count_items, offset=offset_bytes
            )
            assert_array_equal(
                y, x.flat[offset_items:offset_items+count_items]
            )

            # subsequent seeks should stack
            offset_bytes = x.dtype.itemsize
            z = np.fromfile(f, dtype=x.dtype, offset=offset_bytes)
            assert_array_equal(z, x.flat[offset_items+count_items+1:])

        with open(tmp_filename, 'wb') as f:
            x.tofile(f, sep=",")

        with open(tmp_filename, 'rb') as f:
            assert_raises_regex(
                    TypeError,
                    "'offset' argument only permitted for binary files",
                    np.fromfile, tmp_filename, dtype=x.dtype,
                    sep=",", offset=1)

    @pytest.mark.skipif(IS_PYPY, reason="bug in PyPy's PyNumber_AsSsize_t")
    def test_fromfile_bad_dup(self, x, tmp_filename):
        def dup_str(fd):
            return 'abc'

        def dup_bigint(fd):
            return 2**68

        old_dup = os.dup
        try:
            with open(tmp_filename, 'wb') as f:
                x.tofile(f)
                for dup, exc in ((dup_str, TypeError), (dup_bigint, OSError)):
                    os.dup = dup
                    assert_raises(exc, np.fromfile, f)
        finally:
            os.dup = old_dup

    def _check_from(self, s, value, filename, **kw):
        if 'sep' not in kw:
            y = np.frombuffer(s, **kw)
        else:
            y = np.fromstring(s, **kw)
        assert_array_equal(y, value)

        with open(filename, 'wb') as f:
            f.write(s)
        y = np.fromfile(filename, **kw)
        assert_array_equal(y, value)

    @pytest.fixture(params=["period", "comma"])
    def decimal_sep_localization(self, request):
        """
        Including this fixture in a test will automatically
        execute it with both types of decimal separator.

        So::

            def test_decimal(decimal_sep_localization):
                pass

        is equivalent to the following two tests::

            def test_decimal_period_separator():
                pass

            def test_decimal_comma_separator():
                with CommaDecimalPointLocale():
                    pass
        """
        if request.param == "period":
            yield
        elif request.param == "comma":
            with CommaDecimalPointLocale():
                yield
        else:
            assert False, request.param

    def test_nan(self, tmp_filename, decimal_sep_localization):
        self._check_from(
            b"nan +nan -nan NaN nan(foo) +NaN(BAR) -NAN(q_u_u_x_)",
            [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
            tmp_filename,
            sep=' ')

    def test_inf(self, tmp_filename, decimal_sep_localization):
        self._check_from(
            b"inf +inf -inf infinity -Infinity iNfInItY -inF",
            [np.inf, np.inf, -np.inf, np.inf, -np.inf, np.inf, -np.inf],
            tmp_filename,
            sep=' ')

    def test_numbers(self, tmp_filename, decimal_sep_localization):
        self._check_from(
            b"1.234 -1.234 .3 .3e55 -123133.1231e+133",
            [1.234, -1.234, .3, .3e55, -123133.1231e+133],
            tmp_filename,
            sep=' ')

    def test_binary(self, tmp_filename):
        self._check_from(
            b'\x00\x00\x80?\x00\x00\x00@\x00\x00@@\x00\x00\x80@',
            np.array([1, 2, 3, 4]),
            tmp_filename,
            dtype='<f4')

    @pytest.mark.slow  # takes > 1 minute on mechanical hard drive
    def test_big_binary(self):
        """Test workarounds for 32-bit limit for MSVC fwrite, fseek, and ftell

        These normally would hang doing something like this.
        See : https://github.com/numpy/numpy/issues/2256
        """
        if sys.platform != 'win32' or '[GCC ' in sys.version:
            return
        try:
            # before workarounds, only up to 2**32-1 worked
            fourgbplus = 2**32 + 2**16
            testbytes = np.arange(8, dtype=np.int8)
            n = len(testbytes)
            flike = tempfile.NamedTemporaryFile()
            f = flike.file
            np.tile(testbytes, fourgbplus // testbytes.nbytes).tofile(f)
            flike.seek(0)
            a = np.fromfile(f, dtype=np.int8)
            flike.close()
            assert_(len(a) == fourgbplus)
            # check only start and end for speed:
            assert_((a[:n] == testbytes).all())
            assert_((a[-n:] == testbytes).all())
        except (MemoryError, ValueError):
            pass

    def test_string(self, tmp_filename):
        self._check_from(b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, sep=',')

    def test_counted_string(self, tmp_filename, decimal_sep_localization):
        self._check_from(
            b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, count=4, sep=',')
        self._check_from(
            b'1,2,3,4', [1., 2., 3.], tmp_filename, count=3, sep=',')
        self._check_from(
            b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, count=-1, sep=',')

    def test_string_with_ws(self, tmp_filename):
        self._check_from(
            b'1 2  3     4   ', [1, 2, 3, 4], tmp_filename, dtype=int, sep=' ')

    def test_counted_string_with_ws(self, tmp_filename):
        self._check_from(
            b'1 2  3     4   ', [1, 2, 3], tmp_filename, count=3, dtype=int,
            sep=' ')

    def test_ascii(self, tmp_filename, decimal_sep_localization):
        self._check_from(
            b'1 , 2 , 3 , 4', [1., 2., 3., 4.], tmp_filename, sep=',')
        self._check_from(
            b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, dtype=float, sep=',')

    def test_malformed(self, tmp_filename, decimal_sep_localization):
        with assert_warns(DeprecationWarning):
            self._check_from(
                b'1.234 1,234', [1.234, 1.], tmp_filename, sep=' ')

    def test_long_sep(self, tmp_filename):
        self._check_from(
            b'1_x_3_x_4_x_5', [1, 3, 4, 5], tmp_filename, sep='_x_')

    def test_dtype(self, tmp_filename):
        v = np.array([1, 2, 3, 4], dtype=np.int_)
        self._check_from(b'1,2,3,4', v, tmp_filename, sep=',', dtype=np.int_)

    def test_dtype_bool(self, tmp_filename):
        # can't use _check_from because fromstring can't handle True/False
        v = np.array([True, False, True, False], dtype=np.bool_)
        s = b'1,0,-2.3,0'
        with open(tmp_filename, 'wb') as f:
            f.write(s)
        y = np.fromfile(tmp_filename, sep=',', dtype=np.bool_)
        assert_(y.dtype == '?')
        assert_array_equal(y, v)

    def test_tofile_sep(self, tmp_filename, decimal_sep_localization):
        x = np.array([1.51, 2, 3.51, 4], dtype=float)
        with open(tmp_filename, 'w') as f:
            x.tofile(f, sep=',')
        with open(tmp_filename, 'r') as f:
            s = f.read()
        #assert_equal(s, '1.51,2.0,3.51,4.0')
        y = np.array([float(p) for p in s.split(',')])
        assert_array_equal(x,y)

    def test_tofile_format(self, tmp_filename, decimal_sep_localization):
        x = np.array([1.51, 2, 3.51, 4], dtype=float)
        with open(tmp_filename, 'w') as f:
            x.tofile(f, sep=',', format='%.2f')
        with open(tmp_filename, 'r') as f:
            s = f.read()
        assert_equal(s, '1.51,2.00,3.51,4.00')

    def test_tofile_cleanup(self, tmp_filename):
        x = np.zeros((10), dtype=object)
        with open(tmp_filename, 'wb') as f:
            assert_raises(OSError, lambda: x.tofile(f, sep=''))
        # Dup-ed file handle should be closed or remove will fail on Windows OS
        os.remove(tmp_filename)

        # Also make sure that we close the Python handle
        assert_raises(OSError, lambda: x.tofile(tmp_filename))
        os.remove(tmp_filename)

    def test_fromfile_subarray_binary(self, tmp_filename):
        # Test subarray dtypes which are absorbed into the shape
        x = np.arange(24, dtype="i4").reshape(2, 3, 4)
        x.tofile(tmp_filename)
        res = np.fromfile(tmp_filename, dtype="(3,4)i4")
        assert_array_equal(x, res)

        x_str = x.tobytes()
        with assert_warns(DeprecationWarning):
            # binary fromstring is deprecated
            res = np.fromstring(x_str, dtype="(3,4)i4")
            assert_array_equal(x, res)

    def test_parsing_subarray_unsupported(self, tmp_filename):
        # We currently do not support parsing subarray dtypes
        data = "12,42,13," * 50
        with pytest.raises(ValueError):
            expected = np.fromstring(data, dtype="(3,)i", sep=",")

        with open(tmp_filename, "w") as f:
            f.write(data)

        with pytest.raises(ValueError):
            np.fromfile(tmp_filename, dtype="(3,)i", sep=",")

    def test_read_shorter_than_count_subarray(self, tmp_filename):
        # Test that requesting more values does not cause any problems
        # in conjunction with subarray dimensions being absorbed into the
        # array dimension.
        expected = np.arange(511 * 10, dtype="i").reshape(-1, 10)

        binary = expected.tobytes()
        with pytest.raises(ValueError):
            with pytest.warns(DeprecationWarning):
                np.fromstring(binary, dtype="(10,)i", count=10000)

        expected.tofile(tmp_filename)
        res = np.fromfile(tmp_filename, dtype="(10,)i", count=10000)
        assert_array_equal(res, expected)


class TestFromBuffer:
    @pytest.mark.parametrize('byteorder', ['<', '>'])
    @pytest.mark.parametrize('dtype', [float, int, complex])
    def test_basic(self, byteorder, dtype):
        dt = np.dtype(dtype).newbyteorder(byteorder)
        x = (np.random.random((4, 7)) * 5).astype(dt)
        buf = x.tobytes()
        assert_array_equal(np.frombuffer(buf, dtype=dt), x.flat)

    @pytest.mark.parametrize("obj", [np.arange(10), b"12345678"])
    def test_array_base(self, obj):
        # Objects (including NumPy arrays), which do not use the
        # `release_buffer` slot should be directly used as a base object.
        # See also gh-21612
        new = np.frombuffer(obj)
        assert new.base is obj

    def test_empty(self):
        assert_array_equal(np.frombuffer(b''), np.array([]))

    @pytest.mark.skipif(IS_PYPY,
            reason="PyPy's memoryview currently does not track exports. See: "
                   "https://foss.heptapod.net/pypy/pypy/-/issues/3724")
    def test_mmap_close(self):
        # The old buffer protocol was not safe for some things that the new
        # one is.  But `frombuffer` always used the old one for a long time.
        # Checks that it is safe with the new one (using memoryviews)
        with tempfile.TemporaryFile(mode='wb') as tmp:
            tmp.write(b"asdf")
            tmp.flush()
            mm = mmap.mmap(tmp.fileno(), 0)
            arr = np.frombuffer(mm, dtype=np.uint8)
            with pytest.raises(BufferError):
                mm.close()  # cannot close while array uses the buffer
            del arr
            mm.close()

class TestFlat:
    def setup_method(self):
        a0 = np.arange(20.0)
        a = a0.reshape(4, 5)
        a0.shape = (4, 5)
        a.flags.writeable = False
        self.a = a
        self.b = a[::2, ::2]
        self.a0 = a0
        self.b0 = a0[::2, ::2]

    def test_contiguous(self):
        testpassed = False
        try:
            self.a.flat[12] = 100.0
        except ValueError:
            testpassed = True
        assert_(testpassed)
        assert_(self.a.flat[12] == 12.0)

    def test_discontiguous(self):
        testpassed = False
        try:
            self.b.flat[4] = 100.0
        except ValueError:
            testpassed = True
        assert_(testpassed)
        assert_(self.b.flat[4] == 12.0)

    def test___array__(self):
        c = self.a.flat.__array__()
        d = self.b.flat.__array__()
        e = self.a0.flat.__array__()
        f = self.b0.flat.__array__()

        assert_(c.flags.writeable is False)
        assert_(d.flags.writeable is False)
        assert_(e.flags.writeable is True)
        assert_(f.flags.writeable is False)
        assert_(c.flags.writebackifcopy is False)
        assert_(d.flags.writebackifcopy is False)
        assert_(e.flags.writebackifcopy is False)
        assert_(f.flags.writebackifcopy is False)

    @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
    def test_refcount(self):
        # includes regression test for reference count error gh-13165
        inds = [np.intp(0), np.array([True]*self.a.size), np.array([0]), None]
        indtype = np.dtype(np.intp)
        rc_indtype = sys.getrefcount(indtype)
        for ind in inds:
            rc_ind = sys.getrefcount(ind)
            for _ in range(100):
                try:
                    self.a.flat[ind]
                except IndexError:
                    pass
            assert_(abs(sys.getrefcount(ind) - rc_ind) < 50)
            assert_(abs(sys.getrefcount(indtype) - rc_indtype) < 50)

    def test_index_getset(self):
        it = np.arange(10).reshape(2, 1, 5).flat
        with pytest.raises(AttributeError):
            it.index = 10

        for _ in it:
            pass
        # Check the value of `.index` is updated correctly (see also gh-19153)
        # If the type was incorrect, this would show up on big-endian machines
        assert it.index == it.base.size


class TestResize:

    @_no_tracing
    def test_basic(self):
        x = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
        if IS_PYPY:
            x.resize((5, 5), refcheck=False)
        else:
            x.resize((5, 5))
        assert_array_equal(x.flat[:9],
                np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]).flat)
        assert_array_equal(x[9:].flat, 0)

    def test_check_reference(self):
        x = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
        y = x
        assert_raises(ValueError, x.resize, (5, 1))
        del y  # avoid pyflakes unused variable warning.

    @_no_tracing
    def test_int_shape(self):
        x = np.eye(3)
        if IS_PYPY:
            x.resize(3, refcheck=False)
        else:
            x.resize(3)
        assert_array_equal(x, np.eye(3)[0,:])

    def test_none_shape(self):
        x = np.eye(3)
        x.resize(None)
        assert_array_equal(x, np.eye(3))
        x.resize()
        assert_array_equal(x, np.eye(3))

    def test_0d_shape(self):
        # to it multiple times to test it does not break alloc cache gh-9216
        for i in range(10):
            x = np.empty((1,))
            x.resize(())
            assert_equal(x.shape, ())
            assert_equal(x.size, 1)
            x = np.empty(())
            x.resize((1,))
            assert_equal(x.shape, (1,))
            assert_equal(x.size, 1)

    def test_invalid_arguments(self):
        assert_raises(TypeError, np.eye(3).resize, 'hi')
        assert_raises(ValueError, np.eye(3).resize, -1)
        assert_raises(TypeError, np.eye(3).resize, order=1)
        assert_raises(TypeError, np.eye(3).resize, refcheck='hi')

    @_no_tracing
    def test_freeform_shape(self):
        x = np.eye(3)
        if IS_PYPY:
            x.resize(3, 2, 1, refcheck=False)
        else:
            x.resize(3, 2, 1)
        assert_(x.shape == (3, 2, 1))

    @_no_tracing
    def test_zeros_appended(self):
        x = np.eye(3)
        if IS_PYPY:
            x.resize(2, 3, 3, refcheck=False)
        else:
            x.resize(2, 3, 3)
        assert_array_equal(x[0], np.eye(3))
        assert_array_equal(x[1], np.zeros((3, 3)))

    @_no_tracing
    def test_obj_obj(self):
        # check memory is initialized on resize, gh-4857
        a = np.ones(10, dtype=[('k', object, 2)])
        if IS_PYPY:
            a.resize(15, refcheck=False)
        else:
            a.resize(15,)
        assert_equal(a.shape, (15,))
        assert_array_equal(a['k'][-5:], 0)
        assert_array_equal(a['k'][:-5], 1)

    def test_empty_view(self):
        # check that sizes containing a zero don't trigger a reallocate for
        # already empty arrays
        x = np.zeros((10, 0), int)
        x_view = x[...]
        x_view.resize((0, 10))
        x_view.resize((0, 100))

    def test_check_weakref(self):
        x = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
        xref = weakref.ref(x)
        assert_raises(ValueError, x.resize, (5, 1))
        del xref  # avoid pyflakes unused variable warning.


class TestRecord:
    def test_field_rename(self):
        dt = np.dtype([('f', float), ('i', int)])
        dt.names = ['p', 'q']
        assert_equal(dt.names, ['p', 'q'])

    def test_multiple_field_name_occurrence(self):
        def test_dtype_init():
            np.dtype([("A", "f8"), ("B", "f8"), ("A", "f8")])

        # Error raised when multiple fields have the same name
        assert_raises(ValueError, test_dtype_init)

    def test_bytes_fields(self):
        # Bytes are not allowed in field names and not recognized in titles
        # on Py3
        assert_raises(TypeError, np.dtype, [(b'a', int)])
        assert_raises(TypeError, np.dtype, [(('b', b'a'), int)])

        dt = np.dtype([((b'a', 'b'), int)])
        assert_raises(TypeError, dt.__getitem__, b'a')

        x = np.array([(1,), (2,), (3,)], dtype=dt)
        assert_raises(IndexError, x.__getitem__, b'a')

        y = x[0]
        assert_raises(IndexError, y.__getitem__, b'a')

    def test_multiple_field_name_unicode(self):
        def test_dtype_unicode():
            np.dtype([("\u20B9", "f8"), ("B", "f8"), ("\u20B9", "f8")])

        # Error raised when multiple fields have the same name(unicode included)
        assert_raises(ValueError, test_dtype_unicode)

    def test_fromarrays_unicode(self):
        # A single name string provided to fromarrays() is allowed to be unicode
        # on both Python 2 and 3:
        x = np.core.records.fromarrays(
            [[0], [1]], names='a,b', formats='i4,i4')
        assert_equal(x['a'][0], 0)
        assert_equal(x['b'][0], 1)

    def test_unicode_order(self):
        # Test that we can sort with order as a unicode field name in both Python 2 and
        # 3:
        name = 'b'
        x = np.array([1, 3, 2], dtype=[(name, int)])
        x.sort(order=name)
        assert_equal(x['b'], np.array([1, 2, 3]))

    def test_field_names(self):
        # Test unicode and 8-bit / byte strings can be used
        a = np.zeros((1,), dtype=[('f1', 'i4'),
                                  ('f2', 'i4'),
                                  ('f3', [('sf1', 'i4')])])
        # byte string indexing fails gracefully
        assert_raises(IndexError, a.__setitem__, b'f1', 1)
        assert_raises(IndexError, a.__getitem__, b'f1')
        assert_raises(IndexError, a['f1'].__setitem__, b'sf1', 1)
        assert_raises(IndexError, a['f1'].__getitem__, b'sf1')
        b = a.copy()
        fn1 = str('f1')
        b[fn1] = 1
        assert_equal(b[fn1], 1)
        fnn = str('not at all')
        assert_raises(ValueError, b.__setitem__, fnn, 1)
        assert_raises(ValueError, b.__getitem__, fnn)
        b[0][fn1] = 2
        assert_equal(b[fn1], 2)
        # Subfield
        assert_raises(ValueError, b[0].__setitem__, fnn, 1)
        assert_raises(ValueError, b[0].__getitem__, fnn)
        # Subfield
        fn3 = str('f3')
        sfn1 = str('sf1')
        b[fn3][sfn1] = 1
        assert_equal(b[fn3][sfn1], 1)
        assert_raises(ValueError, b[fn3].__setitem__, fnn, 1)
        assert_raises(ValueError, b[fn3].__getitem__, fnn)
        # multiple subfields
        fn2 = str('f2')
        b[fn2] = 3

        assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
        assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
        assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))

        # non-ascii unicode field indexing is well behaved
        assert_raises(ValueError, a.__setitem__, '\u03e0', 1)
        assert_raises(ValueError, a.__getitem__, '\u03e0')

    def test_record_hash(self):
        a = np.array([(1, 2), (1, 2)], dtype='i1,i2')
        a.flags.writeable = False
        b = np.array([(1, 2), (3, 4)], dtype=[('num1', 'i1'), ('num2', 'i2')])
        b.flags.writeable = False
        c = np.array([(1, 2), (3, 4)], dtype='i1,i2')
        c.flags.writeable = False
        assert_(hash(a[0]) == hash(a[1]))
        assert_(hash(a[0]) == hash(b[0]))
        assert_(hash(a[0]) != hash(b[1]))
        assert_(hash(c[0]) == hash(a[0]) and c[0] == a[0])

    def test_record_no_hash(self):
        a = np.array([(1, 2), (1, 2)], dtype='i1,i2')
        assert_raises(TypeError, hash, a[0])

    def test_empty_structure_creation(self):
        # make sure these do not raise errors (gh-5631)
        np.array([()], dtype={'names': [], 'formats': [],
                           'offsets': [], 'itemsize': 12})
        np.array([(), (), (), (), ()], dtype={'names': [], 'formats': [],
                                           'offsets': [], 'itemsize': 12})

    def test_multifield_indexing_view(self):
        a = np.ones(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u4')])
        v = a[['a', 'c']]
        assert_(v.base is a)
        assert_(v.dtype == np.dtype({'names': ['a', 'c'],
                                     'formats': ['i4', 'u4'],
                                     'offsets': [0, 8]}))
        v[:] = (4,5)
        assert_equal(a[0].item(), (4, 1, 5))

class TestView:
    def test_basic(self):
        x = np.array([(1, 2, 3, 4), (5, 6, 7, 8)],
                     dtype=[('r', np.int8), ('g', np.int8),
                            ('b', np.int8), ('a', np.int8)])
        # We must be specific about the endianness here:
        y = x.view(dtype='<i4')
        # ... and again without the keyword.
        z = x.view('<i4')
        assert_array_equal(y, z)
        assert_array_equal(y, [67305985, 134678021])


def _mean(a, **args):
    return a.mean(**args)


def _var(a, **args):
    return a.var(**args)


def _std(a, **args):
    return a.std(**args)


class TestStats:

    funcs = [_mean, _var, _std]

    def setup_method(self):
        np.random.seed(range(3))
        self.rmat = np.random.random((4, 5))
        self.cmat = self.rmat + 1j * self.rmat
        self.omat = np.array([Decimal(repr(r)) for r in self.rmat.flat])
        self.omat = self.omat.reshape(4, 5)

    def test_python_type(self):
        for x in (np.float16(1.), 1, 1., 1+0j):
            assert_equal(np.mean([x]), 1.)
            assert_equal(np.std([x]), 0.)
            assert_equal(np.var([x]), 0.)

    def test_keepdims(self):
        mat = np.eye(3)
        for f in self.funcs:
            for axis in [0, 1]:
                res = f(mat, axis=axis, keepdims=True)
                assert_(res.ndim == mat.ndim)
                assert_(res.shape[axis] == 1)
            for axis in [None]:
                res = f(mat, axis=axis, keepdims=True)
                assert_(res.shape == (1, 1))

    def test_out(self):
        mat = np.eye(3)
        for f in self.funcs:
            out = np.zeros(3)
            tgt = f(mat, axis=1)
            res = f(mat, axis=1, out=out)
            assert_almost_equal(res, out)
            assert_almost_equal(res, tgt)
        out = np.empty(2)
        assert_raises(ValueError, f, mat, axis=1, out=out)
        out = np.empty((2, 2))
        assert_raises(ValueError, f, mat, axis=1, out=out)

    def test_dtype_from_input(self):

        icodes = np.typecodes['AllInteger']
        fcodes = np.typecodes['AllFloat']

        # object type
        for f in self.funcs:
            mat = np.array([[Decimal(1)]*3]*3)
            tgt = mat.dtype.type
            res = f(mat, axis=1).dtype.type
            assert_(res is tgt)
            # scalar case
            res = type(f(mat, axis=None))
            assert_(res is Decimal)

        # integer types
        for f in self.funcs:
            for c in icodes:
                mat = np.eye(3, dtype=c)
                tgt = np.float64
                res = f(mat, axis=1).dtype.type
                assert_(res is tgt)
                # scalar case
                res = f(mat, axis=None).dtype.type
                assert_(res is tgt)

        # mean for float types
        for f in [_mean]:
            for c in fcodes:
                mat = np.eye(3, dtype=c)
                tgt = mat.dtype.type
                res = f(mat, axis=1).dtype.type
                assert_(res is tgt)
                # scalar case
                res = f(mat, axis=None).dtype.type
                assert_(res is tgt)

        # var, std for float types
        for f in [_var, _std]:
            for c in fcodes:
                mat = np.eye(3, dtype=c)
                # deal with complex types
                tgt = mat.real.dtype.type
                res = f(mat, axis=1).dtype.type
                assert_(res is tgt)
                # scalar case
                res = f(mat, axis=None).dtype.type
                assert_(res is tgt)

    def test_dtype_from_dtype(self):
        mat = np.eye(3)

        # stats for integer types
        # FIXME:
        # this needs definition as there are lots places along the line
        # where type casting may take place.

        # for f in self.funcs:
        #    for c in np.typecodes['AllInteger']:
        #        tgt = np.dtype(c).type
        #        res = f(mat, axis=1, dtype=c).dtype.type
        #        assert_(res is tgt)
        #        # scalar case
        #        res = f(mat, axis=None, dtype=c).dtype.type
        #        assert_(res is tgt)

        # stats for float types
        for f in self.funcs:
            for c in np.typecodes['AllFloat']:
                tgt = np.dtype(c).type
                res = f(mat, axis=1, dtype=c).dtype.type
                assert_(res is tgt)
                # scalar case
                res = f(mat, axis=None, dtype=c).dtype.type
                assert_(res is tgt)

    def test_ddof(self):
        for f in [_var]:
            for ddof in range(3):
                dim = self.rmat.shape[1]
                tgt = f(self.rmat, axis=1) * dim
                res = f(self.rmat, axis=1, ddof=ddof) * (dim - ddof)
        for f in [_std]:
            for ddof in range(3):
                dim = self.rmat.shape[1]
                tgt = f(self.rmat, axis=1) * np.sqrt(dim)
                res = f(self.rmat, axis=1, ddof=ddof) * np.sqrt(dim - ddof)
                assert_almost_equal(res, tgt)
                assert_almost_equal(res, tgt)

    def test_ddof_too_big(self):
        dim = self.rmat.shape[1]
        for f in [_var, _std]:
            for ddof in range(dim, dim + 2):
                with warnings.catch_warnings(record=True) as w:
                    warnings.simplefilter('always')
                    res = f(self.rmat, axis=1, ddof=ddof)
                    assert_(not (res < 0).any())
                    assert_(len(w) > 0)
                    assert_(issubclass(w[0].category, RuntimeWarning))

    def test_empty(self):
        A = np.zeros((0, 3))
        for f in self.funcs:
            for axis in [0, None]:
                with warnings.catch_warnings(record=True) as w:
                    warnings.simplefilter('always')
                    assert_(np.isnan(f(A, axis=axis)).all())
                    assert_(len(w) > 0)
                    assert_(issubclass(w[0].category, RuntimeWarning))
            for axis in [1]:
                with warnings.catch_warnings(record=True) as w:
                    warnings.simplefilter('always')
                    assert_equal(f(A, axis=axis), np.zeros([]))

    def test_mean_values(self):
        for mat in [self.rmat, self.cmat, self.omat]:
            for axis in [0, 1]:
                tgt = mat.sum(axis=axis)
                res = _mean(mat, axis=axis) * mat.shape[axis]
                assert_almost_equal(res, tgt)
            for axis in [None]:
                tgt = mat.sum(axis=axis)
                res = _mean(mat, axis=axis) * np.prod(mat.shape)
                assert_almost_equal(res, tgt)

    def test_mean_float16(self):
        # This fail if the sum inside mean is done in float16 instead
        # of float32.
        assert_(_mean(np.ones(100000, dtype='float16')) == 1)

    def test_mean_axis_error(self):
        # Ensure that AxisError is raised instead of IndexError when axis is
        # out of bounds, see gh-15817.
        with assert_raises(np.core._exceptions.AxisError):
            np.arange(10).mean(axis=2)

    def test_mean_where(self):
        a = np.arange(16).reshape((4, 4))
        wh_full = np.array([[False, True, False, True],
                            [True, False, True, False],
                            [True, True, False, False],
                            [False, False, True, True]])
        wh_partial = np.array([[False],
                               [True],
                               [True],
                               [False]])
        _cases = [(1, True, [1.5, 5.5, 9.5, 13.5]),
                  (0, wh_full, [6., 5., 10., 9.]),
                  (1, wh_full, [2., 5., 8.5, 14.5]),
                  (0, wh_partial, [6., 7., 8., 9.])]
        for _ax, _wh, _res in _cases:
            assert_allclose(a.mean(axis=_ax, where=_wh),
                            np.array(_res))
            assert_allclose(np.mean(a, axis=_ax, where=_wh),
                            np.array(_res))

        a3d = np.arange(16).reshape((2, 2, 4))
        _wh_partial = np.array([False, True, True, False])
        _res = [[1.5, 5.5], [9.5, 13.5]]
        assert_allclose(a3d.mean(axis=2, where=_wh_partial),
                        np.array(_res))
        assert_allclose(np.mean(a3d, axis=2, where=_wh_partial),
                        np.array(_res))

        with pytest.warns(RuntimeWarning) as w:
            assert_allclose(a.mean(axis=1, where=wh_partial),
                            np.array([np.nan, 5.5, 9.5, np.nan]))
        with pytest.warns(RuntimeWarning) as w:
            assert_equal(a.mean(where=False), np.nan)
        with pytest.warns(RuntimeWarning) as w:
            assert_equal(np.mean(a, where=False), np.nan)

    def test_var_values(self):
        for mat in [self.rmat, self.cmat, self.omat]:
            for axis in [0, 1, None]:
                msqr = _mean(mat * mat.conj(), axis=axis)
                mean = _mean(mat, axis=axis)
                tgt = msqr - mean * mean.conjugate()
                res = _var(mat, axis=axis)
                assert_almost_equal(res, tgt)

    @pytest.mark.parametrize(('complex_dtype', 'ndec'), (
        ('complex64', 6),
        ('complex128', 7),
        ('clongdouble', 7),
    ))
    def test_var_complex_values(self, complex_dtype, ndec):
        # Test fast-paths for every builtin complex type
        for axis in [0, 1, None]:
            mat = self.cmat.copy().astype(complex_dtype)
            msqr = _mean(mat * mat.conj(), axis=axis)
            mean = _mean(mat, axis=axis)
            tgt = msqr - mean * mean.conjugate()
            res = _var(mat, axis=axis)
            assert_almost_equal(res, tgt, decimal=ndec)

    def test_var_dimensions(self):
        # _var paths for complex number introduce additions on views that
        # increase dimensions. Ensure this generalizes to higher dims
        mat = np.stack([self.cmat]*3)
        for axis in [0, 1, 2, -1, None]:
            msqr = _mean(mat * mat.conj(), axis=axis)
            mean = _mean(mat, axis=axis)
            tgt = msqr - mean * mean.conjugate()
            res = _var(mat, axis=axis)
            assert_almost_equal(res, tgt)

    def test_var_complex_byteorder(self):
        # Test that var fast-path does not cause failures for complex arrays
        # with non-native byteorder
        cmat = self.cmat.copy().astype('complex128')
        cmat_swapped = cmat.astype(cmat.dtype.newbyteorder())
        assert_almost_equal(cmat.var(), cmat_swapped.var())

    def test_var_axis_error(self):
        # Ensure that AxisError is raised instead of IndexError when axis is
        # out of bounds, see gh-15817.
        with assert_raises(np.core._exceptions.AxisError):
            np.arange(10).var(axis=2)

    def test_var_where(self):
        a = np.arange(25).reshape((5, 5))
        wh_full = np.array([[False, True, False, True, True],
                            [True, False, True, True, False],
                            [True, True, False, False, True],
                            [False, True, True, False, True],
                            [True, False, True, True, False]])
        wh_partial = np.array([[False],
                               [True],
                               [True],
                               [False],
                               [True]])
        _cases = [(0, True, [50., 50., 50., 50., 50.]),
                  (1, True, [2., 2., 2., 2., 2.])]
        for _ax, _wh, _res in _cases:
            assert_allclose(a.var(axis=_ax, where=_wh),
                            np.array(_res))
            assert_allclose(np.var(a, axis=_ax, where=_wh),
                            np.array(_res))

        a3d = np.arange(16).reshape((2, 2, 4))
        _wh_partial = np.array([False, True, True, False])
        _res = [[0.25, 0.25], [0.25, 0.25]]
        assert_allclose(a3d.var(axis=2, where=_wh_partial),
                        np.array(_res))
        assert_allclose(np.var(a3d, axis=2, where=_wh_partial),
                        np.array(_res))

        assert_allclose(np.var(a, axis=1, where=wh_full),
                        np.var(a[wh_full].reshape((5, 3)), axis=1))
        assert_allclose(np.var(a, axis=0, where=wh_partial),
                        np.var(a[wh_partial[:,0]], axis=0))
        with pytest.warns(RuntimeWarning) as w:
            assert_equal(a.var(where=False), np.nan)
        with pytest.warns(RuntimeWarning) as w:
            assert_equal(np.var(a, where=False), np.nan)

    def test_std_values(self):
        for mat in [self.rmat, self.cmat, self.omat]:
            for axis in [0, 1, None]:
                tgt = np.sqrt(_var(mat, axis=axis))
                res = _std(mat, axis=axis)
                assert_almost_equal(res, tgt)

    def test_std_where(self):
        a = np.arange(25).reshape((5,5))[::-1]
        whf = np.array([[False, True, False, True, True],
                        [True, False, True, False, True],
                        [True, True, False, True, False],
                        [True, False, True, True, False],
                        [False, True, False, True, True]])
        whp = np.array([[False],
                        [False],
                        [True],
                        [True],
                        [False]])
        _cases = [
            (0, True, 7.07106781*np.ones((5))),
            (1, True, 1.41421356*np.ones((5))),
            (0, whf,
             np.array([4.0824829 , 8.16496581, 5., 7.39509973, 8.49836586])),
            (0, whp, 2.5*np.ones((5)))
        ]
        for _ax, _wh, _res in _cases:
            assert_allclose(a.std(axis=_ax, where=_wh), _res)
            assert_allclose(np.std(a, axis=_ax, where=_wh), _res)

        a3d = np.arange(16).reshape((2, 2, 4))
        _wh_partial = np.array([False, True, True, False])
        _res = [[0.5, 0.5], [0.5, 0.5]]
        assert_allclose(a3d.std(axis=2, where=_wh_partial),
                        np.array(_res))
        assert_allclose(np.std(a3d, axis=2, where=_wh_partial),
                        np.array(_res))

        assert_allclose(a.std(axis=1, where=whf),
                        np.std(a[whf].reshape((5,3)), axis=1))
        assert_allclose(np.std(a, axis=1, where=whf),
                        (a[whf].reshape((5,3))).std(axis=1))
        assert_allclose(a.std(axis=0, where=whp),
                        np.std(a[whp[:,0]], axis=0))
        assert_allclose(np.std(a, axis=0, where=whp),
                        (a[whp[:,0]]).std(axis=0))
        with pytest.warns(RuntimeWarning) as w:
            assert_equal(a.std(where=False), np.nan)
        with pytest.warns(RuntimeWarning) as w:
            assert_equal(np.std(a, where=False), np.nan)

    def test_subclass(self):
        class TestArray(np.ndarray):
            def __new__(cls, data, info):
                result = np.array(data)
                result = result.view(cls)
                result.info = info
                return result

            def __array_finalize__(self, obj):
                self.info = getattr(obj, "info", '')

        dat = TestArray([[1, 2, 3, 4], [5, 6, 7, 8]], 'jubba')
        res = dat.mean(1)
        assert_(res.info == dat.info)
        res = dat.std(1)
        assert_(res.info == dat.info)
        res = dat.var(1)
        assert_(res.info == dat.info)


class TestVdot:
    def test_basic(self):
        dt_numeric = np.typecodes['AllFloat'] + np.typecodes['AllInteger']
        dt_complex = np.typecodes['Complex']

        # test real
        a = np.eye(3)
        for dt in dt_numeric + 'O':
            b = a.astype(dt)
            res = np.vdot(b, b)
            assert_(np.isscalar(res))
            assert_equal(np.vdot(b, b), 3)

        # test complex
        a = np.eye(3) * 1j
        for dt in dt_complex + 'O':
            b = a.astype(dt)
            res = np.vdot(b, b)
            assert_(np.isscalar(res))
            assert_equal(np.vdot(b, b), 3)

        # test boolean
        b = np.eye(3, dtype=bool)
        res = np.vdot(b, b)
        assert_(np.isscalar(res))
        assert_equal(np.vdot(b, b), True)

    def test_vdot_array_order(self):
        a = np.array([[1, 2], [3, 4]], order='C')
        b = np.array([[1, 2], [3, 4]], order='F')
        res = np.vdot(a, a)

        # integer arrays are exact
        assert_equal(np.vdot(a, b), res)
        assert_equal(np.vdot(b, a), res)
        assert_equal(np.vdot(b, b), res)

    def test_vdot_uncontiguous(self):
        for size in [2, 1000]:
            # Different sizes match different branches in vdot.
            a = np.zeros((size, 2, 2))
            b = np.zeros((size, 2, 2))
            a[:, 0, 0] = np.arange(size)
            b[:, 0, 0] = np.arange(size) + 1
            # Make a and b uncontiguous:
            a = a[..., 0]
            b = b[..., 0]

            assert_equal(np.vdot(a, b),
                         np.vdot(a.flatten(), b.flatten()))
            assert_equal(np.vdot(a, b.copy()),
                         np.vdot(a.flatten(), b.flatten()))
            assert_equal(np.vdot(a.copy(), b),
                         np.vdot(a.flatten(), b.flatten()))
            assert_equal(np.vdot(a.copy('F'), b),
                         np.vdot(a.flatten(), b.flatten()))
            assert_equal(np.vdot(a, b.copy('F')),
                         np.vdot(a.flatten(), b.flatten()))


class TestDot:
    def setup_method(self):
        np.random.seed(128)
        self.A = np.random.rand(4, 2)
        self.b1 = np.random.rand(2, 1)
        self.b2 = np.random.rand(2)
        self.b3 = np.random.rand(1, 2)
        self.b4 = np.random.rand(4)
        self.N = 7

    def test_dotmatmat(self):
        A = self.A
        res = np.dot(A.transpose(), A)
        tgt = np.array([[1.45046013, 0.86323640],
                        [0.86323640, 0.84934569]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotmatvec(self):
        A, b1 = self.A, self.b1
        res = np.dot(A, b1)
        tgt = np.array([[0.32114320], [0.04889721],
                        [0.15696029], [0.33612621]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotmatvec2(self):
        A, b2 = self.A, self.b2
        res = np.dot(A, b2)
        tgt = np.array([0.29677940, 0.04518649, 0.14468333, 0.31039293])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecmat(self):
        A, b4 = self.A, self.b4
        res = np.dot(b4, A)
        tgt = np.array([1.23495091, 1.12222648])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecmat2(self):
        b3, A = self.b3, self.A
        res = np.dot(b3, A.transpose())
        tgt = np.array([[0.58793804, 0.08957460, 0.30605758, 0.62716383]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecmat3(self):
        A, b4 = self.A, self.b4
        res = np.dot(A.transpose(), b4)
        tgt = np.array([1.23495091, 1.12222648])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecvecouter(self):
        b1, b3 = self.b1, self.b3
        res = np.dot(b1, b3)
        tgt = np.array([[0.20128610, 0.08400440], [0.07190947, 0.03001058]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecvecinner(self):
        b1, b3 = self.b1, self.b3
        res = np.dot(b3, b1)
        tgt = np.array([[ 0.23129668]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotcolumnvect1(self):
        b1 = np.ones((3, 1))
        b2 = [5.3]
        res = np.dot(b1, b2)
        tgt = np.array([5.3, 5.3, 5.3])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotcolumnvect2(self):
        b1 = np.ones((3, 1)).transpose()
        b2 = [6.2]
        res = np.dot(b2, b1)
        tgt = np.array([6.2, 6.2, 6.2])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecscalar(self):
        np.random.seed(100)
        b1 = np.random.rand(1, 1)
        b2 = np.random.rand(1, 4)
        res = np.dot(b1, b2)
        tgt = np.array([[0.15126730, 0.23068496, 0.45905553, 0.00256425]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_dotvecscalar2(self):
        np.random.seed(100)
        b1 = np.random.rand(4, 1)
        b2 = np.random.rand(1, 1)
        res = np.dot(b1, b2)
        tgt = np.array([[0.00256425],[0.00131359],[0.00200324],[ 0.00398638]])
        assert_almost_equal(res, tgt, decimal=self.N)

    def test_all(self):
        dims = [(), (1,), (1, 1)]
        dout = [(), (1,), (1, 1), (1,), (), (1,), (1, 1), (1,), (1, 1)]
        for dim, (dim1, dim2) in zip(dout, itertools.product(dims, dims)):
            b1 = np.zeros(dim1)
            b2 = np.zeros(dim2)
            res = np.dot(b1, b2)
            tgt = np.zeros(dim)
            assert_(res.shape == tgt.shape)
            assert_almost_equal(res, tgt, decimal=self.N)

    def test_vecobject(self):
        class Vec:
            def __init__(self, sequence=None):
                if sequence is None:
                    sequence = []
                self.array = np.array(sequence)

            def __add__(self, other):
                out = Vec()
                out.array = self.array + other.array
                return out

            def __sub__(self, other):
                out = Vec()
                out.array = self.array - other.array
                return out

            def __mul__(self, other):  # with scalar
                out = Vec(self.array.copy())
                out.array *= other
                return out

            def __rmul__(self, other):
                return self*other

        U_non_cont = np.transpose([[1., 1.], [1., 2.]])
        U_cont = np.ascontiguousarray(U_non_cont)
        x = np.array([Vec([1., 0.]), Vec([0., 1.])])
        zeros = np.array([Vec([0., 0.]), Vec([0., 0.])])
        zeros_test = np.dot(U_cont, x) - np.dot(U_non_cont, x)
        assert_equal(zeros[0].array, zeros_test[0].array)
        assert_equal(zeros[1].array, zeros_test[1].array)

    def test_dot_2args(self):
        from numpy.core.multiarray import dot

        a = np.array([[1, 2], [3, 4]], dtype=float)
        b = np.array([[1, 0], [1, 1]], dtype=float)
        c = np.array([[3, 2], [7, 4]], dtype=float)

        d = dot(a, b)
        assert_allclose(c, d)

    def test_dot_3args(self):
        from numpy.core.multiarray import dot

        np.random.seed(22)
        f = np.random.random_sample((1024, 16))
        v = np.random.random_sample((16, 32))

        r = np.empty((1024, 32))
        for i in range(12):
            dot(f, v, r)
        if HAS_REFCOUNT:
            assert_equal(sys.getrefcount(r), 2)
        r2 = dot(f, v, out=None)
        assert_array_equal(r2, r)
        assert_(r is dot(f, v, out=r))

        v = v[:, 0].copy()  # v.shape == (16,)
        r = r[:, 0].copy()  # r.shape == (1024,)
        r2 = dot(f, v)
        assert_(r is dot(f, v, r))
        assert_array_equal(r2, r)

    def test_dot_3args_errors(self):
        from numpy.core.multiarray import dot

        np.random.seed(22)
        f = np.random.random_sample((1024, 16))
        v = np.random.random_sample((16, 32))

        r = np.empty((1024, 31))
        assert_raises(ValueError, dot, f, v, r)

        r = np.empty((1024,))
        assert_raises(ValueError, dot, f, v, r)

        r = np.empty((32,))
        assert_raises(ValueError, dot, f, v, r)

        r = np.empty((32, 1024))
        assert_raises(ValueError, dot, f, v, r)
        assert_raises(ValueError, dot, f, v, r.T)

        r = np.empty((1024, 64))
        assert_raises(ValueError, dot, f, v, r[:, ::2])
        assert_raises(ValueError, dot, f, v, r[:, :32])

        r = np.empty((1024, 32), dtype=np.float32)
        assert_raises(ValueError, dot, f, v, r)

        r = np.empty((1024, 32), dtype=int)
        assert_raises(ValueError, dot, f, v, r)

    def test_dot_array_order(self):
        a = np.array([[1, 2], [3, 4]], order='C')
        b = np.array([[1, 2], [3, 4]], order='F')
        res = np.dot(a, a)

        # integer arrays are exact
        assert_equal(np.dot(a, b), res)
        assert_equal(np.dot(b, a), res)
        assert_equal(np.dot(b, b), res)

    def test_accelerate_framework_sgemv_fix(self):

        def aligned_array(shape, align, dtype, order='C'):
            d = dtype(0)
            N = np.prod(shape)
            tmp = np.zeros(N * d.nbytes + align, dtype=np.uint8)
            address = tmp.__array_interface__["data"][0]
            for offset in range(align):
                if (address + offset) % align == 0:
                    break
            tmp = tmp[offset:offset+N*d.nbytes].view(dtype=dtype)
            return tmp.reshape(shape, order=order)

        def as_aligned(arr, align, dtype, order='C'):
            aligned = aligned_array(arr.shape, align, dtype, order)
            aligned[:] = arr[:]
            return aligned

        def assert_dot_close(A, X, desired):
            assert_allclose(np.dot(A, X), desired, rtol=1e-5, atol=1e-7)

        m = aligned_array(100, 15, np.float32)
        s = aligned_array((100, 100), 15, np.float32)
        np.dot(s, m)  # this will always segfault if the bug is present

        testdata = itertools.product((15, 32), (10000,), (200, 89), ('C', 'F'))
        for align, m, n, a_order in testdata:
            # Calculation in double precision
            A_d = np.random.rand(m, n)
            X_d = np.random.rand(n)
            desired = np.dot(A_d, X_d)
            # Calculation with aligned single precision
            A_f = as_aligned(A_d, align, np.float32, order=a_order)
            X_f = as_aligned(X_d, align, np.float32)
            assert_dot_close(A_f, X_f, desired)
            # Strided A rows
            A_d_2 = A_d[::2]
            desired = np.dot(A_d_2, X_d)
            A_f_2 = A_f[::2]
            assert_dot_close(A_f_2, X_f, desired)
            # Strided A columns, strided X vector
            A_d_22 = A_d_2[:, ::2]
            X_d_2 = X_d[::2]
            desired = np.dot(A_d_22, X_d_2)
            A_f_22 = A_f_2[:, ::2]
            X_f_2 = X_f[::2]
            assert_dot_close(A_f_22, X_f_2, desired)
            # Check the strides are as expected
            if a_order == 'F':
                assert_equal(A_f_22.strides, (8, 8 * m))
            else:
                assert_equal(A_f_22.strides, (8 * n, 8))
            assert_equal(X_f_2.strides, (8,))
            # Strides in A rows + cols only
            X_f_2c = as_aligned(X_f_2, align, np.float32)
            assert_dot_close(A_f_22, X_f_2c, desired)
            # Strides just in A cols
            A_d_12 = A_d[:, ::2]
            desired = np.dot(A_d_12, X_d_2)
            A_f_12 = A_f[:, ::2]
            assert_dot_close(A_f_12, X_f_2c, desired)
            # Strides in A cols and X
            assert_dot_close(A_f_12, X_f_2, desired)

    @pytest.mark.slow
    @pytest.mark.parametrize("dtype", [np.float64, np.complex128])
    @requires_memory(free_bytes=18e9)  # complex case needs 18GiB+
    def test_huge_vectordot(self, dtype):
        # Large vector multiplications are chunked with 32bit BLAS
        # Test that the chunking does the right thing, see also gh-22262
        data = np.ones(2**30+100, dtype=dtype)
        res = np.dot(data, data)
        assert res == 2**30+100

    def test_dtype_discovery_fails(self):
        # See gh-14247, error checking was missing for failed dtype discovery
        class BadObject(object):
            def __array__(self):
                raise TypeError("just this tiny mint leaf")

        with pytest.raises(TypeError):
            np.dot(BadObject(), BadObject())

        with pytest.raises(TypeError):
            np.dot(3.0, BadObject())


class MatmulCommon:
    """Common tests for '@' operator and numpy.matmul.

    """
    # Should work with these types. Will want to add
    # "O" at some point
    types = "?bhilqBHILQefdgFDGO"

    def test_exceptions(self):
        dims = [
            ((1,), (2,)),            # mismatched vector vector
            ((2, 1,), (2,)),         # mismatched matrix vector
            ((2,), (1, 2)),          # mismatched vector matrix
            ((1, 2), (3, 1)),        # mismatched matrix matrix
            ((1,), ()),              # vector scalar
            ((), (1)),               # scalar vector
            ((1, 1), ()),            # matrix scalar
            ((), (1, 1)),            # scalar matrix
            ((2, 2, 1), (3, 1, 2)),  # cannot broadcast
            ]

        for dt, (dm1, dm2) in itertools.product(self.types, dims):
            a = np.ones(dm1, dtype=dt)
            b = np.ones(dm2, dtype=dt)
            assert_raises(ValueError, self.matmul, a, b)

    def test_shapes(self):
        dims = [
            ((1, 1), (2, 1, 1)),     # broadcast first argument
            ((2, 1, 1), (1, 1)),     # broadcast second argument
            ((2, 1, 1), (2, 1, 1)),  # matrix stack sizes match
            ]

        for dt, (dm1, dm2) in itertools.product(self.types, dims):
            a = np.ones(dm1, dtype=dt)
            b = np.ones(dm2, dtype=dt)
            res = self.matmul(a, b)
            assert_(res.shape == (2, 1, 1))

        # vector vector returns scalars.
        for dt in self.types:
            a = np.ones((2,), dtype=dt)
            b = np.ones((2,), dtype=dt)
            c = self.matmul(a, b)
            assert_(np.array(c).shape == ())

    def test_result_types(self):
        mat = np.ones((1,1))
        vec = np.ones((1,))
        for dt in self.types:
            m = mat.astype(dt)
            v = vec.astype(dt)
            for arg in [(m, v), (v, m), (m, m)]:
                res = self.matmul(*arg)
                assert_(res.dtype == dt)

            # vector vector returns scalars
            if dt != "O":
                res = self.matmul(v, v)
                assert_(type(res) is np.dtype(dt).type)

    def test_scalar_output(self):
        vec1 = np.array([2])
        vec2 = np.array([3, 4]).reshape(1, -1)
        tgt = np.array([6, 8])
        for dt in self.types[1:]:
            v1 = vec1.astype(dt)
            v2 = vec2.astype(dt)
            res = self.matmul(v1, v2)
            assert_equal(res, tgt)
            res = self.matmul(v2.T, v1)
            assert_equal(res, tgt)

        # boolean type
        vec = np.array([True, True], dtype='?').reshape(1, -1)
        res = self.matmul(vec[:, 0], vec)
        assert_equal(res, True)

    def test_vector_vector_values(self):
        vec1 = np.array([1, 2])
        vec2 = np.array([3, 4]).reshape(-1, 1)
        tgt1 = np.array([11])
        tgt2 = np.array([[3, 6], [4, 8]])
        for dt in self.types[1:]:
            v1 = vec1.astype(dt)
            v2 = vec2.astype(dt)
            res = self.matmul(v1, v2)
            assert_equal(res, tgt1)
            # no broadcast, we must make v1 into a 2d ndarray
            res = self.matmul(v2, v1.reshape(1, -1))
            assert_equal(res, tgt2)

        # boolean type
        vec = np.array([True, True], dtype='?')
        res = self.matmul(vec, vec)
        assert_equal(res, True)

    def test_vector_matrix_values(self):
        vec = np.array([1, 2])
        mat1 = np.array([[1, 2], [3, 4]])
        mat2 = np.stack([mat1]*2, axis=0)
        tgt1 = np.array([7, 10])
        tgt2 = np.stack([tgt1]*2, axis=0)
        for dt in self.types[1:]:
            v = vec.astype(dt)
            m1 = mat1.astype(dt)
            m2 = mat2.astype(dt)
            res = self.matmul(v, m1)
            assert_equal(res, tgt1)
            res = self.matmul(v, m2)
            assert_equal(res, tgt2)

        # boolean type
        vec = np.array([True, False])
        mat1 = np.array([[True, False], [False, True]])
        mat2 = np.stack([mat1]*2, axis=0)
        tgt1 = np.array([True, False])
        tgt2 = np.stack([tgt1]*2, axis=0)

        res = self.matmul(vec, mat1)
        assert_equal(res, tgt1)
        res = self.matmul(vec, mat2)
        assert_equal(res, tgt2)

    def test_matrix_vector_values(self):
        vec = np.array([1, 2])
        mat1 = np.array([[1, 2], [3, 4]])
        mat2 = np.stack([mat1]*2, axis=0)
        tgt1 = np.array([5, 11])
        tgt2 = np.stack([tgt1]*2, axis=0)
        for dt in self.types[1:]:
            v = vec.astype(dt)
            m1 = mat1.astype(dt)
            m2 = mat2.astype(dt)
            res = self.matmul(m1, v)
            assert_equal(res, tgt1)
            res = self.matmul(m2, v)
            assert_equal(res, tgt2)

        # boolean type
        vec = np.array([True, False])
        mat1 = np.array([[True, False], [False, True]])
        mat2 = np.stack([mat1]*2, axis=0)
        tgt1 = np.array([True, False])
        tgt2 = np.stack([tgt1]*2, axis=0)

        res = self.matmul(vec, mat1)
        assert_equal(res, tgt1)
        res = self.matmul(vec, mat2)
        assert_equal(res, tgt2)

    def test_matrix_matrix_values(self):
        mat1 = np.array([[1, 2], [3, 4]])
        mat2 = np.array([[1, 0], [1, 1]])
        mat12 = np.stack([mat1, mat2], axis=0)
        mat21 = np.stack([mat2, mat1], axis=0)
        tgt11 = np.array([[7, 10], [15, 22]])
        tgt12 = np.array([[3, 2], [7, 4]])
        tgt21 = np.array([[1, 2], [4, 6]])
        tgt12_21 = np.stack([tgt12, tgt21], axis=0)
        tgt11_12 = np.stack((tgt11, tgt12), axis=0)
        tgt11_21 = np.stack((tgt11, tgt21), axis=0)
        for dt in self.types[1:]:
            m1 = mat1.astype(dt)
            m2 = mat2.astype(dt)
            m12 = mat12.astype(dt)
            m21 = mat21.astype(dt)

            # matrix @ matrix
            res = self.matmul(m1, m2)
            assert_equal(res, tgt12)
            res = self.matmul(m2, m1)
            assert_equal(res, tgt21)

            # stacked @ matrix
            res = self.matmul(m12, m1)
            assert_equal(res, tgt11_21)

            # matrix @ stacked
            res = self.matmul(m1, m12)
            assert_equal(res, tgt11_12)

            # stacked @ stacked
            res = self.matmul(m12, m21)
            assert_equal(res, tgt12_21)

        # boolean type
        m1 = np.array([[1, 1], [0, 0]], dtype=np.bool_)
        m2 = np.array([[1, 0], [1, 1]], dtype=np.bool_)
        m12 = np.stack([m1, m2], axis=0)
        m21 = np.stack([m2, m1], axis=0)
        tgt11 = m1
        tgt12 = m1
        tgt21 = np.array([[1, 1], [1, 1]], dtype=np.bool_)
        tgt12_21 = np.stack([tgt12, tgt21], axis=0)
        tgt11_12 = np.stack((tgt11, tgt12), axis=0)
        tgt11_21 = np.stack((tgt11, tgt21), axis=0)

        # matrix @ matrix
        res = self.matmul(m1, m2)
        assert_equal(res, tgt12)
        res = self.matmul(m2, m1)
        assert_equal(res, tgt21)

        # stacked @ matrix
        res = self.matmul(m12, m1)
        assert_equal(res, tgt11_21)

        # matrix @ stacked
        res = self.matmul(m1, m12)
        assert_equal(res, tgt11_12)

        # stacked @ stacked
        res = self.matmul(m12, m21)
        assert_equal(res, tgt12_21)


class TestMatmul(MatmulCommon):
    matmul = np.matmul

    def test_out_arg(self):
        a = np.ones((5, 2), dtype=float)
        b = np.array([[1, 3], [5, 7]], dtype=float)
        tgt = np.dot(a, b)

        # test as positional argument
        msg = "out positional argument"
        out = np.zeros((5, 2), dtype=float)
        self.matmul(a, b, out)
        assert_array_equal(out, tgt, err_msg=msg)

        # test as keyword argument
        msg = "out keyword argument"
        out = np.zeros((5, 2), dtype=float)
        self.matmul(a, b, out=out)
        assert_array_equal(out, tgt, err_msg=msg)

        # test out with not allowed type cast (safe casting)
        msg = "Cannot cast ufunc .* output"
        out = np.zeros((5, 2), dtype=np.int32)
        assert_raises_regex(TypeError, msg, self.matmul, a, b, out=out)

        # test out with type upcast to complex
        out = np.zeros((5, 2), dtype=np.complex128)
        c = self.matmul(a, b, out=out)
        assert_(c is out)
        with suppress_warnings() as sup:
            sup.filter(np.ComplexWarning, '')
            c = c.astype(tgt.dtype)
        assert_array_equal(c, tgt)

    def test_empty_out(self):
        # Check that the output cannot be broadcast, so that it cannot be
        # size zero when the outer dimensions (iterator size) has size zero.
        arr = np.ones((0, 1, 1))
        out = np.ones((1, 1, 1))
        assert self.matmul(arr, arr).shape == (0, 1, 1)

        with pytest.raises(ValueError, match=r"non-broadcastable"):
            self.matmul(arr, arr, out=out)

    def test_out_contiguous(self):
        a = np.ones((5, 2), dtype=float)
        b = np.array([[1, 3], [5, 7]], dtype=float)
        v = np.array([1, 3], dtype=float)
        tgt = np.dot(a, b)
        tgt_mv = np.dot(a, v)

        # test out non-contiguous
        out = np.ones((5, 2, 2), dtype=float)
        c = self.matmul(a, b, out=out[..., 0])
        assert c.base is out
        assert_array_equal(c, tgt)
        c = self.matmul(a, v, out=out[:, 0, 0])
        assert_array_equal(c, tgt_mv)
        c = self.matmul(v, a.T, out=out[:, 0, 0])
        assert_array_equal(c, tgt_mv)

        # test out contiguous in only last dim
        out = np.ones((10, 2), dtype=float)
        c = self.matmul(a, b, out=out[::2, :])
        assert_array_equal(c, tgt)

        # test transposes of out, args
        out = np.ones((5, 2), dtype=float)
        c = self.matmul(b.T, a.T, out=out.T)
        assert_array_equal(out, tgt)

    m1 = np.arange(15.).reshape(5, 3)
    m2 = np.arange(21.).reshape(3, 7)
    m3 = np.arange(30.).reshape(5, 6)[:, ::2]  # non-contiguous
    vc = np.arange(10.)
    vr = np.arange(6.)
    m0 = np.zeros((3, 0))
    @pytest.mark.parametrize('args', (
            # matrix-matrix
            (m1, m2), (m2.T, m1.T), (m2.T.copy(), m1.T), (m2.T, m1.T.copy()),
            # matrix-matrix-transpose, contiguous and non
            (m1, m1.T), (m1.T, m1), (m1, m3.T), (m3, m1.T),
            (m3, m3.T), (m3.T, m3),
            # matrix-matrix non-contiguous
            (m3, m2), (m2.T, m3.T), (m2.T.copy(), m3.T),
            # vector-matrix, matrix-vector, contiguous
            (m1, vr[:3]), (vc[:5], m1), (m1.T, vc[:5]), (vr[:3], m1.T),
            # vector-matrix, matrix-vector, vector non-contiguous
            (m1, vr[::2]), (vc[::2], m1), (m1.T, vc[::2]), (vr[::2], m1.T),
            # vector-matrix, matrix-vector, matrix non-contiguous
            (m3, vr[:3]), (vc[:5], m3), (m3.T, vc[:5]), (vr[:3], m3.T),
            # vector-matrix, matrix-vector, both non-contiguous
            (m3, vr[::2]), (vc[::2], m3), (m3.T, vc[::2]), (vr[::2], m3.T),
            # size == 0
            (m0, m0.T), (m0.T, m0), (m1, m0), (m0.T, m1.T),
        ))
    def test_dot_equivalent(self, args):
        r1 = np.matmul(*args)
        r2 = np.dot(*args)
        assert_equal(r1, r2)

        r3 = np.matmul(args[0].copy(), args[1].copy())
        assert_equal(r1, r3)

    def test_matmul_object(self):
        import fractions

        f = np.vectorize(fractions.Fraction)
        def random_ints():
            return np.random.randint(1, 1000, size=(10, 3, 3))
        M1 = f(random_ints(), random_ints())
        M2 = f(random_ints(), random_ints())

        M3 = self.matmul(M1, M2)

        [N1, N2, N3] = [a.astype(float) for a in [M1, M2, M3]]

        assert_allclose(N3, self.matmul(N1, N2))

    def test_matmul_object_type_scalar(self):
        from fractions import Fraction as F
        v = np.array([F(2,3), F(5,7)])
        res = self.matmul(v, v)
        assert_(type(res) is F)

    def test_matmul_empty(self):
        a = np.empty((3, 0), dtype=object)
        b = np.empty((0, 3), dtype=object)
        c = np.zeros((3, 3))
        assert_array_equal(np.matmul(a, b), c)

    def test_matmul_exception_multiply(self):
        # test that matmul fails if `__mul__` is missing
        class add_not_multiply():
            def __add__(self, other):
                return self
        a = np.full((3,3), add_not_multiply())
        with assert_raises(TypeError):
            b = np.matmul(a, a)

    def test_matmul_exception_add(self):
        # test that matmul fails if `__add__` is missing
        class multiply_not_add():
            def __mul__(self, other):
                return self
        a = np.full((3,3), multiply_not_add())
        with assert_raises(TypeError):
            b = np.matmul(a, a)

    def test_matmul_bool(self):
        # gh-14439
        a = np.array([[1, 0],[1, 1]], dtype=bool)
        assert np.max(a.view(np.uint8)) == 1
        b = np.matmul(a, a)
        # matmul with boolean output should always be 0, 1
        assert np.max(b.view(np.uint8)) == 1

        rg = np.random.default_rng(np.random.PCG64(43))
        d = rg.integers(2, size=4*5, dtype=np.int8)
        d = d.reshape(4, 5) > 0
        out1 = np.matmul(d, d.reshape(5, 4))
        out2 = np.dot(d, d.reshape(5, 4))
        assert_equal(out1, out2)

        c = np.matmul(np.zeros((2, 0), dtype=bool), np.zeros(0, dtype=bool))
        assert not np.any(c)


class TestMatmulOperator(MatmulCommon):
    import operator
    matmul = operator.matmul

    def test_array_priority_override(self):

        class A:
            __array_priority__ = 1000

            def __matmul__(self, other):
                return "A"

            def __rmatmul__(self, other):
                return "A"

        a = A()
        b = np.ones(2)
        assert_equal(self.matmul(a, b), "A")
        assert_equal(self.matmul(b, a), "A")

    def test_matmul_raises(self):
        assert_raises(TypeError, self.matmul, np.int8(5), np.int8(5))
        assert_raises(TypeError, self.matmul, np.void(b'abc'), np.void(b'abc'))
        assert_raises(TypeError, self.matmul, np.arange(10), np.void(b'abc'))

def test_matmul_inplace():
    # It would be nice to support in-place matmul eventually, but for now
    # we don't have a working implementation, so better just to error out
    # and nudge people to writing "a = a @ b".
    a = np.eye(3)
    b = np.eye(3)
    assert_raises(TypeError, a.__imatmul__, b)
    import operator
    assert_raises(TypeError, operator.imatmul, a, b)
    assert_raises(TypeError, exec, "a @= b", globals(), locals())

def test_matmul_axes():
    a = np.arange(3*4*5).reshape(3, 4, 5)
    c = np.matmul(a, a, axes=[(-2, -1), (-1, -2), (1, 2)])
    assert c.shape == (3, 4, 4)
    d = np.matmul(a, a, axes=[(-2, -1), (-1, -2), (0, 1)])
    assert d.shape == (4, 4, 3)
    e = np.swapaxes(d, 0, 2)
    assert_array_equal(e, c)
    f = np.matmul(a, np.arange(3), axes=[(1, 0), (0), (0)])
    assert f.shape == (4, 5)


class TestInner:

    def test_inner_type_mismatch(self):
        c = 1.
        A = np.array((1,1), dtype='i,i')

        assert_raises(TypeError, np.inner, c, A)
        assert_raises(TypeError, np.inner, A, c)

    def test_inner_scalar_and_vector(self):
        for dt in np.typecodes['AllInteger'] + np.typecodes['AllFloat'] + '?':
            sca = np.array(3, dtype=dt)[()]
            vec = np.array([1, 2], dtype=dt)
            desired = np.array([3, 6], dtype=dt)
            assert_equal(np.inner(vec, sca), desired)
            assert_equal(np.inner(sca, vec), desired)

    def test_vecself(self):
        # Ticket 844.
        # Inner product of a vector with itself segfaults or give
        # meaningless result
        a = np.zeros(shape=(1, 80), dtype=np.float64)
        p = np.inner(a, a)
        assert_almost_equal(p, 0, decimal=14)

    def test_inner_product_with_various_contiguities(self):
        # github issue 6532
        for dt in np.typecodes['AllInteger'] + np.typecodes['AllFloat'] + '?':
            # check an inner product involving a matrix transpose
            A = np.array([[1, 2], [3, 4]], dtype=dt)
            B = np.array([[1, 3], [2, 4]], dtype=dt)
            C = np.array([1, 1], dtype=dt)
            desired = np.array([4, 6], dtype=dt)
            assert_equal(np.inner(A.T, C), desired)
            assert_equal(np.inner(C, A.T), desired)
            assert_equal(np.inner(B, C), desired)
            assert_equal(np.inner(C, B), desired)
            # check a matrix product
            desired = np.array([[7, 10], [15, 22]], dtype=dt)
            assert_equal(np.inner(A, B), desired)
            # check the syrk vs. gemm paths
            desired = np.array([[5, 11], [11, 25]], dtype=dt)
            assert_equal(np.inner(A, A), desired)
            assert_equal(np.inner(A, A.copy()), desired)
            # check an inner product involving an aliased and reversed view
            a = np.arange(5).astype(dt)
            b = a[::-1]
            desired = np.array(10, dtype=dt).item()
            assert_equal(np.inner(b, a), desired)

    def test_3d_tensor(self):
        for dt in np.typecodes['AllInteger'] + np.typecodes['AllFloat'] + '?':
            a = np.arange(24).reshape(2,3,4).astype(dt)
            b = np.arange(24, 48).reshape(2,3,4).astype(dt)
            desired = np.array(
                [[[[ 158,  182,  206],
                   [ 230,  254,  278]],

                  [[ 566,  654,  742],
                   [ 830,  918, 1006]],

                  [[ 974, 1126, 1278],
                   [1430, 1582, 1734]]],

                 [[[1382, 1598, 1814],
                   [2030, 2246, 2462]],

                  [[1790, 2070, 2350],
                   [2630, 2910, 3190]],

                  [[2198, 2542, 2886],
                   [3230, 3574, 3918]]]]
            ).astype(dt)
            assert_equal(np.inner(a, b), desired)
            assert_equal(np.inner(b, a).transpose(2,3,0,1), desired)


class TestChoose:
    def setup_method(self):
        self.x = 2*np.ones((3,), dtype=int)
        self.y = 3*np.ones((3,), dtype=int)
        self.x2 = 2*np.ones((2, 3), dtype=int)
        self.y2 = 3*np.ones((2, 3), dtype=int)
        self.ind = [0, 0, 1]

    def test_basic(self):
        A = np.choose(self.ind, (self.x, self.y))
        assert_equal(A, [2, 2, 3])

    def test_broadcast1(self):
        A = np.choose(self.ind, (self.x2, self.y2))
        assert_equal(A, [[2, 2, 3], [2, 2, 3]])

    def test_broadcast2(self):
        A = np.choose(self.ind, (self.x, self.y2))
        assert_equal(A, [[2, 2, 3], [2, 2, 3]])

    @pytest.mark.parametrize("ops",
        [(1000, np.array([1], dtype=np.uint8)),
         (-1, np.array([1], dtype=np.uint8)),
         (1., np.float32(3)),
         (1., np.array([3], dtype=np.float32))],)
    def test_output_dtype(self, ops):
        expected_dt = np.result_type(*ops)
        assert(np.choose([0], ops).dtype == expected_dt)


class TestRepeat:
    def setup_method(self):
        self.m = np.array([1, 2, 3, 4, 5, 6])
        self.m_rect = self.m.reshape((2, 3))

    def test_basic(self):
        A = np.repeat(self.m, [1, 3, 2, 1, 1, 2])
        assert_equal(A, [1, 2, 2, 2, 3,
                         3, 4, 5, 6, 6])

    def test_broadcast1(self):
        A = np.repeat(self.m, 2)
        assert_equal(A, [1, 1, 2, 2, 3, 3,
                         4, 4, 5, 5, 6, 6])

    def test_axis_spec(self):
        A = np.repeat(self.m_rect, [2, 1], axis=0)
        assert_equal(A, [[1, 2, 3],
                         [1, 2, 3],
                         [4, 5, 6]])

        A = np.repeat(self.m_rect, [1, 3, 2], axis=1)
        assert_equal(A, [[1, 2, 2, 2, 3, 3],
                         [4, 5, 5, 5, 6, 6]])

    def test_broadcast2(self):
        A = np.repeat(self.m_rect, 2, axis=0)
        assert_equal(A, [[1, 2, 3],
                         [1, 2, 3],
                         [4, 5, 6],
                         [4, 5, 6]])

        A = np.repeat(self.m_rect, 2, axis=1)
        assert_equal(A, [[1, 1, 2, 2, 3, 3],
                         [4, 4, 5, 5, 6, 6]])


# TODO: test for multidimensional
NEIGH_MODE = {'zero': 0, 'one': 1, 'constant': 2, 'circular': 3, 'mirror': 4}


@pytest.mark.parametrize('dt', [float, Decimal], ids=['float', 'object'])
class TestNeighborhoodIter:
    # Simple, 2d tests
    def test_simple2d(self, dt):
        # Test zero and one padding for simple data type
        x = np.array([[0, 1], [2, 3]], dtype=dt)
        r = [np.array([[0, 0, 0], [0, 0, 1]], dtype=dt),
             np.array([[0, 0, 0], [0, 1, 0]], dtype=dt),
             np.array([[0, 0, 1], [0, 2, 3]], dtype=dt),
             np.array([[0, 1, 0], [2, 3, 0]], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 0, -1, 1], x[0], NEIGH_MODE['zero'])
        assert_array_equal(l, r)

        r = [np.array([[1, 1, 1], [1, 0, 1]], dtype=dt),
             np.array([[1, 1, 1], [0, 1, 1]], dtype=dt),
             np.array([[1, 0, 1], [1, 2, 3]], dtype=dt),
             np.array([[0, 1, 1], [2, 3, 1]], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 0, -1, 1], x[0], NEIGH_MODE['one'])
        assert_array_equal(l, r)

        r = [np.array([[4, 4, 4], [4, 0, 1]], dtype=dt),
             np.array([[4, 4, 4], [0, 1, 4]], dtype=dt),
             np.array([[4, 0, 1], [4, 2, 3]], dtype=dt),
             np.array([[0, 1, 4], [2, 3, 4]], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 0, -1, 1], 4, NEIGH_MODE['constant'])
        assert_array_equal(l, r)

        # Test with start in the middle
        r = [np.array([[4, 0, 1], [4, 2, 3]], dtype=dt),
             np.array([[0, 1, 4], [2, 3, 4]], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 0, -1, 1], 4, NEIGH_MODE['constant'], 2)
        assert_array_equal(l, r)

    def test_mirror2d(self, dt):
        x = np.array([[0, 1], [2, 3]], dtype=dt)
        r = [np.array([[0, 0, 1], [0, 0, 1]], dtype=dt),
             np.array([[0, 1, 1], [0, 1, 1]], dtype=dt),
             np.array([[0, 0, 1], [2, 2, 3]], dtype=dt),
             np.array([[0, 1, 1], [2, 3, 3]], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 0, -1, 1], x[0], NEIGH_MODE['mirror'])
        assert_array_equal(l, r)

    # Simple, 1d tests
    def test_simple(self, dt):
        # Test padding with constant values
        x = np.linspace(1, 5, 5).astype(dt)
        r = [[0, 1, 2], [1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 0]]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 1], x[0], NEIGH_MODE['zero'])
        assert_array_equal(l, r)

        r = [[1, 1, 2], [1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 1]]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 1], x[0], NEIGH_MODE['one'])
        assert_array_equal(l, r)

        r = [[x[4], 1, 2], [1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, x[4]]]
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-1, 1], x[4], NEIGH_MODE['constant'])
        assert_array_equal(l, r)

    # Test mirror modes
    def test_mirror(self, dt):
        x = np.linspace(1, 5, 5).astype(dt)
        r = np.array([[2, 1, 1, 2, 3], [1, 1, 2, 3, 4], [1, 2, 3, 4, 5],
                [2, 3, 4, 5, 5], [3, 4, 5, 5, 4]], dtype=dt)
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-2, 2], x[1], NEIGH_MODE['mirror'])
        assert_([i.dtype == dt for i in l])
        assert_array_equal(l, r)

    # Circular mode
    def test_circular(self, dt):
        x = np.linspace(1, 5, 5).astype(dt)
        r = np.array([[4, 5, 1, 2, 3], [5, 1, 2, 3, 4], [1, 2, 3, 4, 5],
                [2, 3, 4, 5, 1], [3, 4, 5, 1, 2]], dtype=dt)
        l = _multiarray_tests.test_neighborhood_iterator(
                x, [-2, 2], x[0], NEIGH_MODE['circular'])
        assert_array_equal(l, r)


# Test stacking neighborhood iterators
class TestStackedNeighborhoodIter:
    # Simple, 1d test: stacking 2 constant-padded neigh iterators
    def test_simple_const(self):
        dt = np.float64
        # Test zero and one padding for simple data type
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([0], dtype=dt),
             np.array([0], dtype=dt),
             np.array([1], dtype=dt),
             np.array([2], dtype=dt),
             np.array([3], dtype=dt),
             np.array([0], dtype=dt),
             np.array([0], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-2, 4], NEIGH_MODE['zero'], [0, 0], NEIGH_MODE['zero'])
        assert_array_equal(l, r)

        r = [np.array([1, 0, 1], dtype=dt),
             np.array([0, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 0], dtype=dt),
             np.array([3, 0, 1], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [-1, 1], NEIGH_MODE['one'])
        assert_array_equal(l, r)

    # 2nd simple, 1d test: stacking 2 neigh iterators, mixing const padding and
    # mirror padding
    def test_simple_mirror(self):
        dt = np.float64
        # Stacking zero on top of mirror
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([0, 1, 1], dtype=dt),
             np.array([1, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 3], dtype=dt),
             np.array([3, 3, 0], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['mirror'], [-1, 1], NEIGH_MODE['zero'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([1, 0, 0], dtype=dt),
             np.array([0, 0, 1], dtype=dt),
             np.array([0, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 0], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [-2, 0], NEIGH_MODE['mirror'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero: 2nd
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([0, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 0], dtype=dt),
             np.array([3, 0, 0], dtype=dt),
             np.array([0, 0, 3], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [0, 2], NEIGH_MODE['mirror'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero: 3rd
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([1, 0, 0, 1, 2], dtype=dt),
             np.array([0, 0, 1, 2, 3], dtype=dt),
             np.array([0, 1, 2, 3, 0], dtype=dt),
             np.array([1, 2, 3, 0, 0], dtype=dt),
             np.array([2, 3, 0, 0, 3], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [-2, 2], NEIGH_MODE['mirror'])
        assert_array_equal(l, r)

    # 3rd simple, 1d test: stacking 2 neigh iterators, mixing const padding and
    # circular padding
    def test_simple_circular(self):
        dt = np.float64
        # Stacking zero on top of mirror
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([0, 3, 1], dtype=dt),
             np.array([3, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 1], dtype=dt),
             np.array([3, 1, 0], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['circular'], [-1, 1], NEIGH_MODE['zero'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([3, 0, 0], dtype=dt),
             np.array([0, 0, 1], dtype=dt),
             np.array([0, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 0], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [-2, 0], NEIGH_MODE['circular'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero: 2nd
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([0, 1, 2], dtype=dt),
             np.array([1, 2, 3], dtype=dt),
             np.array([2, 3, 0], dtype=dt),
             np.array([3, 0, 0], dtype=dt),
             np.array([0, 0, 1], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [0, 2], NEIGH_MODE['circular'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero: 3rd
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([3, 0, 0, 1, 2], dtype=dt),
             np.array([0, 0, 1, 2, 3], dtype=dt),
             np.array([0, 1, 2, 3, 0], dtype=dt),
             np.array([1, 2, 3, 0, 0], dtype=dt),
             np.array([2, 3, 0, 0, 1], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [-1, 3], NEIGH_MODE['zero'], [-2, 2], NEIGH_MODE['circular'])
        assert_array_equal(l, r)

    # 4th simple, 1d test: stacking 2 neigh iterators, but with lower iterator
    # being strictly within the array
    def test_simple_strict_within(self):
        dt = np.float64
        # Stacking zero on top of zero, first neighborhood strictly inside the
        # array
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([1, 2, 3, 0], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [1, 1], NEIGH_MODE['zero'], [-1, 2], NEIGH_MODE['zero'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero, first neighborhood strictly inside the
        # array
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([1, 2, 3, 3], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [1, 1], NEIGH_MODE['zero'], [-1, 2], NEIGH_MODE['mirror'])
        assert_array_equal(l, r)

        # Stacking mirror on top of zero, first neighborhood strictly inside the
        # array
        x = np.array([1, 2, 3], dtype=dt)
        r = [np.array([1, 2, 3, 1], dtype=dt)]
        l = _multiarray_tests.test_neighborhood_iterator_oob(
                x, [1, 1], NEIGH_MODE['zero'], [-1, 2], NEIGH_MODE['circular'])
        assert_array_equal(l, r)

class TestWarnings:

    def test_complex_warning(self):
        x = np.array([1, 2])
        y = np.array([1-2j, 1+2j])

        with warnings.catch_warnings():
            warnings.simplefilter("error", np.ComplexWarning)
            assert_raises(np.ComplexWarning, x.__setitem__, slice(None), y)
            assert_equal(x, [1, 2])


class TestMinScalarType:

    def test_usigned_shortshort(self):
        dt = np.min_scalar_type(2**8-1)
        wanted = np.dtype('uint8')
        assert_equal(wanted, dt)

    def test_usigned_short(self):
        dt = np.min_scalar_type(2**16-1)
        wanted = np.dtype('uint16')
        assert_equal(wanted, dt)

    def test_usigned_int(self):
        dt = np.min_scalar_type(2**32-1)
        wanted = np.dtype('uint32')
        assert_equal(wanted, dt)

    def test_usigned_longlong(self):
        dt = np.min_scalar_type(2**63-1)
        wanted = np.dtype('uint64')
        assert_equal(wanted, dt)

    def test_object(self):
        dt = np.min_scalar_type(2**64)
        wanted = np.dtype('O')
        assert_equal(wanted, dt)


from numpy.core._internal import _dtype_from_pep3118


class TestPEP3118Dtype:
    def _check(self, spec, wanted):
        dt = np.dtype(wanted)
        actual = _dtype_from_pep3118(spec)
        assert_equal(actual, dt,
                     err_msg="spec %r != dtype %r" % (spec, wanted))

    def test_native_padding(self):
        align = np.dtype('i').alignment
        for j in range(8):
            if j == 0:
                s = 'bi'
            else:
                s = 'b%dxi' % j
            self._check('@'+s, {'f0': ('i1', 0),
                                'f1': ('i', align*(1 + j//align))})
            self._check('='+s, {'f0': ('i1', 0),
                                'f1': ('i', 1+j)})

    def test_native_padding_2(self):
        # Native padding should work also for structs and sub-arrays
        self._check('x3T{xi}', {'f0': (({'f0': ('i', 4)}, (3,)), 4)})
        self._check('^x3T{xi}', {'f0': (({'f0': ('i', 1)}, (3,)), 1)})

    def test_trailing_padding(self):
        # Trailing padding should be included, *and*, the item size
        # should match the alignment if in aligned mode
        align = np.dtype('i').alignment
        size = np.dtype('i').itemsize

        def aligned(n):
            return align*(1 + (n-1)//align)

        base = dict(formats=['i'], names=['f0'])

        self._check('ix',    dict(itemsize=aligned(size + 1), **base))
        self._check('ixx',   dict(itemsize=aligned(size + 2), **base))
        self._check('ixxx',  dict(itemsize=aligned(size + 3), **base))
        self._check('ixxxx', dict(itemsize=aligned(size + 4), **base))
        self._check('i7x',   dict(itemsize=aligned(size + 7), **base))

        self._check('^ix',    dict(itemsize=size + 1, **base))
        self._check('^ixx',   dict(itemsize=size + 2, **base))
        self._check('^ixxx',  dict(itemsize=size + 3, **base))
        self._check('^ixxxx', dict(itemsize=size + 4, **base))
        self._check('^i7x',   dict(itemsize=size + 7, **base))

    def test_native_padding_3(self):
        dt = np.dtype(
                [('a', 'b'), ('b', 'i'),
                    ('sub', np.dtype('b,i')), ('c', 'i')],
                align=True)
        self._check("T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxxi:c:}", dt)

        dt = np.dtype(
                [('a', 'b'), ('b', 'i'), ('c', 'b'), ('d', 'b'),
                    ('e', 'b'), ('sub', np.dtype('b,i', align=True))])
        self._check("T{b:a:=i:b:b:c:b:d:b:e:T{b:f0:xxxi:f1:}:sub:}", dt)

    def test_padding_with_array_inside_struct(self):
        dt = np.dtype(
                [('a', 'b'), ('b', 'i'), ('c', 'b', (3,)),
                    ('d', 'i')],
                align=True)
        self._check("T{b:a:xxxi:b:3b:c:xi:d:}", dt)

    def test_byteorder_inside_struct(self):
        # The byte order after @T{=i} should be '=', not '@'.
        # Check this by noting the absence of native alignment.
        self._check('@T{^i}xi', {'f0': ({'f0': ('i', 0)}, 0),
                                 'f1': ('i', 5)})

    def test_intra_padding(self):
        # Natively aligned sub-arrays may require some internal padding
        align = np.dtype('i').alignment
        size = np.dtype('i').itemsize

        def aligned(n):
            return (align*(1 + (n-1)//align))

        self._check('(3)T{ix}', (dict(
            names=['f0'],
            formats=['i'],
            offsets=[0],
            itemsize=aligned(size + 1)
        ), (3,)))

    def test_char_vs_string(self):
        dt = np.dtype('c')
        self._check('c', dt)

        dt = np.dtype([('f0', 'S1', (4,)), ('f1', 'S4')])
        self._check('4c4s', dt)

    def test_field_order(self):
        # gh-9053 - previously, we relied on dictionary key order
        self._check("(0)I:a:f:b:", [('a', 'I', (0,)), ('b', 'f')])
        self._check("(0)I:b:f:a:", [('b', 'I', (0,)), ('a', 'f')])

    def test_unnamed_fields(self):
        self._check('ii',     [('f0', 'i'), ('f1', 'i')])
        self._check('ii:f0:', [('f1', 'i'), ('f0', 'i')])

        self._check('i', 'i')
        self._check('i:f0:', [('f0', 'i')])


class TestNewBufferProtocol:
    """ Test PEP3118 buffers """

    def _check_roundtrip(self, obj):
        obj = np.asarray(obj)
        x = memoryview(obj)
        y = np.asarray(x)
        y2 = np.array(x)
        assert_(not y.flags.owndata)
        assert_(y2.flags.owndata)

        assert_equal(y.dtype, obj.dtype)
        assert_equal(y.shape, obj.shape)
        assert_array_equal(obj, y)

        assert_equal(y2.dtype, obj.dtype)
        assert_equal(y2.shape, obj.shape)
        assert_array_equal(obj, y2)

    def test_roundtrip(self):
        x = np.array([1, 2, 3, 4, 5], dtype='i4')
        self._check_roundtrip(x)

        x = np.array([[1, 2], [3, 4]], dtype=np.float64)
        self._check_roundtrip(x)

        x = np.zeros((3, 3, 3), dtype=np.float32)[:, 0,:]
        self._check_roundtrip(x)

        dt = [('a', 'b'),
              ('b', 'h'),
              ('c', 'i'),
              ('d', 'l'),
              ('dx', 'q'),
              ('e', 'B'),
              ('f', 'H'),
              ('g', 'I'),
              ('h', 'L'),
              ('hx', 'Q'),
              ('i', np.single),
              ('j', np.double),
              ('k', np.longdouble),
              ('ix', np.csingle),
              ('jx', np.cdouble),
              ('kx', np.clongdouble),
              ('l', 'S4'),
              ('m', 'U4'),
              ('n', 'V3'),
              ('o', '?'),
              ('p', np.half),
              ]
        x = np.array(
                [(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                    b'aaaa', 'bbbb', b'xxx', True, 1.0)],
                dtype=dt)
        self._check_roundtrip(x)

        x = np.array(([[1, 2], [3, 4]],), dtype=[('a', (int, (2, 2)))])
        self._check_roundtrip(x)

        x = np.array([1, 2, 3], dtype='>i2')
        self._check_roundtrip(x)

        x = np.array([1, 2, 3], dtype='<i2')
        self._check_roundtrip(x)

        x = np.array([1, 2, 3], dtype='>i4')
        self._check_roundtrip(x)

        x = np.array([1, 2, 3], dtype='<i4')
        self._check_roundtrip(x)

        # check long long can be represented as non-native
        x = np.array([1, 2, 3], dtype='>q')
        self._check_roundtrip(x)

        # Native-only data types can be passed through the buffer interface
        # only in native byte order
        if sys.byteorder == 'little':
            x = np.array([1, 2, 3], dtype='>g')
            assert_raises(ValueError, self._check_roundtrip, x)
            x = np.array([1, 2, 3], dtype='<g')
            self._check_roundtrip(x)
        else:
            x = np.array([1, 2, 3], dtype='>g')
            self._check_roundtrip(x)
            x = np.array([1, 2, 3], dtype='<g')
            assert_raises(ValueError, self._check_roundtrip, x)

    def test_roundtrip_half(self):
        half_list = [
            1.0,
            -2.0,
            6.5504 * 10**4,  # (max half precision)
            2**-14,  # ~= 6.10352 * 10**-5 (minimum positive normal)
            2**-24,  # ~= 5.96046 * 10**-8 (minimum strictly positive subnormal)
            0.0,
            -0.0,
            float('+inf'),
            float('-inf'),
            0.333251953125,  # ~= 1/3
        ]

        x = np.array(half_list, dtype='>e')
        self._check_roundtrip(x)
        x = np.array(half_list, dtype='<e')
        self._check_roundtrip(x)

    def test_roundtrip_single_types(self):
        for typ in np.sctypeDict.values():
            dtype = np.dtype(typ)

            if dtype.char in 'Mm':
                # datetimes cannot be used in buffers
                continue
            if dtype.char == 'V':
                # skip void
                continue

            x = np.zeros(4, dtype=dtype)
            self._check_roundtrip(x)

            if dtype.char not in 'qQgG':
                dt = dtype.newbyteorder('<')
                x = np.zeros(4, dtype=dt)
                self._check_roundtrip(x)

                dt = dtype.newbyteorder('>')
                x = np.zeros(4, dtype=dt)
                self._check_roundtrip(x)

    def test_roundtrip_scalar(self):
        # Issue #4015.
        self._check_roundtrip(0)

    def test_invalid_buffer_format(self):
        # datetime64 cannot be used fully in a buffer yet
        # Should be fixed in the next Numpy major release
        dt = np.dtype([('a', 'uint16'), ('b', 'M8[s]')])
        a = np.empty(3, dt)
        assert_raises((ValueError, BufferError), memoryview, a)
        assert_raises((ValueError, BufferError), memoryview, np.array((3), 'M8[D]'))

    def test_export_simple_1d(self):
        x = np.array([1, 2, 3, 4, 5], dtype='i')
        y = memoryview(x)
        assert_equal(y.format, 'i')
        assert_equal(y.shape, (5,))
        assert_equal(y.ndim, 1)
        assert_equal(y.strides, (4,))
        assert_equal(y.suboffsets, ())
        assert_equal(y.itemsize, 4)

    def test_export_simple_nd(self):
        x = np.array([[1, 2], [3, 4]], dtype=np.float64)
        y = memoryview(x)
        assert_equal(y.format, 'd')
        assert_equal(y.shape, (2, 2))
        assert_equal(y.ndim, 2)
        assert_equal(y.strides, (16, 8))
        assert_equal(y.suboffsets, ())
        assert_equal(y.itemsize, 8)

    def test_export_discontiguous(self):
        x = np.zeros((3, 3, 3), dtype=np.float32)[:, 0,:]
        y = memoryview(x)
        assert_equal(y.format, 'f')
        assert_equal(y.shape, (3, 3))
        assert_equal(y.ndim, 2)
        assert_equal(y.strides, (36, 4))
        assert_equal(y.suboffsets, ())
        assert_equal(y.itemsize, 4)

    def test_export_record(self):
        dt = [('a', 'b'),
              ('b', 'h'),
              ('c', 'i'),
              ('d', 'l'),
              ('dx', 'q'),
              ('e', 'B'),
              ('f', 'H'),
              ('g', 'I'),
              ('h', 'L'),
              ('hx', 'Q'),
              ('i', np.single),
              ('j', np.double),
              ('k', np.longdouble),
              ('ix', np.csingle),
              ('jx', np.cdouble),
              ('kx', np.clongdouble),
              ('l', 'S4'),
              ('m', 'U4'),
              ('n', 'V3'),
              ('o', '?'),
              ('p', np.half),
              ]
        x = np.array(
                [(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                    b'aaaa', 'bbbb', b'   ', True, 1.0)],
                dtype=dt)
        y = memoryview(x)
        assert_equal(y.shape, (1,))
        assert_equal(y.ndim, 1)
        assert_equal(y.suboffsets, ())

        sz = sum([np.dtype(b).itemsize for a, b in dt])
        if np.dtype('l').itemsize == 4:
            assert_equal(y.format, 'T{b:a:=h:b:i:c:l:d:q:dx:B:e:@H:f:=I:g:L:h:Q:hx:f:i:d:j:^g:k:=Zf:ix:Zd:jx:^Zg:kx:4s:l:=4w:m:3x:n:?:o:@e:p:}')
        else:
            assert_equal(y.format, 'T{b:a:=h:b:i:c:q:d:q:dx:B:e:@H:f:=I:g:Q:h:Q:hx:f:i:d:j:^g:k:=Zf:ix:Zd:jx:^Zg:kx:4s:l:=4w:m:3x:n:?:o:@e:p:}')
        # Cannot test if NPY_RELAXED_STRIDES_DEBUG changes the strides
        if not (np.ones(1).strides[0] == np.iinfo(np.intp).max):
            assert_equal(y.strides, (sz,))
        assert_equal(y.itemsize, sz)

    def test_export_subarray(self):
        x = np.array(([[1, 2], [3, 4]],), dtype=[('a', ('i', (2, 2)))])
        y = memoryview(x)
        assert_equal(y.format, 'T{(2,2)i:a:}')
        assert_equal(y.shape, ())
        assert_equal(y.ndim, 0)
        assert_equal(y.strides, ())
        assert_equal(y.suboffsets, ())
        assert_equal(y.itemsize, 16)

    def test_export_endian(self):
        x = np.array([1, 2, 3], dtype='>i')
        y = memoryview(x)
        if sys.byteorder == 'little':
            assert_equal(y.format, '>i')
        else:
            assert_equal(y.format, 'i')

        x = np.array([1, 2, 3], dtype='<i')
        y = memoryview(x)
        if sys.byteorder == 'little':
            assert_equal(y.format, 'i')
        else:
            assert_equal(y.format, '<i')

    def test_export_flags(self):
        # Check SIMPLE flag, see also gh-3613 (exception should be BufferError)
        assert_raises(ValueError,
                      _multiarray_tests.get_buffer_info,
                       np.arange(5)[::2], ('SIMPLE',))

    @pytest.mark.parametrize(["obj", "error"], [
            pytest.param(np.array([1, 2], dtype=rational), ValueError, id="array"),
            pytest.param(rational(1, 2), TypeError, id="scalar")])
    def test_export_and_pickle_user_dtype(self, obj, error):
        # User dtypes should export successfully when FORMAT was not requested.
        with pytest.raises(error):
            _multiarray_tests.get_buffer_info(obj, ("STRIDED_RO", "FORMAT"))

        _multiarray_tests.get_buffer_info(obj, ("STRIDED_RO",))

        # This is currently also necessary to implement pickling:
        pickle_obj = pickle.dumps(obj)
        res = pickle.loads(pickle_obj)
        assert_array_equal(res, obj)

    def test_padding(self):
        for j in range(8):
            x = np.array([(1,), (2,)], dtype={'f0': (int, j)})
            self._check_roundtrip(x)

    def test_reference_leak(self):
        if HAS_REFCOUNT:
            count_1 = sys.getrefcount(np.core._internal)
        a = np.zeros(4)
        b = memoryview(a)
        c = np.asarray(b)
        if HAS_REFCOUNT:
            count_2 = sys.getrefcount(np.core._internal)
            assert_equal(count_1, count_2)
        del c  # avoid pyflakes unused variable warning.

    def test_padded_struct_array(self):
        dt1 = np.dtype(
                [('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')],
                align=True)
        x1 = np.arange(dt1.itemsize, dtype=np.int8).view(dt1)
        self._check_roundtrip(x1)

        dt2 = np.dtype(
                [('a', 'b'), ('b', 'i'), ('c', 'b', (3,)), ('d', 'i')],
                align=True)
        x2 = np.arange(dt2.itemsize, dtype=np.int8).view(dt2)
        self._check_roundtrip(x2)

        dt3 = np.dtype(
                [('a', 'b'), ('b', 'i'), ('c', 'b'), ('d', 'b'),
                    ('e', 'b'), ('sub', np.dtype('b,i', align=True))])
        x3 = np.arange(dt3.itemsize, dtype=np.int8).view(dt3)
        self._check_roundtrip(x3)

    @pytest.mark.valgrind_error(reason="leaks buffer info cache temporarily.")
    def test_relaxed_strides(self, c=np.ones((1, 10, 10), dtype='i8')):
        # Note: c defined as parameter so that it is persistent and leak
        # checks will notice gh-16934 (buffer info cache leak).
        c.strides = (-1, 80, 8)  # strides need to be fixed at export

        assert_(memoryview(c).strides == (800, 80, 8))

        # Writing C-contiguous data to a BytesIO buffer should work
        fd = io.BytesIO()
        fd.write(c.data)

        fortran = c.T
        assert_(memoryview(fortran).strides == (8, 80, 800))

        arr = np.ones((1, 10))
        if arr.flags.f_contiguous:
            shape, strides = _multiarray_tests.get_buffer_info(
                    arr, ['F_CONTIGUOUS'])
            assert_(strides[0] == 8)
            arr = np.ones((10, 1), order='F')
            shape, strides = _multiarray_tests.get_buffer_info(
                    arr, ['C_CONTIGUOUS'])
            assert_(strides[-1] == 8)

    @pytest.mark.valgrind_error(reason="leaks buffer info cache temporarily.")
    @pytest.mark.skipif(not np.ones((10, 1), order="C").flags.f_contiguous,
            reason="Test is unnecessary (but fails) without relaxed strides.")
    def test_relaxed_strides_buffer_info_leak(self, arr=np.ones((1, 10))):
        """Test that alternating export of C- and F-order buffers from
        an array which is both C- and F-order when relaxed strides is
        active works.
        This test defines array in the signature to ensure leaking more
        references every time the test is run (catching the leak with
        pytest-leaks).
        """
        for i in range(10):
            _, s = _multiarray_tests.get_buffer_info(arr, ['F_CONTIGUOUS'])
            assert s == (8, 8)
            _, s = _multiarray_tests.get_buffer_info(arr, ['C_CONTIGUOUS'])
            assert s == (80, 8)

    def test_out_of_order_fields(self):
        dt = np.dtype(dict(
            formats=['<i4', '<i4'],
            names=['one', 'two'],
            offsets=[4, 0],
            itemsize=8
        ))

        # overlapping fields cannot be represented by PEP3118
        arr = np.empty(1, dt)
        with assert_raises(ValueError):
            memoryview(arr)

    def test_max_dims(self):
        a = np.ones((1,) * 32)
        self._check_roundtrip(a)

    @pytest.mark.slow
    def test_error_too_many_dims(self):
        def make_ctype(shape, scalar_type):
            t = scalar_type
            for dim in shape[::-1]:
                t = dim * t
            return t

        # construct a memoryview with 33 dimensions
        c_u8_33d = make_ctype((1,)*33, ctypes.c_uint8)
        m = memoryview(c_u8_33d())
        assert_equal(m.ndim, 33)

        assert_raises_regex(
            RuntimeError, "ndim",
            np.array, m)

        # The above seems to create some deep cycles, clean them up for
        # easier reference count debugging:
        del c_u8_33d, m
        for i in range(33):
            if gc.collect() == 0:
                break

    def test_error_pointer_type(self):
        # gh-6741
        m = memoryview(ctypes.pointer(ctypes.c_uint8()))
        assert_('&' in m.format)

        assert_raises_regex(
            ValueError, "format string",
            np.array, m)

    def test_error_message_unsupported(self):
        # wchar has no corresponding numpy type - if this changes in future, we
        # need a better way to construct an invalid memoryview format.
        t = ctypes.c_wchar * 4
        with assert_raises(ValueError) as cm:
            np.array(t())

        exc = cm.exception
        with assert_raises_regex(
            NotImplementedError,
            r"Unrepresentable .* 'u' \(UCS-2 strings\)"
        ):
            raise exc.__cause__

    def test_ctypes_integer_via_memoryview(self):
        # gh-11150, due to bpo-10746
        for c_integer in {ctypes.c_int, ctypes.c_long, ctypes.c_longlong}:
            value = c_integer(42)
            with warnings.catch_warnings(record=True):
                warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
                np.asarray(value)

    def test_ctypes_struct_via_memoryview(self):
        # gh-10528
        class foo(ctypes.Structure):
            _fields_ = [('a', ctypes.c_uint8), ('b', ctypes.c_uint32)]
        f = foo(a=1, b=2)

        with warnings.catch_warnings(record=True):
            warnings.filterwarnings('always', r'.*\bctypes\b', RuntimeWarning)
            arr = np.asarray(f)

        assert_equal(arr['a'], 1)
        assert_equal(arr['b'], 2)
        f.a = 3
        assert_equal(arr['a'], 3)

    @pytest.mark.parametrize("obj", [np.ones(3), np.ones(1, dtype="i,i")[()]])
    def test_error_if_stored_buffer_info_is_corrupted(self, obj):
        """
        If a user extends a NumPy array before 1.20 and then runs it
        on NumPy 1.20+. A C-subclassed array might in theory modify
        the new buffer-info field. This checks that an error is raised
        if this happens (for buffer export), an error is written on delete.
        This is a sanity check to help users transition to safe code, it
        may be deleted at any point.
        """
        # corrupt buffer info:
        _multiarray_tests.corrupt_or_fix_bufferinfo(obj)
        name = type(obj)
        with pytest.raises(RuntimeError,
                    match=f".*{name} appears to be C subclassed"):
            memoryview(obj)
        # Fix buffer info again before we delete (or we lose the memory)
        _multiarray_tests.corrupt_or_fix_bufferinfo(obj)

    def test_no_suboffsets(self):
        try:
            import _testbuffer
        except ImportError:
            raise pytest.skip("_testbuffer is not available")

        for shape in [(2, 3), (2, 3, 4)]:
            data = list(range(np.prod(shape)))
            buffer = _testbuffer.ndarray(data, shape, format='i',
                                         flags=_testbuffer.ND_PIL)
            msg = "NumPy currently does not support.*suboffsets"
            with pytest.raises(BufferError, match=msg):
                np.asarray(buffer)
            with pytest.raises(BufferError, match=msg):
                np.asarray([buffer])

            # Also check (unrelated and more limited but similar) frombuffer:
            with pytest.raises(BufferError):
                np.frombuffer(buffer)


class TestArrayCreationCopyArgument(object):

    class RaiseOnBool:

        def __bool__(self):
            raise ValueError

    true_vals = [True, np._CopyMode.ALWAYS, np.True_]
    false_vals = [False, np._CopyMode.IF_NEEDED, np.False_]

    def test_scalars(self):
        # Test both numpy and python scalars
        for dtype in np.typecodes["All"]:
            arr = np.zeros((), dtype=dtype)
            scalar = arr[()]
            pyscalar = arr.item(0)

            # Test never-copy raises error:
            assert_raises(ValueError, np.array, scalar,
                            copy=np._CopyMode.NEVER)
            assert_raises(ValueError, np.array, pyscalar,
                            copy=np._CopyMode.NEVER)
            assert_raises(ValueError, np.array, pyscalar,
                            copy=self.RaiseOnBool())
            assert_raises(ValueError, _multiarray_tests.npy_ensurenocopy,
                            [1])
            # Casting with a dtype (to unsigned integers) can be special:
            with pytest.raises(ValueError):
                np.array(pyscalar, dtype=np.int64, copy=np._CopyMode.NEVER)

    def test_compatible_cast(self):

        # Some types are compatible even though they are different, no
        # copy is necessary for them. This is mostly true for some integers
        def int_types(byteswap=False):
            int_types = (np.typecodes["Integer"] +
                         np.typecodes["UnsignedInteger"])
            for int_type in int_types:
                yield np.dtype(int_type)
                if byteswap:
                    yield np.dtype(int_type).newbyteorder()

        for int1 in int_types():
            for int2 in int_types(True):
                arr = np.arange(10, dtype=int1)

                for copy in self.true_vals:
                    res = np.array(arr, copy=copy, dtype=int2)
                    assert res is not arr and res.flags.owndata
                    assert_array_equal(res, arr)

                if int1 == int2:
                    # Casting is not necessary, base check is sufficient here
                    for copy in self.false_vals:
                        res = np.array(arr, copy=copy, dtype=int2)
                        assert res is arr or res.base is arr

                    res = np.array(arr,
                                   copy=np._CopyMode.NEVER,
                                   dtype=int2)
                    assert res is arr or res.base is arr

                else:
                    # Casting is necessary, assert copy works:
                    for copy in self.false_vals:
                        res = np.array(arr, copy=copy, dtype=int2)
                        assert res is not arr and res.flags.owndata
                        assert_array_equal(res, arr)

                    assert_raises(ValueError, np.array,
                                  arr, copy=np._CopyMode.NEVER,
                                  dtype=int2)
                    assert_raises(ValueError, np.array,
                                  arr, copy=None,
                                  dtype=int2)

    def test_buffer_interface(self):

        # Buffer interface gives direct memory access (no copy)
        arr = np.arange(10)
        view = memoryview(arr)

        # Checking bases is a bit tricky since numpy creates another
        # memoryview, so use may_share_memory.
        for copy in self.true_vals:
            res = np.array(view, copy=copy)
            assert not np.may_share_memory(arr, res)
        for copy in self.false_vals:
            res = np.array(view, copy=copy)
            assert np.may_share_memory(arr, res)
        res = np.array(view, copy=np._CopyMode.NEVER)
        assert np.may_share_memory(arr, res)

    def test_array_interfaces(self):
        # Array interface gives direct memory access (much like a memoryview)
        base_arr = np.arange(10)

        class ArrayLike:
            __array_interface__ = base_arr.__array_interface__

        arr = ArrayLike()

        for copy, val in [(True, None), (np._CopyMode.ALWAYS, None),
                          (False, arr), (np._CopyMode.IF_NEEDED, arr),
                          (np._CopyMode.NEVER, arr)]:
            res = np.array(arr, copy=copy)
            assert res.base is val

    def test___array__(self):
        base_arr = np.arange(10)

        class ArrayLike:
            def __array__(self):
                # __array__ should return a copy, numpy cannot know this
                # however.
                return base_arr

        arr = ArrayLike()

        for copy in self.true_vals:
            res = np.array(arr, copy=copy)
            assert_array_equal(res, base_arr)
            # An additional copy is currently forced by numpy in this case,
            # you could argue, numpy does not trust the ArrayLike. This
            # may be open for change:
            assert res is not base_arr

        for copy in self.false_vals:
            res = np.array(arr, copy=False)
            assert_array_equal(res, base_arr)
            assert res is base_arr  # numpy trusts the ArrayLike

        with pytest.raises(ValueError):
            np.array(arr, copy=np._CopyMode.NEVER)

    @pytest.mark.parametrize(
            "arr", [np.ones(()), np.arange(81).reshape((9, 9))])
    @pytest.mark.parametrize("order1", ["C", "F", None])
    @pytest.mark.parametrize("order2", ["C", "F", "A", "K"])
    def test_order_mismatch(self, arr, order1, order2):
        # The order is the main (python side) reason that can cause
        # a never-copy to fail.
        # Prepare C-order, F-order and non-contiguous arrays:
        arr = arr.copy(order1)
        if order1 == "C":
            assert arr.flags.c_contiguous
        elif order1 == "F":
            assert arr.flags.f_contiguous
        elif arr.ndim != 0:
            # Make array non-contiguous
            arr = arr[::2, ::2]
            assert not arr.flags.forc

        # Whether a copy is necessary depends on the order of arr:
        if order2 == "C":
            no_copy_necessary = arr.flags.c_contiguous
        elif order2 == "F":
            no_copy_necessary = arr.flags.f_contiguous
        else:
            # Keeporder and Anyorder are OK with non-contiguous output.
            # This is not consistent with the `astype` behaviour which
            # enforces contiguity for "A". It is probably historic from when
            # "K" did not exist.
            no_copy_necessary = True

        # Test it for both the array and a memoryview
        for view in [arr, memoryview(arr)]:
            for copy in self.true_vals:
                res = np.array(view, copy=copy, order=order2)
                assert res is not arr and res.flags.owndata
                assert_array_equal(arr, res)

            if no_copy_necessary:
                for copy in self.false_vals:
                    res = np.array(view, copy=copy, order=order2)
                    # res.base.obj refers to the memoryview
                    if not IS_PYPY:
                        assert res is arr or res.base.obj is arr

                res = np.array(view, copy=np._CopyMode.NEVER,
                               order=order2)
                if not IS_PYPY:
                    assert res is arr or res.base.obj is arr
            else:
                for copy in self.false_vals:
                    res = np.array(arr, copy=copy, order=order2)
                    assert_array_equal(arr, res)
                assert_raises(ValueError, np.array,
                              view, copy=np._CopyMode.NEVER,
                              order=order2)
                assert_raises(ValueError, np.array,
                              view, copy=None,
                              order=order2)

    def test_striding_not_ok(self):
        arr = np.array([[1, 2, 4], [3, 4, 5]])
        assert_raises(ValueError, np.array,
                      arr.T, copy=np._CopyMode.NEVER,
                      order='C')
        assert_raises(ValueError, np.array,
                      arr.T, copy=np._CopyMode.NEVER,
                      order='C', dtype=np.int64)
        assert_raises(ValueError, np.array,
                      arr, copy=np._CopyMode.NEVER,
                      order='F')
        assert_raises(ValueError, np.array,
                      arr, copy=np._CopyMode.NEVER,
                      order='F', dtype=np.int64)


class TestArrayAttributeDeletion:

    def test_multiarray_writable_attributes_deletion(self):
        # ticket #2046, should not seqfault, raise AttributeError
        a = np.ones(2)
        attr = ['shape', 'strides', 'data', 'dtype', 'real', 'imag', 'flat']
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, "Assigning the 'data' attribute")
            for s in attr:
                assert_raises(AttributeError, delattr, a, s)

    def test_multiarray_not_writable_attributes_deletion(self):
        a = np.ones(2)
        attr = ["ndim", "flags", "itemsize", "size", "nbytes", "base",
                "ctypes", "T", "__array_interface__", "__array_struct__",
                "__array_priority__", "__array_finalize__"]
        for s in attr:
            assert_raises(AttributeError, delattr, a, s)

    def test_multiarray_flags_writable_attribute_deletion(self):
        a = np.ones(2).flags
        attr = ['writebackifcopy', 'updateifcopy', 'aligned', 'writeable']
        for s in attr:
            assert_raises(AttributeError, delattr, a, s)

    def test_multiarray_flags_not_writable_attribute_deletion(self):
        a = np.ones(2).flags
        attr = ["contiguous", "c_contiguous", "f_contiguous", "fortran",
                "owndata", "fnc", "forc", "behaved", "carray", "farray",
                "num"]
        for s in attr:
            assert_raises(AttributeError, delattr, a, s)


class TestArrayInterface():
    class Foo:
        def __init__(self, value):
            self.value = value
            self.iface = {'typestr': 'f8'}

        def __float__(self):
            return float(self.value)

        @property
        def __array_interface__(self):
            return self.iface


    f = Foo(0.5)

    @pytest.mark.parametrize('val, iface, expected', [
        (f, {}, 0.5),
        ([f], {}, [0.5]),
        ([f, f], {}, [0.5, 0.5]),
        (f, {'shape': ()}, 0.5),
        (f, {'shape': None}, TypeError),
        (f, {'shape': (1, 1)}, [[0.5]]),
        (f, {'shape': (2,)}, ValueError),
        (f, {'strides': ()}, 0.5),
        (f, {'strides': (2,)}, ValueError),
        (f, {'strides': 16}, TypeError),
        ])
    def test_scalar_interface(self, val, iface, expected):
        # Test scalar coercion within the array interface
        self.f.iface = {'typestr': 'f8'}
        self.f.iface.update(iface)
        if HAS_REFCOUNT:
            pre_cnt = sys.getrefcount(np.dtype('f8'))
        if isinstance(expected, type):
            assert_raises(expected, np.array, val)
        else:
            result = np.array(val)
            assert_equal(np.array(val), expected)
            assert result.dtype == 'f8'
            del result
        if HAS_REFCOUNT:
            post_cnt = sys.getrefcount(np.dtype('f8'))
            assert_equal(pre_cnt, post_cnt)

def test_interface_no_shape():
    class ArrayLike:
        array = np.array(1)
        __array_interface__ = array.__array_interface__
    assert_equal(np.array(ArrayLike()), 1)


def test_array_interface_itemsize():
    # See gh-6361
    my_dtype = np.dtype({'names': ['A', 'B'], 'formats': ['f4', 'f4'],
                         'offsets': [0, 8], 'itemsize': 16})
    a = np.ones(10, dtype=my_dtype)
    descr_t = np.dtype(a.__array_interface__['descr'])
    typestr_t = np.dtype(a.__array_interface__['typestr'])
    assert_equal(descr_t.itemsize, typestr_t.itemsize)


def test_array_interface_empty_shape():
    # See gh-7994
    arr = np.array([1, 2, 3])
    interface1 = dict(arr.__array_interface__)
    interface1['shape'] = ()

    class DummyArray1:
        __array_interface__ = interface1

    # NOTE: Because Py2 str/Py3 bytes supports the buffer interface, setting
    # the interface data to bytes would invoke the bug this tests for, that
    # __array_interface__ with shape=() is not allowed if the data is an object
    # exposing the buffer interface
    interface2 = dict(interface1)
    interface2['data'] = arr[0].tobytes()

    class DummyArray2:
        __array_interface__ = interface2

    arr1 = np.asarray(DummyArray1())
    arr2 = np.asarray(DummyArray2())
    arr3 = arr[:1].reshape(())
    assert_equal(arr1, arr2)
    assert_equal(arr1, arr3)

def test_array_interface_offset():
    arr = np.array([1, 2, 3], dtype='int32')
    interface = dict(arr.__array_interface__)
    interface['data'] = memoryview(arr)
    interface['shape'] = (2,)
    interface['offset'] = 4


    class DummyArray:
        __array_interface__ = interface

    arr1 = np.asarray(DummyArray())
    assert_equal(arr1, arr[1:])

def test_array_interface_unicode_typestr():
    arr = np.array([1, 2, 3], dtype='int32')
    interface = dict(arr.__array_interface__)
    interface['typestr'] = '\N{check mark}'

    class DummyArray:
        __array_interface__ = interface

    # should not be UnicodeEncodeError
    with pytest.raises(TypeError):
        np.asarray(DummyArray())

def test_flat_element_deletion():
    it = np.ones(3).flat
    try:
        del it[1]
        del it[1:2]
    except TypeError:
        pass
    except Exception:
        raise AssertionError


def test_scalar_element_deletion():
    a = np.zeros(2, dtype=[('x', 'int'), ('y', 'int')])
    assert_raises(ValueError, a[0].__delitem__, 'x')


class TestMapIter:
    def test_mapiter(self):
        # The actual tests are within the C code in
        # multiarray/_multiarray_tests.c.src

        a = np.arange(12).reshape((3, 4)).astype(float)
        index = ([1, 1, 2, 0],
                 [0, 0, 2, 3])
        vals = [50, 50, 30, 16]

        _multiarray_tests.test_inplace_increment(a, index, vals)
        assert_equal(a, [[0.00, 1., 2.0, 19.],
                         [104., 5., 6.0, 7.0],
                         [8.00, 9., 40., 11.]])

        b = np.arange(6).astype(float)
        index = (np.array([1, 2, 0]),)
        vals = [50, 4, 100.1]
        _multiarray_tests.test_inplace_increment(b, index, vals)
        assert_equal(b, [100.1,  51.,   6.,   3.,   4.,   5.])


class TestAsCArray:
    def test_1darray(self):
        array = np.arange(24, dtype=np.double)
        from_c = _multiarray_tests.test_as_c_array(array, 3)
        assert_equal(array[3], from_c)

    def test_2darray(self):
        array = np.arange(24, dtype=np.double).reshape(3, 8)
        from_c = _multiarray_tests.test_as_c_array(array, 2, 4)
        assert_equal(array[2, 4], from_c)

    def test_3darray(self):
        array = np.arange(24, dtype=np.double).reshape(2, 3, 4)
        from_c = _multiarray_tests.test_as_c_array(array, 1, 2, 3)
        assert_equal(array[1, 2, 3], from_c)


class TestConversion:
    def test_array_scalar_relational_operation(self):
        # All integer
        for dt1 in np.typecodes['AllInteger']:
            assert_(1 > np.array(0, dtype=dt1), "type %s failed" % (dt1,))
            assert_(not 1 < np.array(0, dtype=dt1), "type %s failed" % (dt1,))

            for dt2 in np.typecodes['AllInteger']:
                assert_(np.array(1, dtype=dt1) > np.array(0, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))
                assert_(not np.array(1, dtype=dt1) < np.array(0, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))

        # Unsigned integers
        for dt1 in 'BHILQP':
            assert_(-1 < np.array(1, dtype=dt1), "type %s failed" % (dt1,))
            assert_(not -1 > np.array(1, dtype=dt1), "type %s failed" % (dt1,))
            assert_(-1 != np.array(1, dtype=dt1), "type %s failed" % (dt1,))

            # Unsigned vs signed
            for dt2 in 'bhilqp':
                assert_(np.array(1, dtype=dt1) > np.array(-1, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))
                assert_(not np.array(1, dtype=dt1) < np.array(-1, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))
                assert_(np.array(1, dtype=dt1) != np.array(-1, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))

        # Signed integers and floats
        for dt1 in 'bhlqp' + np.typecodes['Float']:
            assert_(1 > np.array(-1, dtype=dt1), "type %s failed" % (dt1,))
            assert_(not 1 < np.array(-1, dtype=dt1), "type %s failed" % (dt1,))
            assert_(-1 == np.array(-1, dtype=dt1), "type %s failed" % (dt1,))

            for dt2 in 'bhlqp' + np.typecodes['Float']:
                assert_(np.array(1, dtype=dt1) > np.array(-1, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))
                assert_(not np.array(1, dtype=dt1) < np.array(-1, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))
                assert_(np.array(-1, dtype=dt1) == np.array(-1, dtype=dt2),
                        "type %s and %s failed" % (dt1, dt2))

    def test_to_bool_scalar(self):
        assert_equal(bool(np.array([False])), False)
        assert_equal(bool(np.array([True])), True)
        assert_equal(bool(np.array([[42]])), True)
        assert_raises(ValueError, bool, np.array([1, 2]))

        class NotConvertible:
            def __bool__(self):
                raise NotImplementedError

        assert_raises(NotImplementedError, bool, np.array(NotConvertible()))
        assert_raises(NotImplementedError, bool, np.array([NotConvertible()]))
        if IS_PYSTON:
            pytest.skip("Pyston disables recursion checking")

        self_containing = np.array([None])
        self_containing[0] = self_containing

        Error = RecursionError

        assert_raises(Error, bool, self_containing)  # previously stack overflow
        self_containing[0] = None  # resolve circular reference

    def test_to_int_scalar(self):
        # gh-9972 means that these aren't always the same
        int_funcs = (int, lambda x: x.__int__())
        for int_func in int_funcs:
            assert_equal(int_func(np.array(0)), 0)
            assert_equal(int_func(np.array([1])), 1)
            assert_equal(int_func(np.array([[42]])), 42)
            assert_raises(TypeError, int_func, np.array([1, 2]))

            # gh-9972
            assert_equal(4, int_func(np.array('4')))
            assert_equal(5, int_func(np.bytes_(b'5')))
            assert_equal(6, int_func(np.unicode_('6')))

            # The delegation of int() to __trunc__ was deprecated in
            # Python 3.11.
            if sys.version_info < (3, 11):
                class HasTrunc:
                    def __trunc__(self):
                        return 3
                assert_equal(3, int_func(np.array(HasTrunc())))
                assert_equal(3, int_func(np.array([HasTrunc()])))
            else:
                pass

            class NotConvertible:
                def __int__(self):
                    raise NotImplementedError
            assert_raises(NotImplementedError,
                int_func, np.array(NotConvertible()))
            assert_raises(NotImplementedError,
                int_func, np.array([NotConvertible()]))


class TestWhere:
    def test_basic(self):
        dts = [bool, np.int16, np.int32, np.int64, np.double, np.complex128,
               np.longdouble, np.clongdouble]
        for dt in dts:
            c = np.ones(53, dtype=bool)
            assert_equal(np.where( c, dt(0), dt(1)), dt(0))
            assert_equal(np.where(~c, dt(0), dt(1)), dt(1))
            assert_equal(np.where(True, dt(0), dt(1)), dt(0))
            assert_equal(np.where(False, dt(0), dt(1)), dt(1))
            d = np.ones_like(c).astype(dt)
            e = np.zeros_like(d)
            r = d.astype(dt)
            c[7] = False
            r[7] = e[7]
            assert_equal(np.where(c, e, e), e)
            assert_equal(np.where(c, d, e), r)
            assert_equal(np.where(c, d, e[0]), r)
            assert_equal(np.where(c, d[0], e), r)
            assert_equal(np.where(c[::2], d[::2], e[::2]), r[::2])
            assert_equal(np.where(c[1::2], d[1::2], e[1::2]), r[1::2])
            assert_equal(np.where(c[::3], d[::3], e[::3]), r[::3])
            assert_equal(np.where(c[1::3], d[1::3], e[1::3]), r[1::3])
            assert_equal(np.where(c[::-2], d[::-2], e[::-2]), r[::-2])
            assert_equal(np.where(c[::-3], d[::-3], e[::-3]), r[::-3])
            assert_equal(np.where(c[1::-3], d[1::-3], e[1::-3]), r[1::-3])

    def test_exotic(self):
        # object
        assert_array_equal(np.where(True, None, None), np.array(None))
        # zero sized
        m = np.array([], dtype=bool).reshape(0, 3)
        b = np.array([], dtype=np.float64).reshape(0, 3)
        assert_array_equal(np.where(m, 0, b), np.array([]).reshape(0, 3))

        # object cast
        d = np.array([-1.34, -0.16, -0.54, -0.31, -0.08, -0.95, 0.000, 0.313,
                      0.547, -0.18, 0.876, 0.236, 1.969, 0.310, 0.699, 1.013,
                      1.267, 0.229, -1.39, 0.487])
        nan = float('NaN')
        e = np.array(['5z', '0l', nan, 'Wz', nan, nan, 'Xq', 'cs', nan, nan,
                     'QN', nan, nan, 'Fd', nan, nan, 'kp', nan, '36', 'i1'],
                     dtype=object)
        m = np.array([0, 0, 1, 0, 1, 1, 0, 0, 1, 1,
                      0, 1, 1, 0, 1, 1, 0, 1, 0, 0], dtype=bool)

        r = e[:]
        r[np.where(m)] = d[np.where(m)]
        assert_array_equal(np.where(m, d, e), r)

        r = e[:]
        r[np.where(~m)] = d[np.where(~m)]
        assert_array_equal(np.where(m, e, d), r)

        assert_array_equal(np.where(m, e, e), e)

        # minimal dtype result with NaN scalar (e.g required by pandas)
        d = np.array([1., 2.], dtype=np.float32)
        e = float('NaN')
        assert_equal(np.where(True, d, e).dtype, np.float32)
        e = float('Infinity')
        assert_equal(np.where(True, d, e).dtype, np.float32)
        e = float('-Infinity')
        assert_equal(np.where(True, d, e).dtype, np.float32)
        # also check upcast
        e = float(1e150)
        assert_equal(np.where(True, d, e).dtype, np.float64)

    def test_ndim(self):
        c = [True, False]
        a = np.zeros((2, 25))
        b = np.ones((2, 25))
        r = np.where(np.array(c)[:,np.newaxis], a, b)
        assert_array_equal(r[0], a[0])
        assert_array_equal(r[1], b[0])

        a = a.T
        b = b.T
        r = np.where(c, a, b)
        assert_array_equal(r[:,0], a[:,0])
        assert_array_equal(r[:,1], b[:,0])

    def test_dtype_mix(self):
        c = np.array([False, True, False, False, False, False, True, False,
                     False, False, True, False])
        a = np.uint32(1)
        b = np.array([5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.],
                      dtype=np.float64)
        r = np.array([5., 1., 3., 2., -1., -4., 1., -10., 10., 1., 1., 3.],
                     dtype=np.float64)
        assert_equal(np.where(c, a, b), r)

        a = a.astype(np.float32)
        b = b.astype(np.int64)
        assert_equal(np.where(c, a, b), r)

        # non bool mask
        c = c.astype(int)
        c[c != 0] = 34242324
        assert_equal(np.where(c, a, b), r)
        # invert
        tmpmask = c != 0
        c[c == 0] = 41247212
        c[tmpmask] = 0
        assert_equal(np.where(c, b, a), r)

    def test_foreign(self):
        c = np.array([False, True, False, False, False, False, True, False,
                     False, False, True, False])
        r = np.array([5., 1., 3., 2., -1., -4., 1., -10., 10., 1., 1., 3.],
                     dtype=np.float64)
        a = np.ones(1, dtype='>i4')
        b = np.array([5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.],
                     dtype=np.float64)
        assert_equal(np.where(c, a, b), r)

        b = b.astype('>f8')
        assert_equal(np.where(c, a, b), r)

        a = a.astype('<i4')
        assert_equal(np.where(c, a, b), r)

        c = c.astype('>i4')
        assert_equal(np.where(c, a, b), r)

    def test_error(self):
        c = [True, True]
        a = np.ones((4, 5))
        b = np.ones((5, 5))
        assert_raises(ValueError, np.where, c, a, a)
        assert_raises(ValueError, np.where, c[0], a, b)

    def test_string(self):
        # gh-4778 check strings are properly filled with nulls
        a = np.array("abc")
        b = np.array("x" * 753)
        assert_equal(np.where(True, a, b), "abc")
        assert_equal(np.where(False, b, a), "abc")

        # check native datatype sized strings
        a = np.array("abcd")
        b = np.array("x" * 8)
        assert_equal(np.where(True, a, b), "abcd")
        assert_equal(np.where(False, b, a), "abcd")

    def test_empty_result(self):
        # pass empty where result through an assignment which reads the data of
        # empty arrays, error detectable with valgrind, see gh-8922
        x = np.zeros((1, 1))
        ibad = np.vstack(np.where(x == 99.))
        assert_array_equal(ibad,
                           np.atleast_2d(np.array([[],[]], dtype=np.intp)))

    def test_largedim(self):
        # invalid read regression gh-9304
        shape = [10, 2, 3, 4, 5, 6]
        np.random.seed(2)
        array = np.random.rand(*shape)

        for i in range(10):
            benchmark = array.nonzero()
            result = array.nonzero()
            assert_array_equal(benchmark, result)


if not IS_PYPY:
    # sys.getsizeof() is not valid on PyPy
    class TestSizeOf:

        def test_empty_array(self):
            x = np.array([])
            assert_(sys.getsizeof(x) > 0)

        def check_array(self, dtype):
            elem_size = dtype(0).itemsize

            for length in [10, 50, 100, 500]:
                x = np.arange(length, dtype=dtype)
                assert_(sys.getsizeof(x) > length * elem_size)

        def test_array_int32(self):
            self.check_array(np.int32)

        def test_array_int64(self):
            self.check_array(np.int64)

        def test_array_float32(self):
            self.check_array(np.float32)

        def test_array_float64(self):
            self.check_array(np.float64)

        def test_view(self):
            d = np.ones(100)
            assert_(sys.getsizeof(d[...]) < sys.getsizeof(d))

        def test_reshape(self):
            d = np.ones(100)
            assert_(sys.getsizeof(d) < sys.getsizeof(d.reshape(100, 1, 1).copy()))

        @_no_tracing
        def test_resize(self):
            d = np.ones(100)
            old = sys.getsizeof(d)
            d.resize(50)
            assert_(old > sys.getsizeof(d))
            d.resize(150)
            assert_(old < sys.getsizeof(d))

        def test_error(self):
            d = np.ones(100)
            assert_raises(TypeError, d.__sizeof__, "a")


class TestHashing:

    def test_arrays_not_hashable(self):
        x = np.ones(3)
        assert_raises(TypeError, hash, x)

    def test_collections_hashable(self):
        x = np.array([])
        assert_(not isinstance(x, collections.abc.Hashable))


class TestArrayPriority:
    # This will go away when __array_priority__ is settled, meanwhile
    # it serves to check unintended changes.
    op = operator
    binary_ops = [
        op.pow, op.add, op.sub, op.mul, op.floordiv, op.truediv, op.mod,
        op.and_, op.or_, op.xor, op.lshift, op.rshift, op.mod, op.gt,
        op.ge, op.lt, op.le, op.ne, op.eq
        ]

    class Foo(np.ndarray):
        __array_priority__ = 100.

        def __new__(cls, *args, **kwargs):
            return np.array(*args, **kwargs).view(cls)

    class Bar(np.ndarray):
        __array_priority__ = 101.

        def __new__(cls, *args, **kwargs):
            return np.array(*args, **kwargs).view(cls)

    class Other:
        __array_priority__ = 1000.

        def _all(self, other):
            return self.__class__()

        __add__ = __radd__ = _all
        __sub__ = __rsub__ = _all
        __mul__ = __rmul__ = _all
        __pow__ = __rpow__ = _all
        __div__ = __rdiv__ = _all
        __mod__ = __rmod__ = _all
        __truediv__ = __rtruediv__ = _all
        __floordiv__ = __rfloordiv__ = _all
        __and__ = __rand__ = _all
        __xor__ = __rxor__ = _all
        __or__ = __ror__ = _all
        __lshift__ = __rlshift__ = _all
        __rshift__ = __rrshift__ = _all
        __eq__ = _all
        __ne__ = _all
        __gt__ = _all
        __ge__ = _all
        __lt__ = _all
        __le__ = _all

    def test_ndarray_subclass(self):
        a = np.array([1, 2])
        b = self.Bar([1, 2])
        for f in self.binary_ops:
            msg = repr(f)
            assert_(isinstance(f(a, b), self.Bar), msg)
            assert_(isinstance(f(b, a), self.Bar), msg)

    def test_ndarray_other(self):
        a = np.array([1, 2])
        b = self.Other()
        for f in self.binary_ops:
            msg = repr(f)
            assert_(isinstance(f(a, b), self.Other), msg)
            assert_(isinstance(f(b, a), self.Other), msg)

    def test_subclass_subclass(self):
        a = self.Foo([1, 2])
        b = self.Bar([1, 2])
        for f in self.binary_ops:
            msg = repr(f)
            assert_(isinstance(f(a, b), self.Bar), msg)
            assert_(isinstance(f(b, a), self.Bar), msg)

    def test_subclass_other(self):
        a = self.Foo([1, 2])
        b = self.Other()
        for f in self.binary_ops:
            msg = repr(f)
            assert_(isinstance(f(a, b), self.Other), msg)
            assert_(isinstance(f(b, a), self.Other), msg)


class TestBytestringArrayNonzero:

    def test_empty_bstring_array_is_falsey(self):
        assert_(not np.array([''], dtype=str))

    def test_whitespace_bstring_array_is_falsey(self):
        a = np.array(['spam'], dtype=str)
        a[0] = '  \0\0'
        assert_(not a)

    def test_all_null_bstring_array_is_falsey(self):
        a = np.array(['spam'], dtype=str)
        a[0] = '\0\0\0\0'
        assert_(not a)

    def test_null_inside_bstring_array_is_truthy(self):
        a = np.array(['spam'], dtype=str)
        a[0] = ' \0 \0'
        assert_(a)


class TestUnicodeEncoding:
    """
    Tests for encoding related bugs, such as UCS2 vs UCS4, round-tripping
    issues, etc
    """
    def test_round_trip(self):
        """ Tests that GETITEM, SETITEM, and PyArray_Scalar roundtrip """
        # gh-15363
        arr = np.zeros(shape=(), dtype="U1")
        for i in range(1, sys.maxunicode + 1):
            expected = chr(i)
            arr[()] = expected
            assert arr[()] == expected
            assert arr.item() == expected

    def test_assign_scalar(self):
        # gh-3258
        l = np.array(['aa', 'bb'])
        l[:] = np.unicode_('cc')
        assert_equal(l, ['cc', 'cc'])

    def test_fill_scalar(self):
        # gh-7227
        l = np.array(['aa', 'bb'])
        l.fill(np.unicode_('cc'))
        assert_equal(l, ['cc', 'cc'])


class TestUnicodeArrayNonzero:

    def test_empty_ustring_array_is_falsey(self):
        assert_(not np.array([''], dtype=np.unicode_))

    def test_whitespace_ustring_array_is_falsey(self):
        a = np.array(['eggs'], dtype=np.unicode_)
        a[0] = '  \0\0'
        assert_(not a)

    def test_all_null_ustring_array_is_falsey(self):
        a = np.array(['eggs'], dtype=np.unicode_)
        a[0] = '\0\0\0\0'
        assert_(not a)

    def test_null_inside_ustring_array_is_truthy(self):
        a = np.array(['eggs'], dtype=np.unicode_)
        a[0] = ' \0 \0'
        assert_(a)


class TestFormat:

    def test_0d(self):
        a = np.array(np.pi)
        assert_equal('{:0.3g}'.format(a), '3.14')
        assert_equal('{:0.3g}'.format(a[()]), '3.14')

    def test_1d_no_format(self):
        a = np.array([np.pi])
        assert_equal('{}'.format(a), str(a))

    def test_1d_format(self):
        # until gh-5543, ensure that the behaviour matches what it used to be
        a = np.array([np.pi])
        assert_raises(TypeError, '{:30}'.format, a)

from numpy.testing import IS_PYPY

class TestCTypes:

    def test_ctypes_is_available(self):
        test_arr = np.array([[1, 2, 3], [4, 5, 6]])

        assert_equal(ctypes, test_arr.ctypes._ctypes)
        assert_equal(tuple(test_arr.ctypes.shape), (2, 3))

    def test_ctypes_is_not_available(self):
        from numpy.core import _internal
        _internal.ctypes = None
        try:
            test_arr = np.array([[1, 2, 3], [4, 5, 6]])

            assert_(isinstance(test_arr.ctypes._ctypes,
                               _internal._missing_ctypes))
            assert_equal(tuple(test_arr.ctypes.shape), (2, 3))
        finally:
            _internal.ctypes = ctypes

    def _make_readonly(x):
        x.flags.writeable = False
        return x

    @pytest.mark.parametrize('arr', [
        np.array([1, 2, 3]),
        np.array([['one', 'two'], ['three', 'four']]),
        np.array((1, 2), dtype='i4,i4'),
        np.zeros((2,), dtype=
            np.dtype(dict(
                formats=['<i4', '<i4'],
                names=['a', 'b'],
                offsets=[0, 2],
                itemsize=6
            ))
        ),
        np.array([None], dtype=object),
        np.array([]),
        np.empty((0, 0)),
        _make_readonly(np.array([1, 2, 3])),
    ], ids=[
        '1d',
        '2d',
        'structured',
        'overlapping',
        'object',
        'empty',
        'empty-2d',
        'readonly'
    ])
    def test_ctypes_data_as_holds_reference(self, arr):
        # gh-9647
        # create a copy to ensure that pytest does not mess with the refcounts
        arr = arr.copy()

        arr_ref = weakref.ref(arr)

        ctypes_ptr = arr.ctypes.data_as(ctypes.c_void_p)

        # `ctypes_ptr` should hold onto `arr`
        del arr
        break_cycles()
        assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference")

        # but when the `ctypes_ptr` object dies, so should `arr`
        del ctypes_ptr
        if IS_PYPY:
            # Pypy does not recycle arr objects immediately. Trigger gc to
            # release arr. Cpython uses refcounts. An explicit call to gc
            # should not be needed here.
            break_cycles()
        assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference")

    def test_ctypes_as_parameter_holds_reference(self):
        arr = np.array([None]).copy()

        arr_ref = weakref.ref(arr)

        ctypes_ptr = arr.ctypes._as_parameter_

        # `ctypes_ptr` should hold onto `arr`
        del arr
        break_cycles()
        assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference")

        # but when the `ctypes_ptr` object dies, so should `arr`
        del ctypes_ptr
        if IS_PYPY:
            break_cycles()
        assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference")


class TestWritebackIfCopy:
    # all these tests use the WRITEBACKIFCOPY mechanism
    def test_argmax_with_out(self):
        mat = np.eye(5)
        out = np.empty(5, dtype='i2')
        res = np.argmax(mat, 0, out=out)
        assert_equal(res, range(5))

    def test_argmin_with_out(self):
        mat = -np.eye(5)
        out = np.empty(5, dtype='i2')
        res = np.argmin(mat, 0, out=out)
        assert_equal(res, range(5))

    def test_insert_noncontiguous(self):
        a = np.arange(6).reshape(2,3).T # force non-c-contiguous
        # uses arr_insert
        np.place(a, a>2, [44, 55])
        assert_equal(a, np.array([[0, 44], [1, 55], [2, 44]]))
        # hit one of the failing paths
        assert_raises(ValueError, np.place, a, a>20, [])

    def test_put_noncontiguous(self):
        a = np.arange(6).reshape(2,3).T # force non-c-contiguous
        np.put(a, [0, 2], [44, 55])
        assert_equal(a, np.array([[44, 3], [55, 4], [2, 5]]))

    def test_putmask_noncontiguous(self):
        a = np.arange(6).reshape(2,3).T # force non-c-contiguous
        # uses arr_putmask
        np.putmask(a, a>2, a**2)
        assert_equal(a, np.array([[0, 9], [1, 16], [2, 25]]))

    def test_take_mode_raise(self):
        a = np.arange(6, dtype='int')
        out = np.empty(2, dtype='int')
        np.take(a, [0, 2], out=out, mode='raise')
        assert_equal(out, np.array([0, 2]))

    def test_choose_mod_raise(self):
        a = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]])
        out = np.empty((3,3), dtype='int')
        choices = [-10, 10]
        np.choose(a, choices, out=out, mode='raise')
        assert_equal(out, np.array([[ 10, -10,  10],
                                    [-10,  10, -10],
                                    [ 10, -10,  10]]))

    def test_flatiter__array__(self):
        a = np.arange(9).reshape(3,3)
        b = a.T.flat
        c = b.__array__()
        # triggers the WRITEBACKIFCOPY resolution, assuming refcount semantics
        del c

    def test_dot_out(self):
        # if HAVE_CBLAS, will use WRITEBACKIFCOPY
        a = np.arange(9, dtype=float).reshape(3,3)
        b = np.dot(a, a, out=a)
        assert_equal(b, np.array([[15, 18, 21], [42, 54, 66], [69, 90, 111]]))

    def test_view_assign(self):
        from numpy.core._multiarray_tests import npy_create_writebackifcopy, npy_resolve

        arr = np.arange(9).reshape(3, 3).T
        arr_wb = npy_create_writebackifcopy(arr)
        assert_(arr_wb.flags.writebackifcopy)
        assert_(arr_wb.base is arr)
        arr_wb[...] = -100
        npy_resolve(arr_wb)
        # arr changes after resolve, even though we assigned to arr_wb
        assert_equal(arr, -100)
        # after resolve, the two arrays no longer reference each other
        assert_(arr_wb.ctypes.data != 0)
        assert_equal(arr_wb.base, None)
        # assigning to arr_wb does not get transferred to arr
        arr_wb[...] = 100
        assert_equal(arr, -100)

    @pytest.mark.leaks_references(
            reason="increments self in dealloc; ignore since deprecated path.")
    def test_dealloc_warning(self):
        with suppress_warnings() as sup:
            sup.record(RuntimeWarning)
            arr = np.arange(9).reshape(3, 3)
            v = arr.T
            _multiarray_tests.npy_abuse_writebackifcopy(v)
            assert len(sup.log) == 1

    def test_view_discard_refcount(self):
        from numpy.core._multiarray_tests import npy_create_writebackifcopy, npy_discard

        arr = np.arange(9).reshape(3, 3).T
        orig = arr.copy()
        if HAS_REFCOUNT:
            arr_cnt = sys.getrefcount(arr)
        arr_wb = npy_create_writebackifcopy(arr)
        assert_(arr_wb.flags.writebackifcopy)
        assert_(arr_wb.base is arr)
        arr_wb[...] = -100
        npy_discard(arr_wb)
        # arr remains unchanged after discard
        assert_equal(arr, orig)
        # after discard, the two arrays no longer reference each other
        assert_(arr_wb.ctypes.data != 0)
        assert_equal(arr_wb.base, None)
        if HAS_REFCOUNT:
            assert_equal(arr_cnt, sys.getrefcount(arr))
        # assigning to arr_wb does not get transferred to arr
        arr_wb[...] = 100
        assert_equal(arr, orig)


class TestArange:
    def test_infinite(self):
        assert_raises_regex(
            ValueError, "size exceeded",
            np.arange, 0, np.inf
        )

    def test_nan_step(self):
        assert_raises_regex(
            ValueError, "cannot compute length",
            np.arange, 0, 1, np.nan
        )

    def test_zero_step(self):
        assert_raises(ZeroDivisionError, np.arange, 0, 10, 0)
        assert_raises(ZeroDivisionError, np.arange, 0.0, 10.0, 0.0)

        # empty range
        assert_raises(ZeroDivisionError, np.arange, 0, 0, 0)
        assert_raises(ZeroDivisionError, np.arange, 0.0, 0.0, 0.0)

    def test_require_range(self):
        assert_raises(TypeError, np.arange)
        assert_raises(TypeError, np.arange, step=3)
        assert_raises(TypeError, np.arange, dtype='int64')
        assert_raises(TypeError, np.arange, start=4)

    def test_start_stop_kwarg(self):
        keyword_stop = np.arange(stop=3)
        keyword_zerotostop = np.arange(start=0, stop=3)
        keyword_start_stop = np.arange(start=3, stop=9)

        assert len(keyword_stop) == 3
        assert len(keyword_zerotostop) == 3
        assert len(keyword_start_stop) == 6
        assert_array_equal(keyword_stop, keyword_zerotostop)

    def test_arange_booleans(self):
        # Arange makes some sense for booleans and works up to length 2.
        # But it is weird since `arange(2, 4, dtype=bool)` works.
        # Arguably, much or all of this could be deprecated/removed.
        res = np.arange(False, dtype=bool)
        assert_array_equal(res, np.array([], dtype="bool"))

        res = np.arange(True, dtype="bool")
        assert_array_equal(res, [False])

        res = np.arange(2, dtype="bool")
        assert_array_equal(res, [False, True])

        # This case is especially weird, but drops out without special case:
        res = np.arange(6, 8, dtype="bool")
        assert_array_equal(res, [True, True])

        with pytest.raises(TypeError):
            np.arange(3, dtype="bool")

    @pytest.mark.parametrize("dtype", ["S3", "U", "5i"])
    def test_rejects_bad_dtypes(self, dtype):
        dtype = np.dtype(dtype)
        DType_name = re.escape(str(type(dtype)))
        with pytest.raises(TypeError,
                match=rf"arange\(\) not supported for inputs .* {DType_name}"):
            np.arange(2, dtype=dtype)

    def test_rejects_strings(self):
        # Explicitly test error for strings which may call "b" - "a":
        DType_name = re.escape(str(type(np.array("a").dtype)))
        with pytest.raises(TypeError,
                match=rf"arange\(\) not supported for inputs .* {DType_name}"):
            np.arange("a", "b")

    def test_byteswapped(self):
        res_be = np.arange(1, 1000, dtype=">i4")
        res_le = np.arange(1, 1000, dtype="<i4")
        assert res_be.dtype == ">i4"
        assert res_le.dtype == "<i4"
        assert_array_equal(res_le, res_be)

    @pytest.mark.parametrize("which", [0, 1, 2])
    def test_error_paths_and_promotion(self, which):
        args = [0, 1, 2]  # start, stop, and step
        args[which] = np.float64(2.)  # should ensure float64 output

        assert np.arange(*args).dtype == np.float64

        # Cover stranger error path, test only to achieve code coverage!
        args[which] = [None, []]
        with pytest.raises(ValueError):
            # Fails discovering start dtype
            np.arange(*args)


class TestArrayFinalize:
    """ Tests __array_finalize__ """

    def test_receives_base(self):
        # gh-11237
        class SavesBase(np.ndarray):
            def __array_finalize__(self, obj):
                self.saved_base = self.base

        a = np.array(1).view(SavesBase)
        assert_(a.saved_base is a.base)

    def test_bad_finalize1(self):
        class BadAttributeArray(np.ndarray):
            @property
            def __array_finalize__(self):
                raise RuntimeError("boohoo!")

        with pytest.raises(TypeError, match="not callable"):
            np.arange(10).view(BadAttributeArray)

    def test_bad_finalize2(self):
        class BadAttributeArray(np.ndarray):
            def __array_finalize__(self):
                raise RuntimeError("boohoo!")

        with pytest.raises(TypeError, match="takes 1 positional"):
            np.arange(10).view(BadAttributeArray)

    def test_bad_finalize3(self):
        class BadAttributeArray(np.ndarray):
            def __array_finalize__(self, obj):
                raise RuntimeError("boohoo!")

        with pytest.raises(RuntimeError, match="boohoo!"):
            np.arange(10).view(BadAttributeArray)

    def test_lifetime_on_error(self):
        # gh-11237
        class RaisesInFinalize(np.ndarray):
            def __array_finalize__(self, obj):
                # crash, but keep this object alive
                raise Exception(self)

        # a plain object can't be weakref'd
        class Dummy: pass

        # get a weak reference to an object within an array
        obj_arr = np.array(Dummy())
        obj_ref = weakref.ref(obj_arr[()])

        # get an array that crashed in __array_finalize__
        with assert_raises(Exception) as e:
            obj_arr.view(RaisesInFinalize)

        obj_subarray = e.exception.args[0]
        del e
        assert_(isinstance(obj_subarray, RaisesInFinalize))

        # reference should still be held by obj_arr
        break_cycles()
        assert_(obj_ref() is not None, "object should not already be dead")

        del obj_arr
        break_cycles()
        assert_(obj_ref() is not None, "obj_arr should not hold the last reference")

        del obj_subarray
        break_cycles()
        assert_(obj_ref() is None, "no references should remain")

    def test_can_use_super(self):
        class SuperFinalize(np.ndarray):
            def __array_finalize__(self, obj):
                self.saved_result = super().__array_finalize__(obj)

        a = np.array(1).view(SuperFinalize)
        assert_(a.saved_result is None)


def test_orderconverter_with_nonASCII_unicode_ordering():
    # gh-7475
    a = np.arange(5)
    assert_raises(ValueError, a.flatten, order='\xe2')


def test_equal_override():
    # gh-9153: ndarray.__eq__ uses special logic for structured arrays, which
    # did not respect overrides with __array_priority__ or __array_ufunc__.
    # The PR fixed this for __array_priority__ and __array_ufunc__ = None.
    class MyAlwaysEqual:
        def __eq__(self, other):
            return "eq"

        def __ne__(self, other):
            return "ne"

    class MyAlwaysEqualOld(MyAlwaysEqual):
        __array_priority__ = 10000

    class MyAlwaysEqualNew(MyAlwaysEqual):
        __array_ufunc__ = None

    array = np.array([(0, 1), (2, 3)], dtype='i4,i4')
    for my_always_equal_cls in MyAlwaysEqualOld, MyAlwaysEqualNew:
        my_always_equal = my_always_equal_cls()
        assert_equal(my_always_equal == array, 'eq')
        assert_equal(array == my_always_equal, 'eq')
        assert_equal(my_always_equal != array, 'ne')
        assert_equal(array != my_always_equal, 'ne')


@pytest.mark.parametrize(
    ["fun", "npfun"],
    [
        (_multiarray_tests.npy_cabs, np.absolute),
        (_multiarray_tests.npy_carg, np.angle)
    ]
)
@pytest.mark.parametrize("x", [1, np.inf, -np.inf, np.nan])
@pytest.mark.parametrize("y", [1, np.inf, -np.inf, np.nan])
@pytest.mark.parametrize("test_dtype", np.complexfloating.__subclasses__())
def test_npymath_complex(fun, npfun, x, y, test_dtype):
    # Smoketest npymath functions
    z = test_dtype(complex(x, y))
    got = fun(z)
    expected = npfun(z)
    assert_allclose(got, expected)


def test_npymath_real():
    # Smoketest npymath functions
    from numpy.core._multiarray_tests import (
        npy_log10, npy_cosh, npy_sinh, npy_tan, npy_tanh)

    funcs = {npy_log10: np.log10,
             npy_cosh: np.cosh,
             npy_sinh: np.sinh,
             npy_tan: np.tan,
             npy_tanh: np.tanh}
    vals = (1, np.inf, -np.inf, np.nan)
    types = (np.float32, np.float64, np.longdouble)

    with np.errstate(all='ignore'):
        for fun, npfun in funcs.items():
            for x, t in itertools.product(vals, types):
                z = t(x)
                got = fun(z)
                expected = npfun(z)
                assert_allclose(got, expected)

def test_uintalignment_and_alignment():
    # alignment code needs to satisfy these requirements:
    #  1. numpy structs match C struct layout
    #  2. ufuncs/casting is safe wrt to aligned access
    #  3. copy code is safe wrt to "uint alidned" access
    #
    # Complex types are the main problem, whose alignment may not be the same
    # as their "uint alignment".
    #
    # This test might only fail on certain platforms, where uint64 alignment is
    # not equal to complex64 alignment. The second 2 tests will only fail
    # for DEBUG=1.

    d1 = np.dtype('u1,c8', align=True)
    d2 = np.dtype('u4,c8', align=True)
    d3 = np.dtype({'names': ['a', 'b'], 'formats': ['u1', d1]}, align=True)

    assert_equal(np.zeros(1, dtype=d1)['f1'].flags['ALIGNED'], True)
    assert_equal(np.zeros(1, dtype=d2)['f1'].flags['ALIGNED'], True)
    assert_equal(np.zeros(1, dtype='u1,c8')['f1'].flags['ALIGNED'], False)

    # check that C struct matches numpy struct size
    s = _multiarray_tests.get_struct_alignments()
    for d, (alignment, size) in zip([d1,d2,d3], s):
        assert_equal(d.alignment, alignment)
        assert_equal(d.itemsize, size)

    # check that ufuncs don't complain in debug mode
    # (this is probably OK if the aligned flag is true above)
    src = np.zeros((2,2), dtype=d1)['f1']  # 4-byte aligned, often
    np.exp(src)  # assert fails?

    # check that copy code doesn't complain in debug mode
    dst = np.zeros((2,2), dtype='c8')
    dst[:,1] = src[:,1]  # assert in lowlevel_strided_loops fails?

class TestAlignment:
    # adapted from scipy._lib.tests.test__util.test__aligned_zeros
    # Checks that unusual memory alignments don't trip up numpy.
    # In particular, check RELAXED_STRIDES don't trip alignment assertions in
    # NDEBUG mode for size-0 arrays (gh-12503)

    def check(self, shape, dtype, order, align):
        err_msg = repr((shape, dtype, order, align))
        x = _aligned_zeros(shape, dtype, order, align=align)
        if align is None:
            align = np.dtype(dtype).alignment
        assert_equal(x.__array_interface__['data'][0] % align, 0)
        if hasattr(shape, '__len__'):
            assert_equal(x.shape, shape, err_msg)
        else:
            assert_equal(x.shape, (shape,), err_msg)
        assert_equal(x.dtype, dtype)
        if order == "C":
            assert_(x.flags.c_contiguous, err_msg)
        elif order == "F":
            if x.size > 0:
                assert_(x.flags.f_contiguous, err_msg)
        elif order is None:
            assert_(x.flags.c_contiguous, err_msg)
        else:
            raise ValueError()

    def test_various_alignments(self):
        for align in [1, 2, 3, 4, 8, 12, 16, 32, 64, None]:
            for n in [0, 1, 3, 11]:
                for order in ["C", "F", None]:
                    for dtype in list(np.typecodes["All"]) + ['i4,i4,i4']:
                        if dtype == 'O':
                            # object dtype can't be misaligned
                            continue
                        for shape in [n, (1, 2, 3, n)]:
                            self.check(shape, np.dtype(dtype), order, align)

    def test_strided_loop_alignments(self):
        # particularly test that complex64 and float128 use right alignment
        # code-paths, since these are particularly problematic. It is useful to
        # turn on USE_DEBUG for this test, so lowlevel-loop asserts are run.
        for align in [1, 2, 4, 8, 12, 16, None]:
            xf64 = _aligned_zeros(3, np.float64)

            xc64 = _aligned_zeros(3, np.complex64, align=align)
            xf128 = _aligned_zeros(3, np.longdouble, align=align)

            # test casting, both to and from misaligned
            with suppress_warnings() as sup:
                sup.filter(np.ComplexWarning, "Casting complex values")
                xc64.astype('f8')
            xf64.astype(np.complex64)
            test = xc64 + xf64

            xf128.astype('f8')
            xf64.astype(np.longdouble)
            test = xf128 + xf64

            test = xf128 + xc64

            # test copy, both to and from misaligned
            # contig copy
            xf64[:] = xf64.copy()
            xc64[:] = xc64.copy()
            xf128[:] = xf128.copy()
            # strided copy
            xf64[::2] = xf64[::2].copy()
            xc64[::2] = xc64[::2].copy()
            xf128[::2] = xf128[::2].copy()

def test_getfield():
    a = np.arange(32, dtype='uint16')
    if sys.byteorder == 'little':
        i = 0
        j = 1
    else:
        i = 1
        j = 0
    b = a.getfield('int8', i)
    assert_equal(b, a)
    b = a.getfield('int8', j)
    assert_equal(b, 0)
    pytest.raises(ValueError, a.getfield, 'uint8', -1)
    pytest.raises(ValueError, a.getfield, 'uint8', 16)
    pytest.raises(ValueError, a.getfield, 'uint64', 0)


class TestViewDtype:
    """
    Verify that making a view of a non-contiguous array works as expected.
    """
    def test_smaller_dtype_multiple(self):
        # x is non-contiguous
        x = np.arange(10, dtype='<i4')[::2]
        with pytest.raises(ValueError,
                           match='the last axis must be contiguous'):
            x.view('<i2')
        expected = [[0, 0], [2, 0], [4, 0], [6, 0], [8, 0]]
        assert_array_equal(x[:, np.newaxis].view('<i2'), expected)

    def test_smaller_dtype_not_multiple(self):
        # x is non-contiguous
        x = np.arange(5, dtype='<i4')[::2]

        with pytest.raises(ValueError,
                           match='the last axis must be contiguous'):
            x.view('S3')
        with pytest.raises(ValueError,
                           match='When changing to a smaller dtype'):
            x[:, np.newaxis].view('S3')

        # Make sure the problem is because of the dtype size
        expected = [[b''], [b'\x02'], [b'\x04']]
        assert_array_equal(x[:, np.newaxis].view('S4'), expected)

    def test_larger_dtype_multiple(self):
        # x is non-contiguous in the first dimension, contiguous in the last
        x = np.arange(20, dtype='<i2').reshape(10, 2)[::2, :]
        expected = np.array([[65536], [327684], [589832],
                             [851980], [1114128]], dtype='<i4')
        assert_array_equal(x.view('<i4'), expected)

    def test_larger_dtype_not_multiple(self):
        # x is non-contiguous in the first dimension, contiguous in the last
        x = np.arange(20, dtype='<i2').reshape(10, 2)[::2, :]
        with pytest.raises(ValueError,
                           match='When changing to a larger dtype'):
            x.view('S3')
        # Make sure the problem is because of the dtype size
        expected = [[b'\x00\x00\x01'], [b'\x04\x00\x05'], [b'\x08\x00\t'],
                    [b'\x0c\x00\r'], [b'\x10\x00\x11']]
        assert_array_equal(x.view('S4'), expected)

    def test_f_contiguous(self):
        # x is F-contiguous
        x = np.arange(4 * 3, dtype='<i4').reshape(4, 3).T
        with pytest.raises(ValueError,
                           match='the last axis must be contiguous'):
            x.view('<i2')

    def test_non_c_contiguous(self):
        # x is contiguous in axis=-1, but not C-contiguous in other axes
        x = np.arange(2 * 3 * 4, dtype='i1').\
                    reshape(2, 3, 4).transpose(1, 0, 2)
        expected = [[[256, 770], [3340, 3854]],
                    [[1284, 1798], [4368, 4882]],
                    [[2312, 2826], [5396, 5910]]]
        assert_array_equal(x.view('<i2'), expected)


# Test various array sizes that hit different code paths in quicksort-avx512
@pytest.mark.parametrize("N", [8, 16, 24, 32, 48, 64, 96, 128, 151, 191,
                               256, 383, 512, 1023, 2047])
def test_sort_float(N):
    # Regular data with nan sprinkled
    np.random.seed(42)
    arr = -0.5 + np.random.sample(N).astype('f')
    arr[np.random.choice(arr.shape[0], 3)] = np.nan
    assert_equal(np.sort(arr, kind='quick'), np.sort(arr, kind='heap'))

    # (2) with +INF
    infarr = np.inf*np.ones(N, dtype='f')
    infarr[np.random.choice(infarr.shape[0], 5)] = -1.0
    assert_equal(np.sort(infarr, kind='quick'), np.sort(infarr, kind='heap'))

    # (3) with -INF
    neginfarr = -np.inf*np.ones(N, dtype='f')
    neginfarr[np.random.choice(neginfarr.shape[0], 5)] = 1.0
    assert_equal(np.sort(neginfarr, kind='quick'),
                 np.sort(neginfarr, kind='heap'))

    # (4) with +/-INF
    infarr = np.inf*np.ones(N, dtype='f')
    infarr[np.random.choice(infarr.shape[0], (int)(N/2))] = -np.inf
    assert_equal(np.sort(infarr, kind='quick'), np.sort(infarr, kind='heap'))


def test_sort_int():
    # Random data with NPY_MAX_INT32 and NPY_MIN_INT32 sprinkled
    rng = np.random.default_rng(42)
    N = 2047
    minv = np.iinfo(np.int32).min
    maxv = np.iinfo(np.int32).max
    arr = rng.integers(low=minv, high=maxv, size=N).astype('int32')
    arr[np.random.choice(arr.shape[0], 10)] = minv
    arr[np.random.choice(arr.shape[0], 10)] = maxv
    assert_equal(np.sort(arr, kind='quick'), np.sort(arr, kind='heap'))


def test_sort_uint():
    # Random data with NPY_MAX_UINT32 sprinkled
    rng = np.random.default_rng(42)
    N = 2047
    maxv = np.iinfo(np.uint32).max
    arr = rng.integers(low=0, high=maxv, size=N).astype('uint32')
    arr[np.random.choice(arr.shape[0], 10)] = maxv
    assert_equal(np.sort(arr, kind='quick'), np.sort(arr, kind='heap'))

env_pyrecodes/lib/python3.9/site-packages/numpy/core/tests/test_multiarray.py
NikolaBlagojevic/pyrecodes
# This file is generated by numpy's setup.py
# It contains system_info results at the time of building this package.
__all__ = ["get_info","show"]


import os
import sys

extra_dll_dir = os.path.join(os.path.dirname(__file__), '.libs')

if sys.platform == 'win32' and os.path.isdir(extra_dll_dir):
    os.add_dll_directory(extra_dll_dir)

openblas64__info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None)], 'runtime_library_dirs': ['/usr/local/lib']}
blas_ilp64_opt_info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None)], 'runtime_library_dirs': ['/usr/local/lib']}
openblas64__lapack_info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None), ('HAVE_LAPACKE', None)], 'runtime_library_dirs': ['/usr/local/lib']}
lapack_ilp64_opt_info={'libraries': ['openblas64_', 'openblas64_'], 'library_dirs': ['/usr/local/lib'], 'language': 'c', 'define_macros': [('HAVE_CBLAS', None), ('BLAS_SYMBOL_SUFFIX', '64_'), ('HAVE_BLAS_ILP64', None), ('HAVE_LAPACKE', None)], 'runtime_library_dirs': ['/usr/local/lib']}

def get_info(name):
    g = globals()
    return g.get(name, g.get(name + "_info", {}))

def show():
    """
    Show libraries in the system on which NumPy was built.

    Print information about various resources (libraries, library
    directories, include directories, etc.) in the system on which
    NumPy was built.

    See Also
    --------
    get_include : Returns the directory containing NumPy C
                  header files.

    Notes
    -----
    1. Classes specifying the information to be printed are defined
       in the `numpy.distutils.system_info` module.

       Information may include:

       * ``language``: language used to write the libraries (mostly
         C or f77)
       * ``libraries``: names of libraries found in the system
       * ``library_dirs``: directories containing the libraries
       * ``include_dirs``: directories containing library header files
       * ``src_dirs``: directories containing library source files
       * ``define_macros``: preprocessor macros used by
         ``distutils.setup``
       * ``baseline``: minimum CPU features required
       * ``found``: dispatched features supported in the system
       * ``not found``: dispatched features that are not supported
         in the system

    2. NumPy BLAS/LAPACK Installation Notes

       Installing a numpy wheel (``pip install numpy`` or force it
       via ``pip install numpy --only-binary :numpy: numpy``) includes
       an OpenBLAS implementation of the BLAS and LAPACK linear algebra
       APIs. In this case, ``library_dirs`` reports the original build
       time configuration as compiled with gcc/gfortran; at run time
       the OpenBLAS library is in
       ``site-packages/numpy.libs/`` (linux), or
       ``site-packages/numpy/.dylibs/`` (macOS), or
       ``site-packages/numpy/.libs/`` (windows).

       Installing numpy from source
       (``pip install numpy --no-binary numpy``) searches for BLAS and
       LAPACK dynamic link libraries at build time as influenced by
       environment variables NPY_BLAS_LIBS, NPY_CBLAS_LIBS, and
       NPY_LAPACK_LIBS; or NPY_BLAS_ORDER and NPY_LAPACK_ORDER;
       or the optional file ``~/.numpy-site.cfg``.
       NumPy remembers those locations and expects to load the same
       libraries at run-time.
       In NumPy 1.21+ on macOS, 'accelerate' (Apple's Accelerate BLAS
       library) is in the default build-time search order after
       'openblas'.

    Examples
    --------
    >>> import numpy as np
    >>> np.show_config()
    blas_opt_info:
        language = c
        define_macros = [('HAVE_CBLAS', None)]
        libraries = ['openblas', 'openblas']
        library_dirs = ['/usr/local/lib']
    """
    from numpy.core._multiarray_umath import (
        __cpu_features__, __cpu_baseline__, __cpu_dispatch__
    )
    for name,info_dict in globals().items():
        if name[0] == "_" or type(info_dict) is not type({}): continue
        print(name + ":")
        if not info_dict:
            print("  NOT AVAILABLE")
        for k,v in info_dict.items():
            v = str(v)
            if k == "sources" and len(v) > 200:
                v = v[:60] + " ...\n... " + v[-60:]
            print("    %s = %s" % (k,v))

    features_found, features_not_found = [], []
    for feature in __cpu_dispatch__:
        if __cpu_features__[feature]:
            features_found.append(feature)
        else:
            features_not_found.append(feature)

    print("Supported SIMD extensions in this NumPy install:")
    print("    baseline = %s" % (','.join(__cpu_baseline__)))
    print("    found = %s" % (','.join(features_found)))
    print("    not found = %s" % (','.join(features_not_found)))


env_pyrecodes/lib/python3.9/site-packages/numpy/distutils/__config__.py
NikolaBlagojevic/pyrecodes
import operator
import warnings
import sys
import decimal
from fractions import Fraction
import math
import pytest
import hypothesis
from hypothesis.extra.numpy import arrays
import hypothesis.strategies as st


import numpy as np
from numpy import ma
from numpy.testing import (
    assert_, assert_equal, assert_array_equal, assert_almost_equal,
    assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY,
    assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, IS_WASM
    )
import numpy.lib.function_base as nfb
from numpy.random import rand
from numpy.lib import (
    add_newdoc_ufunc, angle, average, bartlett, blackman, corrcoef, cov,
    delete, diff, digitize, extract, flipud, gradient, hamming, hanning,
    i0, insert, interp, kaiser, meshgrid, msort, piecewise, place, rot90,
    select, setxor1d, sinc, trapz, trim_zeros, unwrap, unique, vectorize
    )
from numpy.core.numeric import normalize_axis_tuple


def get_mat(n):
    data = np.arange(n)
    data = np.add.outer(data, data)
    return data


def _make_complex(real, imag):
    """
    Like real + 1j * imag, but behaves as expected when imag contains non-finite
    values
    """
    ret = np.zeros(np.broadcast(real, imag).shape, np.complex_)
    ret.real = real
    ret.imag = imag
    return ret


class TestRot90:
    def test_basic(self):
        assert_raises(ValueError, rot90, np.ones(4))
        assert_raises(ValueError, rot90, np.ones((2,2,2)), axes=(0,1,2))
        assert_raises(ValueError, rot90, np.ones((2,2)), axes=(0,2))
        assert_raises(ValueError, rot90, np.ones((2,2)), axes=(1,1))
        assert_raises(ValueError, rot90, np.ones((2,2,2)), axes=(-2,1))

        a = [[0, 1, 2],
             [3, 4, 5]]
        b1 = [[2, 5],
              [1, 4],
              [0, 3]]
        b2 = [[5, 4, 3],
              [2, 1, 0]]
        b3 = [[3, 0],
              [4, 1],
              [5, 2]]
        b4 = [[0, 1, 2],
              [3, 4, 5]]

        for k in range(-3, 13, 4):
            assert_equal(rot90(a, k=k), b1)
        for k in range(-2, 13, 4):
            assert_equal(rot90(a, k=k), b2)
        for k in range(-1, 13, 4):
            assert_equal(rot90(a, k=k), b3)
        for k in range(0, 13, 4):
            assert_equal(rot90(a, k=k), b4)

        assert_equal(rot90(rot90(a, axes=(0,1)), axes=(1,0)), a)
        assert_equal(rot90(a, k=1, axes=(1,0)), rot90(a, k=-1, axes=(0,1)))

    def test_axes(self):
        a = np.ones((50, 40, 3))
        assert_equal(rot90(a).shape, (40, 50, 3))
        assert_equal(rot90(a, axes=(0,2)), rot90(a, axes=(0,-1)))
        assert_equal(rot90(a, axes=(1,2)), rot90(a, axes=(-2,-1)))

    def test_rotation_axes(self):
        a = np.arange(8).reshape((2,2,2))

        a_rot90_01 = [[[2, 3],
                       [6, 7]],
                      [[0, 1],
                       [4, 5]]]
        a_rot90_12 = [[[1, 3],
                       [0, 2]],
                      [[5, 7],
                       [4, 6]]]
        a_rot90_20 = [[[4, 0],
                       [6, 2]],
                      [[5, 1],
                       [7, 3]]]
        a_rot90_10 = [[[4, 5],
                       [0, 1]],
                      [[6, 7],
                       [2, 3]]]

        assert_equal(rot90(a, axes=(0, 1)), a_rot90_01)
        assert_equal(rot90(a, axes=(1, 0)), a_rot90_10)
        assert_equal(rot90(a, axes=(1, 2)), a_rot90_12)

        for k in range(1,5):
            assert_equal(rot90(a, k=k, axes=(2, 0)),
                         rot90(a_rot90_20, k=k-1, axes=(2, 0)))


class TestFlip:

    def test_axes(self):
        assert_raises(np.AxisError, np.flip, np.ones(4), axis=1)
        assert_raises(np.AxisError, np.flip, np.ones((4, 4)), axis=2)
        assert_raises(np.AxisError, np.flip, np.ones((4, 4)), axis=-3)
        assert_raises(np.AxisError, np.flip, np.ones((4, 4)), axis=(0, 3))

    def test_basic_lr(self):
        a = get_mat(4)
        b = a[:, ::-1]
        assert_equal(np.flip(a, 1), b)
        a = [[0, 1, 2],
             [3, 4, 5]]
        b = [[2, 1, 0],
             [5, 4, 3]]
        assert_equal(np.flip(a, 1), b)

    def test_basic_ud(self):
        a = get_mat(4)
        b = a[::-1, :]
        assert_equal(np.flip(a, 0), b)
        a = [[0, 1, 2],
             [3, 4, 5]]
        b = [[3, 4, 5],
             [0, 1, 2]]
        assert_equal(np.flip(a, 0), b)

    def test_3d_swap_axis0(self):
        a = np.array([[[0, 1],
                       [2, 3]],
                      [[4, 5],
                       [6, 7]]])

        b = np.array([[[4, 5],
                       [6, 7]],
                      [[0, 1],
                       [2, 3]]])

        assert_equal(np.flip(a, 0), b)

    def test_3d_swap_axis1(self):
        a = np.array([[[0, 1],
                       [2, 3]],
                      [[4, 5],
                       [6, 7]]])

        b = np.array([[[2, 3],
                       [0, 1]],
                      [[6, 7],
                       [4, 5]]])

        assert_equal(np.flip(a, 1), b)

    def test_3d_swap_axis2(self):
        a = np.array([[[0, 1],
                       [2, 3]],
                      [[4, 5],
                       [6, 7]]])

        b = np.array([[[1, 0],
                       [3, 2]],
                      [[5, 4],
                       [7, 6]]])

        assert_equal(np.flip(a, 2), b)

    def test_4d(self):
        a = np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5)
        for i in range(a.ndim):
            assert_equal(np.flip(a, i),
                         np.flipud(a.swapaxes(0, i)).swapaxes(i, 0))

    def test_default_axis(self):
        a = np.array([[1, 2, 3],
                      [4, 5, 6]])
        b = np.array([[6, 5, 4],
                      [3, 2, 1]])
        assert_equal(np.flip(a), b)

    def test_multiple_axes(self):
        a = np.array([[[0, 1],
                       [2, 3]],
                      [[4, 5],
                       [6, 7]]])

        assert_equal(np.flip(a, axis=()), a)

        b = np.array([[[5, 4],
                       [7, 6]],
                      [[1, 0],
                       [3, 2]]])

        assert_equal(np.flip(a, axis=(0, 2)), b)

        c = np.array([[[3, 2],
                       [1, 0]],
                      [[7, 6],
                       [5, 4]]])

        assert_equal(np.flip(a, axis=(1, 2)), c)


class TestAny:

    def test_basic(self):
        y1 = [0, 0, 1, 0]
        y2 = [0, 0, 0, 0]
        y3 = [1, 0, 1, 0]
        assert_(np.any(y1))
        assert_(np.any(y3))
        assert_(not np.any(y2))

    def test_nd(self):
        y1 = [[0, 0, 0], [0, 1, 0], [1, 1, 0]]
        assert_(np.any(y1))
        assert_array_equal(np.sometrue(y1, axis=0), [1, 1, 0])
        assert_array_equal(np.sometrue(y1, axis=1), [0, 1, 1])


class TestAll:

    def test_basic(self):
        y1 = [0, 1, 1, 0]
        y2 = [0, 0, 0, 0]
        y3 = [1, 1, 1, 1]
        assert_(not np.all(y1))
        assert_(np.all(y3))
        assert_(not np.all(y2))
        assert_(np.all(~np.array(y2)))

    def test_nd(self):
        y1 = [[0, 0, 1], [0, 1, 1], [1, 1, 1]]
        assert_(not np.all(y1))
        assert_array_equal(np.alltrue(y1, axis=0), [0, 0, 1])
        assert_array_equal(np.alltrue(y1, axis=1), [0, 0, 1])


class TestCopy:

    def test_basic(self):
        a = np.array([[1, 2], [3, 4]])
        a_copy = np.copy(a)
        assert_array_equal(a, a_copy)
        a_copy[0, 0] = 10
        assert_equal(a[0, 0], 1)
        assert_equal(a_copy[0, 0], 10)

    def test_order(self):
        # It turns out that people rely on np.copy() preserving order by
        # default; changing this broke scikit-learn:
        # github.com/scikit-learn/scikit-learn/commit/7842748cf777412c506a8c0ed28090711d3a3783  # noqa
        a = np.array([[1, 2], [3, 4]])
        assert_(a.flags.c_contiguous)
        assert_(not a.flags.f_contiguous)
        a_fort = np.array([[1, 2], [3, 4]], order="F")
        assert_(not a_fort.flags.c_contiguous)
        assert_(a_fort.flags.f_contiguous)
        a_copy = np.copy(a)
        assert_(a_copy.flags.c_contiguous)
        assert_(not a_copy.flags.f_contiguous)
        a_fort_copy = np.copy(a_fort)
        assert_(not a_fort_copy.flags.c_contiguous)
        assert_(a_fort_copy.flags.f_contiguous)

    def test_subok(self):
        mx = ma.ones(5)
        assert_(not ma.isMaskedArray(np.copy(mx, subok=False)))
        assert_(ma.isMaskedArray(np.copy(mx, subok=True)))
        # Default behavior
        assert_(not ma.isMaskedArray(np.copy(mx)))


class TestAverage:

    def test_basic(self):
        y1 = np.array([1, 2, 3])
        assert_(average(y1, axis=0) == 2.)
        y2 = np.array([1., 2., 3.])
        assert_(average(y2, axis=0) == 2.)
        y3 = [0., 0., 0.]
        assert_(average(y3, axis=0) == 0.)

        y4 = np.ones((4, 4))
        y4[0, 1] = 0
        y4[1, 0] = 2
        assert_almost_equal(y4.mean(0), average(y4, 0))
        assert_almost_equal(y4.mean(1), average(y4, 1))

        y5 = rand(5, 5)
        assert_almost_equal(y5.mean(0), average(y5, 0))
        assert_almost_equal(y5.mean(1), average(y5, 1))

    @pytest.mark.parametrize(
        'x, axis, expected_avg, weights, expected_wavg, expected_wsum',
        [([1, 2, 3], None, [2.0], [3, 4, 1], [1.75], [8.0]),
         ([[1, 2, 5], [1, 6, 11]], 0, [[1.0, 4.0, 8.0]],
          [1, 3], [[1.0, 5.0, 9.5]], [[4, 4, 4]])],
    )
    def test_basic_keepdims(self, x, axis, expected_avg,
                            weights, expected_wavg, expected_wsum):
        avg = np.average(x, axis=axis, keepdims=True)
        assert avg.shape == np.shape(expected_avg)
        assert_array_equal(avg, expected_avg)

        wavg = np.average(x, axis=axis, weights=weights, keepdims=True)
        assert wavg.shape == np.shape(expected_wavg)
        assert_array_equal(wavg, expected_wavg)

        wavg, wsum = np.average(x, axis=axis, weights=weights, returned=True,
                                keepdims=True)
        assert wavg.shape == np.shape(expected_wavg)
        assert_array_equal(wavg, expected_wavg)
        assert wsum.shape == np.shape(expected_wsum)
        assert_array_equal(wsum, expected_wsum)

    def test_weights(self):
        y = np.arange(10)
        w = np.arange(10)
        actual = average(y, weights=w)
        desired = (np.arange(10) ** 2).sum() * 1. / np.arange(10).sum()
        assert_almost_equal(actual, desired)

        y1 = np.array([[1, 2, 3], [4, 5, 6]])
        w0 = [1, 2]
        actual = average(y1, weights=w0, axis=0)
        desired = np.array([3., 4., 5.])
        assert_almost_equal(actual, desired)

        w1 = [0, 0, 1]
        actual = average(y1, weights=w1, axis=1)
        desired = np.array([3., 6.])
        assert_almost_equal(actual, desired)

        # This should raise an error. Can we test for that ?
        # assert_equal(average(y1, weights=w1), 9./2.)

        # 2D Case
        w2 = [[0, 0, 1], [0, 0, 2]]
        desired = np.array([3., 6.])
        assert_array_equal(average(y1, weights=w2, axis=1), desired)
        assert_equal(average(y1, weights=w2), 5.)

        y3 = rand(5).astype(np.float32)
        w3 = rand(5).astype(np.float64)

        assert_(np.average(y3, weights=w3).dtype == np.result_type(y3, w3))

        # test weights with `keepdims=False` and `keepdims=True`
        x = np.array([2, 3, 4]).reshape(3, 1)
        w = np.array([4, 5, 6]).reshape(3, 1)

        actual = np.average(x, weights=w, axis=1, keepdims=False)
        desired = np.array([2., 3., 4.])
        assert_array_equal(actual, desired)

        actual = np.average(x, weights=w, axis=1, keepdims=True)
        desired = np.array([[2.], [3.], [4.]])
        assert_array_equal(actual, desired)

    def test_returned(self):
        y = np.array([[1, 2, 3], [4, 5, 6]])

        # No weights
        avg, scl = average(y, returned=True)
        assert_equal(scl, 6.)

        avg, scl = average(y, 0, returned=True)
        assert_array_equal(scl, np.array([2., 2., 2.]))

        avg, scl = average(y, 1, returned=True)
        assert_array_equal(scl, np.array([3., 3.]))

        # With weights
        w0 = [1, 2]
        avg, scl = average(y, weights=w0, axis=0, returned=True)
        assert_array_equal(scl, np.array([3., 3., 3.]))

        w1 = [1, 2, 3]
        avg, scl = average(y, weights=w1, axis=1, returned=True)
        assert_array_equal(scl, np.array([6., 6.]))

        w2 = [[0, 0, 1], [1, 2, 3]]
        avg, scl = average(y, weights=w2, axis=1, returned=True)
        assert_array_equal(scl, np.array([1., 6.]))

    def test_subclasses(self):
        class subclass(np.ndarray):
            pass
        a = np.array([[1,2],[3,4]]).view(subclass)
        w = np.array([[1,2],[3,4]]).view(subclass)

        assert_equal(type(np.average(a)), subclass)
        assert_equal(type(np.average(a, weights=w)), subclass)

    def test_upcasting(self):
        typs = [('i4', 'i4', 'f8'), ('i4', 'f4', 'f8'), ('f4', 'i4', 'f8'),
                 ('f4', 'f4', 'f4'), ('f4', 'f8', 'f8')]
        for at, wt, rt in typs:
            a = np.array([[1,2],[3,4]], dtype=at)
            w = np.array([[1,2],[3,4]], dtype=wt)
            assert_equal(np.average(a, weights=w).dtype, np.dtype(rt))

    def test_object_dtype(self):
        a = np.array([decimal.Decimal(x) for x in range(10)])
        w = np.array([decimal.Decimal(1) for _ in range(10)])
        w /= w.sum()
        assert_almost_equal(a.mean(0), average(a, weights=w))

    def test_average_class_without_dtype(self):
        # see gh-21988
        a = np.array([Fraction(1, 5), Fraction(3, 5)])
        assert_equal(np.average(a), Fraction(2, 5))

class TestSelect:
    choices = [np.array([1, 2, 3]),
               np.array([4, 5, 6]),
               np.array([7, 8, 9])]
    conditions = [np.array([False, False, False]),
                  np.array([False, True, False]),
                  np.array([False, False, True])]

    def _select(self, cond, values, default=0):
        output = []
        for m in range(len(cond)):
            output += [V[m] for V, C in zip(values, cond) if C[m]] or [default]
        return output

    def test_basic(self):
        choices = self.choices
        conditions = self.conditions
        assert_array_equal(select(conditions, choices, default=15),
                           self._select(conditions, choices, default=15))

        assert_equal(len(choices), 3)
        assert_equal(len(conditions), 3)

    def test_broadcasting(self):
        conditions = [np.array(True), np.array([False, True, False])]
        choices = [1, np.arange(12).reshape(4, 3)]
        assert_array_equal(select(conditions, choices), np.ones((4, 3)))
        # default can broadcast too:
        assert_equal(select([True], [0], default=[0]).shape, (1,))

    def test_return_dtype(self):
        assert_equal(select(self.conditions, self.choices, 1j).dtype,
                     np.complex_)
        # But the conditions need to be stronger then the scalar default
        # if it is scalar.
        choices = [choice.astype(np.int8) for choice in self.choices]
        assert_equal(select(self.conditions, choices).dtype, np.int8)

        d = np.array([1, 2, 3, np.nan, 5, 7])
        m = np.isnan(d)
        assert_equal(select([m], [d]), [0, 0, 0, np.nan, 0, 0])

    def test_deprecated_empty(self):
        assert_raises(ValueError, select, [], [], 3j)
        assert_raises(ValueError, select, [], [])

    def test_non_bool_deprecation(self):
        choices = self.choices
        conditions = self.conditions[:]
        conditions[0] = conditions[0].astype(np.int_)
        assert_raises(TypeError, select, conditions, choices)
        conditions[0] = conditions[0].astype(np.uint8)
        assert_raises(TypeError, select, conditions, choices)
        assert_raises(TypeError, select, conditions, choices)

    def test_many_arguments(self):
        # This used to be limited by NPY_MAXARGS == 32
        conditions = [np.array([False])] * 100
        choices = [np.array([1])] * 100
        select(conditions, choices)


class TestInsert:

    def test_basic(self):
        a = [1, 2, 3]
        assert_equal(insert(a, 0, 1), [1, 1, 2, 3])
        assert_equal(insert(a, 3, 1), [1, 2, 3, 1])
        assert_equal(insert(a, [1, 1, 1], [1, 2, 3]), [1, 1, 2, 3, 2, 3])
        assert_equal(insert(a, 1, [1, 2, 3]), [1, 1, 2, 3, 2, 3])
        assert_equal(insert(a, [1, -1, 3], 9), [1, 9, 2, 9, 3, 9])
        assert_equal(insert(a, slice(-1, None, -1), 9), [9, 1, 9, 2, 9, 3])
        assert_equal(insert(a, [-1, 1, 3], [7, 8, 9]), [1, 8, 2, 7, 3, 9])
        b = np.array([0, 1], dtype=np.float64)
        assert_equal(insert(b, 0, b[0]), [0., 0., 1.])
        assert_equal(insert(b, [], []), b)
        # Bools will be treated differently in the future:
        # assert_equal(insert(a, np.array([True]*4), 9), [9, 1, 9, 2, 9, 3, 9])
        with warnings.catch_warnings(record=True) as w:
            warnings.filterwarnings('always', '', FutureWarning)
            assert_equal(
                insert(a, np.array([True] * 4), 9), [1, 9, 9, 9, 9, 2, 3])
            assert_(w[0].category is FutureWarning)

    def test_multidim(self):
        a = [[1, 1, 1]]
        r = [[2, 2, 2],
             [1, 1, 1]]
        assert_equal(insert(a, 0, [1]), [1, 1, 1, 1])
        assert_equal(insert(a, 0, [2, 2, 2], axis=0), r)
        assert_equal(insert(a, 0, 2, axis=0), r)
        assert_equal(insert(a, 2, 2, axis=1), [[1, 1, 2, 1]])

        a = np.array([[1, 1], [2, 2], [3, 3]])
        b = np.arange(1, 4).repeat(3).reshape(3, 3)
        c = np.concatenate(
            (a[:, 0:1], np.arange(1, 4).repeat(3).reshape(3, 3).T,
             a[:, 1:2]), axis=1)
        assert_equal(insert(a, [1], [[1], [2], [3]], axis=1), b)
        assert_equal(insert(a, [1], [1, 2, 3], axis=1), c)
        # scalars behave differently, in this case exactly opposite:
        assert_equal(insert(a, 1, [1, 2, 3], axis=1), b)
        assert_equal(insert(a, 1, [[1], [2], [3]], axis=1), c)

        a = np.arange(4).reshape(2, 2)
        assert_equal(insert(a[:, :1], 1, a[:, 1], axis=1), a)
        assert_equal(insert(a[:1,:], 1, a[1,:], axis=0), a)

        # negative axis value
        a = np.arange(24).reshape((2, 3, 4))
        assert_equal(insert(a, 1, a[:,:, 3], axis=-1),
                     insert(a, 1, a[:,:, 3], axis=2))
        assert_equal(insert(a, 1, a[:, 2,:], axis=-2),
                     insert(a, 1, a[:, 2,:], axis=1))

        # invalid axis value
        assert_raises(np.AxisError, insert, a, 1, a[:, 2, :], axis=3)
        assert_raises(np.AxisError, insert, a, 1, a[:, 2, :], axis=-4)

        # negative axis value
        a = np.arange(24).reshape((2, 3, 4))
        assert_equal(insert(a, 1, a[:, :, 3], axis=-1),
                     insert(a, 1, a[:, :, 3], axis=2))
        assert_equal(insert(a, 1, a[:, 2, :], axis=-2),
                     insert(a, 1, a[:, 2, :], axis=1))

    def test_0d(self):
        a = np.array(1)
        with pytest.raises(np.AxisError):
            insert(a, [], 2, axis=0)
        with pytest.raises(TypeError):
            insert(a, [], 2, axis="nonsense")

    def test_subclass(self):
        class SubClass(np.ndarray):
            pass
        a = np.arange(10).view(SubClass)
        assert_(isinstance(np.insert(a, 0, [0]), SubClass))
        assert_(isinstance(np.insert(a, [], []), SubClass))
        assert_(isinstance(np.insert(a, [0, 1], [1, 2]), SubClass))
        assert_(isinstance(np.insert(a, slice(1, 2), [1, 2]), SubClass))
        assert_(isinstance(np.insert(a, slice(1, -2, -1), []), SubClass))
        # This is an error in the future:
        a = np.array(1).view(SubClass)
        assert_(isinstance(np.insert(a, 0, [0]), SubClass))

    def test_index_array_copied(self):
        x = np.array([1, 1, 1])
        np.insert([0, 1, 2], x, [3, 4, 5])
        assert_equal(x, np.array([1, 1, 1]))

    def test_structured_array(self):
        a = np.array([(1, 'a'), (2, 'b'), (3, 'c')],
                     dtype=[('foo', 'i'), ('bar', 'a1')])
        val = (4, 'd')
        b = np.insert(a, 0, val)
        assert_array_equal(b[0], np.array(val, dtype=b.dtype))
        val = [(4, 'd')] * 2
        b = np.insert(a, [0, 2], val)
        assert_array_equal(b[[0, 3]], np.array(val, dtype=b.dtype))

    def test_index_floats(self):
        with pytest.raises(IndexError):
            np.insert([0, 1, 2], np.array([1.0, 2.0]), [10, 20])
        with pytest.raises(IndexError):
            np.insert([0, 1, 2], np.array([], dtype=float), [])

    @pytest.mark.parametrize('idx', [4, -4])
    def test_index_out_of_bounds(self, idx):
        with pytest.raises(IndexError, match='out of bounds'):
            np.insert([0, 1, 2], [idx], [3, 4])


class TestAmax:

    def test_basic(self):
        a = [3, 4, 5, 10, -3, -5, 6.0]
        assert_equal(np.amax(a), 10.0)
        b = [[3, 6.0, 9.0],
             [4, 10.0, 5.0],
             [8, 3.0, 2.0]]
        assert_equal(np.amax(b, axis=0), [8.0, 10.0, 9.0])
        assert_equal(np.amax(b, axis=1), [9.0, 10.0, 8.0])


class TestAmin:

    def test_basic(self):
        a = [3, 4, 5, 10, -3, -5, 6.0]
        assert_equal(np.amin(a), -5.0)
        b = [[3, 6.0, 9.0],
             [4, 10.0, 5.0],
             [8, 3.0, 2.0]]
        assert_equal(np.amin(b, axis=0), [3.0, 3.0, 2.0])
        assert_equal(np.amin(b, axis=1), [3.0, 4.0, 2.0])


class TestPtp:

    def test_basic(self):
        a = np.array([3, 4, 5, 10, -3, -5, 6.0])
        assert_equal(a.ptp(axis=0), 15.0)
        b = np.array([[3, 6.0, 9.0],
                      [4, 10.0, 5.0],
                      [8, 3.0, 2.0]])
        assert_equal(b.ptp(axis=0), [5.0, 7.0, 7.0])
        assert_equal(b.ptp(axis=-1), [6.0, 6.0, 6.0])

        assert_equal(b.ptp(axis=0, keepdims=True), [[5.0, 7.0, 7.0]])
        assert_equal(b.ptp(axis=(0,1), keepdims=True), [[8.0]])


class TestCumsum:

    def test_basic(self):
        ba = [1, 2, 10, 11, 6, 5, 4]
        ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
        for ctype in [np.int8, np.uint8, np.int16, np.uint16, np.int32,
                      np.uint32, np.float32, np.float64, np.complex64,
                      np.complex128]:
            a = np.array(ba, ctype)
            a2 = np.array(ba2, ctype)

            tgt = np.array([1, 3, 13, 24, 30, 35, 39], ctype)
            assert_array_equal(np.cumsum(a, axis=0), tgt)

            tgt = np.array(
                [[1, 2, 3, 4], [6, 8, 10, 13], [16, 11, 14, 18]], ctype)
            assert_array_equal(np.cumsum(a2, axis=0), tgt)

            tgt = np.array(
                [[1, 3, 6, 10], [5, 11, 18, 27], [10, 13, 17, 22]], ctype)
            assert_array_equal(np.cumsum(a2, axis=1), tgt)


class TestProd:

    def test_basic(self):
        ba = [1, 2, 10, 11, 6, 5, 4]
        ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
        for ctype in [np.int16, np.uint16, np.int32, np.uint32,
                      np.float32, np.float64, np.complex64, np.complex128]:
            a = np.array(ba, ctype)
            a2 = np.array(ba2, ctype)
            if ctype in ['1', 'b']:
                assert_raises(ArithmeticError, np.prod, a)
                assert_raises(ArithmeticError, np.prod, a2, 1)
            else:
                assert_equal(a.prod(axis=0), 26400)
                assert_array_equal(a2.prod(axis=0),
                                   np.array([50, 36, 84, 180], ctype))
                assert_array_equal(a2.prod(axis=-1),
                                   np.array([24, 1890, 600], ctype))


class TestCumprod:

    def test_basic(self):
        ba = [1, 2, 10, 11, 6, 5, 4]
        ba2 = [[1, 2, 3, 4], [5, 6, 7, 9], [10, 3, 4, 5]]
        for ctype in [np.int16, np.uint16, np.int32, np.uint32,
                      np.float32, np.float64, np.complex64, np.complex128]:
            a = np.array(ba, ctype)
            a2 = np.array(ba2, ctype)
            if ctype in ['1', 'b']:
                assert_raises(ArithmeticError, np.cumprod, a)
                assert_raises(ArithmeticError, np.cumprod, a2, 1)
                assert_raises(ArithmeticError, np.cumprod, a)
            else:
                assert_array_equal(np.cumprod(a, axis=-1),
                                   np.array([1, 2, 20, 220,
                                             1320, 6600, 26400], ctype))
                assert_array_equal(np.cumprod(a2, axis=0),
                                   np.array([[1, 2, 3, 4],
                                             [5, 12, 21, 36],
                                             [50, 36, 84, 180]], ctype))
                assert_array_equal(np.cumprod(a2, axis=-1),
                                   np.array([[1, 2, 6, 24],
                                             [5, 30, 210, 1890],
                                             [10, 30, 120, 600]], ctype))


class TestDiff:

    def test_basic(self):
        x = [1, 4, 6, 7, 12]
        out = np.array([3, 2, 1, 5])
        out2 = np.array([-1, -1, 4])
        out3 = np.array([0, 5])
        assert_array_equal(diff(x), out)
        assert_array_equal(diff(x, n=2), out2)
        assert_array_equal(diff(x, n=3), out3)

        x = [1.1, 2.2, 3.0, -0.2, -0.1]
        out = np.array([1.1, 0.8, -3.2, 0.1])
        assert_almost_equal(diff(x), out)

        x = [True, True, False, False]
        out = np.array([False, True, False])
        out2 = np.array([True, True])
        assert_array_equal(diff(x), out)
        assert_array_equal(diff(x, n=2), out2)

    def test_axis(self):
        x = np.zeros((10, 20, 30))
        x[:, 1::2, :] = 1
        exp = np.ones((10, 19, 30))
        exp[:, 1::2, :] = -1
        assert_array_equal(diff(x), np.zeros((10, 20, 29)))
        assert_array_equal(diff(x, axis=-1), np.zeros((10, 20, 29)))
        assert_array_equal(diff(x, axis=0), np.zeros((9, 20, 30)))
        assert_array_equal(diff(x, axis=1), exp)
        assert_array_equal(diff(x, axis=-2), exp)
        assert_raises(np.AxisError, diff, x, axis=3)
        assert_raises(np.AxisError, diff, x, axis=-4)

        x = np.array(1.11111111111, np.float64)
        assert_raises(ValueError, diff, x)

    def test_nd(self):
        x = 20 * rand(10, 20, 30)
        out1 = x[:, :, 1:] - x[:, :, :-1]
        out2 = out1[:, :, 1:] - out1[:, :, :-1]
        out3 = x[1:, :, :] - x[:-1, :, :]
        out4 = out3[1:, :, :] - out3[:-1, :, :]
        assert_array_equal(diff(x), out1)
        assert_array_equal(diff(x, n=2), out2)
        assert_array_equal(diff(x, axis=0), out3)
        assert_array_equal(diff(x, n=2, axis=0), out4)

    def test_n(self):
        x = list(range(3))
        assert_raises(ValueError, diff, x, n=-1)
        output = [diff(x, n=n) for n in range(1, 5)]
        expected = [[1, 1], [0], [], []]
        assert_(diff(x, n=0) is x)
        for n, (expected, out) in enumerate(zip(expected, output), start=1):
            assert_(type(out) is np.ndarray)
            assert_array_equal(out, expected)
            assert_equal(out.dtype, np.int_)
            assert_equal(len(out), max(0, len(x) - n))

    def test_times(self):
        x = np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64)
        expected = [
            np.array([1, 1], dtype='timedelta64[D]'),
            np.array([0], dtype='timedelta64[D]'),
        ]
        expected.extend([np.array([], dtype='timedelta64[D]')] * 3)
        for n, exp in enumerate(expected, start=1):
            out = diff(x, n=n)
            assert_array_equal(out, exp)
            assert_equal(out.dtype, exp.dtype)

    def test_subclass(self):
        x = ma.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]],
                     mask=[[False, False], [True, False],
                           [False, True], [True, True], [False, False]])
        out = diff(x)
        assert_array_equal(out.data, [[1], [1], [1], [1], [1]])
        assert_array_equal(out.mask, [[False], [True],
                                      [True], [True], [False]])
        assert_(type(out) is type(x))

        out3 = diff(x, n=3)
        assert_array_equal(out3.data, [[], [], [], [], []])
        assert_array_equal(out3.mask, [[], [], [], [], []])
        assert_(type(out3) is type(x))

    def test_prepend(self):
        x = np.arange(5) + 1
        assert_array_equal(diff(x, prepend=0), np.ones(5))
        assert_array_equal(diff(x, prepend=[0]), np.ones(5))
        assert_array_equal(np.cumsum(np.diff(x, prepend=0)), x)
        assert_array_equal(diff(x, prepend=[-1, 0]), np.ones(6))

        x = np.arange(4).reshape(2, 2)
        result = np.diff(x, axis=1, prepend=0)
        expected = [[0, 1], [2, 1]]
        assert_array_equal(result, expected)
        result = np.diff(x, axis=1, prepend=[[0], [0]])
        assert_array_equal(result, expected)

        result = np.diff(x, axis=0, prepend=0)
        expected = [[0, 1], [2, 2]]
        assert_array_equal(result, expected)
        result = np.diff(x, axis=0, prepend=[[0, 0]])
        assert_array_equal(result, expected)

        assert_raises(ValueError, np.diff, x, prepend=np.zeros((3,3)))

        assert_raises(np.AxisError, diff, x, prepend=0, axis=3)

    def test_append(self):
        x = np.arange(5)
        result = diff(x, append=0)
        expected = [1, 1, 1, 1, -4]
        assert_array_equal(result, expected)
        result = diff(x, append=[0])
        assert_array_equal(result, expected)
        result = diff(x, append=[0, 2])
        expected = expected + [2]
        assert_array_equal(result, expected)

        x = np.arange(4).reshape(2, 2)
        result = np.diff(x, axis=1, append=0)
        expected = [[1, -1], [1, -3]]
        assert_array_equal(result, expected)
        result = np.diff(x, axis=1, append=[[0], [0]])
        assert_array_equal(result, expected)

        result = np.diff(x, axis=0, append=0)
        expected = [[2, 2], [-2, -3]]
        assert_array_equal(result, expected)
        result = np.diff(x, axis=0, append=[[0, 0]])
        assert_array_equal(result, expected)

        assert_raises(ValueError, np.diff, x, append=np.zeros((3,3)))

        assert_raises(np.AxisError, diff, x, append=0, axis=3)


class TestDelete:

    def setup_method(self):
        self.a = np.arange(5)
        self.nd_a = np.arange(5).repeat(2).reshape(1, 5, 2)

    def _check_inverse_of_slicing(self, indices):
        a_del = delete(self.a, indices)
        nd_a_del = delete(self.nd_a, indices, axis=1)
        msg = 'Delete failed for obj: %r' % indices
        assert_array_equal(setxor1d(a_del, self.a[indices, ]), self.a,
                           err_msg=msg)
        xor = setxor1d(nd_a_del[0,:, 0], self.nd_a[0, indices, 0])
        assert_array_equal(xor, self.nd_a[0,:, 0], err_msg=msg)

    def test_slices(self):
        lims = [-6, -2, 0, 1, 2, 4, 5]
        steps = [-3, -1, 1, 3]
        for start in lims:
            for stop in lims:
                for step in steps:
                    s = slice(start, stop, step)
                    self._check_inverse_of_slicing(s)

    def test_fancy(self):
        self._check_inverse_of_slicing(np.array([[0, 1], [2, 1]]))
        with pytest.raises(IndexError):
            delete(self.a, [100])
        with pytest.raises(IndexError):
            delete(self.a, [-100])

        self._check_inverse_of_slicing([0, -1, 2, 2])

        self._check_inverse_of_slicing([True, False, False, True, False])

        # not legal, indexing with these would change the dimension
        with pytest.raises(ValueError):
            delete(self.a, True)
        with pytest.raises(ValueError):
            delete(self.a, False)

        # not enough items
        with pytest.raises(ValueError):
            delete(self.a, [False]*4)

    def test_single(self):
        self._check_inverse_of_slicing(0)
        self._check_inverse_of_slicing(-4)

    def test_0d(self):
        a = np.array(1)
        with pytest.raises(np.AxisError):
            delete(a, [], axis=0)
        with pytest.raises(TypeError):
            delete(a, [], axis="nonsense")

    def test_subclass(self):
        class SubClass(np.ndarray):
            pass
        a = self.a.view(SubClass)
        assert_(isinstance(delete(a, 0), SubClass))
        assert_(isinstance(delete(a, []), SubClass))
        assert_(isinstance(delete(a, [0, 1]), SubClass))
        assert_(isinstance(delete(a, slice(1, 2)), SubClass))
        assert_(isinstance(delete(a, slice(1, -2)), SubClass))

    def test_array_order_preserve(self):
        # See gh-7113
        k = np.arange(10).reshape(2, 5, order='F')
        m = delete(k, slice(60, None), axis=1)

        # 'k' is Fortran ordered, and 'm' should have the
        # same ordering as 'k' and NOT become C ordered
        assert_equal(m.flags.c_contiguous, k.flags.c_contiguous)
        assert_equal(m.flags.f_contiguous, k.flags.f_contiguous)

    def test_index_floats(self):
        with pytest.raises(IndexError):
            np.delete([0, 1, 2], np.array([1.0, 2.0]))
        with pytest.raises(IndexError):
            np.delete([0, 1, 2], np.array([], dtype=float))

    @pytest.mark.parametrize("indexer", [np.array([1]), [1]])
    def test_single_item_array(self, indexer):
        a_del_int = delete(self.a, 1)
        a_del = delete(self.a, indexer)
        assert_equal(a_del_int, a_del)

        nd_a_del_int = delete(self.nd_a, 1, axis=1)
        nd_a_del = delete(self.nd_a, np.array([1]), axis=1)
        assert_equal(nd_a_del_int, nd_a_del)

    def test_single_item_array_non_int(self):
        # Special handling for integer arrays must not affect non-integer ones.
        # If `False` was cast to `0` it would delete the element:
        res = delete(np.ones(1), np.array([False]))
        assert_array_equal(res, np.ones(1))

        # Test the more complicated (with axis) case from gh-21840
        x = np.ones((3, 1))
        false_mask = np.array([False], dtype=bool)
        true_mask = np.array([True], dtype=bool)

        res = delete(x, false_mask, axis=-1)
        assert_array_equal(res, x)
        res = delete(x, true_mask, axis=-1)
        assert_array_equal(res, x[:, :0])

        # Object or e.g. timedeltas should *not* be allowed
        with pytest.raises(IndexError):
            delete(np.ones(2), np.array([0], dtype=object))

        with pytest.raises(IndexError):
            # timedeltas are sometimes "integral, but clearly not allowed:
            delete(np.ones(2), np.array([0], dtype="m8[ns]"))


class TestGradient:

    def test_basic(self):
        v = [[1, 1], [3, 4]]
        x = np.array(v)
        dx = [np.array([[2., 3.], [2., 3.]]),
              np.array([[0., 0.], [1., 1.]])]
        assert_array_equal(gradient(x), dx)
        assert_array_equal(gradient(v), dx)

    def test_args(self):
        dx = np.cumsum(np.ones(5))
        dx_uneven = [1., 2., 5., 9., 11.]
        f_2d = np.arange(25).reshape(5, 5)

        # distances must be scalars or have size equal to gradient[axis]
        gradient(np.arange(5), 3.)
        gradient(np.arange(5), np.array(3.))
        gradient(np.arange(5), dx)
        # dy is set equal to dx because scalar
        gradient(f_2d, 1.5)
        gradient(f_2d, np.array(1.5))

        gradient(f_2d, dx_uneven, dx_uneven)
        # mix between even and uneven spaces and
        # mix between scalar and vector
        gradient(f_2d, dx, 2)

        # 2D but axis specified
        gradient(f_2d, dx, axis=1)

        # 2d coordinate arguments are not yet allowed
        assert_raises_regex(ValueError, '.*scalars or 1d',
            gradient, f_2d, np.stack([dx]*2, axis=-1), 1)

    def test_badargs(self):
        f_2d = np.arange(25).reshape(5, 5)
        x = np.cumsum(np.ones(5))

        # wrong sizes
        assert_raises(ValueError, gradient, f_2d, x, np.ones(2))
        assert_raises(ValueError, gradient, f_2d, 1, np.ones(2))
        assert_raises(ValueError, gradient, f_2d, np.ones(2), np.ones(2))
        # wrong number of arguments
        assert_raises(TypeError, gradient, f_2d, x)
        assert_raises(TypeError, gradient, f_2d, x, axis=(0,1))
        assert_raises(TypeError, gradient, f_2d, x, x, x)
        assert_raises(TypeError, gradient, f_2d, 1, 1, 1)
        assert_raises(TypeError, gradient, f_2d, x, x, axis=1)
        assert_raises(TypeError, gradient, f_2d, 1, 1, axis=1)

    def test_datetime64(self):
        # Make sure gradient() can handle special types like datetime64
        x = np.array(
            ['1910-08-16', '1910-08-11', '1910-08-10', '1910-08-12',
             '1910-10-12', '1910-12-12', '1912-12-12'],
            dtype='datetime64[D]')
        dx = np.array(
            [-5, -3, 0, 31, 61, 396, 731],
            dtype='timedelta64[D]')
        assert_array_equal(gradient(x), dx)
        assert_(dx.dtype == np.dtype('timedelta64[D]'))

    def test_masked(self):
        # Make sure that gradient supports subclasses like masked arrays
        x = np.ma.array([[1, 1], [3, 4]],
                        mask=[[False, False], [False, False]])
        out = gradient(x)[0]
        assert_equal(type(out), type(x))
        # And make sure that the output and input don't have aliased mask
        # arrays
        assert_(x._mask is not out._mask)
        # Also check that edge_order=2 doesn't alter the original mask
        x2 = np.ma.arange(5)
        x2[2] = np.ma.masked
        np.gradient(x2, edge_order=2)
        assert_array_equal(x2.mask, [False, False, True, False, False])

    def test_second_order_accurate(self):
        # Testing that the relative numerical error is less that 3% for
        # this example problem. This corresponds to second order
        # accurate finite differences for all interior and boundary
        # points.
        x = np.linspace(0, 1, 10)
        dx = x[1] - x[0]
        y = 2 * x ** 3 + 4 * x ** 2 + 2 * x
        analytical = 6 * x ** 2 + 8 * x + 2
        num_error = np.abs((np.gradient(y, dx, edge_order=2) / analytical) - 1)
        assert_(np.all(num_error < 0.03) == True)

        # test with unevenly spaced
        np.random.seed(0)
        x = np.sort(np.random.random(10))
        y = 2 * x ** 3 + 4 * x ** 2 + 2 * x
        analytical = 6 * x ** 2 + 8 * x + 2
        num_error = np.abs((np.gradient(y, x, edge_order=2) / analytical) - 1)
        assert_(np.all(num_error < 0.03) == True)

    def test_spacing(self):
        f = np.array([0, 2., 3., 4., 5., 5.])
        f = np.tile(f, (6,1)) + f.reshape(-1, 1)
        x_uneven = np.array([0., 0.5, 1., 3., 5., 7.])
        x_even = np.arange(6.)

        fdx_even_ord1 = np.tile([2., 1.5, 1., 1., 0.5, 0.], (6,1))
        fdx_even_ord2 = np.tile([2.5, 1.5, 1., 1., 0.5, -0.5], (6,1))
        fdx_uneven_ord1 = np.tile([4., 3., 1.7, 0.5, 0.25, 0.], (6,1))
        fdx_uneven_ord2 = np.tile([5., 3., 1.7, 0.5, 0.25, -0.25], (6,1))

        # evenly spaced
        for edge_order, exp_res in [(1, fdx_even_ord1), (2, fdx_even_ord2)]:
            res1 = gradient(f, 1., axis=(0,1), edge_order=edge_order)
            res2 = gradient(f, x_even, x_even,
                            axis=(0,1), edge_order=edge_order)
            res3 = gradient(f, x_even, x_even,
                            axis=None, edge_order=edge_order)
            assert_array_equal(res1, res2)
            assert_array_equal(res2, res3)
            assert_almost_equal(res1[0], exp_res.T)
            assert_almost_equal(res1[1], exp_res)

            res1 = gradient(f, 1., axis=0, edge_order=edge_order)
            res2 = gradient(f, x_even, axis=0, edge_order=edge_order)
            assert_(res1.shape == res2.shape)
            assert_almost_equal(res2, exp_res.T)

            res1 = gradient(f, 1., axis=1, edge_order=edge_order)
            res2 = gradient(f, x_even, axis=1, edge_order=edge_order)
            assert_(res1.shape == res2.shape)
            assert_array_equal(res2, exp_res)

        # unevenly spaced
        for edge_order, exp_res in [(1, fdx_uneven_ord1), (2, fdx_uneven_ord2)]:
            res1 = gradient(f, x_uneven, x_uneven,
                            axis=(0,1), edge_order=edge_order)
            res2 = gradient(f, x_uneven, x_uneven,
                            axis=None, edge_order=edge_order)
            assert_array_equal(res1, res2)
            assert_almost_equal(res1[0], exp_res.T)
            assert_almost_equal(res1[1], exp_res)

            res1 = gradient(f, x_uneven, axis=0, edge_order=edge_order)
            assert_almost_equal(res1, exp_res.T)

            res1 = gradient(f, x_uneven, axis=1, edge_order=edge_order)
            assert_almost_equal(res1, exp_res)

        # mixed
        res1 = gradient(f, x_even, x_uneven, axis=(0,1), edge_order=1)
        res2 = gradient(f, x_uneven, x_even, axis=(1,0), edge_order=1)
        assert_array_equal(res1[0], res2[1])
        assert_array_equal(res1[1], res2[0])
        assert_almost_equal(res1[0], fdx_even_ord1.T)
        assert_almost_equal(res1[1], fdx_uneven_ord1)

        res1 = gradient(f, x_even, x_uneven, axis=(0,1), edge_order=2)
        res2 = gradient(f, x_uneven, x_even, axis=(1,0), edge_order=2)
        assert_array_equal(res1[0], res2[1])
        assert_array_equal(res1[1], res2[0])
        assert_almost_equal(res1[0], fdx_even_ord2.T)
        assert_almost_equal(res1[1], fdx_uneven_ord2)

    def test_specific_axes(self):
        # Testing that gradient can work on a given axis only
        v = [[1, 1], [3, 4]]
        x = np.array(v)
        dx = [np.array([[2., 3.], [2., 3.]]),
              np.array([[0., 0.], [1., 1.]])]
        assert_array_equal(gradient(x, axis=0), dx[0])
        assert_array_equal(gradient(x, axis=1), dx[1])
        assert_array_equal(gradient(x, axis=-1), dx[1])
        assert_array_equal(gradient(x, axis=(1, 0)), [dx[1], dx[0]])

        # test axis=None which means all axes
        assert_almost_equal(gradient(x, axis=None), [dx[0], dx[1]])
        # and is the same as no axis keyword given
        assert_almost_equal(gradient(x, axis=None), gradient(x))

        # test vararg order
        assert_array_equal(gradient(x, 2, 3, axis=(1, 0)),
                           [dx[1]/2.0, dx[0]/3.0])
        # test maximal number of varargs
        assert_raises(TypeError, gradient, x, 1, 2, axis=1)

        assert_raises(np.AxisError, gradient, x, axis=3)
        assert_raises(np.AxisError, gradient, x, axis=-3)
        # assert_raises(TypeError, gradient, x, axis=[1,])

    def test_timedelta64(self):
        # Make sure gradient() can handle special types like timedelta64
        x = np.array(
            [-5, -3, 10, 12, 61, 321, 300],
            dtype='timedelta64[D]')
        dx = np.array(
            [2, 7, 7, 25, 154, 119, -21],
            dtype='timedelta64[D]')
        assert_array_equal(gradient(x), dx)
        assert_(dx.dtype == np.dtype('timedelta64[D]'))

    def test_inexact_dtypes(self):
        for dt in [np.float16, np.float32, np.float64]:
            # dtypes should not be promoted in a different way to what diff does
            x = np.array([1, 2, 3], dtype=dt)
            assert_equal(gradient(x).dtype, np.diff(x).dtype)

    def test_values(self):
        # needs at least 2 points for edge_order ==1
        gradient(np.arange(2), edge_order=1)
        # needs at least 3 points for edge_order ==1
        gradient(np.arange(3), edge_order=2)

        assert_raises(ValueError, gradient, np.arange(0), edge_order=1)
        assert_raises(ValueError, gradient, np.arange(0), edge_order=2)
        assert_raises(ValueError, gradient, np.arange(1), edge_order=1)
        assert_raises(ValueError, gradient, np.arange(1), edge_order=2)
        assert_raises(ValueError, gradient, np.arange(2), edge_order=2)

    @pytest.mark.parametrize('f_dtype', [np.uint8, np.uint16,
                                         np.uint32, np.uint64])
    def test_f_decreasing_unsigned_int(self, f_dtype):
        f = np.array([5, 4, 3, 2, 1], dtype=f_dtype)
        g = gradient(f)
        assert_array_equal(g, [-1]*len(f))

    @pytest.mark.parametrize('f_dtype', [np.int8, np.int16,
                                         np.int32, np.int64])
    def test_f_signed_int_big_jump(self, f_dtype):
        maxint = np.iinfo(f_dtype).max
        x = np.array([1, 3])
        f = np.array([-1, maxint], dtype=f_dtype)
        dfdx = gradient(f, x)
        assert_array_equal(dfdx, [(maxint + 1) // 2]*2)

    @pytest.mark.parametrize('x_dtype', [np.uint8, np.uint16,
                                         np.uint32, np.uint64])
    def test_x_decreasing_unsigned(self, x_dtype):
        x = np.array([3, 2, 1], dtype=x_dtype)
        f = np.array([0, 2, 4])
        dfdx = gradient(f, x)
        assert_array_equal(dfdx, [-2]*len(x))

    @pytest.mark.parametrize('x_dtype', [np.int8, np.int16,
                                         np.int32, np.int64])
    def test_x_signed_int_big_jump(self, x_dtype):
        minint = np.iinfo(x_dtype).min
        maxint = np.iinfo(x_dtype).max
        x = np.array([-1, maxint], dtype=x_dtype)
        f = np.array([minint // 2, 0])
        dfdx = gradient(f, x)
        assert_array_equal(dfdx, [0.5, 0.5])


class TestAngle:

    def test_basic(self):
        x = [1 + 3j, np.sqrt(2) / 2.0 + 1j * np.sqrt(2) / 2,
             1, 1j, -1, -1j, 1 - 3j, -1 + 3j]
        y = angle(x)
        yo = [
            np.arctan(3.0 / 1.0),
            np.arctan(1.0), 0, np.pi / 2, np.pi, -np.pi / 2.0,
            -np.arctan(3.0 / 1.0), np.pi - np.arctan(3.0 / 1.0)]
        z = angle(x, deg=True)
        zo = np.array(yo) * 180 / np.pi
        assert_array_almost_equal(y, yo, 11)
        assert_array_almost_equal(z, zo, 11)

    def test_subclass(self):
        x = np.ma.array([1 + 3j, 1, np.sqrt(2)/2 * (1 + 1j)])
        x[1] = np.ma.masked
        expected = np.ma.array([np.arctan(3.0 / 1.0), 0, np.arctan(1.0)])
        expected[1] = np.ma.masked
        actual = angle(x)
        assert_equal(type(actual), type(expected))
        assert_equal(actual.mask, expected.mask)
        assert_equal(actual, expected)


class TestTrimZeros:

    a = np.array([0, 0, 1, 0, 2, 3, 4, 0])
    b = a.astype(float)
    c = a.astype(complex)
    d = a.astype(object)

    def values(self):
        attr_names = ('a', 'b', 'c', 'd')
        return (getattr(self, name) for name in attr_names)

    def test_basic(self):
        slc = np.s_[2:-1]
        for arr in self.values():
            res = trim_zeros(arr)
            assert_array_equal(res, arr[slc])

    def test_leading_skip(self):
        slc = np.s_[:-1]
        for arr in self.values():
            res = trim_zeros(arr, trim='b')
            assert_array_equal(res, arr[slc])

    def test_trailing_skip(self):
        slc = np.s_[2:]
        for arr in self.values():
            res = trim_zeros(arr, trim='F')
            assert_array_equal(res, arr[slc])

    def test_all_zero(self):
        for _arr in self.values():
            arr = np.zeros_like(_arr, dtype=_arr.dtype)

            res1 = trim_zeros(arr, trim='B')
            assert len(res1) == 0

            res2 = trim_zeros(arr, trim='f')
            assert len(res2) == 0

    def test_size_zero(self):
        arr = np.zeros(0)
        res = trim_zeros(arr)
        assert_array_equal(arr, res)

    @pytest.mark.parametrize(
        'arr',
        [np.array([0, 2**62, 0]),
         np.array([0, 2**63, 0]),
         np.array([0, 2**64, 0])]
    )
    def test_overflow(self, arr):
        slc = np.s_[1:2]
        res = trim_zeros(arr)
        assert_array_equal(res, arr[slc])

    def test_no_trim(self):
        arr = np.array([None, 1, None])
        res = trim_zeros(arr)
        assert_array_equal(arr, res)

    def test_list_to_list(self):
        res = trim_zeros(self.a.tolist())
        assert isinstance(res, list)


class TestExtins:

    def test_basic(self):
        a = np.array([1, 3, 2, 1, 2, 3, 3])
        b = extract(a > 1, a)
        assert_array_equal(b, [3, 2, 2, 3, 3])

    def test_place(self):
        # Make sure that non-np.ndarray objects
        # raise an error instead of doing nothing
        assert_raises(TypeError, place, [1, 2, 3], [True, False], [0, 1])

        a = np.array([1, 4, 3, 2, 5, 8, 7])
        place(a, [0, 1, 0, 1, 0, 1, 0], [2, 4, 6])
        assert_array_equal(a, [1, 2, 3, 4, 5, 6, 7])

        place(a, np.zeros(7), [])
        assert_array_equal(a, np.arange(1, 8))

        place(a, [1, 0, 1, 0, 1, 0, 1], [8, 9])
        assert_array_equal(a, [8, 2, 9, 4, 8, 6, 9])
        assert_raises_regex(ValueError, "Cannot insert from an empty array",
                            lambda: place(a, [0, 0, 0, 0, 0, 1, 0], []))

        # See Issue #6974
        a = np.array(['12', '34'])
        place(a, [0, 1], '9')
        assert_array_equal(a, ['12', '9'])

    def test_both(self):
        a = rand(10)
        mask = a > 0.5
        ac = a.copy()
        c = extract(mask, a)
        place(a, mask, 0)
        place(a, mask, c)
        assert_array_equal(a, ac)


# _foo1 and _foo2 are used in some tests in TestVectorize.

def _foo1(x, y=1.0):
    return y*math.floor(x)


def _foo2(x, y=1.0, z=0.0):
    return y*math.floor(x) + z


class TestVectorize:

    def test_simple(self):
        def addsubtract(a, b):
            if a > b:
                return a - b
            else:
                return a + b

        f = vectorize(addsubtract)
        r = f([0, 3, 6, 9], [1, 3, 5, 7])
        assert_array_equal(r, [1, 6, 1, 2])

    def test_scalar(self):
        def addsubtract(a, b):
            if a > b:
                return a - b
            else:
                return a + b

        f = vectorize(addsubtract)
        r = f([0, 3, 6, 9], 5)
        assert_array_equal(r, [5, 8, 1, 4])

    def test_large(self):
        x = np.linspace(-3, 2, 10000)
        f = vectorize(lambda x: x)
        y = f(x)
        assert_array_equal(y, x)

    def test_ufunc(self):
        f = vectorize(math.cos)
        args = np.array([0, 0.5 * np.pi, np.pi, 1.5 * np.pi, 2 * np.pi])
        r1 = f(args)
        r2 = np.cos(args)
        assert_array_almost_equal(r1, r2)

    def test_keywords(self):

        def foo(a, b=1):
            return a + b

        f = vectorize(foo)
        args = np.array([1, 2, 3])
        r1 = f(args)
        r2 = np.array([2, 3, 4])
        assert_array_equal(r1, r2)
        r1 = f(args, 2)
        r2 = np.array([3, 4, 5])
        assert_array_equal(r1, r2)

    def test_keywords_with_otypes_order1(self):
        # gh-1620: The second call of f would crash with
        # `ValueError: invalid number of arguments`.
        f = vectorize(_foo1, otypes=[float])
        # We're testing the caching of ufuncs by vectorize, so the order
        # of these function calls is an important part of the test.
        r1 = f(np.arange(3.0), 1.0)
        r2 = f(np.arange(3.0))
        assert_array_equal(r1, r2)

    def test_keywords_with_otypes_order2(self):
        # gh-1620: The second call of f would crash with
        # `ValueError: non-broadcastable output operand with shape ()
        # doesn't match the broadcast shape (3,)`.
        f = vectorize(_foo1, otypes=[float])
        # We're testing the caching of ufuncs by vectorize, so the order
        # of these function calls is an important part of the test.
        r1 = f(np.arange(3.0))
        r2 = f(np.arange(3.0), 1.0)
        assert_array_equal(r1, r2)

    def test_keywords_with_otypes_order3(self):
        # gh-1620: The third call of f would crash with
        # `ValueError: invalid number of arguments`.
        f = vectorize(_foo1, otypes=[float])
        # We're testing the caching of ufuncs by vectorize, so the order
        # of these function calls is an important part of the test.
        r1 = f(np.arange(3.0))
        r2 = f(np.arange(3.0), y=1.0)
        r3 = f(np.arange(3.0))
        assert_array_equal(r1, r2)
        assert_array_equal(r1, r3)

    def test_keywords_with_otypes_several_kwd_args1(self):
        # gh-1620 Make sure different uses of keyword arguments
        # don't break the vectorized function.
        f = vectorize(_foo2, otypes=[float])
        # We're testing the caching of ufuncs by vectorize, so the order
        # of these function calls is an important part of the test.
        r1 = f(10.4, z=100)
        r2 = f(10.4, y=-1)
        r3 = f(10.4)
        assert_equal(r1, _foo2(10.4, z=100))
        assert_equal(r2, _foo2(10.4, y=-1))
        assert_equal(r3, _foo2(10.4))

    def test_keywords_with_otypes_several_kwd_args2(self):
        # gh-1620 Make sure different uses of keyword arguments
        # don't break the vectorized function.
        f = vectorize(_foo2, otypes=[float])
        # We're testing the caching of ufuncs by vectorize, so the order
        # of these function calls is an important part of the test.
        r1 = f(z=100, x=10.4, y=-1)
        r2 = f(1, 2, 3)
        assert_equal(r1, _foo2(z=100, x=10.4, y=-1))
        assert_equal(r2, _foo2(1, 2, 3))

    def test_keywords_no_func_code(self):
        # This needs to test a function that has keywords but
        # no func_code attribute, since otherwise vectorize will
        # inspect the func_code.
        import random
        try:
            vectorize(random.randrange)  # Should succeed
        except Exception:
            raise AssertionError()

    def test_keywords2_ticket_2100(self):
        # Test kwarg support: enhancement ticket 2100

        def foo(a, b=1):
            return a + b

        f = vectorize(foo)
        args = np.array([1, 2, 3])
        r1 = f(a=args)
        r2 = np.array([2, 3, 4])
        assert_array_equal(r1, r2)
        r1 = f(b=1, a=args)
        assert_array_equal(r1, r2)
        r1 = f(args, b=2)
        r2 = np.array([3, 4, 5])
        assert_array_equal(r1, r2)

    def test_keywords3_ticket_2100(self):
        # Test excluded with mixed positional and kwargs: ticket 2100
        def mypolyval(x, p):
            _p = list(p)
            res = _p.pop(0)
            while _p:
                res = res * x + _p.pop(0)
            return res

        vpolyval = np.vectorize(mypolyval, excluded=['p', 1])
        ans = [3, 6]
        assert_array_equal(ans, vpolyval(x=[0, 1], p=[1, 2, 3]))
        assert_array_equal(ans, vpolyval([0, 1], p=[1, 2, 3]))
        assert_array_equal(ans, vpolyval([0, 1], [1, 2, 3]))

    def test_keywords4_ticket_2100(self):
        # Test vectorizing function with no positional args.
        @vectorize
        def f(**kw):
            res = 1.0
            for _k in kw:
                res *= kw[_k]
            return res

        assert_array_equal(f(a=[1, 2], b=[3, 4]), [3, 8])

    def test_keywords5_ticket_2100(self):
        # Test vectorizing function with no kwargs args.
        @vectorize
        def f(*v):
            return np.prod(v)

        assert_array_equal(f([1, 2], [3, 4]), [3, 8])

    def test_coverage1_ticket_2100(self):
        def foo():
            return 1

        f = vectorize(foo)
        assert_array_equal(f(), 1)

    def test_assigning_docstring(self):
        def foo(x):
            """Original documentation"""
            return x

        f = vectorize(foo)
        assert_equal(f.__doc__, foo.__doc__)

        doc = "Provided documentation"
        f = vectorize(foo, doc=doc)
        assert_equal(f.__doc__, doc)

    def test_UnboundMethod_ticket_1156(self):
        # Regression test for issue 1156
        class Foo:
            b = 2

            def bar(self, a):
                return a ** self.b

        assert_array_equal(vectorize(Foo().bar)(np.arange(9)),
                           np.arange(9) ** 2)
        assert_array_equal(vectorize(Foo.bar)(Foo(), np.arange(9)),
                           np.arange(9) ** 2)

    def test_execution_order_ticket_1487(self):
        # Regression test for dependence on execution order: issue 1487
        f1 = vectorize(lambda x: x)
        res1a = f1(np.arange(3))
        res1b = f1(np.arange(0.1, 3))
        f2 = vectorize(lambda x: x)
        res2b = f2(np.arange(0.1, 3))
        res2a = f2(np.arange(3))
        assert_equal(res1a, res2a)
        assert_equal(res1b, res2b)

    def test_string_ticket_1892(self):
        # Test vectorization over strings: issue 1892.
        f = np.vectorize(lambda x: x)
        s = '0123456789' * 10
        assert_equal(s, f(s))

    def test_cache(self):
        # Ensure that vectorized func called exactly once per argument.
        _calls = [0]

        @vectorize
        def f(x):
            _calls[0] += 1
            return x ** 2

        f.cache = True
        x = np.arange(5)
        assert_array_equal(f(x), x * x)
        assert_equal(_calls[0], len(x))

    def test_otypes(self):
        f = np.vectorize(lambda x: x)
        f.otypes = 'i'
        x = np.arange(5)
        assert_array_equal(f(x), x)

    def test_parse_gufunc_signature(self):
        assert_equal(nfb._parse_gufunc_signature('(x)->()'), ([('x',)], [()]))
        assert_equal(nfb._parse_gufunc_signature('(x,y)->()'),
                     ([('x', 'y')], [()]))
        assert_equal(nfb._parse_gufunc_signature('(x),(y)->()'),
                     ([('x',), ('y',)], [()]))
        assert_equal(nfb._parse_gufunc_signature('(x)->(y)'),
                     ([('x',)], [('y',)]))
        assert_equal(nfb._parse_gufunc_signature('(x)->(y),()'),
                     ([('x',)], [('y',), ()]))
        assert_equal(nfb._parse_gufunc_signature('(),(a,b,c),(d)->(d,e)'),
                     ([(), ('a', 'b', 'c'), ('d',)], [('d', 'e')]))

        # Tests to check if whitespaces are ignored
        assert_equal(nfb._parse_gufunc_signature('(x )->()'), ([('x',)], [()]))
        assert_equal(nfb._parse_gufunc_signature('( x , y )->(  )'),
                     ([('x', 'y')], [()]))
        assert_equal(nfb._parse_gufunc_signature('(x),( y) ->()'),
                     ([('x',), ('y',)], [()]))
        assert_equal(nfb._parse_gufunc_signature('(  x)-> (y )  '),
                     ([('x',)], [('y',)]))
        assert_equal(nfb._parse_gufunc_signature(' (x)->( y),( )'),
                     ([('x',)], [('y',), ()]))
        assert_equal(nfb._parse_gufunc_signature(
                     '(  ), ( a,  b,c )  ,(  d)   ->   (d  ,  e)'),
                     ([(), ('a', 'b', 'c'), ('d',)], [('d', 'e')]))

        with assert_raises(ValueError):
            nfb._parse_gufunc_signature('(x)(y)->()')
        with assert_raises(ValueError):
            nfb._parse_gufunc_signature('(x),(y)->')
        with assert_raises(ValueError):
            nfb._parse_gufunc_signature('((x))->(x)')

    def test_signature_simple(self):
        def addsubtract(a, b):
            if a > b:
                return a - b
            else:
                return a + b

        f = vectorize(addsubtract, signature='(),()->()')
        r = f([0, 3, 6, 9], [1, 3, 5, 7])
        assert_array_equal(r, [1, 6, 1, 2])

    def test_signature_mean_last(self):
        def mean(a):
            return a.mean()

        f = vectorize(mean, signature='(n)->()')
        r = f([[1, 3], [2, 4]])
        assert_array_equal(r, [2, 3])

    def test_signature_center(self):
        def center(a):
            return a - a.mean()

        f = vectorize(center, signature='(n)->(n)')
        r = f([[1, 3], [2, 4]])
        assert_array_equal(r, [[-1, 1], [-1, 1]])

    def test_signature_two_outputs(self):
        f = vectorize(lambda x: (x, x), signature='()->(),()')
        r = f([1, 2, 3])
        assert_(isinstance(r, tuple) and len(r) == 2)
        assert_array_equal(r[0], [1, 2, 3])
        assert_array_equal(r[1], [1, 2, 3])

    def test_signature_outer(self):
        f = vectorize(np.outer, signature='(a),(b)->(a,b)')
        r = f([1, 2], [1, 2, 3])
        assert_array_equal(r, [[1, 2, 3], [2, 4, 6]])

        r = f([[[1, 2]]], [1, 2, 3])
        assert_array_equal(r, [[[[1, 2, 3], [2, 4, 6]]]])

        r = f([[1, 0], [2, 0]], [1, 2, 3])
        assert_array_equal(r, [[[1, 2, 3], [0, 0, 0]],
                               [[2, 4, 6], [0, 0, 0]]])

        r = f([1, 2], [[1, 2, 3], [0, 0, 0]])
        assert_array_equal(r, [[[1, 2, 3], [2, 4, 6]],
                               [[0, 0, 0], [0, 0, 0]]])

    def test_signature_computed_size(self):
        f = vectorize(lambda x: x[:-1], signature='(n)->(m)')
        r = f([1, 2, 3])
        assert_array_equal(r, [1, 2])

        r = f([[1, 2, 3], [2, 3, 4]])
        assert_array_equal(r, [[1, 2], [2, 3]])

    def test_signature_excluded(self):

        def foo(a, b=1):
            return a + b

        f = vectorize(foo, signature='()->()', excluded={'b'})
        assert_array_equal(f([1, 2, 3]), [2, 3, 4])
        assert_array_equal(f([1, 2, 3], b=0), [1, 2, 3])

    def test_signature_otypes(self):
        f = vectorize(lambda x: x, signature='(n)->(n)', otypes=['float64'])
        r = f([1, 2, 3])
        assert_equal(r.dtype, np.dtype('float64'))
        assert_array_equal(r, [1, 2, 3])

    def test_signature_invalid_inputs(self):
        f = vectorize(operator.add, signature='(n),(n)->(n)')
        with assert_raises_regex(TypeError, 'wrong number of positional'):
            f([1, 2])
        with assert_raises_regex(
                ValueError, 'does not have enough dimensions'):
            f(1, 2)
        with assert_raises_regex(
                ValueError, 'inconsistent size for core dimension'):
            f([1, 2], [1, 2, 3])

        f = vectorize(operator.add, signature='()->()')
        with assert_raises_regex(TypeError, 'wrong number of positional'):
            f(1, 2)

    def test_signature_invalid_outputs(self):

        f = vectorize(lambda x: x[:-1], signature='(n)->(n)')
        with assert_raises_regex(
                ValueError, 'inconsistent size for core dimension'):
            f([1, 2, 3])

        f = vectorize(lambda x: x, signature='()->(),()')
        with assert_raises_regex(ValueError, 'wrong number of outputs'):
            f(1)

        f = vectorize(lambda x: (x, x), signature='()->()')
        with assert_raises_regex(ValueError, 'wrong number of outputs'):
            f([1, 2])

    def test_size_zero_output(self):
        # see issue 5868
        f = np.vectorize(lambda x: x)
        x = np.zeros([0, 5], dtype=int)
        with assert_raises_regex(ValueError, 'otypes'):
            f(x)

        f.otypes = 'i'
        assert_array_equal(f(x), x)

        f = np.vectorize(lambda x: x, signature='()->()')
        with assert_raises_regex(ValueError, 'otypes'):
            f(x)

        f = np.vectorize(lambda x: x, signature='()->()', otypes='i')
        assert_array_equal(f(x), x)

        f = np.vectorize(lambda x: x, signature='(n)->(n)', otypes='i')
        assert_array_equal(f(x), x)

        f = np.vectorize(lambda x: x, signature='(n)->(n)')
        assert_array_equal(f(x.T), x.T)

        f = np.vectorize(lambda x: [x], signature='()->(n)', otypes='i')
        with assert_raises_regex(ValueError, 'new output dimensions'):
            f(x)

    def test_subclasses(self):
        class subclass(np.ndarray):
            pass

        m = np.array([[1., 0., 0.],
                      [0., 0., 1.],
                      [0., 1., 0.]]).view(subclass)
        v = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]).view(subclass)
        # generalized (gufunc)
        matvec = np.vectorize(np.matmul, signature='(m,m),(m)->(m)')
        r = matvec(m, v)
        assert_equal(type(r), subclass)
        assert_equal(r, [[1., 3., 2.], [4., 6., 5.], [7., 9., 8.]])

        # element-wise (ufunc)
        mult = np.vectorize(lambda x, y: x*y)
        r = mult(m, v)
        assert_equal(type(r), subclass)
        assert_equal(r, m * v)


class TestLeaks:
    class A:
        iters = 20

        def bound(self, *args):
            return 0

        @staticmethod
        def unbound(*args):
            return 0

    @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
    @pytest.mark.parametrize('name, incr', [
            ('bound', A.iters),
            ('unbound', 0),
            ])
    def test_frompyfunc_leaks(self, name, incr):
        # exposed in gh-11867 as np.vectorized, but the problem stems from
        # frompyfunc.
        # class.attribute = np.frompyfunc(<method>) creates a
        # reference cycle if <method> is a bound class method. It requires a
        # gc collection cycle to break the cycle (on CPython 3)
        import gc
        A_func = getattr(self.A, name)
        gc.disable()
        try:
            refcount = sys.getrefcount(A_func)
            for i in range(self.A.iters):
                a = self.A()
                a.f = np.frompyfunc(getattr(a, name), 1, 1)
                out = a.f(np.arange(10))
            a = None
            # A.func is part of a reference cycle if incr is non-zero
            assert_equal(sys.getrefcount(A_func), refcount + incr)
            for i in range(5):
                gc.collect()
            assert_equal(sys.getrefcount(A_func), refcount)
        finally:
            gc.enable()


class TestDigitize:

    def test_forward(self):
        x = np.arange(-6, 5)
        bins = np.arange(-5, 5)
        assert_array_equal(digitize(x, bins), np.arange(11))

    def test_reverse(self):
        x = np.arange(5, -6, -1)
        bins = np.arange(5, -5, -1)
        assert_array_equal(digitize(x, bins), np.arange(11))

    def test_random(self):
        x = rand(10)
        bin = np.linspace(x.min(), x.max(), 10)
        assert_(np.all(digitize(x, bin) != 0))

    def test_right_basic(self):
        x = [1, 5, 4, 10, 8, 11, 0]
        bins = [1, 5, 10]
        default_answer = [1, 2, 1, 3, 2, 3, 0]
        assert_array_equal(digitize(x, bins), default_answer)
        right_answer = [0, 1, 1, 2, 2, 3, 0]
        assert_array_equal(digitize(x, bins, True), right_answer)

    def test_right_open(self):
        x = np.arange(-6, 5)
        bins = np.arange(-6, 4)
        assert_array_equal(digitize(x, bins, True), np.arange(11))

    def test_right_open_reverse(self):
        x = np.arange(5, -6, -1)
        bins = np.arange(4, -6, -1)
        assert_array_equal(digitize(x, bins, True), np.arange(11))

    def test_right_open_random(self):
        x = rand(10)
        bins = np.linspace(x.min(), x.max(), 10)
        assert_(np.all(digitize(x, bins, True) != 10))

    def test_monotonic(self):
        x = [-1, 0, 1, 2]
        bins = [0, 0, 1]
        assert_array_equal(digitize(x, bins, False), [0, 2, 3, 3])
        assert_array_equal(digitize(x, bins, True), [0, 0, 2, 3])
        bins = [1, 1, 0]
        assert_array_equal(digitize(x, bins, False), [3, 2, 0, 0])
        assert_array_equal(digitize(x, bins, True), [3, 3, 2, 0])
        bins = [1, 1, 1, 1]
        assert_array_equal(digitize(x, bins, False), [0, 0, 4, 4])
        assert_array_equal(digitize(x, bins, True), [0, 0, 0, 4])
        bins = [0, 0, 1, 0]
        assert_raises(ValueError, digitize, x, bins)
        bins = [1, 1, 0, 1]
        assert_raises(ValueError, digitize, x, bins)

    def test_casting_error(self):
        x = [1, 2, 3 + 1.j]
        bins = [1, 2, 3]
        assert_raises(TypeError, digitize, x, bins)
        x, bins = bins, x
        assert_raises(TypeError, digitize, x, bins)

    def test_return_type(self):
        # Functions returning indices should always return base ndarrays
        class A(np.ndarray):
            pass
        a = np.arange(5).view(A)
        b = np.arange(1, 3).view(A)
        assert_(not isinstance(digitize(b, a, False), A))
        assert_(not isinstance(digitize(b, a, True), A))

    def test_large_integers_increasing(self):
        # gh-11022
        x = 2**54  # loses precision in a float
        assert_equal(np.digitize(x, [x - 1, x + 1]), 1)

    @pytest.mark.xfail(
        reason="gh-11022: np.core.multiarray._monoticity loses precision")
    def test_large_integers_decreasing(self):
        # gh-11022
        x = 2**54  # loses precision in a float
        assert_equal(np.digitize(x, [x + 1, x - 1]), 1)


class TestUnwrap:

    def test_simple(self):
        # check that unwrap removes jumps greater that 2*pi
        assert_array_equal(unwrap([1, 1 + 2 * np.pi]), [1, 1])
        # check that unwrap maintains continuity
        assert_(np.all(diff(unwrap(rand(10) * 100)) < np.pi))

    def test_period(self):
        # check that unwrap removes jumps greater that 255
        assert_array_equal(unwrap([1, 1 + 256], period=255), [1, 2])
        # check that unwrap maintains continuity
        assert_(np.all(diff(unwrap(rand(10) * 1000, period=255)) < 255))
        # check simple case
        simple_seq = np.array([0, 75, 150, 225, 300])
        wrap_seq = np.mod(simple_seq, 255)
        assert_array_equal(unwrap(wrap_seq, period=255), simple_seq)
        # check custom discont value
        uneven_seq = np.array([0, 75, 150, 225, 300, 430])
        wrap_uneven = np.mod(uneven_seq, 250)
        no_discont = unwrap(wrap_uneven, period=250)
        assert_array_equal(no_discont, [0, 75, 150, 225, 300, 180])
        sm_discont = unwrap(wrap_uneven, period=250, discont=140)
        assert_array_equal(sm_discont, [0, 75, 150, 225, 300, 430])
        assert sm_discont.dtype == wrap_uneven.dtype


@pytest.mark.parametrize(
    "dtype", "O" + np.typecodes["AllInteger"] + np.typecodes["Float"]
)
@pytest.mark.parametrize("M", [0, 1, 10])
class TestFilterwindows:

    def test_hanning(self, dtype: str, M: int) -> None:
        scalar = np.array(M, dtype=dtype)[()]

        w = hanning(scalar)
        if dtype == "O":
            ref_dtype = np.float64
        else:
            ref_dtype = np.result_type(scalar.dtype, np.float64)
        assert w.dtype == ref_dtype

        # check symmetry
        assert_equal(w, flipud(w))

        # check known value
        if scalar < 1:
            assert_array_equal(w, np.array([]))
        elif scalar == 1:
            assert_array_equal(w, np.ones(1))
        else:
            assert_almost_equal(np.sum(w, axis=0), 4.500, 4)

    def test_hamming(self, dtype: str, M: int) -> None:
        scalar = np.array(M, dtype=dtype)[()]

        w = hamming(scalar)
        if dtype == "O":
            ref_dtype = np.float64
        else:
            ref_dtype = np.result_type(scalar.dtype, np.float64)
        assert w.dtype == ref_dtype

        # check symmetry
        assert_equal(w, flipud(w))

        # check known value
        if scalar < 1:
            assert_array_equal(w, np.array([]))
        elif scalar == 1:
            assert_array_equal(w, np.ones(1))
        else:
            assert_almost_equal(np.sum(w, axis=0), 4.9400, 4)

    def test_bartlett(self, dtype: str, M: int) -> None:
        scalar = np.array(M, dtype=dtype)[()]

        w = bartlett(scalar)
        if dtype == "O":
            ref_dtype = np.float64
        else:
            ref_dtype = np.result_type(scalar.dtype, np.float64)
        assert w.dtype == ref_dtype

        # check symmetry
        assert_equal(w, flipud(w))

        # check known value
        if scalar < 1:
            assert_array_equal(w, np.array([]))
        elif scalar == 1:
            assert_array_equal(w, np.ones(1))
        else:
            assert_almost_equal(np.sum(w, axis=0), 4.4444, 4)

    def test_blackman(self, dtype: str, M: int) -> None:
        scalar = np.array(M, dtype=dtype)[()]

        w = blackman(scalar)
        if dtype == "O":
            ref_dtype = np.float64
        else:
            ref_dtype = np.result_type(scalar.dtype, np.float64)
        assert w.dtype == ref_dtype

        # check symmetry
        assert_equal(w, flipud(w))

        # check known value
        if scalar < 1:
            assert_array_equal(w, np.array([]))
        elif scalar == 1:
            assert_array_equal(w, np.ones(1))
        else:
            assert_almost_equal(np.sum(w, axis=0), 3.7800, 4)

    def test_kaiser(self, dtype: str, M: int) -> None:
        scalar = np.array(M, dtype=dtype)[()]

        w = kaiser(scalar, 0)
        if dtype == "O":
            ref_dtype = np.float64
        else:
            ref_dtype = np.result_type(scalar.dtype, np.float64)
        assert w.dtype == ref_dtype

        # check symmetry
        assert_equal(w, flipud(w))

        # check known value
        if scalar < 1:
            assert_array_equal(w, np.array([]))
        elif scalar == 1:
            assert_array_equal(w, np.ones(1))
        else:
            assert_almost_equal(np.sum(w, axis=0), 10, 15)


class TestTrapz:

    def test_simple(self):
        x = np.arange(-10, 10, .1)
        r = trapz(np.exp(-.5 * x ** 2) / np.sqrt(2 * np.pi), dx=0.1)
        # check integral of normal equals 1
        assert_almost_equal(r, 1, 7)

    def test_ndim(self):
        x = np.linspace(0, 1, 3)
        y = np.linspace(0, 2, 8)
        z = np.linspace(0, 3, 13)

        wx = np.ones_like(x) * (x[1] - x[0])
        wx[0] /= 2
        wx[-1] /= 2
        wy = np.ones_like(y) * (y[1] - y[0])
        wy[0] /= 2
        wy[-1] /= 2
        wz = np.ones_like(z) * (z[1] - z[0])
        wz[0] /= 2
        wz[-1] /= 2

        q = x[:, None, None] + y[None,:, None] + z[None, None,:]

        qx = (q * wx[:, None, None]).sum(axis=0)
        qy = (q * wy[None, :, None]).sum(axis=1)
        qz = (q * wz[None, None, :]).sum(axis=2)

        # n-d `x`
        r = trapz(q, x=x[:, None, None], axis=0)
        assert_almost_equal(r, qx)
        r = trapz(q, x=y[None,:, None], axis=1)
        assert_almost_equal(r, qy)
        r = trapz(q, x=z[None, None,:], axis=2)
        assert_almost_equal(r, qz)

        # 1-d `x`
        r = trapz(q, x=x, axis=0)
        assert_almost_equal(r, qx)
        r = trapz(q, x=y, axis=1)
        assert_almost_equal(r, qy)
        r = trapz(q, x=z, axis=2)
        assert_almost_equal(r, qz)

    def test_masked(self):
        # Testing that masked arrays behave as if the function is 0 where
        # masked
        x = np.arange(5)
        y = x * x
        mask = x == 2
        ym = np.ma.array(y, mask=mask)
        r = 13.0  # sum(0.5 * (0 + 1) * 1.0 + 0.5 * (9 + 16))
        assert_almost_equal(trapz(ym, x), r)

        xm = np.ma.array(x, mask=mask)
        assert_almost_equal(trapz(ym, xm), r)

        xm = np.ma.array(x, mask=mask)
        assert_almost_equal(trapz(y, xm), r)


class TestSinc:

    def test_simple(self):
        assert_(sinc(0) == 1)
        w = sinc(np.linspace(-1, 1, 100))
        # check symmetry
        assert_array_almost_equal(w, flipud(w), 7)

    def test_array_like(self):
        x = [0, 0.5]
        y1 = sinc(np.array(x))
        y2 = sinc(list(x))
        y3 = sinc(tuple(x))
        assert_array_equal(y1, y2)
        assert_array_equal(y1, y3)


class TestUnique:

    def test_simple(self):
        x = np.array([4, 3, 2, 1, 1, 2, 3, 4, 0])
        assert_(np.all(unique(x) == [0, 1, 2, 3, 4]))
        assert_(unique(np.array([1, 1, 1, 1, 1])) == np.array([1]))
        x = ['widget', 'ham', 'foo', 'bar', 'foo', 'ham']
        assert_(np.all(unique(x) == ['bar', 'foo', 'ham', 'widget']))
        x = np.array([5 + 6j, 1 + 1j, 1 + 10j, 10, 5 + 6j])
        assert_(np.all(unique(x) == [1 + 1j, 1 + 10j, 5 + 6j, 10]))


class TestCheckFinite:

    def test_simple(self):
        a = [1, 2, 3]
        b = [1, 2, np.inf]
        c = [1, 2, np.nan]
        np.lib.asarray_chkfinite(a)
        assert_raises(ValueError, np.lib.asarray_chkfinite, b)
        assert_raises(ValueError, np.lib.asarray_chkfinite, c)

    def test_dtype_order(self):
        # Regression test for missing dtype and order arguments
        a = [1, 2, 3]
        a = np.lib.asarray_chkfinite(a, order='F', dtype=np.float64)
        assert_(a.dtype == np.float64)


class TestCorrCoef:
    A = np.array(
        [[0.15391142, 0.18045767, 0.14197213],
         [0.70461506, 0.96474128, 0.27906989],
         [0.9297531, 0.32296769, 0.19267156]])
    B = np.array(
        [[0.10377691, 0.5417086, 0.49807457],
         [0.82872117, 0.77801674, 0.39226705],
         [0.9314666, 0.66800209, 0.03538394]])
    res1 = np.array(
        [[1., 0.9379533, -0.04931983],
         [0.9379533, 1., 0.30007991],
         [-0.04931983, 0.30007991, 1.]])
    res2 = np.array(
        [[1., 0.9379533, -0.04931983, 0.30151751, 0.66318558, 0.51532523],
         [0.9379533, 1., 0.30007991, -0.04781421, 0.88157256, 0.78052386],
         [-0.04931983, 0.30007991, 1., -0.96717111, 0.71483595, 0.83053601],
         [0.30151751, -0.04781421, -0.96717111, 1., -0.51366032, -0.66173113],
         [0.66318558, 0.88157256, 0.71483595, -0.51366032, 1., 0.98317823],
         [0.51532523, 0.78052386, 0.83053601, -0.66173113, 0.98317823, 1.]])

    def test_non_array(self):
        assert_almost_equal(np.corrcoef([0, 1, 0], [1, 0, 1]),
                            [[1., -1.], [-1.,  1.]])

    def test_simple(self):
        tgt1 = corrcoef(self.A)
        assert_almost_equal(tgt1, self.res1)
        assert_(np.all(np.abs(tgt1) <= 1.0))

        tgt2 = corrcoef(self.A, self.B)
        assert_almost_equal(tgt2, self.res2)
        assert_(np.all(np.abs(tgt2) <= 1.0))

    def test_ddof(self):
        # ddof raises DeprecationWarning
        with suppress_warnings() as sup:
            warnings.simplefilter("always")
            assert_warns(DeprecationWarning, corrcoef, self.A, ddof=-1)
            sup.filter(DeprecationWarning)
            # ddof has no or negligible effect on the function
            assert_almost_equal(corrcoef(self.A, ddof=-1), self.res1)
            assert_almost_equal(corrcoef(self.A, self.B, ddof=-1), self.res2)
            assert_almost_equal(corrcoef(self.A, ddof=3), self.res1)
            assert_almost_equal(corrcoef(self.A, self.B, ddof=3), self.res2)

    def test_bias(self):
        # bias raises DeprecationWarning
        with suppress_warnings() as sup:
            warnings.simplefilter("always")
            assert_warns(DeprecationWarning, corrcoef, self.A, self.B, 1, 0)
            assert_warns(DeprecationWarning, corrcoef, self.A, bias=0)
            sup.filter(DeprecationWarning)
            # bias has no or negligible effect on the function
            assert_almost_equal(corrcoef(self.A, bias=1), self.res1)

    def test_complex(self):
        x = np.array([[1, 2, 3], [1j, 2j, 3j]])
        res = corrcoef(x)
        tgt = np.array([[1., -1.j], [1.j, 1.]])
        assert_allclose(res, tgt)
        assert_(np.all(np.abs(res) <= 1.0))

    def test_xy(self):
        x = np.array([[1, 2, 3]])
        y = np.array([[1j, 2j, 3j]])
        assert_allclose(np.corrcoef(x, y), np.array([[1., -1.j], [1.j, 1.]]))

    def test_empty(self):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            assert_array_equal(corrcoef(np.array([])), np.nan)
            assert_array_equal(corrcoef(np.array([]).reshape(0, 2)),
                               np.array([]).reshape(0, 0))
            assert_array_equal(corrcoef(np.array([]).reshape(2, 0)),
                               np.array([[np.nan, np.nan], [np.nan, np.nan]]))

    def test_extreme(self):
        x = [[1e-100, 1e100], [1e100, 1e-100]]
        with np.errstate(all='raise'):
            c = corrcoef(x)
        assert_array_almost_equal(c, np.array([[1., -1.], [-1., 1.]]))
        assert_(np.all(np.abs(c) <= 1.0))

    @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble])
    def test_corrcoef_dtype(self, test_type):
        cast_A = self.A.astype(test_type)
        res = corrcoef(cast_A, dtype=test_type)
        assert test_type == res.dtype


class TestCov:
    x1 = np.array([[0, 2], [1, 1], [2, 0]]).T
    res1 = np.array([[1., -1.], [-1., 1.]])
    x2 = np.array([0.0, 1.0, 2.0], ndmin=2)
    frequencies = np.array([1, 4, 1])
    x2_repeats = np.array([[0.0], [1.0], [1.0], [1.0], [1.0], [2.0]]).T
    res2 = np.array([[0.4, -0.4], [-0.4, 0.4]])
    unit_frequencies = np.ones(3, dtype=np.int_)
    weights = np.array([1.0, 4.0, 1.0])
    res3 = np.array([[2. / 3., -2. / 3.], [-2. / 3., 2. / 3.]])
    unit_weights = np.ones(3)
    x3 = np.array([0.3942, 0.5969, 0.7730, 0.9918, 0.7964])

    def test_basic(self):
        assert_allclose(cov(self.x1), self.res1)

    def test_complex(self):
        x = np.array([[1, 2, 3], [1j, 2j, 3j]])
        res = np.array([[1., -1.j], [1.j, 1.]])
        assert_allclose(cov(x), res)
        assert_allclose(cov(x, aweights=np.ones(3)), res)

    def test_xy(self):
        x = np.array([[1, 2, 3]])
        y = np.array([[1j, 2j, 3j]])
        assert_allclose(cov(x, y), np.array([[1., -1.j], [1.j, 1.]]))

    def test_empty(self):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            assert_array_equal(cov(np.array([])), np.nan)
            assert_array_equal(cov(np.array([]).reshape(0, 2)),
                               np.array([]).reshape(0, 0))
            assert_array_equal(cov(np.array([]).reshape(2, 0)),
                               np.array([[np.nan, np.nan], [np.nan, np.nan]]))

    def test_wrong_ddof(self):
        with warnings.catch_warnings(record=True):
            warnings.simplefilter('always', RuntimeWarning)
            assert_array_equal(cov(self.x1, ddof=5),
                               np.array([[np.inf, -np.inf],
                                         [-np.inf, np.inf]]))

    def test_1D_rowvar(self):
        assert_allclose(cov(self.x3), cov(self.x3, rowvar=False))
        y = np.array([0.0780, 0.3107, 0.2111, 0.0334, 0.8501])
        assert_allclose(cov(self.x3, y), cov(self.x3, y, rowvar=False))

    def test_1D_variance(self):
        assert_allclose(cov(self.x3, ddof=1), np.var(self.x3, ddof=1))

    def test_fweights(self):
        assert_allclose(cov(self.x2, fweights=self.frequencies),
                        cov(self.x2_repeats))
        assert_allclose(cov(self.x1, fweights=self.frequencies),
                        self.res2)
        assert_allclose(cov(self.x1, fweights=self.unit_frequencies),
                        self.res1)
        nonint = self.frequencies + 0.5
        assert_raises(TypeError, cov, self.x1, fweights=nonint)
        f = np.ones((2, 3), dtype=np.int_)
        assert_raises(RuntimeError, cov, self.x1, fweights=f)
        f = np.ones(2, dtype=np.int_)
        assert_raises(RuntimeError, cov, self.x1, fweights=f)
        f = -1 * np.ones(3, dtype=np.int_)
        assert_raises(ValueError, cov, self.x1, fweights=f)

    def test_aweights(self):
        assert_allclose(cov(self.x1, aweights=self.weights), self.res3)
        assert_allclose(cov(self.x1, aweights=3.0 * self.weights),
                        cov(self.x1, aweights=self.weights))
        assert_allclose(cov(self.x1, aweights=self.unit_weights), self.res1)
        w = np.ones((2, 3))
        assert_raises(RuntimeError, cov, self.x1, aweights=w)
        w = np.ones(2)
        assert_raises(RuntimeError, cov, self.x1, aweights=w)
        w = -1.0 * np.ones(3)
        assert_raises(ValueError, cov, self.x1, aweights=w)

    def test_unit_fweights_and_aweights(self):
        assert_allclose(cov(self.x2, fweights=self.frequencies,
                            aweights=self.unit_weights),
                        cov(self.x2_repeats))
        assert_allclose(cov(self.x1, fweights=self.frequencies,
                            aweights=self.unit_weights),
                        self.res2)
        assert_allclose(cov(self.x1, fweights=self.unit_frequencies,
                            aweights=self.unit_weights),
                        self.res1)
        assert_allclose(cov(self.x1, fweights=self.unit_frequencies,
                            aweights=self.weights),
                        self.res3)
        assert_allclose(cov(self.x1, fweights=self.unit_frequencies,
                            aweights=3.0 * self.weights),
                        cov(self.x1, aweights=self.weights))
        assert_allclose(cov(self.x1, fweights=self.unit_frequencies,
                            aweights=self.unit_weights),
                        self.res1)

    @pytest.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble])
    def test_cov_dtype(self, test_type):
        cast_x1 = self.x1.astype(test_type)
        res = cov(cast_x1, dtype=test_type)
        assert test_type == res.dtype


class Test_I0:

    def test_simple(self):
        assert_almost_equal(
            i0(0.5),
            np.array(1.0634833707413234))

        # need at least one test above 8, as the implementation is piecewise
        A = np.array([0.49842636, 0.6969809, 0.22011976, 0.0155549, 10.0])
        expected = np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049, 2815.71662847])
        assert_almost_equal(i0(A), expected)
        assert_almost_equal(i0(-A), expected)

        B = np.array([[0.827002, 0.99959078],
                      [0.89694769, 0.39298162],
                      [0.37954418, 0.05206293],
                      [0.36465447, 0.72446427],
                      [0.48164949, 0.50324519]])
        assert_almost_equal(
            i0(B),
            np.array([[1.17843223, 1.26583466],
                      [1.21147086, 1.03898290],
                      [1.03633899, 1.00067775],
                      [1.03352052, 1.13557954],
                      [1.05884290, 1.06432317]]))
        # Regression test for gh-11205
        i0_0 = np.i0([0.])
        assert_equal(i0_0.shape, (1,))
        assert_array_equal(np.i0([0.]), np.array([1.]))

    def test_non_array(self):
        a = np.arange(4)

        class array_like:
            __array_interface__ = a.__array_interface__

            def __array_wrap__(self, arr):
                return self

        # E.g. pandas series survive ufunc calls through array-wrap:
        assert isinstance(np.abs(array_like()), array_like)
        exp = np.i0(a)
        res = np.i0(array_like())

        assert_array_equal(exp, res)

    def test_complex(self):
        a = np.array([0, 1 + 2j])
        with pytest.raises(TypeError, match="i0 not supported for complex values"):
            res = i0(a)


class TestKaiser:

    def test_simple(self):
        assert_(np.isfinite(kaiser(1, 1.0)))
        assert_almost_equal(kaiser(0, 1.0),
                            np.array([]))
        assert_almost_equal(kaiser(2, 1.0),
                            np.array([0.78984831, 0.78984831]))
        assert_almost_equal(kaiser(5, 1.0),
                            np.array([0.78984831, 0.94503323, 1.,
                                      0.94503323, 0.78984831]))
        assert_almost_equal(kaiser(5, 1.56789),
                            np.array([0.58285404, 0.88409679, 1.,
                                      0.88409679, 0.58285404]))

    def test_int_beta(self):
        kaiser(3, 4)


class TestMsort:

    def test_simple(self):
        A = np.array([[0.44567325, 0.79115165, 0.54900530],
                      [0.36844147, 0.37325583, 0.96098397],
                      [0.64864341, 0.52929049, 0.39172155]])
        with pytest.warns(DeprecationWarning, match="msort is deprecated"):
            assert_almost_equal(
                msort(A),
                np.array([[0.36844147, 0.37325583, 0.39172155],
                          [0.44567325, 0.52929049, 0.54900530],
                          [0.64864341, 0.79115165, 0.96098397]]))


class TestMeshgrid:

    def test_simple(self):
        [X, Y] = meshgrid([1, 2, 3], [4, 5, 6, 7])
        assert_array_equal(X, np.array([[1, 2, 3],
                                        [1, 2, 3],
                                        [1, 2, 3],
                                        [1, 2, 3]]))
        assert_array_equal(Y, np.array([[4, 4, 4],
                                        [5, 5, 5],
                                        [6, 6, 6],
                                        [7, 7, 7]]))

    def test_single_input(self):
        [X] = meshgrid([1, 2, 3, 4])
        assert_array_equal(X, np.array([1, 2, 3, 4]))

    def test_no_input(self):
        args = []
        assert_array_equal([], meshgrid(*args))
        assert_array_equal([], meshgrid(*args, copy=False))

    def test_indexing(self):
        x = [1, 2, 3]
        y = [4, 5, 6, 7]
        [X, Y] = meshgrid(x, y, indexing='ij')
        assert_array_equal(X, np.array([[1, 1, 1, 1],
                                        [2, 2, 2, 2],
                                        [3, 3, 3, 3]]))
        assert_array_equal(Y, np.array([[4, 5, 6, 7],
                                        [4, 5, 6, 7],
                                        [4, 5, 6, 7]]))

        # Test expected shapes:
        z = [8, 9]
        assert_(meshgrid(x, y)[0].shape == (4, 3))
        assert_(meshgrid(x, y, indexing='ij')[0].shape == (3, 4))
        assert_(meshgrid(x, y, z)[0].shape == (4, 3, 2))
        assert_(meshgrid(x, y, z, indexing='ij')[0].shape == (3, 4, 2))

        assert_raises(ValueError, meshgrid, x, y, indexing='notvalid')

    def test_sparse(self):
        [X, Y] = meshgrid([1, 2, 3], [4, 5, 6, 7], sparse=True)
        assert_array_equal(X, np.array([[1, 2, 3]]))
        assert_array_equal(Y, np.array([[4], [5], [6], [7]]))

    def test_invalid_arguments(self):
        # Test that meshgrid complains about invalid arguments
        # Regression test for issue #4755:
        # https://github.com/numpy/numpy/issues/4755
        assert_raises(TypeError, meshgrid,
                      [1, 2, 3], [4, 5, 6, 7], indices='ij')

    def test_return_type(self):
        # Test for appropriate dtype in returned arrays.
        # Regression test for issue #5297
        # https://github.com/numpy/numpy/issues/5297
        x = np.arange(0, 10, dtype=np.float32)
        y = np.arange(10, 20, dtype=np.float64)

        X, Y = np.meshgrid(x,y)

        assert_(X.dtype == x.dtype)
        assert_(Y.dtype == y.dtype)

        # copy
        X, Y = np.meshgrid(x,y, copy=True)

        assert_(X.dtype == x.dtype)
        assert_(Y.dtype == y.dtype)

        # sparse
        X, Y = np.meshgrid(x,y, sparse=True)

        assert_(X.dtype == x.dtype)
        assert_(Y.dtype == y.dtype)

    def test_writeback(self):
        # Issue 8561
        X = np.array([1.1, 2.2])
        Y = np.array([3.3, 4.4])
        x, y = np.meshgrid(X, Y, sparse=False, copy=True)

        x[0, :] = 0
        assert_equal(x[0, :], 0)
        assert_equal(x[1, :], X)

    def test_nd_shape(self):
        a, b, c, d, e = np.meshgrid(*([0] * i for i in range(1, 6)))
        expected_shape = (2, 1, 3, 4, 5)
        assert_equal(a.shape, expected_shape)
        assert_equal(b.shape, expected_shape)
        assert_equal(c.shape, expected_shape)
        assert_equal(d.shape, expected_shape)
        assert_equal(e.shape, expected_shape)

    def test_nd_values(self):
        a, b, c = np.meshgrid([0], [1, 2], [3, 4, 5])
        assert_equal(a, [[[0, 0, 0]], [[0, 0, 0]]])
        assert_equal(b, [[[1, 1, 1]], [[2, 2, 2]]])
        assert_equal(c, [[[3, 4, 5]], [[3, 4, 5]]])

    def test_nd_indexing(self):
        a, b, c = np.meshgrid([0], [1, 2], [3, 4, 5], indexing='ij')
        assert_equal(a, [[[0, 0, 0], [0, 0, 0]]])
        assert_equal(b, [[[1, 1, 1], [2, 2, 2]]])
        assert_equal(c, [[[3, 4, 5], [3, 4, 5]]])


class TestPiecewise:

    def test_simple(self):
        # Condition is single bool list
        x = piecewise([0, 0], [True, False], [1])
        assert_array_equal(x, [1, 0])

        # List of conditions: single bool list
        x = piecewise([0, 0], [[True, False]], [1])
        assert_array_equal(x, [1, 0])

        # Conditions is single bool array
        x = piecewise([0, 0], np.array([True, False]), [1])
        assert_array_equal(x, [1, 0])

        # Condition is single int array
        x = piecewise([0, 0], np.array([1, 0]), [1])
        assert_array_equal(x, [1, 0])

        # List of conditions: int array
        x = piecewise([0, 0], [np.array([1, 0])], [1])
        assert_array_equal(x, [1, 0])

        x = piecewise([0, 0], [[False, True]], [lambda x:-1])
        assert_array_equal(x, [0, -1])

        assert_raises_regex(ValueError, '1 or 2 functions are expected',
            piecewise, [0, 0], [[False, True]], [])
        assert_raises_regex(ValueError, '1 or 2 functions are expected',
            piecewise, [0, 0], [[False, True]], [1, 2, 3])

    def test_two_conditions(self):
        x = piecewise([1, 2], [[True, False], [False, True]], [3, 4])
        assert_array_equal(x, [3, 4])

    def test_scalar_domains_three_conditions(self):
        x = piecewise(3, [True, False, False], [4, 2, 0])
        assert_equal(x, 4)

    def test_default(self):
        # No value specified for x[1], should be 0
        x = piecewise([1, 2], [True, False], [2])
        assert_array_equal(x, [2, 0])

        # Should set x[1] to 3
        x = piecewise([1, 2], [True, False], [2, 3])
        assert_array_equal(x, [2, 3])

    def test_0d(self):
        x = np.array(3)
        y = piecewise(x, x > 3, [4, 0])
        assert_(y.ndim == 0)
        assert_(y == 0)

        x = 5
        y = piecewise(x, [True, False], [1, 0])
        assert_(y.ndim == 0)
        assert_(y == 1)

        # With 3 ranges (It was failing, before)
        y = piecewise(x, [False, False, True], [1, 2, 3])
        assert_array_equal(y, 3)

    def test_0d_comparison(self):
        x = 3
        y = piecewise(x, [x <= 3, x > 3], [4, 0])  # Should succeed.
        assert_equal(y, 4)

        # With 3 ranges (It was failing, before)
        x = 4
        y = piecewise(x, [x <= 3, (x > 3) * (x <= 5), x > 5], [1, 2, 3])
        assert_array_equal(y, 2)

        assert_raises_regex(ValueError, '2 or 3 functions are expected',
            piecewise, x, [x <= 3, x > 3], [1])
        assert_raises_regex(ValueError, '2 or 3 functions are expected',
            piecewise, x, [x <= 3, x > 3], [1, 1, 1, 1])

    def test_0d_0d_condition(self):
        x = np.array(3)
        c = np.array(x > 3)
        y = piecewise(x, [c], [1, 2])
        assert_equal(y, 2)

    def test_multidimensional_extrafunc(self):
        x = np.array([[-2.5, -1.5, -0.5],
                      [0.5, 1.5, 2.5]])
        y = piecewise(x, [x < 0, x >= 2], [-1, 1, 3])
        assert_array_equal(y, np.array([[-1., -1., -1.],
                                        [3., 3., 1.]]))

    def test_subclasses(self):
        class subclass(np.ndarray):
            pass
        x = np.arange(5.).view(subclass)
        r = piecewise(x, [x<2., x>=4], [-1., 1., 0.])
        assert_equal(type(r), subclass)
        assert_equal(r, [-1., -1., 0., 0., 1.])


class TestBincount:

    def test_simple(self):
        y = np.bincount(np.arange(4))
        assert_array_equal(y, np.ones(4))

    def test_simple2(self):
        y = np.bincount(np.array([1, 5, 2, 4, 1]))
        assert_array_equal(y, np.array([0, 2, 1, 0, 1, 1]))

    def test_simple_weight(self):
        x = np.arange(4)
        w = np.array([0.2, 0.3, 0.5, 0.1])
        y = np.bincount(x, w)
        assert_array_equal(y, w)

    def test_simple_weight2(self):
        x = np.array([1, 2, 4, 5, 2])
        w = np.array([0.2, 0.3, 0.5, 0.1, 0.2])
        y = np.bincount(x, w)
        assert_array_equal(y, np.array([0, 0.2, 0.5, 0, 0.5, 0.1]))

    def test_with_minlength(self):
        x = np.array([0, 1, 0, 1, 1])
        y = np.bincount(x, minlength=3)
        assert_array_equal(y, np.array([2, 3, 0]))
        x = []
        y = np.bincount(x, minlength=0)
        assert_array_equal(y, np.array([]))

    def test_with_minlength_smaller_than_maxvalue(self):
        x = np.array([0, 1, 1, 2, 2, 3, 3])
        y = np.bincount(x, minlength=2)
        assert_array_equal(y, np.array([1, 2, 2, 2]))
        y = np.bincount(x, minlength=0)
        assert_array_equal(y, np.array([1, 2, 2, 2]))

    def test_with_minlength_and_weights(self):
        x = np.array([1, 2, 4, 5, 2])
        w = np.array([0.2, 0.3, 0.5, 0.1, 0.2])
        y = np.bincount(x, w, 8)
        assert_array_equal(y, np.array([0, 0.2, 0.5, 0, 0.5, 0.1, 0, 0]))

    def test_empty(self):
        x = np.array([], dtype=int)
        y = np.bincount(x)
        assert_array_equal(x, y)

    def test_empty_with_minlength(self):
        x = np.array([], dtype=int)
        y = np.bincount(x, minlength=5)
        assert_array_equal(y, np.zeros(5, dtype=int))

    def test_with_incorrect_minlength(self):
        x = np.array([], dtype=int)
        assert_raises_regex(TypeError,
                            "'str' object cannot be interpreted",
                            lambda: np.bincount(x, minlength="foobar"))
        assert_raises_regex(ValueError,
                            "must not be negative",
                            lambda: np.bincount(x, minlength=-1))

        x = np.arange(5)
        assert_raises_regex(TypeError,
                            "'str' object cannot be interpreted",
                            lambda: np.bincount(x, minlength="foobar"))
        assert_raises_regex(ValueError,
                            "must not be negative",
                            lambda: np.bincount(x, minlength=-1))

    @pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts")
    def test_dtype_reference_leaks(self):
        # gh-6805
        intp_refcount = sys.getrefcount(np.dtype(np.intp))
        double_refcount = sys.getrefcount(np.dtype(np.double))

        for j in range(10):
            np.bincount([1, 2, 3])
        assert_equal(sys.getrefcount(np.dtype(np.intp)), intp_refcount)
        assert_equal(sys.getrefcount(np.dtype(np.double)), double_refcount)

        for j in range(10):
            np.bincount([1, 2, 3], [4, 5, 6])
        assert_equal(sys.getrefcount(np.dtype(np.intp)), intp_refcount)
        assert_equal(sys.getrefcount(np.dtype(np.double)), double_refcount)

    @pytest.mark.parametrize("vals", [[[2, 2]], 2])
    def test_error_not_1d(self, vals):
        # Test that values has to be 1-D (both as array and nested list)
        vals_arr = np.asarray(vals)
        with assert_raises(ValueError):
            np.bincount(vals_arr)
        with assert_raises(ValueError):
            np.bincount(vals)


class TestInterp:

    def test_exceptions(self):
        assert_raises(ValueError, interp, 0, [], [])
        assert_raises(ValueError, interp, 0, [0], [1, 2])
        assert_raises(ValueError, interp, 0, [0, 1], [1, 2], period=0)
        assert_raises(ValueError, interp, 0, [], [], period=360)
        assert_raises(ValueError, interp, 0, [0], [1, 2], period=360)

    def test_basic(self):
        x = np.linspace(0, 1, 5)
        y = np.linspace(0, 1, 5)
        x0 = np.linspace(0, 1, 50)
        assert_almost_equal(np.interp(x0, x, y), x0)

    def test_right_left_behavior(self):
        # Needs range of sizes to test different code paths.
        # size ==1 is special cased, 1 < size < 5 is linear search, and
        # size >= 5 goes through local search and possibly binary search.
        for size in range(1, 10):
            xp = np.arange(size, dtype=np.double)
            yp = np.ones(size, dtype=np.double)
            incpts = np.array([-1, 0, size - 1, size], dtype=np.double)
            decpts = incpts[::-1]

            incres = interp(incpts, xp, yp)
            decres = interp(decpts, xp, yp)
            inctgt = np.array([1, 1, 1, 1], dtype=float)
            dectgt = inctgt[::-1]
            assert_equal(incres, inctgt)
            assert_equal(decres, dectgt)

            incres = interp(incpts, xp, yp, left=0)
            decres = interp(decpts, xp, yp, left=0)
            inctgt = np.array([0, 1, 1, 1], dtype=float)
            dectgt = inctgt[::-1]
            assert_equal(incres, inctgt)
            assert_equal(decres, dectgt)

            incres = interp(incpts, xp, yp, right=2)
            decres = interp(decpts, xp, yp, right=2)
            inctgt = np.array([1, 1, 1, 2], dtype=float)
            dectgt = inctgt[::-1]
            assert_equal(incres, inctgt)
            assert_equal(decres, dectgt)

            incres = interp(incpts, xp, yp, left=0, right=2)
            decres = interp(decpts, xp, yp, left=0, right=2)
            inctgt = np.array([0, 1, 1, 2], dtype=float)
            dectgt = inctgt[::-1]
            assert_equal(incres, inctgt)
            assert_equal(decres, dectgt)

    def test_scalar_interpolation_point(self):
        x = np.linspace(0, 1, 5)
        y = np.linspace(0, 1, 5)
        x0 = 0
        assert_almost_equal(np.interp(x0, x, y), x0)
        x0 = .3
        assert_almost_equal(np.interp(x0, x, y), x0)
        x0 = np.float32(.3)
        assert_almost_equal(np.interp(x0, x, y), x0)
        x0 = np.float64(.3)
        assert_almost_equal(np.interp(x0, x, y), x0)
        x0 = np.nan
        assert_almost_equal(np.interp(x0, x, y), x0)

    def test_non_finite_behavior_exact_x(self):
        x = [1, 2, 2.5, 3, 4]
        xp = [1, 2, 3, 4]
        fp = [1, 2, np.inf, 4]
        assert_almost_equal(np.interp(x, xp, fp), [1, 2, np.inf, np.inf, 4])
        fp = [1, 2, np.nan, 4]
        assert_almost_equal(np.interp(x, xp, fp), [1, 2, np.nan, np.nan, 4])

    @pytest.fixture(params=[
        lambda x: np.float_(x),
        lambda x: _make_complex(x, 0),
        lambda x: _make_complex(0, x),
        lambda x: _make_complex(x, np.multiply(x, -2))
    ], ids=[
        'real',
        'complex-real',
        'complex-imag',
        'complex-both'
    ])
    def sc(self, request):
        """ scale function used by the below tests """
        return request.param

    def test_non_finite_any_nan(self, sc):
        """ test that nans are propagated """
        assert_equal(np.interp(0.5, [np.nan,      1], sc([     0,     10])), sc(np.nan))
        assert_equal(np.interp(0.5, [     0, np.nan], sc([     0,     10])), sc(np.nan))
        assert_equal(np.interp(0.5, [     0,      1], sc([np.nan,     10])), sc(np.nan))
        assert_equal(np.interp(0.5, [     0,      1], sc([     0, np.nan])), sc(np.nan))

    def test_non_finite_inf(self, sc):
        """ Test that interp between opposite infs gives nan """
        assert_equal(np.interp(0.5, [-np.inf, +np.inf], sc([      0,      10])), sc(np.nan))
        assert_equal(np.interp(0.5, [      0,       1], sc([-np.inf, +np.inf])), sc(np.nan))
        assert_equal(np.interp(0.5, [      0,       1], sc([+np.inf, -np.inf])), sc(np.nan))

        # unless the y values are equal
        assert_equal(np.interp(0.5, [-np.inf, +np.inf], sc([     10,      10])), sc(10))

    def test_non_finite_half_inf_xf(self, sc):
        """ Test that interp where both axes have a bound at inf gives nan """
        assert_equal(np.interp(0.5, [-np.inf,       1], sc([-np.inf,      10])), sc(np.nan))
        assert_equal(np.interp(0.5, [-np.inf,       1], sc([+np.inf,      10])), sc(np.nan))
        assert_equal(np.interp(0.5, [-np.inf,       1], sc([      0, -np.inf])), sc(np.nan))
        assert_equal(np.interp(0.5, [-np.inf,       1], sc([      0, +np.inf])), sc(np.nan))
        assert_equal(np.interp(0.5, [      0, +np.inf], sc([-np.inf,      10])), sc(np.nan))
        assert_equal(np.interp(0.5, [      0, +np.inf], sc([+np.inf,      10])), sc(np.nan))
        assert_equal(np.interp(0.5, [      0, +np.inf], sc([      0, -np.inf])), sc(np.nan))
        assert_equal(np.interp(0.5, [      0, +np.inf], sc([      0, +np.inf])), sc(np.nan))

    def test_non_finite_half_inf_x(self, sc):
        """ Test interp where the x axis has a bound at inf """
        assert_equal(np.interp(0.5, [-np.inf, -np.inf], sc([0, 10])), sc(10))
        assert_equal(np.interp(0.5, [-np.inf, 1      ], sc([0, 10])), sc(10))
        assert_equal(np.interp(0.5, [      0, +np.inf], sc([0, 10])), sc(0))
        assert_equal(np.interp(0.5, [+np.inf, +np.inf], sc([0, 10])), sc(0))

    def test_non_finite_half_inf_f(self, sc):
        """ Test interp where the f axis has a bound at inf """
        assert_equal(np.interp(0.5, [0, 1], sc([      0, -np.inf])), sc(-np.inf))
        assert_equal(np.interp(0.5, [0, 1], sc([      0, +np.inf])), sc(+np.inf))
        assert_equal(np.interp(0.5, [0, 1], sc([-np.inf,      10])), sc(-np.inf))
        assert_equal(np.interp(0.5, [0, 1], sc([+np.inf,      10])), sc(+np.inf))
        assert_equal(np.interp(0.5, [0, 1], sc([-np.inf, -np.inf])), sc(-np.inf))
        assert_equal(np.interp(0.5, [0, 1], sc([+np.inf, +np.inf])), sc(+np.inf))

    def test_complex_interp(self):
        # test complex interpolation
        x = np.linspace(0, 1, 5)
        y = np.linspace(0, 1, 5) + (1 + np.linspace(0, 1, 5))*1.0j
        x0 = 0.3
        y0 = x0 + (1+x0)*1.0j
        assert_almost_equal(np.interp(x0, x, y), y0)
        # test complex left and right
        x0 = -1
        left = 2 + 3.0j
        assert_almost_equal(np.interp(x0, x, y, left=left), left)
        x0 = 2.0
        right = 2 + 3.0j
        assert_almost_equal(np.interp(x0, x, y, right=right), right)
        # test complex non finite
        x = [1, 2, 2.5, 3, 4]
        xp = [1, 2, 3, 4]
        fp = [1, 2+1j, np.inf, 4]
        y = [1, 2+1j, np.inf+0.5j, np.inf, 4]
        assert_almost_equal(np.interp(x, xp, fp), y)
        # test complex periodic
        x = [-180, -170, -185, 185, -10, -5, 0, 365]
        xp = [190, -190, 350, -350]
        fp = [5+1.0j, 10+2j, 3+3j, 4+4j]
        y = [7.5+1.5j, 5.+1.0j, 8.75+1.75j, 6.25+1.25j, 3.+3j, 3.25+3.25j,
             3.5+3.5j, 3.75+3.75j]
        assert_almost_equal(np.interp(x, xp, fp, period=360), y)

    def test_zero_dimensional_interpolation_point(self):
        x = np.linspace(0, 1, 5)
        y = np.linspace(0, 1, 5)
        x0 = np.array(.3)
        assert_almost_equal(np.interp(x0, x, y), x0)

        xp = np.array([0, 2, 4])
        fp = np.array([1, -1, 1])

        actual = np.interp(np.array(1), xp, fp)
        assert_equal(actual, 0)
        assert_(isinstance(actual, np.float64))

        actual = np.interp(np.array(4.5), xp, fp, period=4)
        assert_equal(actual, 0.5)
        assert_(isinstance(actual, np.float64))

    def test_if_len_x_is_small(self):
        xp = np.arange(0, 10, 0.0001)
        fp = np.sin(xp)
        assert_almost_equal(np.interp(np.pi, xp, fp), 0.0)

    def test_period(self):
        x = [-180, -170, -185, 185, -10, -5, 0, 365]
        xp = [190, -190, 350, -350]
        fp = [5, 10, 3, 4]
        y = [7.5, 5., 8.75, 6.25, 3., 3.25, 3.5, 3.75]
        assert_almost_equal(np.interp(x, xp, fp, period=360), y)
        x = np.array(x, order='F').reshape(2, -1)
        y = np.array(y, order='C').reshape(2, -1)
        assert_almost_equal(np.interp(x, xp, fp, period=360), y)


class TestPercentile:

    def test_basic(self):
        x = np.arange(8) * 0.5
        assert_equal(np.percentile(x, 0), 0.)
        assert_equal(np.percentile(x, 100), 3.5)
        assert_equal(np.percentile(x, 50), 1.75)
        x[1] = np.nan
        assert_equal(np.percentile(x, 0), np.nan)
        assert_equal(np.percentile(x, 0, method='nearest'), np.nan)

    def test_fraction(self):
        x = [Fraction(i, 2) for i in range(8)]

        p = np.percentile(x, Fraction(0))
        assert_equal(p, Fraction(0))
        assert_equal(type(p), Fraction)

        p = np.percentile(x, Fraction(100))
        assert_equal(p, Fraction(7, 2))
        assert_equal(type(p), Fraction)

        p = np.percentile(x, Fraction(50))
        assert_equal(p, Fraction(7, 4))
        assert_equal(type(p), Fraction)

        p = np.percentile(x, [Fraction(50)])
        assert_equal(p, np.array([Fraction(7, 4)]))
        assert_equal(type(p), np.ndarray)

    def test_api(self):
        d = np.ones(5)
        np.percentile(d, 5, None, None, False)
        np.percentile(d, 5, None, None, False, 'linear')
        o = np.ones((1,))
        np.percentile(d, 5, None, o, False, 'linear')

    def test_2D(self):
        x = np.array([[1, 1, 1],
                      [1, 1, 1],
                      [4, 4, 3],
                      [1, 1, 1],
                      [1, 1, 1]])
        assert_array_equal(np.percentile(x, 50, axis=0), [1, 1, 1])

    @pytest.mark.parametrize("dtype", np.typecodes["AllFloat"])
    def test_linear_nan_1D(self, dtype):
        # METHOD 1 of H&F
        arr = np.asarray([15.0, np.NAN, 35.0, 40.0, 50.0], dtype=dtype)
        res = np.percentile(
            arr,
            40.0,
            method="linear")
        np.testing.assert_equal(res, np.NAN)
        np.testing.assert_equal(res.dtype, arr.dtype)

    H_F_TYPE_CODES = [(int_type, np.float64)
                      for int_type in np.typecodes["AllInteger"]
                      ] + [(np.float16, np.float16),
                           (np.float32, np.float32),
                           (np.float64, np.float64),
                           (np.longdouble, np.longdouble),
                           (np.complex64, np.complex64),
                           (np.complex128, np.complex128),
                           (np.clongdouble, np.clongdouble),
                           (np.dtype("O"), np.float64)]

    @pytest.mark.parametrize(["input_dtype", "expected_dtype"], H_F_TYPE_CODES)
    @pytest.mark.parametrize(["method", "expected"],
                             [("inverted_cdf", 20),
                              ("averaged_inverted_cdf", 27.5),
                              ("closest_observation", 20),
                              ("interpolated_inverted_cdf", 20),
                              ("hazen", 27.5),
                              ("weibull", 26),
                              ("linear", 29),
                              ("median_unbiased", 27),
                              ("normal_unbiased", 27.125),
                              ])
    def test_linear_interpolation(self,
                                  method,
                                  expected,
                                  input_dtype,
                                  expected_dtype):
        expected_dtype = np.dtype(expected_dtype)
        if np._get_promotion_state() == "legacy":
            expected_dtype = np.promote_types(expected_dtype, np.float64)

        arr = np.asarray([15.0, 20.0, 35.0, 40.0, 50.0], dtype=input_dtype)
        actual = np.percentile(arr, 40.0, method=method)

        np.testing.assert_almost_equal(
            actual, expected_dtype.type(expected), 14)

        if method in ["inverted_cdf", "closest_observation"]:
            if input_dtype == "O":
                np.testing.assert_equal(np.asarray(actual).dtype, np.float64)
            else:
                np.testing.assert_equal(np.asarray(actual).dtype,
                                        np.dtype(input_dtype))
        else:
            np.testing.assert_equal(np.asarray(actual).dtype,
                                    np.dtype(expected_dtype))

    TYPE_CODES = np.typecodes["AllInteger"] + np.typecodes["AllFloat"] + "O"

    @pytest.mark.parametrize("dtype", TYPE_CODES)
    def test_lower_higher(self, dtype):
        assert_equal(np.percentile(np.arange(10, dtype=dtype), 50,
                                   method='lower'), 4)
        assert_equal(np.percentile(np.arange(10, dtype=dtype), 50,
                                   method='higher'), 5)

    @pytest.mark.parametrize("dtype", TYPE_CODES)
    def test_midpoint(self, dtype):
        assert_equal(np.percentile(np.arange(10, dtype=dtype), 51,
                                   method='midpoint'), 4.5)
        assert_equal(np.percentile(np.arange(9, dtype=dtype) + 1, 50,
                                   method='midpoint'), 5)
        assert_equal(np.percentile(np.arange(11, dtype=dtype), 51,
                                   method='midpoint'), 5.5)
        assert_equal(np.percentile(np.arange(11, dtype=dtype), 50,
                                   method='midpoint'), 5)

    @pytest.mark.parametrize("dtype", TYPE_CODES)
    def test_nearest(self, dtype):
        assert_equal(np.percentile(np.arange(10, dtype=dtype), 51,
                                   method='nearest'), 5)
        assert_equal(np.percentile(np.arange(10, dtype=dtype), 49,
                                   method='nearest'), 4)

    def test_linear_interpolation_extrapolation(self):
        arr = np.random.rand(5)

        actual = np.percentile(arr, 100)
        np.testing.assert_equal(actual, arr.max())

        actual = np.percentile(arr, 0)
        np.testing.assert_equal(actual, arr.min())

    def test_sequence(self):
        x = np.arange(8) * 0.5
        assert_equal(np.percentile(x, [0, 100, 50]), [0, 3.5, 1.75])

    def test_axis(self):
        x = np.arange(12).reshape(3, 4)

        assert_equal(np.percentile(x, (25, 50, 100)), [2.75, 5.5, 11.0])

        r0 = [[2, 3, 4, 5], [4, 5, 6, 7], [8, 9, 10, 11]]
        assert_equal(np.percentile(x, (25, 50, 100), axis=0), r0)

        r1 = [[0.75, 1.5, 3], [4.75, 5.5, 7], [8.75, 9.5, 11]]
        assert_equal(np.percentile(x, (25, 50, 100), axis=1), np.array(r1).T)

        # ensure qth axis is always first as with np.array(old_percentile(..))
        x = np.arange(3 * 4 * 5 * 6).reshape(3, 4, 5, 6)
        assert_equal(np.percentile(x, (25, 50)).shape, (2,))
        assert_equal(np.percentile(x, (25, 50, 75)).shape, (3,))
        assert_equal(np.percentile(x, (25, 50), axis=0).shape, (2, 4, 5, 6))
        assert_equal(np.percentile(x, (25, 50), axis=1).shape, (2, 3, 5, 6))
        assert_equal(np.percentile(x, (25, 50), axis=2).shape, (2, 3, 4, 6))
        assert_equal(np.percentile(x, (25, 50), axis=3).shape, (2, 3, 4, 5))
        assert_equal(
            np.percentile(x, (25, 50, 75), axis=1).shape, (3, 3, 5, 6))
        assert_equal(np.percentile(x, (25, 50),
                                   method="higher").shape, (2,))
        assert_equal(np.percentile(x, (25, 50, 75),
                                   method="higher").shape, (3,))
        assert_equal(np.percentile(x, (25, 50), axis=0,
                                   method="higher").shape, (2, 4, 5, 6))
        assert_equal(np.percentile(x, (25, 50), axis=1,
                                   method="higher").shape, (2, 3, 5, 6))
        assert_equal(np.percentile(x, (25, 50), axis=2,
                                   method="higher").shape, (2, 3, 4, 6))
        assert_equal(np.percentile(x, (25, 50), axis=3,
                                   method="higher").shape, (2, 3, 4, 5))
        assert_equal(np.percentile(x, (25, 50, 75), axis=1,
                                   method="higher").shape, (3, 3, 5, 6))

    def test_scalar_q(self):
        # test for no empty dimensions for compatibility with old percentile
        x = np.arange(12).reshape(3, 4)
        assert_equal(np.percentile(x, 50), 5.5)
        assert_(np.isscalar(np.percentile(x, 50)))
        r0 = np.array([4.,  5.,  6.,  7.])
        assert_equal(np.percentile(x, 50, axis=0), r0)
        assert_equal(np.percentile(x, 50, axis=0).shape, r0.shape)
        r1 = np.array([1.5,  5.5,  9.5])
        assert_almost_equal(np.percentile(x, 50, axis=1), r1)
        assert_equal(np.percentile(x, 50, axis=1).shape, r1.shape)

        out = np.empty(1)
        assert_equal(np.percentile(x, 50, out=out), 5.5)
        assert_equal(out, 5.5)
        out = np.empty(4)
        assert_equal(np.percentile(x, 50, axis=0, out=out), r0)
        assert_equal(out, r0)
        out = np.empty(3)
        assert_equal(np.percentile(x, 50, axis=1, out=out), r1)
        assert_equal(out, r1)

        # test for no empty dimensions for compatibility with old percentile
        x = np.arange(12).reshape(3, 4)
        assert_equal(np.percentile(x, 50, method='lower'), 5.)
        assert_(np.isscalar(np.percentile(x, 50)))
        r0 = np.array([4.,  5.,  6.,  7.])
        c0 = np.percentile(x, 50, method='lower', axis=0)
        assert_equal(c0, r0)
        assert_equal(c0.shape, r0.shape)
        r1 = np.array([1.,  5.,  9.])
        c1 = np.percentile(x, 50, method='lower', axis=1)
        assert_almost_equal(c1, r1)
        assert_equal(c1.shape, r1.shape)

        out = np.empty((), dtype=x.dtype)
        c = np.percentile(x, 50, method='lower', out=out)
        assert_equal(c, 5)
        assert_equal(out, 5)
        out = np.empty(4, dtype=x.dtype)
        c = np.percentile(x, 50, method='lower', axis=0, out=out)
        assert_equal(c, r0)
        assert_equal(out, r0)
        out = np.empty(3, dtype=x.dtype)
        c = np.percentile(x, 50, method='lower', axis=1, out=out)
        assert_equal(c, r1)
        assert_equal(out, r1)

    def test_exception(self):
        assert_raises(ValueError, np.percentile, [1, 2], 56,
                      method='foobar')
        assert_raises(ValueError, np.percentile, [1], 101)
        assert_raises(ValueError, np.percentile, [1], -1)
        assert_raises(ValueError, np.percentile, [1], list(range(50)) + [101])
        assert_raises(ValueError, np.percentile, [1], list(range(50)) + [-0.1])

    def test_percentile_list(self):
        assert_equal(np.percentile([1, 2, 3], 0), 1)

    def test_percentile_out(self):
        x = np.array([1, 2, 3])
        y = np.zeros((3,))
        p = (1, 2, 3)
        np.percentile(x, p, out=y)
        assert_equal(np.percentile(x, p), y)

        x = np.array([[1, 2, 3],
                      [4, 5, 6]])

        y = np.zeros((3, 3))
        np.percentile(x, p, axis=0, out=y)
        assert_equal(np.percentile(x, p, axis=0), y)

        y = np.zeros((3, 2))
        np.percentile(x, p, axis=1, out=y)
        assert_equal(np.percentile(x, p, axis=1), y)

        x = np.arange(12).reshape(3, 4)
        # q.dim > 1, float
        r0 = np.array([[2.,  3.,  4., 5.], [4., 5., 6., 7.]])
        out = np.empty((2, 4))
        assert_equal(np.percentile(x, (25, 50), axis=0, out=out), r0)
        assert_equal(out, r0)
        r1 = np.array([[0.75,  4.75,  8.75], [1.5,  5.5,  9.5]])
        out = np.empty((2, 3))
        assert_equal(np.percentile(x, (25, 50), axis=1, out=out), r1)
        assert_equal(out, r1)

        # q.dim > 1, int
        r0 = np.array([[0,  1,  2, 3], [4, 5, 6, 7]])
        out = np.empty((2, 4), dtype=x.dtype)
        c = np.percentile(x, (25, 50), method='lower', axis=0, out=out)
        assert_equal(c, r0)
        assert_equal(out, r0)
        r1 = np.array([[0,  4,  8], [1,  5,  9]])
        out = np.empty((2, 3), dtype=x.dtype)
        c = np.percentile(x, (25, 50), method='lower', axis=1, out=out)
        assert_equal(c, r1)
        assert_equal(out, r1)

    def test_percentile_empty_dim(self):
        # empty dims are preserved
        d = np.arange(11 * 2).reshape(11, 1, 2, 1)
        assert_array_equal(np.percentile(d, 50, axis=0).shape, (1, 2, 1))
        assert_array_equal(np.percentile(d, 50, axis=1).shape, (11, 2, 1))
        assert_array_equal(np.percentile(d, 50, axis=2).shape, (11, 1, 1))
        assert_array_equal(np.percentile(d, 50, axis=3).shape, (11, 1, 2))
        assert_array_equal(np.percentile(d, 50, axis=-1).shape, (11, 1, 2))
        assert_array_equal(np.percentile(d, 50, axis=-2).shape, (11, 1, 1))
        assert_array_equal(np.percentile(d, 50, axis=-3).shape, (11, 2, 1))
        assert_array_equal(np.percentile(d, 50, axis=-4).shape, (1, 2, 1))

        assert_array_equal(np.percentile(d, 50, axis=2,
                                         method='midpoint').shape,
                           (11, 1, 1))
        assert_array_equal(np.percentile(d, 50, axis=-2,
                                         method='midpoint').shape,
                           (11, 1, 1))

        assert_array_equal(np.array(np.percentile(d, [10, 50], axis=0)).shape,
                           (2, 1, 2, 1))
        assert_array_equal(np.array(np.percentile(d, [10, 50], axis=1)).shape,
                           (2, 11, 2, 1))
        assert_array_equal(np.array(np.percentile(d, [10, 50], axis=2)).shape,
                           (2, 11, 1, 1))
        assert_array_equal(np.array(np.percentile(d, [10, 50], axis=3)).shape,
                           (2, 11, 1, 2))

    def test_percentile_no_overwrite(self):
        a = np.array([2, 3, 4, 1])
        np.percentile(a, [50], overwrite_input=False)
        assert_equal(a, np.array([2, 3, 4, 1]))

        a = np.array([2, 3, 4, 1])
        np.percentile(a, [50])
        assert_equal(a, np.array([2, 3, 4, 1]))

    def test_no_p_overwrite(self):
        p = np.linspace(0., 100., num=5)
        np.percentile(np.arange(100.), p, method="midpoint")
        assert_array_equal(p, np.linspace(0., 100., num=5))
        p = np.linspace(0., 100., num=5).tolist()
        np.percentile(np.arange(100.), p, method="midpoint")
        assert_array_equal(p, np.linspace(0., 100., num=5).tolist())

    def test_percentile_overwrite(self):
        a = np.array([2, 3, 4, 1])
        b = np.percentile(a, [50], overwrite_input=True)
        assert_equal(b, np.array([2.5]))

        b = np.percentile([2, 3, 4, 1], [50], overwrite_input=True)
        assert_equal(b, np.array([2.5]))

    def test_extended_axis(self):
        o = np.random.normal(size=(71, 23))
        x = np.dstack([o] * 10)
        assert_equal(np.percentile(x, 30, axis=(0, 1)), np.percentile(o, 30))
        x = np.moveaxis(x, -1, 0)
        assert_equal(np.percentile(x, 30, axis=(-2, -1)), np.percentile(o, 30))
        x = x.swapaxes(0, 1).copy()
        assert_equal(np.percentile(x, 30, axis=(0, -1)), np.percentile(o, 30))
        x = x.swapaxes(0, 1).copy()

        assert_equal(np.percentile(x, [25, 60], axis=(0, 1, 2)),
                     np.percentile(x, [25, 60], axis=None))
        assert_equal(np.percentile(x, [25, 60], axis=(0,)),
                     np.percentile(x, [25, 60], axis=0))

        d = np.arange(3 * 5 * 7 * 11).reshape((3, 5, 7, 11))
        np.random.shuffle(d.ravel())
        assert_equal(np.percentile(d, 25,  axis=(0, 1, 2))[0],
                     np.percentile(d[:,:,:, 0].flatten(), 25))
        assert_equal(np.percentile(d, [10, 90], axis=(0, 1, 3))[:, 1],
                     np.percentile(d[:,:, 1,:].flatten(), [10, 90]))
        assert_equal(np.percentile(d, 25, axis=(3, 1, -4))[2],
                     np.percentile(d[:,:, 2,:].flatten(), 25))
        assert_equal(np.percentile(d, 25, axis=(3, 1, 2))[2],
                     np.percentile(d[2,:,:,:].flatten(), 25))
        assert_equal(np.percentile(d, 25, axis=(3, 2))[2, 1],
                     np.percentile(d[2, 1,:,:].flatten(), 25))
        assert_equal(np.percentile(d, 25, axis=(1, -2))[2, 1],
                     np.percentile(d[2,:,:, 1].flatten(), 25))
        assert_equal(np.percentile(d, 25, axis=(1, 3))[2, 2],
                     np.percentile(d[2,:, 2,:].flatten(), 25))

    def test_extended_axis_invalid(self):
        d = np.ones((3, 5, 7, 11))
        assert_raises(np.AxisError, np.percentile, d, axis=-5, q=25)
        assert_raises(np.AxisError, np.percentile, d, axis=(0, -5), q=25)
        assert_raises(np.AxisError, np.percentile, d, axis=4, q=25)
        assert_raises(np.AxisError, np.percentile, d, axis=(0, 4), q=25)
        # each of these refers to the same axis twice
        assert_raises(ValueError, np.percentile, d, axis=(1, 1), q=25)
        assert_raises(ValueError, np.percentile, d, axis=(-1, -1), q=25)
        assert_raises(ValueError, np.percentile, d, axis=(3, -1), q=25)

    def test_keepdims(self):
        d = np.ones((3, 5, 7, 11))
        assert_equal(np.percentile(d, 7, axis=None, keepdims=True).shape,
                     (1, 1, 1, 1))
        assert_equal(np.percentile(d, 7, axis=(0, 1), keepdims=True).shape,
                     (1, 1, 7, 11))
        assert_equal(np.percentile(d, 7, axis=(0, 3), keepdims=True).shape,
                     (1, 5, 7, 1))
        assert_equal(np.percentile(d, 7, axis=(1,), keepdims=True).shape,
                     (3, 1, 7, 11))
        assert_equal(np.percentile(d, 7, (0, 1, 2, 3), keepdims=True).shape,
                     (1, 1, 1, 1))
        assert_equal(np.percentile(d, 7, axis=(0, 1, 3), keepdims=True).shape,
                     (1, 1, 7, 1))

        assert_equal(np.percentile(d, [1, 7], axis=(0, 1, 3),
                                   keepdims=True).shape, (2, 1, 1, 7, 1))
        assert_equal(np.percentile(d, [1, 7], axis=(0, 3),
                                   keepdims=True).shape, (2, 1, 5, 7, 1))

    @pytest.mark.parametrize('q', [7, [1, 7]])
    @pytest.mark.parametrize(
        argnames='axis',
        argvalues=[
            None,
            1,
            (1,),
            (0, 1),
            (-3, -1),
        ]
    )
    def test_keepdims_out(self, q, axis):
        d = np.ones((3, 5, 7, 11))
        if axis is None:
            shape_out = (1,) * d.ndim
        else:
            axis_norm = normalize_axis_tuple(axis, d.ndim)
            shape_out = tuple(
                1 if i in axis_norm else d.shape[i] for i in range(d.ndim))
        shape_out = np.shape(q) + shape_out

        out = np.empty(shape_out)
        result = np.percentile(d, q, axis=axis, keepdims=True, out=out)
        assert result is out
        assert_equal(result.shape, shape_out)

    def test_out(self):
        o = np.zeros((4,))
        d = np.ones((3, 4))
        assert_equal(np.percentile(d, 0, 0, out=o), o)
        assert_equal(np.percentile(d, 0, 0, method='nearest', out=o), o)
        o = np.zeros((3,))
        assert_equal(np.percentile(d, 1, 1, out=o), o)
        assert_equal(np.percentile(d, 1, 1, method='nearest', out=o), o)

        o = np.zeros(())
        assert_equal(np.percentile(d, 2, out=o), o)
        assert_equal(np.percentile(d, 2, method='nearest', out=o), o)

    def test_out_nan(self):
        with warnings.catch_warnings(record=True):
            warnings.filterwarnings('always', '', RuntimeWarning)
            o = np.zeros((4,))
            d = np.ones((3, 4))
            d[2, 1] = np.nan
            assert_equal(np.percentile(d, 0, 0, out=o), o)
            assert_equal(
                np.percentile(d, 0, 0, method='nearest', out=o), o)
            o = np.zeros((3,))
            assert_equal(np.percentile(d, 1, 1, out=o), o)
            assert_equal(
                np.percentile(d, 1, 1, method='nearest', out=o), o)
            o = np.zeros(())
            assert_equal(np.percentile(d, 1, out=o), o)
            assert_equal(
                np.percentile(d, 1, method='nearest', out=o), o)

    def test_nan_behavior(self):
        a = np.arange(24, dtype=float)
        a[2] = np.nan
        assert_equal(np.percentile(a, 0.3), np.nan)
        assert_equal(np.percentile(a, 0.3, axis=0), np.nan)
        assert_equal(np.percentile(a, [0.3, 0.6], axis=0),
                     np.array([np.nan] * 2))

        a = np.arange(24, dtype=float).reshape(2, 3, 4)
        a[1, 2, 3] = np.nan
        a[1, 1, 2] = np.nan

        # no axis
        assert_equal(np.percentile(a, 0.3), np.nan)
        assert_equal(np.percentile(a, 0.3).ndim, 0)

        # axis0 zerod
        b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), 0.3, 0)
        b[2, 3] = np.nan
        b[1, 2] = np.nan
        assert_equal(np.percentile(a, 0.3, 0), b)

        # axis0 not zerod
        b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4),
                          [0.3, 0.6], 0)
        b[:, 2, 3] = np.nan
        b[:, 1, 2] = np.nan
        assert_equal(np.percentile(a, [0.3, 0.6], 0), b)

        # axis1 zerod
        b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4), 0.3, 1)
        b[1, 3] = np.nan
        b[1, 2] = np.nan
        assert_equal(np.percentile(a, 0.3, 1), b)
        # axis1 not zerod
        b = np.percentile(
            np.arange(24, dtype=float).reshape(2, 3, 4), [0.3, 0.6], 1)
        b[:, 1, 3] = np.nan
        b[:, 1, 2] = np.nan
        assert_equal(np.percentile(a, [0.3, 0.6], 1), b)

        # axis02 zerod
        b = np.percentile(
            np.arange(24, dtype=float).reshape(2, 3, 4), 0.3, (0, 2))
        b[1] = np.nan
        b[2] = np.nan
        assert_equal(np.percentile(a, 0.3, (0, 2)), b)
        # axis02 not zerod
        b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4),
                          [0.3, 0.6], (0, 2))
        b[:, 1] = np.nan
        b[:, 2] = np.nan
        assert_equal(np.percentile(a, [0.3, 0.6], (0, 2)), b)
        # axis02 not zerod with method='nearest'
        b = np.percentile(np.arange(24, dtype=float).reshape(2, 3, 4),
                          [0.3, 0.6], (0, 2), method='nearest')
        b[:, 1] = np.nan
        b[:, 2] = np.nan
        assert_equal(np.percentile(
            a, [0.3, 0.6], (0, 2), method='nearest'), b)

    def test_nan_q(self):
        # GH18830
        with pytest.raises(ValueError, match="Percentiles must be in"):
            np.percentile([1, 2, 3, 4.0], np.nan)
        with pytest.raises(ValueError, match="Percentiles must be in"):
            np.percentile([1, 2, 3, 4.0], [np.nan])
        q = np.linspace(1.0, 99.0, 16)
        q[0] = np.nan
        with pytest.raises(ValueError, match="Percentiles must be in"):
            np.percentile([1, 2, 3, 4.0], q)


class TestQuantile:
    # most of this is already tested by TestPercentile

    def test_max_ulp(self):
        x = [0.0, 0.2, 0.4]
        a = np.quantile(x, 0.45)
        # The default linear method would result in 0 + 0.2 * (0.45/2) = 0.18.
        # 0.18 is not exactly representable and the formula leads to a 1 ULP
        # different result. Ensure it is this exact within 1 ULP, see gh-20331.
        np.testing.assert_array_max_ulp(a, 0.18, maxulp=1)

    def test_basic(self):
        x = np.arange(8) * 0.5
        assert_equal(np.quantile(x, 0), 0.)
        assert_equal(np.quantile(x, 1), 3.5)
        assert_equal(np.quantile(x, 0.5), 1.75)

    @pytest.mark.xfail(reason="See gh-19154")
    def test_correct_quantile_value(self):
        a = np.array([True])
        tf_quant = np.quantile(True, False)
        assert_equal(tf_quant, a[0])
        assert_equal(type(tf_quant), a.dtype)
        a = np.array([False, True, True])
        quant_res = np.quantile(a, a)
        assert_array_equal(quant_res, a)
        assert_equal(quant_res.dtype, a.dtype)

    def test_fraction(self):
        # fractional input, integral quantile
        x = [Fraction(i, 2) for i in range(8)]
        q = np.quantile(x, 0)
        assert_equal(q, 0)
        assert_equal(type(q), Fraction)

        q = np.quantile(x, 1)
        assert_equal(q, Fraction(7, 2))
        assert_equal(type(q), Fraction)

        q = np.quantile(x, Fraction(1, 2))
        assert_equal(q, Fraction(7, 4))
        assert_equal(type(q), Fraction)

        q = np.quantile(x, [Fraction(1, 2)])
        assert_equal(q, np.array([Fraction(7, 4)]))
        assert_equal(type(q), np.ndarray)

        q = np.quantile(x, [[Fraction(1, 2)]])
        assert_equal(q, np.array([[Fraction(7, 4)]]))
        assert_equal(type(q), np.ndarray)

        # repeat with integral input but fractional quantile
        x = np.arange(8)
        assert_equal(np.quantile(x, Fraction(1, 2)), Fraction(7, 2))

    def test_no_p_overwrite(self):
        # this is worth retesting, because quantile does not make a copy
        p0 = np.array([0, 0.75, 0.25, 0.5, 1.0])
        p = p0.copy()
        np.quantile(np.arange(100.), p, method="midpoint")
        assert_array_equal(p, p0)

        p0 = p0.tolist()
        p = p.tolist()
        np.quantile(np.arange(100.), p, method="midpoint")
        assert_array_equal(p, p0)

    @pytest.mark.parametrize("dtype", np.typecodes["AllInteger"])
    def test_quantile_preserve_int_type(self, dtype):
        res = np.quantile(np.array([1, 2], dtype=dtype), [0.5],
                          method="nearest")
        assert res.dtype == dtype

    @pytest.mark.parametrize("method",
             ['inverted_cdf', 'averaged_inverted_cdf', 'closest_observation',
              'interpolated_inverted_cdf', 'hazen', 'weibull', 'linear',
              'median_unbiased', 'normal_unbiased',
              'nearest', 'lower', 'higher', 'midpoint'])
    def test_quantile_monotonic(self, method):
        # GH 14685
        # test that the return value of quantile is monotonic if p0 is ordered
        # Also tests that the boundary values are not mishandled.
        p0 = np.linspace(0, 1, 101)
        quantile = np.quantile(np.array([0, 1, 1, 2, 2, 3, 3, 4, 5, 5, 1, 1, 9, 9, 9,
                                         8, 8, 7]) * 0.1, p0, method=method)
        assert_equal(np.sort(quantile), quantile)

        # Also test one where the number of data points is clearly divisible:
        quantile = np.quantile([0., 1., 2., 3.], p0, method=method)
        assert_equal(np.sort(quantile), quantile)

    @hypothesis.given(
            arr=arrays(dtype=np.float64,
                       shape=st.integers(min_value=3, max_value=1000),
                       elements=st.floats(allow_infinity=False, allow_nan=False,
                                          min_value=-1e300, max_value=1e300)))
    def test_quantile_monotonic_hypo(self, arr):
        p0 = np.arange(0, 1, 0.01)
        quantile = np.quantile(arr, p0)
        assert_equal(np.sort(quantile), quantile)

    def test_quantile_scalar_nan(self):
        a = np.array([[10., 7., 4.], [3., 2., 1.]])
        a[0][1] = np.nan
        actual = np.quantile(a, 0.5)
        assert np.isscalar(actual)
        assert_equal(np.quantile(a, 0.5), np.nan)


class TestLerp:
    @hypothesis.given(t0=st.floats(allow_nan=False, allow_infinity=False,
                                   min_value=0, max_value=1),
                      t1=st.floats(allow_nan=False, allow_infinity=False,
                                   min_value=0, max_value=1),
                      a = st.floats(allow_nan=False, allow_infinity=False,
                                    min_value=-1e300, max_value=1e300),
                      b = st.floats(allow_nan=False, allow_infinity=False,
                                    min_value=-1e300, max_value=1e300))
    def test_linear_interpolation_formula_monotonic(self, t0, t1, a, b):
        l0 = nfb._lerp(a, b, t0)
        l1 = nfb._lerp(a, b, t1)
        if t0 == t1 or a == b:
            assert l0 == l1  # uninteresting
        elif (t0 < t1) == (a < b):
            assert l0 <= l1
        else:
            assert l0 >= l1

    @hypothesis.given(t=st.floats(allow_nan=False, allow_infinity=False,
                                  min_value=0, max_value=1),
                      a=st.floats(allow_nan=False, allow_infinity=False,
                                  min_value=-1e300, max_value=1e300),
                      b=st.floats(allow_nan=False, allow_infinity=False,
                                  min_value=-1e300, max_value=1e300))
    def test_linear_interpolation_formula_bounded(self, t, a, b):
        if a <= b:
            assert a <= nfb._lerp(a, b, t) <= b
        else:
            assert b <= nfb._lerp(a, b, t) <= a

    @hypothesis.given(t=st.floats(allow_nan=False, allow_infinity=False,
                                  min_value=0, max_value=1),
                      a=st.floats(allow_nan=False, allow_infinity=False,
                                  min_value=-1e300, max_value=1e300),
                      b=st.floats(allow_nan=False, allow_infinity=False,
                                  min_value=-1e300, max_value=1e300))
    def test_linear_interpolation_formula_symmetric(self, t, a, b):
        # double subtraction is needed to remove the extra precision of t < 0.5
        left = nfb._lerp(a, b, 1 - (1 - t))
        right = nfb._lerp(b, a, 1 - t)
        assert_allclose(left, right)

    def test_linear_interpolation_formula_0d_inputs(self):
        a = np.array(2)
        b = np.array(5)
        t = np.array(0.2)
        assert nfb._lerp(a, b, t) == 2.6


class TestMedian:

    def test_basic(self):
        a0 = np.array(1)
        a1 = np.arange(2)
        a2 = np.arange(6).reshape(2, 3)
        assert_equal(np.median(a0), 1)
        assert_allclose(np.median(a1), 0.5)
        assert_allclose(np.median(a2), 2.5)
        assert_allclose(np.median(a2, axis=0), [1.5,  2.5,  3.5])
        assert_equal(np.median(a2, axis=1), [1, 4])
        assert_allclose(np.median(a2, axis=None), 2.5)

        a = np.array([0.0444502, 0.0463301, 0.141249, 0.0606775])
        assert_almost_equal((a[1] + a[3]) / 2., np.median(a))
        a = np.array([0.0463301, 0.0444502, 0.141249])
        assert_equal(a[0], np.median(a))
        a = np.array([0.0444502, 0.141249, 0.0463301])
        assert_equal(a[-1], np.median(a))
        # check array scalar result
        assert_equal(np.median(a).ndim, 0)
        a[1] = np.nan
        assert_equal(np.median(a).ndim, 0)

    def test_axis_keyword(self):
        a3 = np.array([[2, 3],
                       [0, 1],
                       [6, 7],
                       [4, 5]])
        for a in [a3, np.random.randint(0, 100, size=(2, 3, 4))]:
            orig = a.copy()
            np.median(a, axis=None)
            for ax in range(a.ndim):
                np.median(a, axis=ax)
            assert_array_equal(a, orig)

        assert_allclose(np.median(a3, axis=0), [3,  4])
        assert_allclose(np.median(a3.T, axis=1), [3,  4])
        assert_allclose(np.median(a3), 3.5)
        assert_allclose(np.median(a3, axis=None), 3.5)
        assert_allclose(np.median(a3.T), 3.5)

    def test_overwrite_keyword(self):
        a3 = np.array([[2, 3],
                       [0, 1],
                       [6, 7],
                       [4, 5]])
        a0 = np.array(1)
        a1 = np.arange(2)
        a2 = np.arange(6).reshape(2, 3)
        assert_allclose(np.median(a0.copy(), overwrite_input=True), 1)
        assert_allclose(np.median(a1.copy(), overwrite_input=True), 0.5)
        assert_allclose(np.median(a2.copy(), overwrite_input=True), 2.5)
        assert_allclose(np.median(a2.copy(), overwrite_input=True, axis=0),
                        [1.5,  2.5,  3.5])
        assert_allclose(
            np.median(a2.copy(), overwrite_input=True, axis=1), [1, 4])
        assert_allclose(
            np.median(a2.copy(), overwrite_input=True, axis=None), 2.5)
        assert_allclose(
            np.median(a3.copy(), overwrite_input=True, axis=0), [3,  4])
        assert_allclose(np.median(a3.T.copy(), overwrite_input=True, axis=1),
                        [3,  4])

        a4 = np.arange(3 * 4 * 5, dtype=np.float32).reshape((3, 4, 5))
        np.random.shuffle(a4.ravel())
        assert_allclose(np.median(a4, axis=None),
                        np.median(a4.copy(), axis=None, overwrite_input=True))
        assert_allclose(np.median(a4, axis=0),
                        np.median(a4.copy(), axis=0, overwrite_input=True))
        assert_allclose(np.median(a4, axis=1),
                        np.median(a4.copy(), axis=1, overwrite_input=True))
        assert_allclose(np.median(a4, axis=2),
                        np.median(a4.copy(), axis=2, overwrite_input=True))

    def test_array_like(self):
        x = [1, 2, 3]
        assert_almost_equal(np.median(x), 2)
        x2 = [x]
        assert_almost_equal(np.median(x2), 2)
        assert_allclose(np.median(x2, axis=0), x)

    def test_subclass(self):
        # gh-3846
        class MySubClass(np.ndarray):

            def __new__(cls, input_array, info=None):
                obj = np.asarray(input_array).view(cls)
                obj.info = info
                return obj

            def mean(self, axis=None, dtype=None, out=None):
                return -7

        a = MySubClass([1, 2, 3])
        assert_equal(np.median(a), -7)

    @pytest.mark.parametrize('arr',
                             ([1., 2., 3.], [1., np.nan, 3.], np.nan, 0.))
    def test_subclass2(self, arr):
        """Check that we return subclasses, even if a NaN scalar."""
        class MySubclass(np.ndarray):
            pass

        m = np.median(np.array(arr).view(MySubclass))
        assert isinstance(m, MySubclass)

    def test_out(self):
        o = np.zeros((4,))
        d = np.ones((3, 4))
        assert_equal(np.median(d, 0, out=o), o)
        o = np.zeros((3,))
        assert_equal(np.median(d, 1, out=o), o)
        o = np.zeros(())
        assert_equal(np.median(d, out=o), o)

    def test_out_nan(self):
        with warnings.catch_warnings(record=True):
            warnings.filterwarnings('always', '', RuntimeWarning)
            o = np.zeros((4,))
            d = np.ones((3, 4))
            d[2, 1] = np.nan
            assert_equal(np.median(d, 0, out=o), o)
            o = np.zeros((3,))
            assert_equal(np.median(d, 1, out=o), o)
            o = np.zeros(())
            assert_equal(np.median(d, out=o), o)

    def test_nan_behavior(self):
        a = np.arange(24, dtype=float)
        a[2] = np.nan
        assert_equal(np.median(a), np.nan)
        assert_equal(np.median(a, axis=0), np.nan)

        a = np.arange(24, dtype=float).reshape(2, 3, 4)
        a[1, 2, 3] = np.nan
        a[1, 1, 2] = np.nan

        # no axis
        assert_equal(np.median(a), np.nan)
        assert_equal(np.median(a).ndim, 0)

        # axis0
        b = np.median(np.arange(24, dtype=float).reshape(2, 3, 4), 0)
        b[2, 3] = np.nan
        b[1, 2] = np.nan
        assert_equal(np.median(a, 0), b)

        # axis1
        b = np.median(np.arange(24, dtype=float).reshape(2, 3, 4), 1)
        b[1, 3] = np.nan
        b[1, 2] = np.nan
        assert_equal(np.median(a, 1), b)

        # axis02
        b = np.median(np.arange(24, dtype=float).reshape(2, 3, 4), (0, 2))
        b[1] = np.nan
        b[2] = np.nan
        assert_equal(np.median(a, (0, 2)), b)

    @pytest.mark.skipif(IS_WASM, reason="fp errors don't work correctly")
    def test_empty(self):
        # mean(empty array) emits two warnings: empty slice and divide by 0
        a = np.array([], dtype=float)
        with warnings.catch_warnings(record=True) as w:
            warnings.filterwarnings('always', '', RuntimeWarning)
            assert_equal(np.median(a), np.nan)
            assert_(w[0].category is RuntimeWarning)
            assert_equal(len(w), 2)

        # multiple dimensions
        a = np.array([], dtype=float, ndmin=3)
        # no axis
        with warnings.catch_warnings(record=True) as w:
            warnings.filterwarnings('always', '', RuntimeWarning)
            assert_equal(np.median(a), np.nan)
            assert_(w[0].category is RuntimeWarning)

        # axis 0 and 1
        b = np.array([], dtype=float, ndmin=2)
        assert_equal(np.median(a, axis=0), b)
        assert_equal(np.median(a, axis=1), b)

        # axis 2
        b = np.array(np.nan, dtype=float, ndmin=2)
        with warnings.catch_warnings(record=True) as w:
            warnings.filterwarnings('always', '', RuntimeWarning)
            assert_equal(np.median(a, axis=2), b)
            assert_(w[0].category is RuntimeWarning)

    def test_object(self):
        o = np.arange(7.)
        assert_(type(np.median(o.astype(object))), float)
        o[2] = np.nan
        assert_(type(np.median(o.astype(object))), float)

    def test_extended_axis(self):
        o = np.random.normal(size=(71, 23))
        x = np.dstack([o] * 10)
        assert_equal(np.median(x, axis=(0, 1)), np.median(o))
        x = np.moveaxis(x, -1, 0)
        assert_equal(np.median(x, axis=(-2, -1)), np.median(o))
        x = x.swapaxes(0, 1).copy()
        assert_equal(np.median(x, axis=(0, -1)), np.median(o))

        assert_equal(np.median(x, axis=(0, 1, 2)), np.median(x, axis=None))
        assert_equal(np.median(x, axis=(0, )), np.median(x, axis=0))
        assert_equal(np.median(x, axis=(-1, )), np.median(x, axis=-1))

        d = np.arange(3 * 5 * 7 * 11).reshape((3, 5, 7, 11))
        np.random.shuffle(d.ravel())
        assert_equal(np.median(d, axis=(0, 1, 2))[0],
                     np.median(d[:,:,:, 0].flatten()))
        assert_equal(np.median(d, axis=(0, 1, 3))[1],
                     np.median(d[:,:, 1,:].flatten()))
        assert_equal(np.median(d, axis=(3, 1, -4))[2],
                     np.median(d[:,:, 2,:].flatten()))
        assert_equal(np.median(d, axis=(3, 1, 2))[2],
                     np.median(d[2,:,:,:].flatten()))
        assert_equal(np.median(d, axis=(3, 2))[2, 1],
                     np.median(d[2, 1,:,:].flatten()))
        assert_equal(np.median(d, axis=(1, -2))[2, 1],
                     np.median(d[2,:,:, 1].flatten()))
        assert_equal(np.median(d, axis=(1, 3))[2, 2],
                     np.median(d[2,:, 2,:].flatten()))

    def test_extended_axis_invalid(self):
        d = np.ones((3, 5, 7, 11))
        assert_raises(np.AxisError, np.median, d, axis=-5)
        assert_raises(np.AxisError, np.median, d, axis=(0, -5))
        assert_raises(np.AxisError, np.median, d, axis=4)
        assert_raises(np.AxisError, np.median, d, axis=(0, 4))
        assert_raises(ValueError, np.median, d, axis=(1, 1))

    def test_keepdims(self):
        d = np.ones((3, 5, 7, 11))
        assert_equal(np.median(d, axis=None, keepdims=True).shape,
                     (1, 1, 1, 1))
        assert_equal(np.median(d, axis=(0, 1), keepdims=True).shape,
                     (1, 1, 7, 11))
        assert_equal(np.median(d, axis=(0, 3), keepdims=True).shape,
                     (1, 5, 7, 1))
        assert_equal(np.median(d, axis=(1,), keepdims=True).shape,
                     (3, 1, 7, 11))
        assert_equal(np.median(d, axis=(0, 1, 2, 3), keepdims=True).shape,
                     (1, 1, 1, 1))
        assert_equal(np.median(d, axis=(0, 1, 3), keepdims=True).shape,
                     (1, 1, 7, 1))

    @pytest.mark.parametrize(
        argnames='axis',
        argvalues=[
            None,
            1,
            (1, ),
            (0, 1),
            (-3, -1),
        ]
    )
    def test_keepdims_out(self, axis):
        d = np.ones((3, 5, 7, 11))
        if axis is None:
            shape_out = (1,) * d.ndim
        else:
            axis_norm = normalize_axis_tuple(axis, d.ndim)
            shape_out = tuple(
                1 if i in axis_norm else d.shape[i] for i in range(d.ndim))
        out = np.empty(shape_out)
        result = np.median(d, axis=axis, keepdims=True, out=out)
        assert result is out
        assert_equal(result.shape, shape_out)


class TestAdd_newdoc_ufunc:

    def test_ufunc_arg(self):
        assert_raises(TypeError, add_newdoc_ufunc, 2, "blah")
        assert_raises(ValueError, add_newdoc_ufunc, np.add, "blah")

    def test_string_arg(self):
        assert_raises(TypeError, add_newdoc_ufunc, np.add, 3)


class TestAdd_newdoc:

    @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO")
    @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
    def test_add_doc(self):
        # test that np.add_newdoc did attach a docstring successfully:
        tgt = "Current flat index into the array."
        assert_equal(np.core.flatiter.index.__doc__[:len(tgt)], tgt)
        assert_(len(np.core.ufunc.identity.__doc__) > 300)
        assert_(len(np.lib.index_tricks.mgrid.__doc__) > 300)

    @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO")
    def test_errors_are_ignored(self):
        prev_doc = np.core.flatiter.index.__doc__
        # nothing changed, but error ignored, this should probably
        # give a warning (or even error) in the future.
        np.add_newdoc("numpy.core", "flatiter", ("index", "bad docstring"))
        assert prev_doc == np.core.flatiter.index.__doc__


class TestAddDocstring():
    # Test should possibly be moved, but it also fits to be close to
    # the newdoc tests...
    @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO")
    @pytest.mark.skipif(IS_PYPY, reason="PyPy does not modify tp_doc")
    def test_add_same_docstring(self):
        # test for attributes (which are C-level defined)
        np.add_docstring(np.ndarray.flat, np.ndarray.flat.__doc__)
        # And typical functions:
        def func():
            """docstring"""
            return

        np.add_docstring(func, func.__doc__)

    @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO")
    def test_different_docstring_fails(self):
        # test for attributes (which are C-level defined)
        with assert_raises(RuntimeError):
            np.add_docstring(np.ndarray.flat, "different docstring")
        # And typical functions:
        def func():
            """docstring"""
            return

        with assert_raises(RuntimeError):
            np.add_docstring(func, "different docstring")


class TestSortComplex:

    @pytest.mark.parametrize("type_in, type_out", [
        ('l', 'D'),
        ('h', 'F'),
        ('H', 'F'),
        ('b', 'F'),
        ('B', 'F'),
        ('g', 'G'),
        ])
    def test_sort_real(self, type_in, type_out):
        # sort_complex() type casting for real input types
        a = np.array([5, 3, 6, 2, 1], dtype=type_in)
        actual = np.sort_complex(a)
        expected = np.sort(a).astype(type_out)
        assert_equal(actual, expected)
        assert_equal(actual.dtype, expected.dtype)

    def test_sort_complex(self):
        # sort_complex() handling of complex input
        a = np.array([2 + 3j, 1 - 2j, 1 - 3j, 2 + 1j], dtype='D')
        expected = np.array([1 - 3j, 1 - 2j, 2 + 1j, 2 + 3j], dtype='D')
        actual = np.sort_complex(a)
        assert_equal(actual, expected)
        assert_equal(actual.dtype, expected.dtype)

env_pyrecodes/lib/python3.9/site-packages/numpy/lib/tests/test_function_base.py
NikolaBlagojevic/pyrecodes
"""
numpy.ma : a package to handle missing or invalid values.

This package was initially written for numarray by Paul F. Dubois
at Lawrence Livermore National Laboratory.
In 2006, the package was completely rewritten by Pierre Gerard-Marchant
(University of Georgia) to make the MaskedArray class a subclass of ndarray,
and to improve support of structured arrays.


Copyright 1999, 2000, 2001 Regents of the University of California.
Released for unlimited redistribution.

* Adapted for numpy_core 2005 by Travis Oliphant and (mainly) Paul Dubois.
* Subclassing of the base `ndarray` 2006 by Pierre Gerard-Marchant
  (pgmdevlist_AT_gmail_DOT_com)
* Improvements suggested by Reggie Dugard (reggie_AT_merfinllc_DOT_com)

.. moduleauthor:: Pierre Gerard-Marchant

"""
# pylint: disable-msg=E1002
import builtins
import inspect
import operator
import warnings
import textwrap
import re
from functools import reduce

import numpy as np
import numpy.core.umath as umath
import numpy.core.numerictypes as ntypes
from numpy.core import multiarray as mu
from numpy import ndarray, amax, amin, iscomplexobj, bool_, _NoValue
from numpy import array as narray
from numpy.lib.function_base import angle
from numpy.compat import (
    getargspec, formatargspec, long, unicode, bytes
    )
from numpy import expand_dims
from numpy.core.numeric import normalize_axis_tuple


__all__ = [
    'MAError', 'MaskError', 'MaskType', 'MaskedArray', 'abs', 'absolute',
    'add', 'all', 'allclose', 'allequal', 'alltrue', 'amax', 'amin',
    'angle', 'anom', 'anomalies', 'any', 'append', 'arange', 'arccos',
    'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh',
    'argmax', 'argmin', 'argsort', 'around', 'array', 'asanyarray',
    'asarray', 'bitwise_and', 'bitwise_or', 'bitwise_xor', 'bool_', 'ceil',
    'choose', 'clip', 'common_fill_value', 'compress', 'compressed',
    'concatenate', 'conjugate', 'convolve', 'copy', 'correlate', 'cos', 'cosh',
    'count', 'cumprod', 'cumsum', 'default_fill_value', 'diag', 'diagonal',
    'diff', 'divide', 'empty', 'empty_like', 'equal', 'exp',
    'expand_dims', 'fabs', 'filled', 'fix_invalid', 'flatten_mask',
    'flatten_structured_array', 'floor', 'floor_divide', 'fmod',
    'frombuffer', 'fromflex', 'fromfunction', 'getdata', 'getmask',
    'getmaskarray', 'greater', 'greater_equal', 'harden_mask', 'hypot',
    'identity', 'ids', 'indices', 'inner', 'innerproduct', 'isMA',
    'isMaskedArray', 'is_mask', 'is_masked', 'isarray', 'left_shift',
    'less', 'less_equal', 'log', 'log10', 'log2',
    'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'make_mask',
    'make_mask_descr', 'make_mask_none', 'mask_or', 'masked',
    'masked_array', 'masked_equal', 'masked_greater',
    'masked_greater_equal', 'masked_inside', 'masked_invalid',
    'masked_less', 'masked_less_equal', 'masked_not_equal',
    'masked_object', 'masked_outside', 'masked_print_option',
    'masked_singleton', 'masked_values', 'masked_where', 'max', 'maximum',
    'maximum_fill_value', 'mean', 'min', 'minimum', 'minimum_fill_value',
    'mod', 'multiply', 'mvoid', 'ndim', 'negative', 'nomask', 'nonzero',
    'not_equal', 'ones', 'ones_like', 'outer', 'outerproduct', 'power', 'prod',
    'product', 'ptp', 'put', 'putmask', 'ravel', 'remainder',
    'repeat', 'reshape', 'resize', 'right_shift', 'round', 'round_',
    'set_fill_value', 'shape', 'sin', 'sinh', 'size', 'soften_mask',
    'sometrue', 'sort', 'sqrt', 'squeeze', 'std', 'subtract', 'sum',
    'swapaxes', 'take', 'tan', 'tanh', 'trace', 'transpose', 'true_divide',
    'var', 'where', 'zeros', 'zeros_like',
    ]

MaskType = np.bool_
nomask = MaskType(0)

class MaskedArrayFutureWarning(FutureWarning):
    pass

def _deprecate_argsort_axis(arr):
    """
    Adjust the axis passed to argsort, warning if necessary

    Parameters
    ----------
    arr
        The array which argsort was called on

    np.ma.argsort has a long-term bug where the default of the axis argument
    is wrong (gh-8701), which now must be kept for backwards compatibility.
    Thankfully, this only makes a difference when arrays are 2- or more-
    dimensional, so we only need a warning then.
    """
    if arr.ndim <= 1:
        # no warning needed - but switch to -1 anyway, to avoid surprising
        # subclasses, which are more likely to implement scalar axes.
        return -1
    else:
        # 2017-04-11, Numpy 1.13.0, gh-8701: warn on axis default
        warnings.warn(
            "In the future the default for argsort will be axis=-1, not the "
            "current None, to match its documentation and np.argsort. "
            "Explicitly pass -1 or None to silence this warning.",
            MaskedArrayFutureWarning, stacklevel=3)
        return None


def doc_note(initialdoc, note):
    """
    Adds a Notes section to an existing docstring.

    """
    if initialdoc is None:
        return
    if note is None:
        return initialdoc

    notesplit = re.split(r'\n\s*?Notes\n\s*?-----', inspect.cleandoc(initialdoc))
    notedoc = "\n\nNotes\n-----\n%s\n" % inspect.cleandoc(note)

    return ''.join(notesplit[:1] + [notedoc] + notesplit[1:])


def get_object_signature(obj):
    """
    Get the signature from obj

    """
    try:
        sig = formatargspec(*getargspec(obj))
    except TypeError:
        sig = ''
    return sig


###############################################################################
#                              Exceptions                                     #
###############################################################################


class MAError(Exception):
    """
    Class for masked array related errors.

    """
    pass


class MaskError(MAError):
    """
    Class for mask related errors.

    """
    pass


###############################################################################
#                           Filling options                                   #
###############################################################################


# b: boolean - c: complex - f: floats - i: integer - O: object - S: string
default_filler = {'b': True,
                  'c': 1.e20 + 0.0j,
                  'f': 1.e20,
                  'i': 999999,
                  'O': '?',
                  'S': b'N/A',
                  'u': 999999,
                  'V': b'???',
                  'U': 'N/A'
                  }

# Add datetime64 and timedelta64 types
for v in ["Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns", "ps",
          "fs", "as"]:
    default_filler["M8[" + v + "]"] = np.datetime64("NaT", v)
    default_filler["m8[" + v + "]"] = np.timedelta64("NaT", v)

float_types_list = [np.half, np.single, np.double, np.longdouble,
                    np.csingle, np.cdouble, np.clongdouble]
max_filler = ntypes._minvals
max_filler.update([(k, -np.inf) for k in float_types_list[:4]])
max_filler.update([(k, complex(-np.inf, -np.inf)) for k in float_types_list[-3:]])

min_filler = ntypes._maxvals
min_filler.update([(k,  +np.inf) for k in float_types_list[:4]])
min_filler.update([(k, complex(+np.inf, +np.inf)) for k in float_types_list[-3:]])

del float_types_list

def _recursive_fill_value(dtype, f):
    """
    Recursively produce a fill value for `dtype`, calling f on scalar dtypes
    """
    if dtype.names is not None:
        # We wrap into `array` here, which ensures we use NumPy cast rules
        # for integer casts, this allows the use of 99999 as a fill value
        # for int8.
        # TODO: This is probably a mess, but should best preserve behavior?
        vals = tuple(
                np.array(_recursive_fill_value(dtype[name], f))
                for name in dtype.names)
        return np.array(vals, dtype=dtype)[()]  # decay to void scalar from 0d
    elif dtype.subdtype:
        subtype, shape = dtype.subdtype
        subval = _recursive_fill_value(subtype, f)
        return np.full(shape, subval)
    else:
        return f(dtype)


def _get_dtype_of(obj):
    """ Convert the argument for *_fill_value into a dtype """
    if isinstance(obj, np.dtype):
        return obj
    elif hasattr(obj, 'dtype'):
        return obj.dtype
    else:
        return np.asanyarray(obj).dtype


def default_fill_value(obj):
    """
    Return the default fill value for the argument object.

    The default filling value depends on the datatype of the input
    array or the type of the input scalar:

       ========  ========
       datatype  default
       ========  ========
       bool      True
       int       999999
       float     1.e20
       complex   1.e20+0j
       object    '?'
       string    'N/A'
       ========  ========

    For structured types, a structured scalar is returned, with each field the
    default fill value for its type.

    For subarray types, the fill value is an array of the same size containing
    the default scalar fill value.

    Parameters
    ----------
    obj : ndarray, dtype or scalar
        The array data-type or scalar for which the default fill value
        is returned.

    Returns
    -------
    fill_value : scalar
        The default fill value.

    Examples
    --------
    >>> np.ma.default_fill_value(1)
    999999
    >>> np.ma.default_fill_value(np.array([1.1, 2., np.pi]))
    1e+20
    >>> np.ma.default_fill_value(np.dtype(complex))
    (1e+20+0j)

    """
    def _scalar_fill_value(dtype):
        if dtype.kind in 'Mm':
            return default_filler.get(dtype.str[1:], '?')
        else:
            return default_filler.get(dtype.kind, '?')

    dtype = _get_dtype_of(obj)
    return _recursive_fill_value(dtype, _scalar_fill_value)


def _extremum_fill_value(obj, extremum, extremum_name):

    def _scalar_fill_value(dtype):
        try:
            return extremum[dtype]
        except KeyError as e:
            raise TypeError(
                f"Unsuitable type {dtype} for calculating {extremum_name}."
            ) from None

    dtype = _get_dtype_of(obj)
    return _recursive_fill_value(dtype, _scalar_fill_value)


def minimum_fill_value(obj):
    """
    Return the maximum value that can be represented by the dtype of an object.

    This function is useful for calculating a fill value suitable for
    taking the minimum of an array with a given dtype.

    Parameters
    ----------
    obj : ndarray, dtype or scalar
        An object that can be queried for it's numeric type.

    Returns
    -------
    val : scalar
        The maximum representable value.

    Raises
    ------
    TypeError
        If `obj` isn't a suitable numeric type.

    See Also
    --------
    maximum_fill_value : The inverse function.
    set_fill_value : Set the filling value of a masked array.
    MaskedArray.fill_value : Return current fill value.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.int8()
    >>> ma.minimum_fill_value(a)
    127
    >>> a = np.int32()
    >>> ma.minimum_fill_value(a)
    2147483647

    An array of numeric data can also be passed.

    >>> a = np.array([1, 2, 3], dtype=np.int8)
    >>> ma.minimum_fill_value(a)
    127
    >>> a = np.array([1, 2, 3], dtype=np.float32)
    >>> ma.minimum_fill_value(a)
    inf

    """
    return _extremum_fill_value(obj, min_filler, "minimum")


def maximum_fill_value(obj):
    """
    Return the minimum value that can be represented by the dtype of an object.

    This function is useful for calculating a fill value suitable for
    taking the maximum of an array with a given dtype.

    Parameters
    ----------
    obj : ndarray, dtype or scalar
        An object that can be queried for it's numeric type.

    Returns
    -------
    val : scalar
        The minimum representable value.

    Raises
    ------
    TypeError
        If `obj` isn't a suitable numeric type.

    See Also
    --------
    minimum_fill_value : The inverse function.
    set_fill_value : Set the filling value of a masked array.
    MaskedArray.fill_value : Return current fill value.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.int8()
    >>> ma.maximum_fill_value(a)
    -128
    >>> a = np.int32()
    >>> ma.maximum_fill_value(a)
    -2147483648

    An array of numeric data can also be passed.

    >>> a = np.array([1, 2, 3], dtype=np.int8)
    >>> ma.maximum_fill_value(a)
    -128
    >>> a = np.array([1, 2, 3], dtype=np.float32)
    >>> ma.maximum_fill_value(a)
    -inf

    """
    return _extremum_fill_value(obj, max_filler, "maximum")


def _recursive_set_fill_value(fillvalue, dt):
    """
    Create a fill value for a structured dtype.

    Parameters
    ----------
    fillvalue : scalar or array_like
        Scalar or array representing the fill value. If it is of shorter
        length than the number of fields in dt, it will be resized.
    dt : dtype
        The structured dtype for which to create the fill value.

    Returns
    -------
    val : tuple
        A tuple of values corresponding to the structured fill value.

    """
    fillvalue = np.resize(fillvalue, len(dt.names))
    output_value = []
    for (fval, name) in zip(fillvalue, dt.names):
        cdtype = dt[name]
        if cdtype.subdtype:
            cdtype = cdtype.subdtype[0]

        if cdtype.names is not None:
            output_value.append(tuple(_recursive_set_fill_value(fval, cdtype)))
        else:
            output_value.append(np.array(fval, dtype=cdtype).item())
    return tuple(output_value)


def _check_fill_value(fill_value, ndtype):
    """
    Private function validating the given `fill_value` for the given dtype.

    If fill_value is None, it is set to the default corresponding to the dtype.

    If fill_value is not None, its value is forced to the given dtype.

    The result is always a 0d array.

    """
    ndtype = np.dtype(ndtype)
    if fill_value is None:
        fill_value = default_fill_value(ndtype)
    elif ndtype.names is not None:
        if isinstance(fill_value, (ndarray, np.void)):
            try:
                fill_value = np.array(fill_value, copy=False, dtype=ndtype)
            except ValueError as e:
                err_msg = "Unable to transform %s to dtype %s"
                raise ValueError(err_msg % (fill_value, ndtype)) from e
        else:
            fill_value = np.asarray(fill_value, dtype=object)
            fill_value = np.array(_recursive_set_fill_value(fill_value, ndtype),
                                  dtype=ndtype)
    else:
        if isinstance(fill_value, str) and (ndtype.char not in 'OSVU'):
            # Note this check doesn't work if fill_value is not a scalar
            err_msg = "Cannot set fill value of string with array of dtype %s"
            raise TypeError(err_msg % ndtype)
        else:
            # In case we want to convert 1e20 to int.
            # Also in case of converting string arrays.
            try:
                fill_value = np.array(fill_value, copy=False, dtype=ndtype)
            except (OverflowError, ValueError) as e:
                # Raise TypeError instead of OverflowError or ValueError.
                # OverflowError is seldom used, and the real problem here is
                # that the passed fill_value is not compatible with the ndtype.
                err_msg = "Cannot convert fill_value %s to dtype %s"
                raise TypeError(err_msg % (fill_value, ndtype)) from e
    return np.array(fill_value)


def set_fill_value(a, fill_value):
    """
    Set the filling value of a, if a is a masked array.

    This function changes the fill value of the masked array `a` in place.
    If `a` is not a masked array, the function returns silently, without
    doing anything.

    Parameters
    ----------
    a : array_like
        Input array.
    fill_value : dtype
        Filling value. A consistency test is performed to make sure
        the value is compatible with the dtype of `a`.

    Returns
    -------
    None
        Nothing returned by this function.

    See Also
    --------
    maximum_fill_value : Return the default fill value for a dtype.
    MaskedArray.fill_value : Return current fill value.
    MaskedArray.set_fill_value : Equivalent method.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(5)
    >>> a
    array([0, 1, 2, 3, 4])
    >>> a = ma.masked_where(a < 3, a)
    >>> a
    masked_array(data=[--, --, --, 3, 4],
                 mask=[ True,  True,  True, False, False],
           fill_value=999999)
    >>> ma.set_fill_value(a, -999)
    >>> a
    masked_array(data=[--, --, --, 3, 4],
                 mask=[ True,  True,  True, False, False],
           fill_value=-999)

    Nothing happens if `a` is not a masked array.

    >>> a = list(range(5))
    >>> a
    [0, 1, 2, 3, 4]
    >>> ma.set_fill_value(a, 100)
    >>> a
    [0, 1, 2, 3, 4]
    >>> a = np.arange(5)
    >>> a
    array([0, 1, 2, 3, 4])
    >>> ma.set_fill_value(a, 100)
    >>> a
    array([0, 1, 2, 3, 4])

    """
    if isinstance(a, MaskedArray):
        a.set_fill_value(fill_value)
    return


def get_fill_value(a):
    """
    Return the filling value of a, if any.  Otherwise, returns the
    default filling value for that type.

    """
    if isinstance(a, MaskedArray):
        result = a.fill_value
    else:
        result = default_fill_value(a)
    return result


def common_fill_value(a, b):
    """
    Return the common filling value of two masked arrays, if any.

    If ``a.fill_value == b.fill_value``, return the fill value,
    otherwise return None.

    Parameters
    ----------
    a, b : MaskedArray
        The masked arrays for which to compare fill values.

    Returns
    -------
    fill_value : scalar or None
        The common fill value, or None.

    Examples
    --------
    >>> x = np.ma.array([0, 1.], fill_value=3)
    >>> y = np.ma.array([0, 1.], fill_value=3)
    >>> np.ma.common_fill_value(x, y)
    3.0

    """
    t1 = get_fill_value(a)
    t2 = get_fill_value(b)
    if t1 == t2:
        return t1
    return None


def filled(a, fill_value=None):
    """
    Return input as an array with masked data replaced by a fill value.

    If `a` is not a `MaskedArray`, `a` itself is returned.
    If `a` is a `MaskedArray` and `fill_value` is None, `fill_value` is set to
    ``a.fill_value``.

    Parameters
    ----------
    a : MaskedArray or array_like
        An input object.
    fill_value : array_like, optional.
        Can be scalar or non-scalar. If non-scalar, the
        resulting filled array should be broadcastable
        over input array. Default is None.

    Returns
    -------
    a : ndarray
        The filled array.

    See Also
    --------
    compressed

    Examples
    --------
    >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[[1, 0, 0],
    ...                                                   [1, 0, 0],
    ...                                                   [0, 0, 0]])
    >>> x.filled()
    array([[999999,      1,      2],
           [999999,      4,      5],
           [     6,      7,      8]])
    >>> x.filled(fill_value=333)
    array([[333,   1,   2],
           [333,   4,   5],
           [  6,   7,   8]])
    >>> x.filled(fill_value=np.arange(3))
    array([[0, 1, 2],
           [0, 4, 5],
           [6, 7, 8]])

    """
    if hasattr(a, 'filled'):
        return a.filled(fill_value)

    elif isinstance(a, ndarray):
        # Should we check for contiguity ? and a.flags['CONTIGUOUS']:
        return a
    elif isinstance(a, dict):
        return np.array(a, 'O')
    else:
        return np.array(a)


def get_masked_subclass(*arrays):
    """
    Return the youngest subclass of MaskedArray from a list of (masked) arrays.

    In case of siblings, the first listed takes over.

    """
    if len(arrays) == 1:
        arr = arrays[0]
        if isinstance(arr, MaskedArray):
            rcls = type(arr)
        else:
            rcls = MaskedArray
    else:
        arrcls = [type(a) for a in arrays]
        rcls = arrcls[0]
        if not issubclass(rcls, MaskedArray):
            rcls = MaskedArray
        for cls in arrcls[1:]:
            if issubclass(cls, rcls):
                rcls = cls
    # Don't return MaskedConstant as result: revert to MaskedArray
    if rcls.__name__ == 'MaskedConstant':
        return MaskedArray
    return rcls


def getdata(a, subok=True):
    """
    Return the data of a masked array as an ndarray.

    Return the data of `a` (if any) as an ndarray if `a` is a ``MaskedArray``,
    else return `a` as a ndarray or subclass (depending on `subok`) if not.

    Parameters
    ----------
    a : array_like
        Input ``MaskedArray``, alternatively a ndarray or a subclass thereof.
    subok : bool
        Whether to force the output to be a `pure` ndarray (False) or to
        return a subclass of ndarray if appropriate (True, default).

    See Also
    --------
    getmask : Return the mask of a masked array, or nomask.
    getmaskarray : Return the mask of a masked array, or full array of False.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = ma.masked_equal([[1,2],[3,4]], 2)
    >>> a
    masked_array(
      data=[[1, --],
            [3, 4]],
      mask=[[False,  True],
            [False, False]],
      fill_value=2)
    >>> ma.getdata(a)
    array([[1, 2],
           [3, 4]])

    Equivalently use the ``MaskedArray`` `data` attribute.

    >>> a.data
    array([[1, 2],
           [3, 4]])

    """
    try:
        data = a._data
    except AttributeError:
        data = np.array(a, copy=False, subok=subok)
    if not subok:
        return data.view(ndarray)
    return data


get_data = getdata


def fix_invalid(a, mask=nomask, copy=True, fill_value=None):
    """
    Return input with invalid data masked and replaced by a fill value.

    Invalid data means values of `nan`, `inf`, etc.

    Parameters
    ----------
    a : array_like
        Input array, a (subclass of) ndarray.
    mask : sequence, optional
        Mask. Must be convertible to an array of booleans with the same
        shape as `data`. True indicates a masked (i.e. invalid) data.
    copy : bool, optional
        Whether to use a copy of `a` (True) or to fix `a` in place (False).
        Default is True.
    fill_value : scalar, optional
        Value used for fixing invalid data. Default is None, in which case
        the ``a.fill_value`` is used.

    Returns
    -------
    b : MaskedArray
        The input array with invalid entries fixed.

    Notes
    -----
    A copy is performed by default.

    Examples
    --------
    >>> x = np.ma.array([1., -1, np.nan, np.inf], mask=[1] + [0]*3)
    >>> x
    masked_array(data=[--, -1.0, nan, inf],
                 mask=[ True, False, False, False],
           fill_value=1e+20)
    >>> np.ma.fix_invalid(x)
    masked_array(data=[--, -1.0, --, --],
                 mask=[ True, False,  True,  True],
           fill_value=1e+20)

    >>> fixed = np.ma.fix_invalid(x)
    >>> fixed.data
    array([ 1.e+00, -1.e+00,  1.e+20,  1.e+20])
    >>> x.data
    array([ 1., -1., nan, inf])

    """
    a = masked_array(a, copy=copy, mask=mask, subok=True)
    invalid = np.logical_not(np.isfinite(a._data))
    if not invalid.any():
        return a
    a._mask |= invalid
    if fill_value is None:
        fill_value = a.fill_value
    a._data[invalid] = fill_value
    return a

def is_string_or_list_of_strings(val):
    return (isinstance(val, str) or
            (isinstance(val, list) and val and
             builtins.all(isinstance(s, str) for s in val)))

###############################################################################
#                                  Ufuncs                                     #
###############################################################################


ufunc_domain = {}
ufunc_fills = {}


class _DomainCheckInterval:
    """
    Define a valid interval, so that :

    ``domain_check_interval(a,b)(x) == True`` where
    ``x < a`` or ``x > b``.

    """

    def __init__(self, a, b):
        "domain_check_interval(a,b)(x) = true where x < a or y > b"
        if a > b:
            (a, b) = (b, a)
        self.a = a
        self.b = b

    def __call__(self, x):
        "Execute the call behavior."
        # nans at masked positions cause RuntimeWarnings, even though
        # they are masked. To avoid this we suppress warnings.
        with np.errstate(invalid='ignore'):
            return umath.logical_or(umath.greater(x, self.b),
                                    umath.less(x, self.a))


class _DomainTan:
    """
    Define a valid interval for the `tan` function, so that:

    ``domain_tan(eps) = True`` where ``abs(cos(x)) < eps``

    """

    def __init__(self, eps):
        "domain_tan(eps) = true where abs(cos(x)) < eps)"
        self.eps = eps

    def __call__(self, x):
        "Executes the call behavior."
        with np.errstate(invalid='ignore'):
            return umath.less(umath.absolute(umath.cos(x)), self.eps)


class _DomainSafeDivide:
    """
    Define a domain for safe division.

    """

    def __init__(self, tolerance=None):
        self.tolerance = tolerance

    def __call__(self, a, b):
        # Delay the selection of the tolerance to here in order to reduce numpy
        # import times. The calculation of these parameters is a substantial
        # component of numpy's import time.
        if self.tolerance is None:
            self.tolerance = np.finfo(float).tiny
        # don't call ma ufuncs from __array_wrap__ which would fail for scalars
        a, b = np.asarray(a), np.asarray(b)
        with np.errstate(invalid='ignore'):
            return umath.absolute(a) * self.tolerance >= umath.absolute(b)


class _DomainGreater:
    """
    DomainGreater(v)(x) is True where x <= v.

    """

    def __init__(self, critical_value):
        "DomainGreater(v)(x) = true where x <= v"
        self.critical_value = critical_value

    def __call__(self, x):
        "Executes the call behavior."
        with np.errstate(invalid='ignore'):
            return umath.less_equal(x, self.critical_value)


class _DomainGreaterEqual:
    """
    DomainGreaterEqual(v)(x) is True where x < v.

    """

    def __init__(self, critical_value):
        "DomainGreaterEqual(v)(x) = true where x < v"
        self.critical_value = critical_value

    def __call__(self, x):
        "Executes the call behavior."
        with np.errstate(invalid='ignore'):
            return umath.less(x, self.critical_value)


class _MaskedUFunc:
    def __init__(self, ufunc):
        self.f = ufunc
        self.__doc__ = ufunc.__doc__
        self.__name__ = ufunc.__name__

    def __str__(self):
        return f"Masked version of {self.f}"


class _MaskedUnaryOperation(_MaskedUFunc):
    """
    Defines masked version of unary operations, where invalid values are
    pre-masked.

    Parameters
    ----------
    mufunc : callable
        The function for which to define a masked version. Made available
        as ``_MaskedUnaryOperation.f``.
    fill : scalar, optional
        Filling value, default is 0.
    domain : class instance
        Domain for the function. Should be one of the ``_Domain*``
        classes. Default is None.

    """

    def __init__(self, mufunc, fill=0, domain=None):
        super().__init__(mufunc)
        self.fill = fill
        self.domain = domain
        ufunc_domain[mufunc] = domain
        ufunc_fills[mufunc] = fill

    def __call__(self, a, *args, **kwargs):
        """
        Execute the call behavior.

        """
        d = getdata(a)
        # Deal with domain
        if self.domain is not None:
            # Case 1.1. : Domained function
            # nans at masked positions cause RuntimeWarnings, even though
            # they are masked. To avoid this we suppress warnings.
            with np.errstate(divide='ignore', invalid='ignore'):
                result = self.f(d, *args, **kwargs)
            # Make a mask
            m = ~umath.isfinite(result)
            m |= self.domain(d)
            m |= getmask(a)
        else:
            # Case 1.2. : Function without a domain
            # Get the result and the mask
            with np.errstate(divide='ignore', invalid='ignore'):
                result = self.f(d, *args, **kwargs)
            m = getmask(a)

        if not result.ndim:
            # Case 2.1. : The result is scalarscalar
            if m:
                return masked
            return result

        if m is not nomask:
            # Case 2.2. The result is an array
            # We need to fill the invalid data back w/ the input Now,
            # that's plain silly: in C, we would just skip the element and
            # keep the original, but we do have to do it that way in Python

            # In case result has a lower dtype than the inputs (as in
            # equal)
            try:
                np.copyto(result, d, where=m)
            except TypeError:
                pass
        # Transform to
        masked_result = result.view(get_masked_subclass(a))
        masked_result._mask = m
        masked_result._update_from(a)
        return masked_result


class _MaskedBinaryOperation(_MaskedUFunc):
    """
    Define masked version of binary operations, where invalid
    values are pre-masked.

    Parameters
    ----------
    mbfunc : function
        The function for which to define a masked version. Made available
        as ``_MaskedBinaryOperation.f``.
    domain : class instance
        Default domain for the function. Should be one of the ``_Domain*``
        classes. Default is None.
    fillx : scalar, optional
        Filling value for the first argument, default is 0.
    filly : scalar, optional
        Filling value for the second argument, default is 0.

    """

    def __init__(self, mbfunc, fillx=0, filly=0):
        """
        abfunc(fillx, filly) must be defined.

        abfunc(x, filly) = x for all x to enable reduce.

        """
        super().__init__(mbfunc)
        self.fillx = fillx
        self.filly = filly
        ufunc_domain[mbfunc] = None
        ufunc_fills[mbfunc] = (fillx, filly)

    def __call__(self, a, b, *args, **kwargs):
        """
        Execute the call behavior.

        """
        # Get the data, as ndarray
        (da, db) = (getdata(a), getdata(b))
        # Get the result
        with np.errstate():
            np.seterr(divide='ignore', invalid='ignore')
            result = self.f(da, db, *args, **kwargs)
        # Get the mask for the result
        (ma, mb) = (getmask(a), getmask(b))
        if ma is nomask:
            if mb is nomask:
                m = nomask
            else:
                m = umath.logical_or(getmaskarray(a), mb)
        elif mb is nomask:
            m = umath.logical_or(ma, getmaskarray(b))
        else:
            m = umath.logical_or(ma, mb)

        # Case 1. : scalar
        if not result.ndim:
            if m:
                return masked
            return result

        # Case 2. : array
        # Revert result to da where masked
        if m is not nomask and m.any():
            # any errors, just abort; impossible to guarantee masked values
            try:
                np.copyto(result, da, casting='unsafe', where=m)
            except Exception:
                pass

        # Transforms to a (subclass of) MaskedArray
        masked_result = result.view(get_masked_subclass(a, b))
        masked_result._mask = m
        if isinstance(a, MaskedArray):
            masked_result._update_from(a)
        elif isinstance(b, MaskedArray):
            masked_result._update_from(b)
        return masked_result

    def reduce(self, target, axis=0, dtype=None):
        """
        Reduce `target` along the given `axis`.

        """
        tclass = get_masked_subclass(target)
        m = getmask(target)
        t = filled(target, self.filly)
        if t.shape == ():
            t = t.reshape(1)
            if m is not nomask:
                m = make_mask(m, copy=True)
                m.shape = (1,)

        if m is nomask:
            tr = self.f.reduce(t, axis)
            mr = nomask
        else:
            tr = self.f.reduce(t, axis, dtype=dtype)
            mr = umath.logical_and.reduce(m, axis)

        if not tr.shape:
            if mr:
                return masked
            else:
                return tr
        masked_tr = tr.view(tclass)
        masked_tr._mask = mr
        return masked_tr

    def outer(self, a, b):
        """
        Return the function applied to the outer product of a and b.

        """
        (da, db) = (getdata(a), getdata(b))
        d = self.f.outer(da, db)
        ma = getmask(a)
        mb = getmask(b)
        if ma is nomask and mb is nomask:
            m = nomask
        else:
            ma = getmaskarray(a)
            mb = getmaskarray(b)
            m = umath.logical_or.outer(ma, mb)
        if (not m.ndim) and m:
            return masked
        if m is not nomask:
            np.copyto(d, da, where=m)
        if not d.shape:
            return d
        masked_d = d.view(get_masked_subclass(a, b))
        masked_d._mask = m
        return masked_d

    def accumulate(self, target, axis=0):
        """Accumulate `target` along `axis` after filling with y fill
        value.

        """
        tclass = get_masked_subclass(target)
        t = filled(target, self.filly)
        result = self.f.accumulate(t, axis)
        masked_result = result.view(tclass)
        return masked_result



class _DomainedBinaryOperation(_MaskedUFunc):
    """
    Define binary operations that have a domain, like divide.

    They have no reduce, outer or accumulate.

    Parameters
    ----------
    mbfunc : function
        The function for which to define a masked version. Made available
        as ``_DomainedBinaryOperation.f``.
    domain : class instance
        Default domain for the function. Should be one of the ``_Domain*``
        classes.
    fillx : scalar, optional
        Filling value for the first argument, default is 0.
    filly : scalar, optional
        Filling value for the second argument, default is 0.

    """

    def __init__(self, dbfunc, domain, fillx=0, filly=0):
        """abfunc(fillx, filly) must be defined.
           abfunc(x, filly) = x for all x to enable reduce.
        """
        super().__init__(dbfunc)
        self.domain = domain
        self.fillx = fillx
        self.filly = filly
        ufunc_domain[dbfunc] = domain
        ufunc_fills[dbfunc] = (fillx, filly)

    def __call__(self, a, b, *args, **kwargs):
        "Execute the call behavior."
        # Get the data
        (da, db) = (getdata(a), getdata(b))
        # Get the result
        with np.errstate(divide='ignore', invalid='ignore'):
            result = self.f(da, db, *args, **kwargs)
        # Get the mask as a combination of the source masks and invalid
        m = ~umath.isfinite(result)
        m |= getmask(a)
        m |= getmask(b)
        # Apply the domain
        domain = ufunc_domain.get(self.f, None)
        if domain is not None:
            m |= domain(da, db)
        # Take care of the scalar case first
        if not m.ndim:
            if m:
                return masked
            else:
                return result
        # When the mask is True, put back da if possible
        # any errors, just abort; impossible to guarantee masked values
        try:
            np.copyto(result, 0, casting='unsafe', where=m)
            # avoid using "*" since this may be overlaid
            masked_da = umath.multiply(m, da)
            # only add back if it can be cast safely
            if np.can_cast(masked_da.dtype, result.dtype, casting='safe'):
                result += masked_da
        except Exception:
            pass

        # Transforms to a (subclass of) MaskedArray
        masked_result = result.view(get_masked_subclass(a, b))
        masked_result._mask = m
        if isinstance(a, MaskedArray):
            masked_result._update_from(a)
        elif isinstance(b, MaskedArray):
            masked_result._update_from(b)
        return masked_result


# Unary ufuncs
exp = _MaskedUnaryOperation(umath.exp)
conjugate = _MaskedUnaryOperation(umath.conjugate)
sin = _MaskedUnaryOperation(umath.sin)
cos = _MaskedUnaryOperation(umath.cos)
arctan = _MaskedUnaryOperation(umath.arctan)
arcsinh = _MaskedUnaryOperation(umath.arcsinh)
sinh = _MaskedUnaryOperation(umath.sinh)
cosh = _MaskedUnaryOperation(umath.cosh)
tanh = _MaskedUnaryOperation(umath.tanh)
abs = absolute = _MaskedUnaryOperation(umath.absolute)
angle = _MaskedUnaryOperation(angle)  # from numpy.lib.function_base
fabs = _MaskedUnaryOperation(umath.fabs)
negative = _MaskedUnaryOperation(umath.negative)
floor = _MaskedUnaryOperation(umath.floor)
ceil = _MaskedUnaryOperation(umath.ceil)
around = _MaskedUnaryOperation(np.round_)
logical_not = _MaskedUnaryOperation(umath.logical_not)

# Domained unary ufuncs
sqrt = _MaskedUnaryOperation(umath.sqrt, 0.0,
                             _DomainGreaterEqual(0.0))
log = _MaskedUnaryOperation(umath.log, 1.0,
                            _DomainGreater(0.0))
log2 = _MaskedUnaryOperation(umath.log2, 1.0,
                             _DomainGreater(0.0))
log10 = _MaskedUnaryOperation(umath.log10, 1.0,
                              _DomainGreater(0.0))
tan = _MaskedUnaryOperation(umath.tan, 0.0,
                            _DomainTan(1e-35))
arcsin = _MaskedUnaryOperation(umath.arcsin, 0.0,
                               _DomainCheckInterval(-1.0, 1.0))
arccos = _MaskedUnaryOperation(umath.arccos, 0.0,
                               _DomainCheckInterval(-1.0, 1.0))
arccosh = _MaskedUnaryOperation(umath.arccosh, 1.0,
                                _DomainGreaterEqual(1.0))
arctanh = _MaskedUnaryOperation(umath.arctanh, 0.0,
                                _DomainCheckInterval(-1.0 + 1e-15, 1.0 - 1e-15))

# Binary ufuncs
add = _MaskedBinaryOperation(umath.add)
subtract = _MaskedBinaryOperation(umath.subtract)
multiply = _MaskedBinaryOperation(umath.multiply, 1, 1)
arctan2 = _MaskedBinaryOperation(umath.arctan2, 0.0, 1.0)
equal = _MaskedBinaryOperation(umath.equal)
equal.reduce = None
not_equal = _MaskedBinaryOperation(umath.not_equal)
not_equal.reduce = None
less_equal = _MaskedBinaryOperation(umath.less_equal)
less_equal.reduce = None
greater_equal = _MaskedBinaryOperation(umath.greater_equal)
greater_equal.reduce = None
less = _MaskedBinaryOperation(umath.less)
less.reduce = None
greater = _MaskedBinaryOperation(umath.greater)
greater.reduce = None
logical_and = _MaskedBinaryOperation(umath.logical_and)
alltrue = _MaskedBinaryOperation(umath.logical_and, 1, 1).reduce
logical_or = _MaskedBinaryOperation(umath.logical_or)
sometrue = logical_or.reduce
logical_xor = _MaskedBinaryOperation(umath.logical_xor)
bitwise_and = _MaskedBinaryOperation(umath.bitwise_and)
bitwise_or = _MaskedBinaryOperation(umath.bitwise_or)
bitwise_xor = _MaskedBinaryOperation(umath.bitwise_xor)
hypot = _MaskedBinaryOperation(umath.hypot)

# Domained binary ufuncs
divide = _DomainedBinaryOperation(umath.divide, _DomainSafeDivide(), 0, 1)
true_divide = _DomainedBinaryOperation(umath.true_divide,
                                       _DomainSafeDivide(), 0, 1)
floor_divide = _DomainedBinaryOperation(umath.floor_divide,
                                        _DomainSafeDivide(), 0, 1)
remainder = _DomainedBinaryOperation(umath.remainder,
                                     _DomainSafeDivide(), 0, 1)
fmod = _DomainedBinaryOperation(umath.fmod, _DomainSafeDivide(), 0, 1)
mod = _DomainedBinaryOperation(umath.mod, _DomainSafeDivide(), 0, 1)


###############################################################################
#                        Mask creation functions                              #
###############################################################################


def _replace_dtype_fields_recursive(dtype, primitive_dtype):
    "Private function allowing recursion in _replace_dtype_fields."
    _recurse = _replace_dtype_fields_recursive

    # Do we have some name fields ?
    if dtype.names is not None:
        descr = []
        for name in dtype.names:
            field = dtype.fields[name]
            if len(field) == 3:
                # Prepend the title to the name
                name = (field[-1], name)
            descr.append((name, _recurse(field[0], primitive_dtype)))
        new_dtype = np.dtype(descr)

    # Is this some kind of composite a la (float,2)
    elif dtype.subdtype:
        descr = list(dtype.subdtype)
        descr[0] = _recurse(dtype.subdtype[0], primitive_dtype)
        new_dtype = np.dtype(tuple(descr))

    # this is a primitive type, so do a direct replacement
    else:
        new_dtype = primitive_dtype

    # preserve identity of dtypes
    if new_dtype == dtype:
        new_dtype = dtype

    return new_dtype


def _replace_dtype_fields(dtype, primitive_dtype):
    """
    Construct a dtype description list from a given dtype.

    Returns a new dtype object, with all fields and subtypes in the given type
    recursively replaced with `primitive_dtype`.

    Arguments are coerced to dtypes first.
    """
    dtype = np.dtype(dtype)
    primitive_dtype = np.dtype(primitive_dtype)
    return _replace_dtype_fields_recursive(dtype, primitive_dtype)


def make_mask_descr(ndtype):
    """
    Construct a dtype description list from a given dtype.

    Returns a new dtype object, with the type of all fields in `ndtype` to a
    boolean type. Field names are not altered.

    Parameters
    ----------
    ndtype : dtype
        The dtype to convert.

    Returns
    -------
    result : dtype
        A dtype that looks like `ndtype`, the type of all fields is boolean.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> dtype = np.dtype({'names':['foo', 'bar'],
    ...                   'formats':[np.float32, np.int64]})
    >>> dtype
    dtype([('foo', '<f4'), ('bar', '<i8')])
    >>> ma.make_mask_descr(dtype)
    dtype([('foo', '|b1'), ('bar', '|b1')])
    >>> ma.make_mask_descr(np.float32)
    dtype('bool')

    """
    return _replace_dtype_fields(ndtype, MaskType)


def getmask(a):
    """
    Return the mask of a masked array, or nomask.

    Return the mask of `a` as an ndarray if `a` is a `MaskedArray` and the
    mask is not `nomask`, else return `nomask`. To guarantee a full array
    of booleans of the same shape as a, use `getmaskarray`.

    Parameters
    ----------
    a : array_like
        Input `MaskedArray` for which the mask is required.

    See Also
    --------
    getdata : Return the data of a masked array as an ndarray.
    getmaskarray : Return the mask of a masked array, or full array of False.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = ma.masked_equal([[1,2],[3,4]], 2)
    >>> a
    masked_array(
      data=[[1, --],
            [3, 4]],
      mask=[[False,  True],
            [False, False]],
      fill_value=2)
    >>> ma.getmask(a)
    array([[False,  True],
           [False, False]])

    Equivalently use the `MaskedArray` `mask` attribute.

    >>> a.mask
    array([[False,  True],
           [False, False]])

    Result when mask == `nomask`

    >>> b = ma.masked_array([[1,2],[3,4]])
    >>> b
    masked_array(
      data=[[1, 2],
            [3, 4]],
      mask=False,
      fill_value=999999)
    >>> ma.nomask
    False
    >>> ma.getmask(b) == ma.nomask
    True
    >>> b.mask == ma.nomask
    True

    """
    return getattr(a, '_mask', nomask)


get_mask = getmask


def getmaskarray(arr):
    """
    Return the mask of a masked array, or full boolean array of False.

    Return the mask of `arr` as an ndarray if `arr` is a `MaskedArray` and
    the mask is not `nomask`, else return a full boolean array of False of
    the same shape as `arr`.

    Parameters
    ----------
    arr : array_like
        Input `MaskedArray` for which the mask is required.

    See Also
    --------
    getmask : Return the mask of a masked array, or nomask.
    getdata : Return the data of a masked array as an ndarray.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = ma.masked_equal([[1,2],[3,4]], 2)
    >>> a
    masked_array(
      data=[[1, --],
            [3, 4]],
      mask=[[False,  True],
            [False, False]],
      fill_value=2)
    >>> ma.getmaskarray(a)
    array([[False,  True],
           [False, False]])

    Result when mask == ``nomask``

    >>> b = ma.masked_array([[1,2],[3,4]])
    >>> b
    masked_array(
      data=[[1, 2],
            [3, 4]],
      mask=False,
      fill_value=999999)
    >>> ma.getmaskarray(b)
    array([[False, False],
           [False, False]])

    """
    mask = getmask(arr)
    if mask is nomask:
        mask = make_mask_none(np.shape(arr), getattr(arr, 'dtype', None))
    return mask


def is_mask(m):
    """
    Return True if m is a valid, standard mask.

    This function does not check the contents of the input, only that the
    type is MaskType. In particular, this function returns False if the
    mask has a flexible dtype.

    Parameters
    ----------
    m : array_like
        Array to test.

    Returns
    -------
    result : bool
        True if `m.dtype.type` is MaskType, False otherwise.

    See Also
    --------
    ma.isMaskedArray : Test whether input is an instance of MaskedArray.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> m = ma.masked_equal([0, 1, 0, 2, 3], 0)
    >>> m
    masked_array(data=[--, 1, --, 2, 3],
                 mask=[ True, False,  True, False, False],
           fill_value=0)
    >>> ma.is_mask(m)
    False
    >>> ma.is_mask(m.mask)
    True

    Input must be an ndarray (or have similar attributes)
    for it to be considered a valid mask.

    >>> m = [False, True, False]
    >>> ma.is_mask(m)
    False
    >>> m = np.array([False, True, False])
    >>> m
    array([False,  True, False])
    >>> ma.is_mask(m)
    True

    Arrays with complex dtypes don't return True.

    >>> dtype = np.dtype({'names':['monty', 'pithon'],
    ...                   'formats':[bool, bool]})
    >>> dtype
    dtype([('monty', '|b1'), ('pithon', '|b1')])
    >>> m = np.array([(True, False), (False, True), (True, False)],
    ...              dtype=dtype)
    >>> m
    array([( True, False), (False,  True), ( True, False)],
          dtype=[('monty', '?'), ('pithon', '?')])
    >>> ma.is_mask(m)
    False

    """
    try:
        return m.dtype.type is MaskType
    except AttributeError:
        return False


def _shrink_mask(m):
    """
    Shrink a mask to nomask if possible
    """
    if m.dtype.names is None and not m.any():
        return nomask
    else:
        return m


def make_mask(m, copy=False, shrink=True, dtype=MaskType):
    """
    Create a boolean mask from an array.

    Return `m` as a boolean mask, creating a copy if necessary or requested.
    The function can accept any sequence that is convertible to integers,
    or ``nomask``.  Does not require that contents must be 0s and 1s, values
    of 0 are interpreted as False, everything else as True.

    Parameters
    ----------
    m : array_like
        Potential mask.
    copy : bool, optional
        Whether to return a copy of `m` (True) or `m` itself (False).
    shrink : bool, optional
        Whether to shrink `m` to ``nomask`` if all its values are False.
    dtype : dtype, optional
        Data-type of the output mask. By default, the output mask has a
        dtype of MaskType (bool). If the dtype is flexible, each field has
        a boolean dtype. This is ignored when `m` is ``nomask``, in which
        case ``nomask`` is always returned.

    Returns
    -------
    result : ndarray
        A boolean mask derived from `m`.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> m = [True, False, True, True]
    >>> ma.make_mask(m)
    array([ True, False,  True,  True])
    >>> m = [1, 0, 1, 1]
    >>> ma.make_mask(m)
    array([ True, False,  True,  True])
    >>> m = [1, 0, 2, -3]
    >>> ma.make_mask(m)
    array([ True, False,  True,  True])

    Effect of the `shrink` parameter.

    >>> m = np.zeros(4)
    >>> m
    array([0., 0., 0., 0.])
    >>> ma.make_mask(m)
    False
    >>> ma.make_mask(m, shrink=False)
    array([False, False, False, False])

    Using a flexible `dtype`.

    >>> m = [1, 0, 1, 1]
    >>> n = [0, 1, 0, 0]
    >>> arr = []
    >>> for man, mouse in zip(m, n):
    ...     arr.append((man, mouse))
    >>> arr
    [(1, 0), (0, 1), (1, 0), (1, 0)]
    >>> dtype = np.dtype({'names':['man', 'mouse'],
    ...                   'formats':[np.int64, np.int64]})
    >>> arr = np.array(arr, dtype=dtype)
    >>> arr
    array([(1, 0), (0, 1), (1, 0), (1, 0)],
          dtype=[('man', '<i8'), ('mouse', '<i8')])
    >>> ma.make_mask(arr, dtype=dtype)
    array([(True, False), (False, True), (True, False), (True, False)],
          dtype=[('man', '|b1'), ('mouse', '|b1')])

    """
    if m is nomask:
        return nomask

    # Make sure the input dtype is valid.
    dtype = make_mask_descr(dtype)

    # legacy boolean special case: "existence of fields implies true"
    if isinstance(m, ndarray) and m.dtype.fields and dtype == np.bool_:
        return np.ones(m.shape, dtype=dtype)

    # Fill the mask in case there are missing data; turn it into an ndarray.
    result = np.array(filled(m, True), copy=copy, dtype=dtype, subok=True)
    # Bas les masques !
    if shrink:
        result = _shrink_mask(result)
    return result


def make_mask_none(newshape, dtype=None):
    """
    Return a boolean mask of the given shape, filled with False.

    This function returns a boolean ndarray with all entries False, that can
    be used in common mask manipulations. If a complex dtype is specified, the
    type of each field is converted to a boolean type.

    Parameters
    ----------
    newshape : tuple
        A tuple indicating the shape of the mask.
    dtype : {None, dtype}, optional
        If None, use a MaskType instance. Otherwise, use a new datatype with
        the same fields as `dtype`, converted to boolean types.

    Returns
    -------
    result : ndarray
        An ndarray of appropriate shape and dtype, filled with False.

    See Also
    --------
    make_mask : Create a boolean mask from an array.
    make_mask_descr : Construct a dtype description list from a given dtype.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> ma.make_mask_none((3,))
    array([False, False, False])

    Defining a more complex dtype.

    >>> dtype = np.dtype({'names':['foo', 'bar'],
    ...                   'formats':[np.float32, np.int64]})
    >>> dtype
    dtype([('foo', '<f4'), ('bar', '<i8')])
    >>> ma.make_mask_none((3,), dtype=dtype)
    array([(False, False), (False, False), (False, False)],
          dtype=[('foo', '|b1'), ('bar', '|b1')])

    """
    if dtype is None:
        result = np.zeros(newshape, dtype=MaskType)
    else:
        result = np.zeros(newshape, dtype=make_mask_descr(dtype))
    return result


def _recursive_mask_or(m1, m2, newmask):
    names = m1.dtype.names
    for name in names:
        current1 = m1[name]
        if current1.dtype.names is not None:
            _recursive_mask_or(current1, m2[name], newmask[name])
        else:
            umath.logical_or(current1, m2[name], newmask[name])


def mask_or(m1, m2, copy=False, shrink=True):
    """
    Combine two masks with the ``logical_or`` operator.

    The result may be a view on `m1` or `m2` if the other is `nomask`
    (i.e. False).

    Parameters
    ----------
    m1, m2 : array_like
        Input masks.
    copy : bool, optional
        If copy is False and one of the inputs is `nomask`, return a view
        of the other input mask. Defaults to False.
    shrink : bool, optional
        Whether to shrink the output to `nomask` if all its values are
        False. Defaults to True.

    Returns
    -------
    mask : output mask
        The result masks values that are masked in either `m1` or `m2`.

    Raises
    ------
    ValueError
        If `m1` and `m2` have different flexible dtypes.

    Examples
    --------
    >>> m1 = np.ma.make_mask([0, 1, 1, 0])
    >>> m2 = np.ma.make_mask([1, 0, 0, 0])
    >>> np.ma.mask_or(m1, m2)
    array([ True,  True,  True, False])

    """

    if (m1 is nomask) or (m1 is False):
        dtype = getattr(m2, 'dtype', MaskType)
        return make_mask(m2, copy=copy, shrink=shrink, dtype=dtype)
    if (m2 is nomask) or (m2 is False):
        dtype = getattr(m1, 'dtype', MaskType)
        return make_mask(m1, copy=copy, shrink=shrink, dtype=dtype)
    if m1 is m2 and is_mask(m1):
        return m1
    (dtype1, dtype2) = (getattr(m1, 'dtype', None), getattr(m2, 'dtype', None))
    if dtype1 != dtype2:
        raise ValueError("Incompatible dtypes '%s'<>'%s'" % (dtype1, dtype2))
    if dtype1.names is not None:
        # Allocate an output mask array with the properly broadcast shape.
        newmask = np.empty(np.broadcast(m1, m2).shape, dtype1)
        _recursive_mask_or(m1, m2, newmask)
        return newmask
    return make_mask(umath.logical_or(m1, m2), copy=copy, shrink=shrink)


def flatten_mask(mask):
    """
    Returns a completely flattened version of the mask, where nested fields
    are collapsed.

    Parameters
    ----------
    mask : array_like
        Input array, which will be interpreted as booleans.

    Returns
    -------
    flattened_mask : ndarray of bools
        The flattened input.

    Examples
    --------
    >>> mask = np.array([0, 0, 1])
    >>> np.ma.flatten_mask(mask)
    array([False, False,  True])

    >>> mask = np.array([(0, 0), (0, 1)], dtype=[('a', bool), ('b', bool)])
    >>> np.ma.flatten_mask(mask)
    array([False, False, False,  True])

    >>> mdtype = [('a', bool), ('b', [('ba', bool), ('bb', bool)])]
    >>> mask = np.array([(0, (0, 0)), (0, (0, 1))], dtype=mdtype)
    >>> np.ma.flatten_mask(mask)
    array([False, False, False, False, False,  True])

    """

    def _flatmask(mask):
        "Flatten the mask and returns a (maybe nested) sequence of booleans."
        mnames = mask.dtype.names
        if mnames is not None:
            return [flatten_mask(mask[name]) for name in mnames]
        else:
            return mask

    def _flatsequence(sequence):
        "Generates a flattened version of the sequence."
        try:
            for element in sequence:
                if hasattr(element, '__iter__'):
                    yield from _flatsequence(element)
                else:
                    yield element
        except TypeError:
            yield sequence

    mask = np.asarray(mask)
    flattened = _flatsequence(_flatmask(mask))
    return np.array([_ for _ in flattened], dtype=bool)


def _check_mask_axis(mask, axis, keepdims=np._NoValue):
    "Check whether there are masked values along the given axis"
    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}
    if mask is not nomask:
        return mask.all(axis=axis, **kwargs)
    return nomask


###############################################################################
#                             Masking functions                               #
###############################################################################

def masked_where(condition, a, copy=True):
    """
    Mask an array where a condition is met.

    Return `a` as an array masked where `condition` is True.
    Any masked values of `a` or `condition` are also masked in the output.

    Parameters
    ----------
    condition : array_like
        Masking condition.  When `condition` tests floating point values for
        equality, consider using ``masked_values`` instead.
    a : array_like
        Array to mask.
    copy : bool
        If True (default) make a copy of `a` in the result.  If False modify
        `a` in place and return a view.

    Returns
    -------
    result : MaskedArray
        The result of masking `a` where `condition` is True.

    See Also
    --------
    masked_values : Mask using floating point equality.
    masked_equal : Mask where equal to a given value.
    masked_not_equal : Mask where `not` equal to a given value.
    masked_less_equal : Mask where less than or equal to a given value.
    masked_greater_equal : Mask where greater than or equal to a given value.
    masked_less : Mask where less than a given value.
    masked_greater : Mask where greater than a given value.
    masked_inside : Mask inside a given interval.
    masked_outside : Mask outside a given interval.
    masked_invalid : Mask invalid values (NaNs or infs).

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_where(a <= 2, a)
    masked_array(data=[--, --, --, 3],
                 mask=[ True,  True,  True, False],
           fill_value=999999)

    Mask array `b` conditional on `a`.

    >>> b = ['a', 'b', 'c', 'd']
    >>> ma.masked_where(a == 2, b)
    masked_array(data=['a', 'b', --, 'd'],
                 mask=[False, False,  True, False],
           fill_value='N/A',
                dtype='<U1')

    Effect of the `copy` argument.

    >>> c = ma.masked_where(a <= 2, a)
    >>> c
    masked_array(data=[--, --, --, 3],
                 mask=[ True,  True,  True, False],
           fill_value=999999)
    >>> c[0] = 99
    >>> c
    masked_array(data=[99, --, --, 3],
                 mask=[False,  True,  True, False],
           fill_value=999999)
    >>> a
    array([0, 1, 2, 3])
    >>> c = ma.masked_where(a <= 2, a, copy=False)
    >>> c[0] = 99
    >>> c
    masked_array(data=[99, --, --, 3],
                 mask=[False,  True,  True, False],
           fill_value=999999)
    >>> a
    array([99,  1,  2,  3])

    When `condition` or `a` contain masked values.

    >>> a = np.arange(4)
    >>> a = ma.masked_where(a == 2, a)
    >>> a
    masked_array(data=[0, 1, --, 3],
                 mask=[False, False,  True, False],
           fill_value=999999)
    >>> b = np.arange(4)
    >>> b = ma.masked_where(b == 0, b)
    >>> b
    masked_array(data=[--, 1, 2, 3],
                 mask=[ True, False, False, False],
           fill_value=999999)
    >>> ma.masked_where(a == 3, b)
    masked_array(data=[--, 1, --, --],
                 mask=[ True, False,  True,  True],
           fill_value=999999)

    """
    # Make sure that condition is a valid standard-type mask.
    cond = make_mask(condition, shrink=False)
    a = np.array(a, copy=copy, subok=True)

    (cshape, ashape) = (cond.shape, a.shape)
    if cshape and cshape != ashape:
        raise IndexError("Inconsistent shape between the condition and the input"
                         " (got %s and %s)" % (cshape, ashape))
    if hasattr(a, '_mask'):
        cond = mask_or(cond, a._mask)
        cls = type(a)
    else:
        cls = MaskedArray
    result = a.view(cls)
    # Assign to *.mask so that structured masks are handled correctly.
    result.mask = _shrink_mask(cond)
    # There is no view of a boolean so when 'a' is a MaskedArray with nomask
    # the update to the result's mask has no effect.
    if not copy and hasattr(a, '_mask') and getmask(a) is nomask:
        a._mask = result._mask.view()
    return result


def masked_greater(x, value, copy=True):
    """
    Mask an array where greater than a given value.

    This function is a shortcut to ``masked_where``, with
    `condition` = (x > value).

    See Also
    --------
    masked_where : Mask where a condition is met.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_greater(a, 2)
    masked_array(data=[0, 1, 2, --],
                 mask=[False, False, False,  True],
           fill_value=999999)

    """
    return masked_where(greater(x, value), x, copy=copy)


def masked_greater_equal(x, value, copy=True):
    """
    Mask an array where greater than or equal to a given value.

    This function is a shortcut to ``masked_where``, with
    `condition` = (x >= value).

    See Also
    --------
    masked_where : Mask where a condition is met.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_greater_equal(a, 2)
    masked_array(data=[0, 1, --, --],
                 mask=[False, False,  True,  True],
           fill_value=999999)

    """
    return masked_where(greater_equal(x, value), x, copy=copy)


def masked_less(x, value, copy=True):
    """
    Mask an array where less than a given value.

    This function is a shortcut to ``masked_where``, with
    `condition` = (x < value).

    See Also
    --------
    masked_where : Mask where a condition is met.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_less(a, 2)
    masked_array(data=[--, --, 2, 3],
                 mask=[ True,  True, False, False],
           fill_value=999999)

    """
    return masked_where(less(x, value), x, copy=copy)


def masked_less_equal(x, value, copy=True):
    """
    Mask an array where less than or equal to a given value.

    This function is a shortcut to ``masked_where``, with
    `condition` = (x <= value).

    See Also
    --------
    masked_where : Mask where a condition is met.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_less_equal(a, 2)
    masked_array(data=[--, --, --, 3],
                 mask=[ True,  True,  True, False],
           fill_value=999999)

    """
    return masked_where(less_equal(x, value), x, copy=copy)


def masked_not_equal(x, value, copy=True):
    """
    Mask an array where `not` equal to a given value.

    This function is a shortcut to ``masked_where``, with
    `condition` = (x != value).

    See Also
    --------
    masked_where : Mask where a condition is met.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_not_equal(a, 2)
    masked_array(data=[--, --, 2, --],
                 mask=[ True,  True, False,  True],
           fill_value=999999)

    """
    return masked_where(not_equal(x, value), x, copy=copy)


def masked_equal(x, value, copy=True):
    """
    Mask an array where equal to a given value.

    Return a MaskedArray, masked where the data in array `x` are
    equal to `value`. The fill_value of the returned MaskedArray
    is set to `value`.

    For floating point arrays, consider using ``masked_values(x, value)``.

    See Also
    --------
    masked_where : Mask where a condition is met.
    masked_values : Mask using floating point equality.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(4)
    >>> a
    array([0, 1, 2, 3])
    >>> ma.masked_equal(a, 2)
    masked_array(data=[0, 1, --, 3],
                 mask=[False, False,  True, False],
           fill_value=2)

    """
    output = masked_where(equal(x, value), x, copy=copy)
    output.fill_value = value
    return output


def masked_inside(x, v1, v2, copy=True):
    """
    Mask an array inside a given interval.

    Shortcut to ``masked_where``, where `condition` is True for `x` inside
    the interval [v1,v2] (v1 <= x <= v2).  The boundaries `v1` and `v2`
    can be given in either order.

    See Also
    --------
    masked_where : Mask where a condition is met.

    Notes
    -----
    The array `x` is prefilled with its filling value.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1]
    >>> ma.masked_inside(x, -0.3, 0.3)
    masked_array(data=[0.31, 1.2, --, --, -0.4, -1.1],
                 mask=[False, False,  True,  True, False, False],
           fill_value=1e+20)

    The order of `v1` and `v2` doesn't matter.

    >>> ma.masked_inside(x, 0.3, -0.3)
    masked_array(data=[0.31, 1.2, --, --, -0.4, -1.1],
                 mask=[False, False,  True,  True, False, False],
           fill_value=1e+20)

    """
    if v2 < v1:
        (v1, v2) = (v2, v1)
    xf = filled(x)
    condition = (xf >= v1) & (xf <= v2)
    return masked_where(condition, x, copy=copy)


def masked_outside(x, v1, v2, copy=True):
    """
    Mask an array outside a given interval.

    Shortcut to ``masked_where``, where `condition` is True for `x` outside
    the interval [v1,v2] (x < v1)|(x > v2).
    The boundaries `v1` and `v2` can be given in either order.

    See Also
    --------
    masked_where : Mask where a condition is met.

    Notes
    -----
    The array `x` is prefilled with its filling value.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = [0.31, 1.2, 0.01, 0.2, -0.4, -1.1]
    >>> ma.masked_outside(x, -0.3, 0.3)
    masked_array(data=[--, --, 0.01, 0.2, --, --],
                 mask=[ True,  True, False, False,  True,  True],
           fill_value=1e+20)

    The order of `v1` and `v2` doesn't matter.

    >>> ma.masked_outside(x, 0.3, -0.3)
    masked_array(data=[--, --, 0.01, 0.2, --, --],
                 mask=[ True,  True, False, False,  True,  True],
           fill_value=1e+20)

    """
    if v2 < v1:
        (v1, v2) = (v2, v1)
    xf = filled(x)
    condition = (xf < v1) | (xf > v2)
    return masked_where(condition, x, copy=copy)


def masked_object(x, value, copy=True, shrink=True):
    """
    Mask the array `x` where the data are exactly equal to value.

    This function is similar to `masked_values`, but only suitable
    for object arrays: for floating point, use `masked_values` instead.

    Parameters
    ----------
    x : array_like
        Array to mask
    value : object
        Comparison value
    copy : {True, False}, optional
        Whether to return a copy of `x`.
    shrink : {True, False}, optional
        Whether to collapse a mask full of False to nomask

    Returns
    -------
    result : MaskedArray
        The result of masking `x` where equal to `value`.

    See Also
    --------
    masked_where : Mask where a condition is met.
    masked_equal : Mask where equal to a given value (integers).
    masked_values : Mask using floating point equality.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> food = np.array(['green_eggs', 'ham'], dtype=object)
    >>> # don't eat spoiled food
    >>> eat = ma.masked_object(food, 'green_eggs')
    >>> eat
    masked_array(data=[--, 'ham'],
                 mask=[ True, False],
           fill_value='green_eggs',
                dtype=object)
    >>> # plain ol` ham is boring
    >>> fresh_food = np.array(['cheese', 'ham', 'pineapple'], dtype=object)
    >>> eat = ma.masked_object(fresh_food, 'green_eggs')
    >>> eat
    masked_array(data=['cheese', 'ham', 'pineapple'],
                 mask=False,
           fill_value='green_eggs',
                dtype=object)

    Note that `mask` is set to ``nomask`` if possible.

    >>> eat
    masked_array(data=['cheese', 'ham', 'pineapple'],
                 mask=False,
           fill_value='green_eggs',
                dtype=object)

    """
    if isMaskedArray(x):
        condition = umath.equal(x._data, value)
        mask = x._mask
    else:
        condition = umath.equal(np.asarray(x), value)
        mask = nomask
    mask = mask_or(mask, make_mask(condition, shrink=shrink))
    return masked_array(x, mask=mask, copy=copy, fill_value=value)


def masked_values(x, value, rtol=1e-5, atol=1e-8, copy=True, shrink=True):
    """
    Mask using floating point equality.

    Return a MaskedArray, masked where the data in array `x` are approximately
    equal to `value`, determined using `isclose`. The default tolerances for
    `masked_values` are the same as those for `isclose`.

    For integer types, exact equality is used, in the same way as
    `masked_equal`.

    The fill_value is set to `value` and the mask is set to ``nomask`` if
    possible.

    Parameters
    ----------
    x : array_like
        Array to mask.
    value : float
        Masking value.
    rtol, atol : float, optional
        Tolerance parameters passed on to `isclose`
    copy : bool, optional
        Whether to return a copy of `x`.
    shrink : bool, optional
        Whether to collapse a mask full of False to ``nomask``.

    Returns
    -------
    result : MaskedArray
        The result of masking `x` where approximately equal to `value`.

    See Also
    --------
    masked_where : Mask where a condition is met.
    masked_equal : Mask where equal to a given value (integers).

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = np.array([1, 1.1, 2, 1.1, 3])
    >>> ma.masked_values(x, 1.1)
    masked_array(data=[1.0, --, 2.0, --, 3.0],
                 mask=[False,  True, False,  True, False],
           fill_value=1.1)

    Note that `mask` is set to ``nomask`` if possible.

    >>> ma.masked_values(x, 2.1)
    masked_array(data=[1. , 1.1, 2. , 1.1, 3. ],
                 mask=False,
           fill_value=2.1)

    Unlike `masked_equal`, `masked_values` can perform approximate equalities. 

    >>> ma.masked_values(x, 2.1, atol=1e-1)
    masked_array(data=[1.0, 1.1, --, 1.1, 3.0],
                 mask=[False, False,  True, False, False],
           fill_value=2.1)

    """
    xnew = filled(x, value)
    if np.issubdtype(xnew.dtype, np.floating):
        mask = np.isclose(xnew, value, atol=atol, rtol=rtol)
    else:
        mask = umath.equal(xnew, value)
    ret = masked_array(xnew, mask=mask, copy=copy, fill_value=value)
    if shrink:
        ret.shrink_mask()
    return ret


def masked_invalid(a, copy=True):
    """
    Mask an array where invalid values occur (NaNs or infs).

    This function is a shortcut to ``masked_where``, with
    `condition` = ~(np.isfinite(a)). Any pre-existing mask is conserved.
    Only applies to arrays with a dtype where NaNs or infs make sense
    (i.e. floating point types), but accepts any array_like object.

    See Also
    --------
    masked_where : Mask where a condition is met.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(5, dtype=float)
    >>> a[2] = np.NaN
    >>> a[3] = np.PINF
    >>> a
    array([ 0.,  1., nan, inf,  4.])
    >>> ma.masked_invalid(a)
    masked_array(data=[0.0, 1.0, --, --, 4.0],
                 mask=[False, False,  True,  True, False],
           fill_value=1e+20)

    """
    a = np.array(a, copy=False, subok=True)
    res = masked_where(~(np.isfinite(a)), a, copy=copy)
    # masked_invalid previously never returned nomask as a mask and doing so
    # threw off matplotlib (gh-22842).  So use shrink=False:
    if res._mask is nomask:
        res._mask = make_mask_none(res.shape, res.dtype)
    return res

###############################################################################
#                            Printing options                                 #
###############################################################################


class _MaskedPrintOption:
    """
    Handle the string used to represent missing data in a masked array.

    """

    def __init__(self, display):
        """
        Create the masked_print_option object.

        """
        self._display = display
        self._enabled = True

    def display(self):
        """
        Display the string to print for masked values.

        """
        return self._display

    def set_display(self, s):
        """
        Set the string to print for masked values.

        """
        self._display = s

    def enabled(self):
        """
        Is the use of the display value enabled?

        """
        return self._enabled

    def enable(self, shrink=1):
        """
        Set the enabling shrink to `shrink`.

        """
        self._enabled = shrink

    def __str__(self):
        return str(self._display)

    __repr__ = __str__

# if you single index into a masked location you get this object.
masked_print_option = _MaskedPrintOption('--')


def _recursive_printoption(result, mask, printopt):
    """
    Puts printoptions in result where mask is True.

    Private function allowing for recursion

    """
    names = result.dtype.names
    if names is not None:
        for name in names:
            curdata = result[name]
            curmask = mask[name]
            _recursive_printoption(curdata, curmask, printopt)
    else:
        np.copyto(result, printopt, where=mask)
    return

# For better or worse, these end in a newline
_legacy_print_templates = dict(
    long_std=textwrap.dedent("""\
        masked_%(name)s(data =
         %(data)s,
        %(nlen)s        mask =
         %(mask)s,
        %(nlen)s  fill_value = %(fill)s)
        """),
    long_flx=textwrap.dedent("""\
        masked_%(name)s(data =
         %(data)s,
        %(nlen)s        mask =
         %(mask)s,
        %(nlen)s  fill_value = %(fill)s,
        %(nlen)s       dtype = %(dtype)s)
        """),
    short_std=textwrap.dedent("""\
        masked_%(name)s(data = %(data)s,
        %(nlen)s        mask = %(mask)s,
        %(nlen)s  fill_value = %(fill)s)
        """),
    short_flx=textwrap.dedent("""\
        masked_%(name)s(data = %(data)s,
        %(nlen)s        mask = %(mask)s,
        %(nlen)s  fill_value = %(fill)s,
        %(nlen)s       dtype = %(dtype)s)
        """)
)

###############################################################################
#                          MaskedArray class                                  #
###############################################################################


def _recursive_filled(a, mask, fill_value):
    """
    Recursively fill `a` with `fill_value`.

    """
    names = a.dtype.names
    for name in names:
        current = a[name]
        if current.dtype.names is not None:
            _recursive_filled(current, mask[name], fill_value[name])
        else:
            np.copyto(current, fill_value[name], where=mask[name])


def flatten_structured_array(a):
    """
    Flatten a structured array.

    The data type of the output is chosen such that it can represent all of the
    (nested) fields.

    Parameters
    ----------
    a : structured array

    Returns
    -------
    output : masked array or ndarray
        A flattened masked array if the input is a masked array, otherwise a
        standard ndarray.

    Examples
    --------
    >>> ndtype = [('a', int), ('b', float)]
    >>> a = np.array([(1, 1), (2, 2)], dtype=ndtype)
    >>> np.ma.flatten_structured_array(a)
    array([[1., 1.],
           [2., 2.]])

    """

    def flatten_sequence(iterable):
        """
        Flattens a compound of nested iterables.

        """
        for elm in iter(iterable):
            if hasattr(elm, '__iter__'):
                yield from flatten_sequence(elm)
            else:
                yield elm

    a = np.asanyarray(a)
    inishape = a.shape
    a = a.ravel()
    if isinstance(a, MaskedArray):
        out = np.array([tuple(flatten_sequence(d.item())) for d in a._data])
        out = out.view(MaskedArray)
        out._mask = np.array([tuple(flatten_sequence(d.item()))
                              for d in getmaskarray(a)])
    else:
        out = np.array([tuple(flatten_sequence(d.item())) for d in a])
    if len(inishape) > 1:
        newshape = list(out.shape)
        newshape[0] = inishape
        out.shape = tuple(flatten_sequence(newshape))
    return out


def _arraymethod(funcname, onmask=True):
    """
    Return a class method wrapper around a basic array method.

    Creates a class method which returns a masked array, where the new
    ``_data`` array is the output of the corresponding basic method called
    on the original ``_data``.

    If `onmask` is True, the new mask is the output of the method called
    on the initial mask. Otherwise, the new mask is just a reference
    to the initial mask.

    Parameters
    ----------
    funcname : str
        Name of the function to apply on data.
    onmask : bool
        Whether the mask must be processed also (True) or left
        alone (False). Default is True. Make available as `_onmask`
        attribute.

    Returns
    -------
    method : instancemethod
        Class method wrapper of the specified basic array method.

    """
    def wrapped_method(self, *args, **params):
        result = getattr(self._data, funcname)(*args, **params)
        result = result.view(type(self))
        result._update_from(self)
        mask = self._mask
        if not onmask:
            result.__setmask__(mask)
        elif mask is not nomask:
            # __setmask__ makes a copy, which we don't want
            result._mask = getattr(mask, funcname)(*args, **params)
        return result
    methdoc = getattr(ndarray, funcname, None) or getattr(np, funcname, None)
    if methdoc is not None:
        wrapped_method.__doc__ = methdoc.__doc__
    wrapped_method.__name__ = funcname
    return wrapped_method


class MaskedIterator:
    """
    Flat iterator object to iterate over masked arrays.

    A `MaskedIterator` iterator is returned by ``x.flat`` for any masked array
    `x`. It allows iterating over the array as if it were a 1-D array,
    either in a for-loop or by calling its `next` method.

    Iteration is done in C-contiguous style, with the last index varying the
    fastest. The iterator can also be indexed using basic slicing or
    advanced indexing.

    See Also
    --------
    MaskedArray.flat : Return a flat iterator over an array.
    MaskedArray.flatten : Returns a flattened copy of an array.

    Notes
    -----
    `MaskedIterator` is not exported by the `ma` module. Instead of
    instantiating a `MaskedIterator` directly, use `MaskedArray.flat`.

    Examples
    --------
    >>> x = np.ma.array(arange(6).reshape(2, 3))
    >>> fl = x.flat
    >>> type(fl)
    <class 'numpy.ma.core.MaskedIterator'>
    >>> for item in fl:
    ...     print(item)
    ...
    0
    1
    2
    3
    4
    5

    Extracting more than a single element b indexing the `MaskedIterator`
    returns a masked array:

    >>> fl[2:4]
    masked_array(data = [2 3],
                 mask = False,
           fill_value = 999999)

    """

    def __init__(self, ma):
        self.ma = ma
        self.dataiter = ma._data.flat

        if ma._mask is nomask:
            self.maskiter = None
        else:
            self.maskiter = ma._mask.flat

    def __iter__(self):
        return self

    def __getitem__(self, indx):
        result = self.dataiter.__getitem__(indx).view(type(self.ma))
        if self.maskiter is not None:
            _mask = self.maskiter.__getitem__(indx)
            if isinstance(_mask, ndarray):
                # set shape to match that of data; this is needed for matrices
                _mask.shape = result.shape
                result._mask = _mask
            elif isinstance(_mask, np.void):
                return mvoid(result, mask=_mask, hardmask=self.ma._hardmask)
            elif _mask:  # Just a scalar, masked
                return masked
        return result

    # This won't work if ravel makes a copy
    def __setitem__(self, index, value):
        self.dataiter[index] = getdata(value)
        if self.maskiter is not None:
            self.maskiter[index] = getmaskarray(value)

    def __next__(self):
        """
        Return the next value, or raise StopIteration.

        Examples
        --------
        >>> x = np.ma.array([3, 2], mask=[0, 1])
        >>> fl = x.flat
        >>> next(fl)
        3
        >>> next(fl)
        masked
        >>> next(fl)
        Traceback (most recent call last):
          ...
        StopIteration

        """
        d = next(self.dataiter)
        if self.maskiter is not None:
            m = next(self.maskiter)
            if isinstance(m, np.void):
                return mvoid(d, mask=m, hardmask=self.ma._hardmask)
            elif m:  # Just a scalar, masked
                return masked
        return d


class MaskedArray(ndarray):
    """
    An array class with possibly masked values.

    Masked values of True exclude the corresponding element from any
    computation.

    Construction::

      x = MaskedArray(data, mask=nomask, dtype=None, copy=False, subok=True,
                      ndmin=0, fill_value=None, keep_mask=True, hard_mask=None,
                      shrink=True, order=None)

    Parameters
    ----------
    data : array_like
        Input data.
    mask : sequence, optional
        Mask. Must be convertible to an array of booleans with the same
        shape as `data`. True indicates a masked (i.e. invalid) data.
    dtype : dtype, optional
        Data type of the output.
        If `dtype` is None, the type of the data argument (``data.dtype``)
        is used. If `dtype` is not None and different from ``data.dtype``,
        a copy is performed.
    copy : bool, optional
        Whether to copy the input data (True), or to use a reference instead.
        Default is False.
    subok : bool, optional
        Whether to return a subclass of `MaskedArray` if possible (True) or a
        plain `MaskedArray`. Default is True.
    ndmin : int, optional
        Minimum number of dimensions. Default is 0.
    fill_value : scalar, optional
        Value used to fill in the masked values when necessary.
        If None, a default based on the data-type is used.
    keep_mask : bool, optional
        Whether to combine `mask` with the mask of the input data, if any
        (True), or to use only `mask` for the output (False). Default is True.
    hard_mask : bool, optional
        Whether to use a hard mask or not. With a hard mask, masked values
        cannot be unmasked. Default is False.
    shrink : bool, optional
        Whether to force compression of an empty mask. Default is True.
    order : {'C', 'F', 'A'}, optional
        Specify the order of the array.  If order is 'C', then the array
        will be in C-contiguous order (last-index varies the fastest).
        If order is 'F', then the returned array will be in
        Fortran-contiguous order (first-index varies the fastest).
        If order is 'A' (default), then the returned array may be
        in any order (either C-, Fortran-contiguous, or even discontiguous),
        unless a copy is required, in which case it will be C-contiguous.

    Examples
    --------

    The ``mask`` can be initialized with an array of boolean values
    with the same shape as ``data``.

    >>> data = np.arange(6).reshape((2, 3))
    >>> np.ma.MaskedArray(data, mask=[[False, True, False],
    ...                               [False, False, True]])
    masked_array(
      data=[[0, --, 2],
            [3, 4, --]],
      mask=[[False,  True, False],
            [False, False,  True]],
      fill_value=999999)

    Alternatively, the ``mask`` can be initialized to homogeneous boolean
    array with the same shape as ``data`` by passing in a scalar
    boolean value:

    >>> np.ma.MaskedArray(data, mask=False)
    masked_array(
      data=[[0, 1, 2],
            [3, 4, 5]],
      mask=[[False, False, False],
            [False, False, False]],
      fill_value=999999)

    >>> np.ma.MaskedArray(data, mask=True)
    masked_array(
      data=[[--, --, --],
            [--, --, --]],
      mask=[[ True,  True,  True],
            [ True,  True,  True]],
      fill_value=999999,
      dtype=int64)

    .. note::
        The recommended practice for initializing ``mask`` with a scalar
        boolean value is to use ``True``/``False`` rather than
        ``np.True_``/``np.False_``. The reason is :attr:`nomask`
        is represented internally as ``np.False_``.

        >>> np.False_ is np.ma.nomask
        True

    """

    __array_priority__ = 15
    _defaultmask = nomask
    _defaulthardmask = False
    _baseclass = ndarray

    # Maximum number of elements per axis used when printing an array. The
    # 1d case is handled separately because we need more values in this case.
    _print_width = 100
    _print_width_1d = 1500

    def __new__(cls, data=None, mask=nomask, dtype=None, copy=False,
                subok=True, ndmin=0, fill_value=None, keep_mask=True,
                hard_mask=None, shrink=True, order=None):
        """
        Create a new masked array from scratch.

        Notes
        -----
        A masked array can also be created by taking a .view(MaskedArray).

        """
        # Process data.
        _data = np.array(data, dtype=dtype, copy=copy,
                         order=order, subok=True, ndmin=ndmin)
        _baseclass = getattr(data, '_baseclass', type(_data))
        # Check that we're not erasing the mask.
        if isinstance(data, MaskedArray) and (data.shape != _data.shape):
            copy = True

        # Here, we copy the _view_, so that we can attach new properties to it
        # we must never do .view(MaskedConstant), as that would create a new
        # instance of np.ma.masked, which make identity comparison fail
        if isinstance(data, cls) and subok and not isinstance(data, MaskedConstant):
            _data = ndarray.view(_data, type(data))
        else:
            _data = ndarray.view(_data, cls)

        # Handle the case where data is not a subclass of ndarray, but
        # still has the _mask attribute like MaskedArrays
        if hasattr(data, '_mask') and not isinstance(data, ndarray):
            _data._mask = data._mask
            # FIXME: should we set `_data._sharedmask = True`?
        # Process mask.
        # Type of the mask
        mdtype = make_mask_descr(_data.dtype)

        if mask is nomask:
            # Case 1. : no mask in input.
            # Erase the current mask ?
            if not keep_mask:
                # With a reduced version
                if shrink:
                    _data._mask = nomask
                # With full version
                else:
                    _data._mask = np.zeros(_data.shape, dtype=mdtype)
            # Check whether we missed something
            elif isinstance(data, (tuple, list)):
                try:
                    # If data is a sequence of masked array
                    mask = np.array(
                        [getmaskarray(np.asanyarray(m, dtype=_data.dtype))
                         for m in data], dtype=mdtype)
                except ValueError:
                    # If data is nested
                    mask = nomask
                # Force shrinking of the mask if needed (and possible)
                if (mdtype == MaskType) and mask.any():
                    _data._mask = mask
                    _data._sharedmask = False
            else:
                _data._sharedmask = not copy
                if copy:
                    _data._mask = _data._mask.copy()
                    # Reset the shape of the original mask
                    if getmask(data) is not nomask:
                        data._mask.shape = data.shape
        else:
            # Case 2. : With a mask in input.
            # If mask is boolean, create an array of True or False
            if mask is True and mdtype == MaskType:
                mask = np.ones(_data.shape, dtype=mdtype)
            elif mask is False and mdtype == MaskType:
                mask = np.zeros(_data.shape, dtype=mdtype)
            else:
                # Read the mask with the current mdtype
                try:
                    mask = np.array(mask, copy=copy, dtype=mdtype)
                # Or assume it's a sequence of bool/int
                except TypeError:
                    mask = np.array([tuple([m] * len(mdtype)) for m in mask],
                                    dtype=mdtype)
            # Make sure the mask and the data have the same shape
            if mask.shape != _data.shape:
                (nd, nm) = (_data.size, mask.size)
                if nm == 1:
                    mask = np.resize(mask, _data.shape)
                elif nm == nd:
                    mask = np.reshape(mask, _data.shape)
                else:
                    msg = "Mask and data not compatible: data size is %i, " + \
                          "mask size is %i."
                    raise MaskError(msg % (nd, nm))
                copy = True
            # Set the mask to the new value
            if _data._mask is nomask:
                _data._mask = mask
                _data._sharedmask = not copy
            else:
                if not keep_mask:
                    _data._mask = mask
                    _data._sharedmask = not copy
                else:
                    if _data.dtype.names is not None:
                        def _recursive_or(a, b):
                            "do a|=b on each field of a, recursively"
                            for name in a.dtype.names:
                                (af, bf) = (a[name], b[name])
                                if af.dtype.names is not None:
                                    _recursive_or(af, bf)
                                else:
                                    af |= bf

                        _recursive_or(_data._mask, mask)
                    else:
                        _data._mask = np.logical_or(mask, _data._mask)
                    _data._sharedmask = False
        # Update fill_value.
        if fill_value is None:
            fill_value = getattr(data, '_fill_value', None)
        # But don't run the check unless we have something to check.
        if fill_value is not None:
            _data._fill_value = _check_fill_value(fill_value, _data.dtype)
        # Process extra options ..
        if hard_mask is None:
            _data._hardmask = getattr(data, '_hardmask', False)
        else:
            _data._hardmask = hard_mask
        _data._baseclass = _baseclass
        return _data


    def _update_from(self, obj):
        """
        Copies some attributes of obj to self.

        """
        if isinstance(obj, ndarray):
            _baseclass = type(obj)
        else:
            _baseclass = ndarray
        # We need to copy the _basedict to avoid backward propagation
        _optinfo = {}
        _optinfo.update(getattr(obj, '_optinfo', {}))
        _optinfo.update(getattr(obj, '_basedict', {}))
        if not isinstance(obj, MaskedArray):
            _optinfo.update(getattr(obj, '__dict__', {}))
        _dict = dict(_fill_value=getattr(obj, '_fill_value', None),
                     _hardmask=getattr(obj, '_hardmask', False),
                     _sharedmask=getattr(obj, '_sharedmask', False),
                     _isfield=getattr(obj, '_isfield', False),
                     _baseclass=getattr(obj, '_baseclass', _baseclass),
                     _optinfo=_optinfo,
                     _basedict=_optinfo)
        self.__dict__.update(_dict)
        self.__dict__.update(_optinfo)
        return

    def __array_finalize__(self, obj):
        """
        Finalizes the masked array.

        """
        # Get main attributes.
        self._update_from(obj)

        # We have to decide how to initialize self.mask, based on
        # obj.mask. This is very difficult.  There might be some
        # correspondence between the elements in the array we are being
        # created from (= obj) and us. Or there might not. This method can
        # be called in all kinds of places for all kinds of reasons -- could
        # be empty_like, could be slicing, could be a ufunc, could be a view.
        # The numpy subclassing interface simply doesn't give us any way
        # to know, which means that at best this method will be based on
        # guesswork and heuristics. To make things worse, there isn't even any
        # clear consensus about what the desired behavior is. For instance,
        # most users think that np.empty_like(marr) -- which goes via this
        # method -- should return a masked array with an empty mask (see
        # gh-3404 and linked discussions), but others disagree, and they have
        # existing code which depends on empty_like returning an array that
        # matches the input mask.
        #
        # Historically our algorithm was: if the template object mask had the
        # same *number of elements* as us, then we used *it's mask object
        # itself* as our mask, so that writes to us would also write to the
        # original array. This is horribly broken in multiple ways.
        #
        # Now what we do instead is, if the template object mask has the same
        # number of elements as us, and we do not have the same base pointer
        # as the template object (b/c views like arr[...] should keep the same
        # mask), then we make a copy of the template object mask and use
        # that. This is also horribly broken but somewhat less so. Maybe.
        if isinstance(obj, ndarray):
            # XX: This looks like a bug -- shouldn't it check self.dtype
            # instead?
            if obj.dtype.names is not None:
                _mask = getmaskarray(obj)
            else:
                _mask = getmask(obj)

            # If self and obj point to exactly the same data, then probably
            # self is a simple view of obj (e.g., self = obj[...]), so they
            # should share the same mask. (This isn't 100% reliable, e.g. self
            # could be the first row of obj, or have strange strides, but as a
            # heuristic it's not bad.) In all other cases, we make a copy of
            # the mask, so that future modifications to 'self' do not end up
            # side-effecting 'obj' as well.
            if (_mask is not nomask and obj.__array_interface__["data"][0]
                    != self.__array_interface__["data"][0]):
                # We should make a copy. But we could get here via astype,
                # in which case the mask might need a new dtype as well
                # (e.g., changing to or from a structured dtype), and the
                # order could have changed. So, change the mask type if
                # needed and use astype instead of copy.
                if self.dtype == obj.dtype:
                    _mask_dtype = _mask.dtype
                else:
                    _mask_dtype = make_mask_descr(self.dtype)

                if self.flags.c_contiguous:
                    order = "C"
                elif self.flags.f_contiguous:
                    order = "F"
                else:
                    order = "K"

                _mask = _mask.astype(_mask_dtype, order)
            else:
                # Take a view so shape changes, etc., do not propagate back.
                _mask = _mask.view()
        else:
            _mask = nomask

        self._mask = _mask
        # Finalize the mask
        if self._mask is not nomask:
            try:
                self._mask.shape = self.shape
            except ValueError:
                self._mask = nomask
            except (TypeError, AttributeError):
                # When _mask.shape is not writable (because it's a void)
                pass

        # Finalize the fill_value
        if self._fill_value is not None:
            self._fill_value = _check_fill_value(self._fill_value, self.dtype)
        elif self.dtype.names is not None:
            # Finalize the default fill_value for structured arrays
            self._fill_value = _check_fill_value(None, self.dtype)

    def __array_wrap__(self, obj, context=None):
        """
        Special hook for ufuncs.

        Wraps the numpy array and sets the mask according to context.

        """
        if obj is self:  # for in-place operations
            result = obj
        else:
            result = obj.view(type(self))
            result._update_from(self)

        if context is not None:
            result._mask = result._mask.copy()
            func, args, out_i = context
            # args sometimes contains outputs (gh-10459), which we don't want
            input_args = args[:func.nin]
            m = reduce(mask_or, [getmaskarray(arg) for arg in input_args])
            # Get the domain mask
            domain = ufunc_domain.get(func, None)
            if domain is not None:
                # Take the domain, and make sure it's a ndarray
                with np.errstate(divide='ignore', invalid='ignore'):
                    d = filled(domain(*input_args), True)

                if d.any():
                    # Fill the result where the domain is wrong
                    try:
                        # Binary domain: take the last value
                        fill_value = ufunc_fills[func][-1]
                    except TypeError:
                        # Unary domain: just use this one
                        fill_value = ufunc_fills[func]
                    except KeyError:
                        # Domain not recognized, use fill_value instead
                        fill_value = self.fill_value

                    np.copyto(result, fill_value, where=d)

                    # Update the mask
                    if m is nomask:
                        m = d
                    else:
                        # Don't modify inplace, we risk back-propagation
                        m = (m | d)

            # Make sure the mask has the proper size
            if result is not self and result.shape == () and m:
                return masked
            else:
                result._mask = m
                result._sharedmask = False

        return result

    def view(self, dtype=None, type=None, fill_value=None):
        """
        Return a view of the MaskedArray data.

        Parameters
        ----------
        dtype : data-type or ndarray sub-class, optional
            Data-type descriptor of the returned view, e.g., float32 or int16.
            The default, None, results in the view having the same data-type
            as `a`. As with ``ndarray.view``, dtype can also be specified as
            an ndarray sub-class, which then specifies the type of the
            returned object (this is equivalent to setting the ``type``
            parameter).
        type : Python type, optional
            Type of the returned view, either ndarray or a subclass.  The
            default None results in type preservation.
        fill_value : scalar, optional
            The value to use for invalid entries (None by default).
            If None, then this argument is inferred from the passed `dtype`, or
            in its absence the original array, as discussed in the notes below.

        See Also
        --------
        numpy.ndarray.view : Equivalent method on ndarray object.

        Notes
        -----

        ``a.view()`` is used two different ways:

        ``a.view(some_dtype)`` or ``a.view(dtype=some_dtype)`` constructs a view
        of the array's memory with a different data-type.  This can cause a
        reinterpretation of the bytes of memory.

        ``a.view(ndarray_subclass)`` or ``a.view(type=ndarray_subclass)`` just
        returns an instance of `ndarray_subclass` that looks at the same array
        (same shape, dtype, etc.)  This does not cause a reinterpretation of the
        memory.

        If `fill_value` is not specified, but `dtype` is specified (and is not
        an ndarray sub-class), the `fill_value` of the MaskedArray will be
        reset. If neither `fill_value` nor `dtype` are specified (or if
        `dtype` is an ndarray sub-class), then the fill value is preserved.
        Finally, if `fill_value` is specified, but `dtype` is not, the fill
        value is set to the specified value.

        For ``a.view(some_dtype)``, if ``some_dtype`` has a different number of
        bytes per entry than the previous dtype (for example, converting a
        regular array to a structured array), then the behavior of the view
        cannot be predicted just from the superficial appearance of ``a`` (shown
        by ``print(a)``). It also depends on exactly how ``a`` is stored in
        memory. Therefore if ``a`` is C-ordered versus fortran-ordered, versus
        defined as a slice or transpose, etc., the view may give different
        results.
        """

        if dtype is None:
            if type is None:
                output = ndarray.view(self)
            else:
                output = ndarray.view(self, type)
        elif type is None:
            try:
                if issubclass(dtype, ndarray):
                    output = ndarray.view(self, dtype)
                    dtype = None
                else:
                    output = ndarray.view(self, dtype)
            except TypeError:
                output = ndarray.view(self, dtype)
        else:
            output = ndarray.view(self, dtype, type)

        # also make the mask be a view (so attr changes to the view's
        # mask do no affect original object's mask)
        # (especially important to avoid affecting np.masked singleton)
        if getmask(output) is not nomask:
            output._mask = output._mask.view()

        # Make sure to reset the _fill_value if needed
        if getattr(output, '_fill_value', None) is not None:
            if fill_value is None:
                if dtype is None:
                    pass  # leave _fill_value as is
                else:
                    output._fill_value = None
            else:
                output.fill_value = fill_value
        return output

    def __getitem__(self, indx):
        """
        x.__getitem__(y) <==> x[y]

        Return the item described by i, as a masked array.

        """
        # We could directly use ndarray.__getitem__ on self.
        # But then we would have to modify __array_finalize__ to prevent the
        # mask of being reshaped if it hasn't been set up properly yet
        # So it's easier to stick to the current version
        dout = self.data[indx]
        _mask = self._mask

        def _is_scalar(m):
            return not isinstance(m, np.ndarray)

        def _scalar_heuristic(arr, elem):
            """
            Return whether `elem` is a scalar result of indexing `arr`, or None
            if undecidable without promoting nomask to a full mask
            """
            # obviously a scalar
            if not isinstance(elem, np.ndarray):
                return True

            # object array scalar indexing can return anything
            elif arr.dtype.type is np.object_:
                if arr.dtype is not elem.dtype:
                    # elem is an array, but dtypes do not match, so must be
                    # an element
                    return True

            # well-behaved subclass that only returns 0d arrays when
            # expected - this is not a scalar
            elif type(arr).__getitem__ == ndarray.__getitem__:
                return False

            return None

        if _mask is not nomask:
            # _mask cannot be a subclass, so it tells us whether we should
            # expect a scalar. It also cannot be of dtype object.
            mout = _mask[indx]
            scalar_expected = _is_scalar(mout)

        else:
            # attempt to apply the heuristic to avoid constructing a full mask
            mout = nomask
            scalar_expected = _scalar_heuristic(self.data, dout)
            if scalar_expected is None:
                # heuristics have failed
                # construct a full array, so we can be certain. This is costly.
                # we could also fall back on ndarray.__getitem__(self.data, indx)
                scalar_expected = _is_scalar(getmaskarray(self)[indx])

        # Did we extract a single item?
        if scalar_expected:
            # A record
            if isinstance(dout, np.void):
                # We should always re-cast to mvoid, otherwise users can
                # change masks on rows that already have masked values, but not
                # on rows that have no masked values, which is inconsistent.
                return mvoid(dout, mask=mout, hardmask=self._hardmask)

            # special case introduced in gh-5962
            elif (self.dtype.type is np.object_ and
                  isinstance(dout, np.ndarray) and
                  dout is not masked):
                # If masked, turn into a MaskedArray, with everything masked.
                if mout:
                    return MaskedArray(dout, mask=True)
                else:
                    return dout

            # Just a scalar
            else:
                if mout:
                    return masked
                else:
                    return dout
        else:
            # Force dout to MA
            dout = dout.view(type(self))
            # Inherit attributes from self
            dout._update_from(self)
            # Check the fill_value
            if is_string_or_list_of_strings(indx):
                if self._fill_value is not None:
                    dout._fill_value = self._fill_value[indx]

                    # Something like gh-15895 has happened if this check fails.
                    # _fill_value should always be an ndarray.
                    if not isinstance(dout._fill_value, np.ndarray):
                        raise RuntimeError('Internal NumPy error.')
                    # If we're indexing a multidimensional field in a
                    # structured array (such as dtype("(2,)i2,(2,)i1")),
                    # dimensionality goes up (M[field].ndim == M.ndim +
                    # M.dtype[field].ndim).  That's fine for
                    # M[field] but problematic for M[field].fill_value
                    # which should have shape () to avoid breaking several
                    # methods. There is no great way out, so set to
                    # first element. See issue #6723.
                    if dout._fill_value.ndim > 0:
                        if not (dout._fill_value ==
                                dout._fill_value.flat[0]).all():
                            warnings.warn(
                                "Upon accessing multidimensional field "
                                f"{indx!s}, need to keep dimensionality "
                                "of fill_value at 0. Discarding "
                                "heterogeneous fill_value and setting "
                                f"all to {dout._fill_value[0]!s}.",
                                stacklevel=2)
                        # Need to use `.flat[0:1].squeeze(...)` instead of just
                        # `.flat[0]` to ensure the result is a 0d array and not
                        # a scalar.
                        dout._fill_value = dout._fill_value.flat[0:1].squeeze(axis=0)
                dout._isfield = True
            # Update the mask if needed
            if mout is not nomask:
                # set shape to match that of data; this is needed for matrices
                dout._mask = reshape(mout, dout.shape)
                dout._sharedmask = True
                # Note: Don't try to check for m.any(), that'll take too long
        return dout

    def __setitem__(self, indx, value):
        """
        x.__setitem__(i, y) <==> x[i]=y

        Set item described by index. If value is masked, masks those
        locations.

        """
        if self is masked:
            raise MaskError('Cannot alter the masked element.')
        _data = self._data
        _mask = self._mask
        if isinstance(indx, str):
            _data[indx] = value
            if _mask is nomask:
                self._mask = _mask = make_mask_none(self.shape, self.dtype)
            _mask[indx] = getmask(value)
            return

        _dtype = _data.dtype

        if value is masked:
            # The mask wasn't set: create a full version.
            if _mask is nomask:
                _mask = self._mask = make_mask_none(self.shape, _dtype)
            # Now, set the mask to its value.
            if _dtype.names is not None:
                _mask[indx] = tuple([True] * len(_dtype.names))
            else:
                _mask[indx] = True
            return

        # Get the _data part of the new value
        dval = getattr(value, '_data', value)
        # Get the _mask part of the new value
        mval = getmask(value)
        if _dtype.names is not None and mval is nomask:
            mval = tuple([False] * len(_dtype.names))
        if _mask is nomask:
            # Set the data, then the mask
            _data[indx] = dval
            if mval is not nomask:
                _mask = self._mask = make_mask_none(self.shape, _dtype)
                _mask[indx] = mval
        elif not self._hardmask:
            # Set the data, then the mask
            if (isinstance(indx, masked_array) and
                    not isinstance(value, masked_array)):
                _data[indx.data] = dval
            else:
                _data[indx] = dval
                _mask[indx] = mval
        elif hasattr(indx, 'dtype') and (indx.dtype == MaskType):
            indx = indx * umath.logical_not(_mask)
            _data[indx] = dval
        else:
            if _dtype.names is not None:
                err_msg = "Flexible 'hard' masks are not yet supported."
                raise NotImplementedError(err_msg)
            mindx = mask_or(_mask[indx], mval, copy=True)
            dindx = self._data[indx]
            if dindx.size > 1:
                np.copyto(dindx, dval, where=~mindx)
            elif mindx is nomask:
                dindx = dval
            _data[indx] = dindx
            _mask[indx] = mindx
        return

    # Define so that we can overwrite the setter.
    @property
    def dtype(self):
        return super().dtype

    @dtype.setter
    def dtype(self, dtype):
        super(MaskedArray, type(self)).dtype.__set__(self, dtype)
        if self._mask is not nomask:
            self._mask = self._mask.view(make_mask_descr(dtype), ndarray)
            # Try to reset the shape of the mask (if we don't have a void).
            # This raises a ValueError if the dtype change won't work.
            try:
                self._mask.shape = self.shape
            except (AttributeError, TypeError):
                pass

    @property
    def shape(self):
        return super().shape

    @shape.setter
    def shape(self, shape):
        super(MaskedArray, type(self)).shape.__set__(self, shape)
        # Cannot use self._mask, since it may not (yet) exist when a
        # masked matrix sets the shape.
        if getmask(self) is not nomask:
            self._mask.shape = self.shape

    def __setmask__(self, mask, copy=False):
        """
        Set the mask.

        """
        idtype = self.dtype
        current_mask = self._mask
        if mask is masked:
            mask = True

        if current_mask is nomask:
            # Make sure the mask is set
            # Just don't do anything if there's nothing to do.
            if mask is nomask:
                return
            current_mask = self._mask = make_mask_none(self.shape, idtype)

        if idtype.names is None:
            # No named fields.
            # Hardmask: don't unmask the data
            if self._hardmask:
                current_mask |= mask
            # Softmask: set everything to False
            # If it's obviously a compatible scalar, use a quick update
            # method.
            elif isinstance(mask, (int, float, np.bool_, np.number)):
                current_mask[...] = mask
            # Otherwise fall back to the slower, general purpose way.
            else:
                current_mask.flat = mask
        else:
            # Named fields w/
            mdtype = current_mask.dtype
            mask = np.array(mask, copy=False)
            # Mask is a singleton
            if not mask.ndim:
                # It's a boolean : make a record
                if mask.dtype.kind == 'b':
                    mask = np.array(tuple([mask.item()] * len(mdtype)),
                                    dtype=mdtype)
                # It's a record: make sure the dtype is correct
                else:
                    mask = mask.astype(mdtype)
            # Mask is a sequence
            else:
                # Make sure the new mask is a ndarray with the proper dtype
                try:
                    mask = np.array(mask, copy=copy, dtype=mdtype)
                # Or assume it's a sequence of bool/int
                except TypeError:
                    mask = np.array([tuple([m] * len(mdtype)) for m in mask],
                                    dtype=mdtype)
            # Hardmask: don't unmask the data
            if self._hardmask:
                for n in idtype.names:
                    current_mask[n] |= mask[n]
            # Softmask: set everything to False
            # If it's obviously a compatible scalar, use a quick update
            # method.
            elif isinstance(mask, (int, float, np.bool_, np.number)):
                current_mask[...] = mask
            # Otherwise fall back to the slower, general purpose way.
            else:
                current_mask.flat = mask
        # Reshape if needed
        if current_mask.shape:
            current_mask.shape = self.shape
        return

    _set_mask = __setmask__

    @property
    def mask(self):
        """ Current mask. """

        # We could try to force a reshape, but that wouldn't work in some
        # cases.
        # Return a view so that the dtype and shape cannot be changed in place
        # This still preserves nomask by identity
        return self._mask.view()

    @mask.setter
    def mask(self, value):
        self.__setmask__(value)

    @property
    def recordmask(self):
        """
        Get or set the mask of the array if it has no named fields. For
        structured arrays, returns a ndarray of booleans where entries are
        ``True`` if **all** the fields are masked, ``False`` otherwise:

        >>> x = np.ma.array([(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)],
        ...         mask=[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)],
        ...        dtype=[('a', int), ('b', int)])
        >>> x.recordmask
        array([False, False,  True, False, False])
        """

        _mask = self._mask.view(ndarray)
        if _mask.dtype.names is None:
            return _mask
        return np.all(flatten_structured_array(_mask), axis=-1)

    @recordmask.setter
    def recordmask(self, mask):
        raise NotImplementedError("Coming soon: setting the mask per records!")

    def harden_mask(self):
        """
        Force the mask to hard, preventing unmasking by assignment.

        Whether the mask of a masked array is hard or soft is determined by
        its `~ma.MaskedArray.hardmask` property. `harden_mask` sets
        `~ma.MaskedArray.hardmask` to ``True`` (and returns the modified
        self).

        See Also
        --------
        ma.MaskedArray.hardmask
        ma.MaskedArray.soften_mask

        """
        self._hardmask = True
        return self

    def soften_mask(self):
        """
        Force the mask to soft (default), allowing unmasking by assignment.

        Whether the mask of a masked array is hard or soft is determined by
        its `~ma.MaskedArray.hardmask` property. `soften_mask` sets
        `~ma.MaskedArray.hardmask` to ``False`` (and returns the modified
        self).

        See Also
        --------
        ma.MaskedArray.hardmask
        ma.MaskedArray.harden_mask

        """
        self._hardmask = False
        return self

    @property
    def hardmask(self):
        """
        Specifies whether values can be unmasked through assignments.

        By default, assigning definite values to masked array entries will
        unmask them.  When `hardmask` is ``True``, the mask will not change
        through assignments.

        See Also
        --------
        ma.MaskedArray.harden_mask
        ma.MaskedArray.soften_mask

        Examples
        --------
        >>> x = np.arange(10)
        >>> m = np.ma.masked_array(x, x>5)
        >>> assert not m.hardmask

        Since `m` has a soft mask, assigning an element value unmasks that
        element:

        >>> m[8] = 42
        >>> m
        masked_array(data=[0, 1, 2, 3, 4, 5, --, --, 42, --],
                     mask=[False, False, False, False, False, False,
                           True, True, False, True],
               fill_value=999999)

        After hardening, the mask is not affected by assignments:

        >>> hardened = np.ma.harden_mask(m)
        >>> assert m.hardmask and hardened is m
        >>> m[:] = 23
        >>> m
        masked_array(data=[23, 23, 23, 23, 23, 23, --, --, 23, --],
                     mask=[False, False, False, False, False, False,
                           True, True, False, True],
               fill_value=999999)

        """
        return self._hardmask

    def unshare_mask(self):
        """
        Copy the mask and set the `sharedmask` flag to ``False``.

        Whether the mask is shared between masked arrays can be seen from
        the `sharedmask` property. `unshare_mask` ensures the mask is not
        shared. A copy of the mask is only made if it was shared.

        See Also
        --------
        sharedmask

        """
        if self._sharedmask:
            self._mask = self._mask.copy()
            self._sharedmask = False
        return self

    @property
    def sharedmask(self):
        """ Share status of the mask (read-only). """
        return self._sharedmask

    def shrink_mask(self):
        """
        Reduce a mask to nomask when possible.

        Parameters
        ----------
        None

        Returns
        -------
        None

        Examples
        --------
        >>> x = np.ma.array([[1,2 ], [3, 4]], mask=[0]*4)
        >>> x.mask
        array([[False, False],
               [False, False]])
        >>> x.shrink_mask()
        masked_array(
          data=[[1, 2],
                [3, 4]],
          mask=False,
          fill_value=999999)
        >>> x.mask
        False

        """
        self._mask = _shrink_mask(self._mask)
        return self

    @property
    def baseclass(self):
        """ Class of the underlying data (read-only). """
        return self._baseclass

    def _get_data(self):
        """
        Returns the underlying data, as a view of the masked array.

        If the underlying data is a subclass of :class:`numpy.ndarray`, it is
        returned as such.

        >>> x = np.ma.array(np.matrix([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]])
        >>> x.data
        matrix([[1, 2],
                [3, 4]])

        The type of the data can be accessed through the :attr:`baseclass`
        attribute.
        """
        return ndarray.view(self, self._baseclass)

    _data = property(fget=_get_data)
    data = property(fget=_get_data)

    @property
    def flat(self):
        """ Return a flat iterator, or set a flattened version of self to value. """
        return MaskedIterator(self)

    @flat.setter
    def flat(self, value):
        y = self.ravel()
        y[:] = value

    @property
    def fill_value(self):
        """
        The filling value of the masked array is a scalar. When setting, None
        will set to a default based on the data type.

        Examples
        --------
        >>> for dt in [np.int32, np.int64, np.float64, np.complex128]:
        ...     np.ma.array([0, 1], dtype=dt).get_fill_value()
        ...
        999999
        999999
        1e+20
        (1e+20+0j)

        >>> x = np.ma.array([0, 1.], fill_value=-np.inf)
        >>> x.fill_value
        -inf
        >>> x.fill_value = np.pi
        >>> x.fill_value
        3.1415926535897931 # may vary

        Reset to default:

        >>> x.fill_value = None
        >>> x.fill_value
        1e+20

        """
        if self._fill_value is None:
            self._fill_value = _check_fill_value(None, self.dtype)

        # Temporary workaround to account for the fact that str and bytes
        # scalars cannot be indexed with (), whereas all other numpy
        # scalars can. See issues #7259 and #7267.
        # The if-block can be removed after #7267 has been fixed.
        if isinstance(self._fill_value, ndarray):
            return self._fill_value[()]
        return self._fill_value

    @fill_value.setter
    def fill_value(self, value=None):
        target = _check_fill_value(value, self.dtype)
        if not target.ndim == 0:
            # 2019-11-12, 1.18.0
            warnings.warn(
                "Non-scalar arrays for the fill value are deprecated. Use "
                "arrays with scalar values instead. The filled function "
                "still supports any array as `fill_value`.",
                DeprecationWarning, stacklevel=2)

        _fill_value = self._fill_value
        if _fill_value is None:
            # Create the attribute if it was undefined
            self._fill_value = target
        else:
            # Don't overwrite the attribute, just fill it (for propagation)
            _fill_value[()] = target

    # kept for compatibility
    get_fill_value = fill_value.fget
    set_fill_value = fill_value.fset

    def filled(self, fill_value=None):
        """
        Return a copy of self, with masked values filled with a given value.
        **However**, if there are no masked values to fill, self will be
        returned instead as an ndarray.

        Parameters
        ----------
        fill_value : array_like, optional
            The value to use for invalid entries. Can be scalar or non-scalar.
            If non-scalar, the resulting ndarray must be broadcastable over
            input array. Default is None, in which case, the `fill_value`
            attribute of the array is used instead.

        Returns
        -------
        filled_array : ndarray
            A copy of ``self`` with invalid entries replaced by *fill_value*
            (be it the function argument or the attribute of ``self``), or
            ``self`` itself as an ndarray if there are no invalid entries to
            be replaced.

        Notes
        -----
        The result is **not** a MaskedArray!

        Examples
        --------
        >>> x = np.ma.array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999)
        >>> x.filled()
        array([   1,    2, -999,    4, -999])
        >>> x.filled(fill_value=1000)
        array([   1,    2, 1000,    4, 1000])
        >>> type(x.filled())
        <class 'numpy.ndarray'>

        Subclassing is preserved. This means that if, e.g., the data part of
        the masked array is a recarray, `filled` returns a recarray:

        >>> x = np.array([(-1, 2), (-3, 4)], dtype='i8,i8').view(np.recarray)
        >>> m = np.ma.array(x, mask=[(True, False), (False, True)])
        >>> m.filled()
        rec.array([(999999,      2), (    -3, 999999)],
                  dtype=[('f0', '<i8'), ('f1', '<i8')])
        """
        m = self._mask
        if m is nomask:
            return self._data

        if fill_value is None:
            fill_value = self.fill_value
        else:
            fill_value = _check_fill_value(fill_value, self.dtype)

        if self is masked_singleton:
            return np.asanyarray(fill_value)

        if m.dtype.names is not None:
            result = self._data.copy('K')
            _recursive_filled(result, self._mask, fill_value)
        elif not m.any():
            return self._data
        else:
            result = self._data.copy('K')
            try:
                np.copyto(result, fill_value, where=m)
            except (TypeError, AttributeError):
                fill_value = narray(fill_value, dtype=object)
                d = result.astype(object)
                result = np.choose(m, (d, fill_value))
            except IndexError:
                # ok, if scalar
                if self._data.shape:
                    raise
                elif m:
                    result = np.array(fill_value, dtype=self.dtype)
                else:
                    result = self._data
        return result

    def compressed(self):
        """
        Return all the non-masked data as a 1-D array.

        Returns
        -------
        data : ndarray
            A new `ndarray` holding the non-masked data is returned.

        Notes
        -----
        The result is **not** a MaskedArray!

        Examples
        --------
        >>> x = np.ma.array(np.arange(5), mask=[0]*2 + [1]*3)
        >>> x.compressed()
        array([0, 1])
        >>> type(x.compressed())
        <class 'numpy.ndarray'>

        """
        data = ndarray.ravel(self._data)
        if self._mask is not nomask:
            data = data.compress(np.logical_not(ndarray.ravel(self._mask)))
        return data

    def compress(self, condition, axis=None, out=None):
        """
        Return `a` where condition is ``True``.

        If condition is a `~ma.MaskedArray`, missing values are considered
        as ``False``.

        Parameters
        ----------
        condition : var
            Boolean 1-d array selecting which entries to return. If len(condition)
            is less than the size of a along the axis, then output is truncated
            to length of condition array.
        axis : {None, int}, optional
            Axis along which the operation must be performed.
        out : {None, ndarray}, optional
            Alternative output array in which to place the result. It must have
            the same shape as the expected output but the type will be cast if
            necessary.

        Returns
        -------
        result : MaskedArray
            A :class:`~ma.MaskedArray` object.

        Notes
        -----
        Please note the difference with :meth:`compressed` !
        The output of :meth:`compress` has a mask, the output of
        :meth:`compressed` does not.

        Examples
        --------
        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)
        >>> x
        masked_array(
          data=[[1, --, 3],
                [--, 5, --],
                [7, --, 9]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)
        >>> x.compress([1, 0, 1])
        masked_array(data=[1, 3],
                     mask=[False, False],
               fill_value=999999)

        >>> x.compress([1, 0, 1], axis=1)
        masked_array(
          data=[[1, 3],
                [--, --],
                [7, 9]],
          mask=[[False, False],
                [ True,  True],
                [False, False]],
          fill_value=999999)

        """
        # Get the basic components
        (_data, _mask) = (self._data, self._mask)

        # Force the condition to a regular ndarray and forget the missing
        # values.
        condition = np.asarray(condition)

        _new = _data.compress(condition, axis=axis, out=out).view(type(self))
        _new._update_from(self)
        if _mask is not nomask:
            _new._mask = _mask.compress(condition, axis=axis)
        return _new

    def _insert_masked_print(self):
        """
        Replace masked values with masked_print_option, casting all innermost
        dtypes to object.
        """
        if masked_print_option.enabled():
            mask = self._mask
            if mask is nomask:
                res = self._data
            else:
                # convert to object array to make filled work
                data = self._data
                # For big arrays, to avoid a costly conversion to the
                # object dtype, extract the corners before the conversion.
                print_width = (self._print_width if self.ndim > 1
                               else self._print_width_1d)
                for axis in range(self.ndim):
                    if data.shape[axis] > print_width:
                        ind = print_width // 2
                        arr = np.split(data, (ind, -ind), axis=axis)
                        data = np.concatenate((arr[0], arr[2]), axis=axis)
                        arr = np.split(mask, (ind, -ind), axis=axis)
                        mask = np.concatenate((arr[0], arr[2]), axis=axis)

                rdtype = _replace_dtype_fields(self.dtype, "O")
                res = data.astype(rdtype)
                _recursive_printoption(res, mask, masked_print_option)
        else:
            res = self.filled(self.fill_value)
        return res

    def __str__(self):
        return str(self._insert_masked_print())

    def __repr__(self):
        """
        Literal string representation.

        """
        if self._baseclass is np.ndarray:
            name = 'array'
        else:
            name = self._baseclass.__name__


        # 2016-11-19: Demoted to legacy format
        if np.core.arrayprint._get_legacy_print_mode() <= 113:
            is_long = self.ndim > 1
            parameters = dict(
                name=name,
                nlen=" " * len(name),
                data=str(self),
                mask=str(self._mask),
                fill=str(self.fill_value),
                dtype=str(self.dtype)
            )
            is_structured = bool(self.dtype.names)
            key = '{}_{}'.format(
                'long' if is_long else 'short',
                'flx' if is_structured else 'std'
            )
            return _legacy_print_templates[key] % parameters

        prefix = f"masked_{name}("

        dtype_needed = (
            not np.core.arrayprint.dtype_is_implied(self.dtype) or
            np.all(self.mask) or
            self.size == 0
        )

        # determine which keyword args need to be shown
        keys = ['data', 'mask', 'fill_value']
        if dtype_needed:
            keys.append('dtype')

        # array has only one row (non-column)
        is_one_row = builtins.all(dim == 1 for dim in self.shape[:-1])

        # choose what to indent each keyword with
        min_indent = 2
        if is_one_row:
            # first key on the same line as the type, remaining keys
            # aligned by equals
            indents = {}
            indents[keys[0]] = prefix
            for k in keys[1:]:
                n = builtins.max(min_indent, len(prefix + keys[0]) - len(k))
                indents[k] = ' ' * n
            prefix = ''  # absorbed into the first indent
        else:
            # each key on its own line, indented by two spaces
            indents = {k: ' ' * min_indent for k in keys}
            prefix = prefix + '\n'  # first key on the next line

        # format the field values
        reprs = {}
        reprs['data'] = np.array2string(
            self._insert_masked_print(),
            separator=", ",
            prefix=indents['data'] + 'data=',
            suffix=',')
        reprs['mask'] = np.array2string(
            self._mask,
            separator=", ",
            prefix=indents['mask'] + 'mask=',
            suffix=',')
        reprs['fill_value'] = repr(self.fill_value)
        if dtype_needed:
            reprs['dtype'] = np.core.arrayprint.dtype_short_repr(self.dtype)

        # join keys with values and indentations
        result = ',\n'.join(
            '{}{}={}'.format(indents[k], k, reprs[k])
            for k in keys
        )
        return prefix + result + ')'

    def _delegate_binop(self, other):
        # This emulates the logic in
        #     private/binop_override.h:forward_binop_should_defer
        if isinstance(other, type(self)):
            return False
        array_ufunc = getattr(other, "__array_ufunc__", False)
        if array_ufunc is False:
            other_priority = getattr(other, "__array_priority__", -1000000)
            return self.__array_priority__ < other_priority
        else:
            # If array_ufunc is not None, it will be called inside the ufunc;
            # None explicitly tells us to not call the ufunc, i.e., defer.
            return array_ufunc is None

    def _comparison(self, other, compare):
        """Compare self with other using operator.eq or operator.ne.

        When either of the elements is masked, the result is masked as well,
        but the underlying boolean data are still set, with self and other
        considered equal if both are masked, and unequal otherwise.

        For structured arrays, all fields are combined, with masked values
        ignored. The result is masked if all fields were masked, with self
        and other considered equal only if both were fully masked.
        """
        omask = getmask(other)
        smask = self.mask
        mask = mask_or(smask, omask, copy=True)

        odata = getdata(other)
        if mask.dtype.names is not None:
            # only == and != are reasonably defined for structured dtypes,
            # so give up early for all other comparisons:
            if compare not in (operator.eq, operator.ne):
                return NotImplemented
            # For possibly masked structured arrays we need to be careful,
            # since the standard structured array comparison will use all
            # fields, masked or not. To avoid masked fields influencing the
            # outcome, we set all masked fields in self to other, so they'll
            # count as equal.  To prepare, we ensure we have the right shape.
            broadcast_shape = np.broadcast(self, odata).shape
            sbroadcast = np.broadcast_to(self, broadcast_shape, subok=True)
            sbroadcast._mask = mask
            sdata = sbroadcast.filled(odata)
            # Now take care of the mask; the merged mask should have an item
            # masked if all fields were masked (in one and/or other).
            mask = (mask == np.ones((), mask.dtype))

        else:
            # For regular arrays, just use the data as they come.
            sdata = self.data

        check = compare(sdata, odata)

        if isinstance(check, (np.bool_, bool)):
            return masked if mask else check

        if mask is not nomask and compare in (operator.eq, operator.ne):
            # Adjust elements that were masked, which should be treated
            # as equal if masked in both, unequal if masked in one.
            # Note that this works automatically for structured arrays too.
            # Ignore this for operations other than `==` and `!=`
            check = np.where(mask, compare(smask, omask), check)
            if mask.shape != check.shape:
                # Guarantee consistency of the shape, making a copy since the
                # the mask may need to get written to later.
                mask = np.broadcast_to(mask, check.shape).copy()

        check = check.view(type(self))
        check._update_from(self)
        check._mask = mask

        # Cast fill value to bool_ if needed. If it cannot be cast, the
        # default boolean fill value is used.
        if check._fill_value is not None:
            try:
                fill = _check_fill_value(check._fill_value, np.bool_)
            except (TypeError, ValueError):
                fill = _check_fill_value(None, np.bool_)
            check._fill_value = fill

        return check

    def __eq__(self, other):
        """Check whether other equals self elementwise.

        When either of the elements is masked, the result is masked as well,
        but the underlying boolean data are still set, with self and other
        considered equal if both are masked, and unequal otherwise.

        For structured arrays, all fields are combined, with masked values
        ignored. The result is masked if all fields were masked, with self
        and other considered equal only if both were fully masked.
        """
        return self._comparison(other, operator.eq)

    def __ne__(self, other):
        """Check whether other does not equal self elementwise.

        When either of the elements is masked, the result is masked as well,
        but the underlying boolean data are still set, with self and other
        considered equal if both are masked, and unequal otherwise.

        For structured arrays, all fields are combined, with masked values
        ignored. The result is masked if all fields were masked, with self
        and other considered equal only if both were fully masked.
        """
        return self._comparison(other, operator.ne)

    # All other comparisons:
    def __le__(self, other):
        return self._comparison(other, operator.le)

    def __lt__(self, other):
        return self._comparison(other, operator.lt)

    def __ge__(self, other):
        return self._comparison(other, operator.ge)

    def __gt__(self, other):
        return self._comparison(other, operator.gt)

    def __add__(self, other):
        """
        Add self to other, and return a new masked array.

        """
        if self._delegate_binop(other):
            return NotImplemented
        return add(self, other)

    def __radd__(self, other):
        """
        Add other to self, and return a new masked array.

        """
        # In analogy with __rsub__ and __rdiv__, use original order:
        # we get here from `other + self`.
        return add(other, self)

    def __sub__(self, other):
        """
        Subtract other from self, and return a new masked array.

        """
        if self._delegate_binop(other):
            return NotImplemented
        return subtract(self, other)

    def __rsub__(self, other):
        """
        Subtract self from other, and return a new masked array.

        """
        return subtract(other, self)

    def __mul__(self, other):
        "Multiply self by other, and return a new masked array."
        if self._delegate_binop(other):
            return NotImplemented
        return multiply(self, other)

    def __rmul__(self, other):
        """
        Multiply other by self, and return a new masked array.

        """
        # In analogy with __rsub__ and __rdiv__, use original order:
        # we get here from `other * self`.
        return multiply(other, self)

    def __div__(self, other):
        """
        Divide other into self, and return a new masked array.

        """
        if self._delegate_binop(other):
            return NotImplemented
        return divide(self, other)

    def __truediv__(self, other):
        """
        Divide other into self, and return a new masked array.

        """
        if self._delegate_binop(other):
            return NotImplemented
        return true_divide(self, other)

    def __rtruediv__(self, other):
        """
        Divide self into other, and return a new masked array.

        """
        return true_divide(other, self)

    def __floordiv__(self, other):
        """
        Divide other into self, and return a new masked array.

        """
        if self._delegate_binop(other):
            return NotImplemented
        return floor_divide(self, other)

    def __rfloordiv__(self, other):
        """
        Divide self into other, and return a new masked array.

        """
        return floor_divide(other, self)

    def __pow__(self, other):
        """
        Raise self to the power other, masking the potential NaNs/Infs

        """
        if self._delegate_binop(other):
            return NotImplemented
        return power(self, other)

    def __rpow__(self, other):
        """
        Raise other to the power self, masking the potential NaNs/Infs

        """
        return power(other, self)

    def __iadd__(self, other):
        """
        Add other to self in-place.

        """
        m = getmask(other)
        if self._mask is nomask:
            if m is not nomask and m.any():
                self._mask = make_mask_none(self.shape, self.dtype)
                self._mask += m
        else:
            if m is not nomask:
                self._mask += m
        other_data = getdata(other)
        other_data = np.where(self._mask, other_data.dtype.type(0), other_data)
        self._data.__iadd__(other_data)
        return self

    def __isub__(self, other):
        """
        Subtract other from self in-place.

        """
        m = getmask(other)
        if self._mask is nomask:
            if m is not nomask and m.any():
                self._mask = make_mask_none(self.shape, self.dtype)
                self._mask += m
        elif m is not nomask:
            self._mask += m
        other_data = getdata(other)
        other_data = np.where(self._mask, other_data.dtype.type(0), other_data)
        self._data.__isub__(other_data)
        return self

    def __imul__(self, other):
        """
        Multiply self by other in-place.

        """
        m = getmask(other)
        if self._mask is nomask:
            if m is not nomask and m.any():
                self._mask = make_mask_none(self.shape, self.dtype)
                self._mask += m
        elif m is not nomask:
            self._mask += m
        other_data = getdata(other)
        other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
        self._data.__imul__(other_data)
        return self

    def __idiv__(self, other):
        """
        Divide self by other in-place.

        """
        other_data = getdata(other)
        dom_mask = _DomainSafeDivide().__call__(self._data, other_data)
        other_mask = getmask(other)
        new_mask = mask_or(other_mask, dom_mask)
        # The following 4 lines control the domain filling
        if dom_mask.any():
            (_, fval) = ufunc_fills[np.divide]
            other_data = np.where(
                    dom_mask, other_data.dtype.type(fval), other_data)
        self._mask |= new_mask
        other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
        self._data.__idiv__(other_data)
        return self

    def __ifloordiv__(self, other):
        """
        Floor divide self by other in-place.

        """
        other_data = getdata(other)
        dom_mask = _DomainSafeDivide().__call__(self._data, other_data)
        other_mask = getmask(other)
        new_mask = mask_or(other_mask, dom_mask)
        # The following 3 lines control the domain filling
        if dom_mask.any():
            (_, fval) = ufunc_fills[np.floor_divide]
            other_data = np.where(
                    dom_mask, other_data.dtype.type(fval), other_data)
        self._mask |= new_mask
        other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
        self._data.__ifloordiv__(other_data)
        return self

    def __itruediv__(self, other):
        """
        True divide self by other in-place.

        """
        other_data = getdata(other)
        dom_mask = _DomainSafeDivide().__call__(self._data, other_data)
        other_mask = getmask(other)
        new_mask = mask_or(other_mask, dom_mask)
        # The following 3 lines control the domain filling
        if dom_mask.any():
            (_, fval) = ufunc_fills[np.true_divide]
            other_data = np.where(
                    dom_mask, other_data.dtype.type(fval), other_data)
        self._mask |= new_mask
        other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
        self._data.__itruediv__(other_data)
        return self

    def __ipow__(self, other):
        """
        Raise self to the power other, in place.

        """
        other_data = getdata(other)
        other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
        other_mask = getmask(other)
        with np.errstate(divide='ignore', invalid='ignore'):
            self._data.__ipow__(other_data)
        invalid = np.logical_not(np.isfinite(self._data))
        if invalid.any():
            if self._mask is not nomask:
                self._mask |= invalid
            else:
                self._mask = invalid
            np.copyto(self._data, self.fill_value, where=invalid)
        new_mask = mask_or(other_mask, invalid)
        self._mask = mask_or(self._mask, new_mask)
        return self

    def __float__(self):
        """
        Convert to float.

        """
        if self.size > 1:
            raise TypeError("Only length-1 arrays can be converted "
                            "to Python scalars")
        elif self._mask:
            warnings.warn("Warning: converting a masked element to nan.", stacklevel=2)
            return np.nan
        return float(self.item())

    def __int__(self):
        """
        Convert to int.

        """
        if self.size > 1:
            raise TypeError("Only length-1 arrays can be converted "
                            "to Python scalars")
        elif self._mask:
            raise MaskError('Cannot convert masked element to a Python int.')
        return int(self.item())

    @property
    def imag(self):
        """
        The imaginary part of the masked array.

        This property is a view on the imaginary part of this `MaskedArray`.

        See Also
        --------
        real

        Examples
        --------
        >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False])
        >>> x.imag
        masked_array(data=[1.0, --, 1.6],
                     mask=[False,  True, False],
               fill_value=1e+20)

        """
        result = self._data.imag.view(type(self))
        result.__setmask__(self._mask)
        return result

    # kept for compatibility
    get_imag = imag.fget

    @property
    def real(self):
        """
        The real part of the masked array.

        This property is a view on the real part of this `MaskedArray`.

        See Also
        --------
        imag

        Examples
        --------
        >>> x = np.ma.array([1+1.j, -2j, 3.45+1.6j], mask=[False, True, False])
        >>> x.real
        masked_array(data=[1.0, --, 3.45],
                     mask=[False,  True, False],
               fill_value=1e+20)

        """
        result = self._data.real.view(type(self))
        result.__setmask__(self._mask)
        return result

    # kept for compatibility
    get_real = real.fget

    def count(self, axis=None, keepdims=np._NoValue):
        """
        Count the non-masked elements of the array along the given axis.

        Parameters
        ----------
        axis : None or int or tuple of ints, optional
            Axis or axes along which the count is performed.
            The default, None, performs the count over all
            the dimensions of the input array. `axis` may be negative, in
            which case it counts from the last to the first axis.

            .. versionadded:: 1.10.0

            If this is a tuple of ints, the count is performed on multiple
            axes, instead of a single axis or all the axes as before.
        keepdims : bool, optional
            If this is set to True, the axes which are reduced are left
            in the result as dimensions with size one. With this option,
            the result will broadcast correctly against the array.

        Returns
        -------
        result : ndarray or scalar
            An array with the same shape as the input array, with the specified
            axis removed. If the array is a 0-d array, or if `axis` is None, a
            scalar is returned.

        See Also
        --------
        ma.count_masked : Count masked elements in array or along a given axis.

        Examples
        --------
        >>> import numpy.ma as ma
        >>> a = ma.arange(6).reshape((2, 3))
        >>> a[1, :] = ma.masked
        >>> a
        masked_array(
          data=[[0, 1, 2],
                [--, --, --]],
          mask=[[False, False, False],
                [ True,  True,  True]],
          fill_value=999999)
        >>> a.count()
        3

        When the `axis` keyword is specified an array of appropriate size is
        returned.

        >>> a.count(axis=0)
        array([1, 1, 1])
        >>> a.count(axis=1)
        array([3, 0])

        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        m = self._mask
        # special case for matrices (we assume no other subclasses modify
        # their dimensions)
        if isinstance(self.data, np.matrix):
            if m is nomask:
                m = np.zeros(self.shape, dtype=np.bool_)
            m = m.view(type(self.data))

        if m is nomask:
            # compare to _count_reduce_items in _methods.py

            if self.shape == ():
                if axis not in (None, 0):
                    raise np.AxisError(axis=axis, ndim=self.ndim)
                return 1
            elif axis is None:
                if kwargs.get('keepdims', False):
                    return np.array(self.size, dtype=np.intp, ndmin=self.ndim)
                return self.size

            axes = normalize_axis_tuple(axis, self.ndim)
            items = 1
            for ax in axes:
                items *= self.shape[ax]

            if kwargs.get('keepdims', False):
                out_dims = list(self.shape)
                for a in axes:
                    out_dims[a] = 1
            else:
                out_dims = [d for n, d in enumerate(self.shape)
                            if n not in axes]
            # make sure to return a 0-d array if axis is supplied
            return np.full(out_dims, items, dtype=np.intp)

        # take care of the masked singleton
        if self is masked:
            return 0

        return (~m).sum(axis=axis, dtype=np.intp, **kwargs)

    def ravel(self, order='C'):
        """
        Returns a 1D version of self, as a view.

        Parameters
        ----------
        order : {'C', 'F', 'A', 'K'}, optional
            The elements of `a` are read using this index order. 'C' means to
            index the elements in C-like order, with the last axis index
            changing fastest, back to the first axis index changing slowest.
            'F' means to index the elements in Fortran-like index order, with
            the first index changing fastest, and the last index changing
            slowest. Note that the 'C' and 'F' options take no account of the
            memory layout of the underlying array, and only refer to the order
            of axis indexing.  'A' means to read the elements in Fortran-like
            index order if `m` is Fortran *contiguous* in memory, C-like order
            otherwise.  'K' means to read the elements in the order they occur
            in memory, except for reversing the data when strides are negative.
            By default, 'C' index order is used.

        Returns
        -------
        MaskedArray
            Output view is of shape ``(self.size,)`` (or
            ``(np.ma.product(self.shape),)``).

        Examples
        --------
        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)
        >>> x
        masked_array(
          data=[[1, --, 3],
                [--, 5, --],
                [7, --, 9]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)
        >>> x.ravel()
        masked_array(data=[1, --, 3, --, 5, --, 7, --, 9],
                     mask=[False,  True, False,  True, False,  True, False,  True,
                           False],
               fill_value=999999)

        """
        r = ndarray.ravel(self._data, order=order).view(type(self))
        r._update_from(self)
        if self._mask is not nomask:
            r._mask = ndarray.ravel(self._mask, order=order).reshape(r.shape)
        else:
            r._mask = nomask
        return r


    def reshape(self, *s, **kwargs):
        """
        Give a new shape to the array without changing its data.

        Returns a masked array containing the same data, but with a new shape.
        The result is a view on the original array; if this is not possible, a
        ValueError is raised.

        Parameters
        ----------
        shape : int or tuple of ints
            The new shape should be compatible with the original shape. If an
            integer is supplied, then the result will be a 1-D array of that
            length.
        order : {'C', 'F'}, optional
            Determines whether the array data should be viewed as in C
            (row-major) or FORTRAN (column-major) order.

        Returns
        -------
        reshaped_array : array
            A new view on the array.

        See Also
        --------
        reshape : Equivalent function in the masked array module.
        numpy.ndarray.reshape : Equivalent method on ndarray object.
        numpy.reshape : Equivalent function in the NumPy module.

        Notes
        -----
        The reshaping operation cannot guarantee that a copy will not be made,
        to modify the shape in place, use ``a.shape = s``

        Examples
        --------
        >>> x = np.ma.array([[1,2],[3,4]], mask=[1,0,0,1])
        >>> x
        masked_array(
          data=[[--, 2],
                [3, --]],
          mask=[[ True, False],
                [False,  True]],
          fill_value=999999)
        >>> x = x.reshape((4,1))
        >>> x
        masked_array(
          data=[[--],
                [2],
                [3],
                [--]],
          mask=[[ True],
                [False],
                [False],
                [ True]],
          fill_value=999999)

        """
        kwargs.update(order=kwargs.get('order', 'C'))
        result = self._data.reshape(*s, **kwargs).view(type(self))
        result._update_from(self)
        mask = self._mask
        if mask is not nomask:
            result._mask = mask.reshape(*s, **kwargs)
        return result

    def resize(self, newshape, refcheck=True, order=False):
        """
        .. warning::

            This method does nothing, except raise a ValueError exception. A
            masked array does not own its data and therefore cannot safely be
            resized in place. Use the `numpy.ma.resize` function instead.

        This method is difficult to implement safely and may be deprecated in
        future releases of NumPy.

        """
        # Note : the 'order' keyword looks broken, let's just drop it
        errmsg = "A masked array does not own its data "\
                 "and therefore cannot be resized.\n" \
                 "Use the numpy.ma.resize function instead."
        raise ValueError(errmsg)

    def put(self, indices, values, mode='raise'):
        """
        Set storage-indexed locations to corresponding values.

        Sets self._data.flat[n] = values[n] for each n in indices.
        If `values` is shorter than `indices` then it will repeat.
        If `values` has some masked values, the initial mask is updated
        in consequence, else the corresponding values are unmasked.

        Parameters
        ----------
        indices : 1-D array_like
            Target indices, interpreted as integers.
        values : array_like
            Values to place in self._data copy at target indices.
        mode : {'raise', 'wrap', 'clip'}, optional
            Specifies how out-of-bounds indices will behave.
            'raise' : raise an error.
            'wrap' : wrap around.
            'clip' : clip to the range.

        Notes
        -----
        `values` can be a scalar or length 1 array.

        Examples
        --------
        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)
        >>> x
        masked_array(
          data=[[1, --, 3],
                [--, 5, --],
                [7, --, 9]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)
        >>> x.put([0,4,8],[10,20,30])
        >>> x
        masked_array(
          data=[[10, --, 3],
                [--, 20, --],
                [7, --, 30]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)

        >>> x.put(4,999)
        >>> x
        masked_array(
          data=[[10, --, 3],
                [--, 999, --],
                [7, --, 30]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)

        """
        # Hard mask: Get rid of the values/indices that fall on masked data
        if self._hardmask and self._mask is not nomask:
            mask = self._mask[indices]
            indices = narray(indices, copy=False)
            values = narray(values, copy=False, subok=True)
            values.resize(indices.shape)
            indices = indices[~mask]
            values = values[~mask]

        self._data.put(indices, values, mode=mode)

        # short circuit if neither self nor values are masked
        if self._mask is nomask and getmask(values) is nomask:
            return

        m = getmaskarray(self)

        if getmask(values) is nomask:
            m.put(indices, False, mode=mode)
        else:
            m.put(indices, values._mask, mode=mode)
        m = make_mask(m, copy=False, shrink=True)
        self._mask = m
        return

    def ids(self):
        """
        Return the addresses of the data and mask areas.

        Parameters
        ----------
        None

        Examples
        --------
        >>> x = np.ma.array([1, 2, 3], mask=[0, 1, 1])
        >>> x.ids()
        (166670640, 166659832) # may vary

        If the array has no mask, the address of `nomask` is returned. This address
        is typically not close to the data in memory:

        >>> x = np.ma.array([1, 2, 3])
        >>> x.ids()
        (166691080, 3083169284) # may vary

        """
        if self._mask is nomask:
            return (self.ctypes.data, id(nomask))
        return (self.ctypes.data, self._mask.ctypes.data)

    def iscontiguous(self):
        """
        Return a boolean indicating whether the data is contiguous.

        Parameters
        ----------
        None

        Examples
        --------
        >>> x = np.ma.array([1, 2, 3])
        >>> x.iscontiguous()
        True

        `iscontiguous` returns one of the flags of the masked array:

        >>> x.flags
          C_CONTIGUOUS : True
          F_CONTIGUOUS : True
          OWNDATA : False
          WRITEABLE : True
          ALIGNED : True
          WRITEBACKIFCOPY : False

        """
        return self.flags['CONTIGUOUS']

    def all(self, axis=None, out=None, keepdims=np._NoValue):
        """
        Returns True if all elements evaluate to True.

        The output array is masked where all the values along the given axis
        are masked: if the output would have been a scalar and that all the
        values are masked, then the output is `masked`.

        Refer to `numpy.all` for full documentation.

        See Also
        --------
        numpy.ndarray.all : corresponding function for ndarrays
        numpy.all : equivalent function

        Examples
        --------
        >>> np.ma.array([1,2,3]).all()
        True
        >>> a = np.ma.array([1,2,3], mask=True)
        >>> (a.all() is np.ma.masked)
        True

        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        mask = _check_mask_axis(self._mask, axis, **kwargs)
        if out is None:
            d = self.filled(True).all(axis=axis, **kwargs).view(type(self))
            if d.ndim:
                d.__setmask__(mask)
            elif mask:
                return masked
            return d
        self.filled(True).all(axis=axis, out=out, **kwargs)
        if isinstance(out, MaskedArray):
            if out.ndim or mask:
                out.__setmask__(mask)
        return out

    def any(self, axis=None, out=None, keepdims=np._NoValue):
        """
        Returns True if any of the elements of `a` evaluate to True.

        Masked values are considered as False during computation.

        Refer to `numpy.any` for full documentation.

        See Also
        --------
        numpy.ndarray.any : corresponding function for ndarrays
        numpy.any : equivalent function

        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        mask = _check_mask_axis(self._mask, axis, **kwargs)
        if out is None:
            d = self.filled(False).any(axis=axis, **kwargs).view(type(self))
            if d.ndim:
                d.__setmask__(mask)
            elif mask:
                d = masked
            return d
        self.filled(False).any(axis=axis, out=out, **kwargs)
        if isinstance(out, MaskedArray):
            if out.ndim or mask:
                out.__setmask__(mask)
        return out

    def nonzero(self):
        """
        Return the indices of unmasked elements that are not zero.

        Returns a tuple of arrays, one for each dimension, containing the
        indices of the non-zero elements in that dimension. The corresponding
        non-zero values can be obtained with::

            a[a.nonzero()]

        To group the indices by element, rather than dimension, use
        instead::

            np.transpose(a.nonzero())

        The result of this is always a 2d array, with a row for each non-zero
        element.

        Parameters
        ----------
        None

        Returns
        -------
        tuple_of_arrays : tuple
            Indices of elements that are non-zero.

        See Also
        --------
        numpy.nonzero :
            Function operating on ndarrays.
        flatnonzero :
            Return indices that are non-zero in the flattened version of the input
            array.
        numpy.ndarray.nonzero :
            Equivalent ndarray method.
        count_nonzero :
            Counts the number of non-zero elements in the input array.

        Examples
        --------
        >>> import numpy.ma as ma
        >>> x = ma.array(np.eye(3))
        >>> x
        masked_array(
          data=[[1., 0., 0.],
                [0., 1., 0.],
                [0., 0., 1.]],
          mask=False,
          fill_value=1e+20)
        >>> x.nonzero()
        (array([0, 1, 2]), array([0, 1, 2]))

        Masked elements are ignored.

        >>> x[1, 1] = ma.masked
        >>> x
        masked_array(
          data=[[1.0, 0.0, 0.0],
                [0.0, --, 0.0],
                [0.0, 0.0, 1.0]],
          mask=[[False, False, False],
                [False,  True, False],
                [False, False, False]],
          fill_value=1e+20)
        >>> x.nonzero()
        (array([0, 2]), array([0, 2]))

        Indices can also be grouped by element.

        >>> np.transpose(x.nonzero())
        array([[0, 0],
               [2, 2]])

        A common use for ``nonzero`` is to find the indices of an array, where
        a condition is True.  Given an array `a`, the condition `a` > 3 is a
        boolean array and since False is interpreted as 0, ma.nonzero(a > 3)
        yields the indices of the `a` where the condition is true.

        >>> a = ma.array([[1,2,3],[4,5,6],[7,8,9]])
        >>> a > 3
        masked_array(
          data=[[False, False, False],
                [ True,  True,  True],
                [ True,  True,  True]],
          mask=False,
          fill_value=True)
        >>> ma.nonzero(a > 3)
        (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))

        The ``nonzero`` method of the condition array can also be called.

        >>> (a > 3).nonzero()
        (array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))

        """
        return narray(self.filled(0), copy=False).nonzero()

    def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None):
        """
        (this docstring should be overwritten)
        """
        #!!!: implement out + test!
        m = self._mask
        if m is nomask:
            result = super().trace(offset=offset, axis1=axis1, axis2=axis2,
                                   out=out)
            return result.astype(dtype)
        else:
            D = self.diagonal(offset=offset, axis1=axis1, axis2=axis2)
            return D.astype(dtype).filled(0).sum(axis=-1, out=out)
    trace.__doc__ = ndarray.trace.__doc__

    def dot(self, b, out=None, strict=False):
        """
        a.dot(b, out=None)

        Masked dot product of two arrays. Note that `out` and `strict` are
        located in different positions than in `ma.dot`. In order to
        maintain compatibility with the functional version, it is
        recommended that the optional arguments be treated as keyword only.
        At some point that may be mandatory.

        .. versionadded:: 1.10.0

        Parameters
        ----------
        b : masked_array_like
            Inputs array.
        out : masked_array, optional
            Output argument. This must have the exact kind that would be
            returned if it was not used. In particular, it must have the
            right type, must be C-contiguous, and its dtype must be the
            dtype that would be returned for `ma.dot(a,b)`. This is a
            performance feature. Therefore, if these conditions are not
            met, an exception is raised, instead of attempting to be
            flexible.
        strict : bool, optional
            Whether masked data are propagated (True) or set to 0 (False)
            for the computation. Default is False.  Propagating the mask
            means that if a masked value appears in a row or column, the
            whole row or column is considered masked.

            .. versionadded:: 1.10.2

        See Also
        --------
        numpy.ma.dot : equivalent function

        """
        return dot(self, b, out=out, strict=strict)

    def sum(self, axis=None, dtype=None, out=None, keepdims=np._NoValue):
        """
        Return the sum of the array elements over the given axis.

        Masked elements are set to 0 internally.

        Refer to `numpy.sum` for full documentation.

        See Also
        --------
        numpy.ndarray.sum : corresponding function for ndarrays
        numpy.sum : equivalent function

        Examples
        --------
        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)
        >>> x
        masked_array(
          data=[[1, --, 3],
                [--, 5, --],
                [7, --, 9]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)
        >>> x.sum()
        25
        >>> x.sum(axis=1)
        masked_array(data=[4, 5, 16],
                     mask=[False, False, False],
               fill_value=999999)
        >>> x.sum(axis=0)
        masked_array(data=[8, 5, 12],
                     mask=[False, False, False],
               fill_value=999999)
        >>> print(type(x.sum(axis=0, dtype=np.int64)[0]))
        <class 'numpy.int64'>

        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        _mask = self._mask
        newmask = _check_mask_axis(_mask, axis, **kwargs)
        # No explicit output
        if out is None:
            result = self.filled(0).sum(axis, dtype=dtype, **kwargs)
            rndim = getattr(result, 'ndim', 0)
            if rndim:
                result = result.view(type(self))
                result.__setmask__(newmask)
            elif newmask:
                result = masked
            return result
        # Explicit output
        result = self.filled(0).sum(axis, dtype=dtype, out=out, **kwargs)
        if isinstance(out, MaskedArray):
            outmask = getmask(out)
            if outmask is nomask:
                outmask = out._mask = make_mask_none(out.shape)
            outmask.flat = newmask
        return out

    def cumsum(self, axis=None, dtype=None, out=None):
        """
        Return the cumulative sum of the array elements over the given axis.

        Masked values are set to 0 internally during the computation.
        However, their position is saved, and the result will be masked at
        the same locations.

        Refer to `numpy.cumsum` for full documentation.

        Notes
        -----
        The mask is lost if `out` is not a valid :class:`ma.MaskedArray` !

        Arithmetic is modular when using integer types, and no error is
        raised on overflow.

        See Also
        --------
        numpy.ndarray.cumsum : corresponding function for ndarrays
        numpy.cumsum : equivalent function

        Examples
        --------
        >>> marr = np.ma.array(np.arange(10), mask=[0,0,0,1,1,1,0,0,0,0])
        >>> marr.cumsum()
        masked_array(data=[0, 1, 3, --, --, --, 9, 16, 24, 33],
                     mask=[False, False, False,  True,  True,  True, False, False,
                           False, False],
               fill_value=999999)

        """
        result = self.filled(0).cumsum(axis=axis, dtype=dtype, out=out)
        if out is not None:
            if isinstance(out, MaskedArray):
                out.__setmask__(self.mask)
            return out
        result = result.view(type(self))
        result.__setmask__(self._mask)
        return result

    def prod(self, axis=None, dtype=None, out=None, keepdims=np._NoValue):
        """
        Return the product of the array elements over the given axis.

        Masked elements are set to 1 internally for computation.

        Refer to `numpy.prod` for full documentation.

        Notes
        -----
        Arithmetic is modular when using integer types, and no error is raised
        on overflow.

        See Also
        --------
        numpy.ndarray.prod : corresponding function for ndarrays
        numpy.prod : equivalent function
        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        _mask = self._mask
        newmask = _check_mask_axis(_mask, axis, **kwargs)
        # No explicit output
        if out is None:
            result = self.filled(1).prod(axis, dtype=dtype, **kwargs)
            rndim = getattr(result, 'ndim', 0)
            if rndim:
                result = result.view(type(self))
                result.__setmask__(newmask)
            elif newmask:
                result = masked
            return result
        # Explicit output
        result = self.filled(1).prod(axis, dtype=dtype, out=out, **kwargs)
        if isinstance(out, MaskedArray):
            outmask = getmask(out)
            if outmask is nomask:
                outmask = out._mask = make_mask_none(out.shape)
            outmask.flat = newmask
        return out
    product = prod

    def cumprod(self, axis=None, dtype=None, out=None):
        """
        Return the cumulative product of the array elements over the given axis.

        Masked values are set to 1 internally during the computation.
        However, their position is saved, and the result will be masked at
        the same locations.

        Refer to `numpy.cumprod` for full documentation.

        Notes
        -----
        The mask is lost if `out` is not a valid MaskedArray !

        Arithmetic is modular when using integer types, and no error is
        raised on overflow.

        See Also
        --------
        numpy.ndarray.cumprod : corresponding function for ndarrays
        numpy.cumprod : equivalent function
        """
        result = self.filled(1).cumprod(axis=axis, dtype=dtype, out=out)
        if out is not None:
            if isinstance(out, MaskedArray):
                out.__setmask__(self._mask)
            return out
        result = result.view(type(self))
        result.__setmask__(self._mask)
        return result

    def mean(self, axis=None, dtype=None, out=None, keepdims=np._NoValue):
        """
        Returns the average of the array elements along given axis.

        Masked entries are ignored, and result elements which are not
        finite will be masked.

        Refer to `numpy.mean` for full documentation.

        See Also
        --------
        numpy.ndarray.mean : corresponding function for ndarrays
        numpy.mean : Equivalent function
        numpy.ma.average : Weighted average.

        Examples
        --------
        >>> a = np.ma.array([1,2,3], mask=[False, False, True])
        >>> a
        masked_array(data=[1, 2, --],
                     mask=[False, False,  True],
               fill_value=999999)
        >>> a.mean()
        1.5

        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}
        if self._mask is nomask:
            result = super().mean(axis=axis, dtype=dtype, **kwargs)[()]
        else:
            is_float16_result = False
            if dtype is None:
                if issubclass(self.dtype.type, (ntypes.integer, ntypes.bool_)):
                    dtype = mu.dtype('f8')
                elif issubclass(self.dtype.type, ntypes.float16):
                    dtype = mu.dtype('f4')
                    is_float16_result = True
            dsum = self.sum(axis=axis, dtype=dtype, **kwargs)
            cnt = self.count(axis=axis, **kwargs)
            if cnt.shape == () and (cnt == 0):
                result = masked
            elif is_float16_result:
                result = self.dtype.type(dsum * 1. / cnt)
            else:
                result = dsum * 1. / cnt
        if out is not None:
            out.flat = result
            if isinstance(out, MaskedArray):
                outmask = getmask(out)
                if outmask is nomask:
                    outmask = out._mask = make_mask_none(out.shape)
                outmask.flat = getmask(result)
            return out
        return result

    def anom(self, axis=None, dtype=None):
        """
        Compute the anomalies (deviations from the arithmetic mean)
        along the given axis.

        Returns an array of anomalies, with the same shape as the input and
        where the arithmetic mean is computed along the given axis.

        Parameters
        ----------
        axis : int, optional
            Axis over which the anomalies are taken.
            The default is to use the mean of the flattened array as reference.
        dtype : dtype, optional
            Type to use in computing the variance. For arrays of integer type
             the default is float32; for arrays of float types it is the same as
             the array type.

        See Also
        --------
        mean : Compute the mean of the array.

        Examples
        --------
        >>> a = np.ma.array([1,2,3])
        >>> a.anom()
        masked_array(data=[-1.,  0.,  1.],
                     mask=False,
               fill_value=1e+20)

        """
        m = self.mean(axis, dtype)
        if not axis:
            return self - m
        else:
            return self - expand_dims(m, axis)

    def var(self, axis=None, dtype=None, out=None, ddof=0,
            keepdims=np._NoValue):
        """
        Returns the variance of the array elements along given axis.

        Masked entries are ignored, and result elements which are not
        finite will be masked.

        Refer to `numpy.var` for full documentation.

        See Also
        --------
        numpy.ndarray.var : corresponding function for ndarrays
        numpy.var : Equivalent function
        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        # Easy case: nomask, business as usual
        if self._mask is nomask:
            ret = super().var(axis=axis, dtype=dtype, out=out, ddof=ddof,
                              **kwargs)[()]
            if out is not None:
                if isinstance(out, MaskedArray):
                    out.__setmask__(nomask)
                return out
            return ret

        # Some data are masked, yay!
        cnt = self.count(axis=axis, **kwargs) - ddof
        danom = self - self.mean(axis, dtype, keepdims=True)
        if iscomplexobj(self):
            danom = umath.absolute(danom) ** 2
        else:
            danom *= danom
        dvar = divide(danom.sum(axis, **kwargs), cnt).view(type(self))
        # Apply the mask if it's not a scalar
        if dvar.ndim:
            dvar._mask = mask_or(self._mask.all(axis, **kwargs), (cnt <= 0))
            dvar._update_from(self)
        elif getmask(dvar):
            # Make sure that masked is returned when the scalar is masked.
            dvar = masked
            if out is not None:
                if isinstance(out, MaskedArray):
                    out.flat = 0
                    out.__setmask__(True)
                elif out.dtype.kind in 'biu':
                    errmsg = "Masked data information would be lost in one or "\
                             "more location."
                    raise MaskError(errmsg)
                else:
                    out.flat = np.nan
                return out
        # In case with have an explicit output
        if out is not None:
            # Set the data
            out.flat = dvar
            # Set the mask if needed
            if isinstance(out, MaskedArray):
                out.__setmask__(dvar.mask)
            return out
        return dvar
    var.__doc__ = np.var.__doc__

    def std(self, axis=None, dtype=None, out=None, ddof=0,
            keepdims=np._NoValue):
        """
        Returns the standard deviation of the array elements along given axis.

        Masked entries are ignored.

        Refer to `numpy.std` for full documentation.

        See Also
        --------
        numpy.ndarray.std : corresponding function for ndarrays
        numpy.std : Equivalent function
        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        dvar = self.var(axis, dtype, out, ddof, **kwargs)
        if dvar is not masked:
            if out is not None:
                np.power(out, 0.5, out=out, casting='unsafe')
                return out
            dvar = sqrt(dvar)
        return dvar

    def round(self, decimals=0, out=None):
        """
        Return each element rounded to the given number of decimals.

        Refer to `numpy.around` for full documentation.

        See Also
        --------
        numpy.ndarray.round : corresponding function for ndarrays
        numpy.around : equivalent function
        """
        result = self._data.round(decimals=decimals, out=out).view(type(self))
        if result.ndim > 0:
            result._mask = self._mask
            result._update_from(self)
        elif self._mask:
            # Return masked when the scalar is masked
            result = masked
        # No explicit output: we're done
        if out is None:
            return result
        if isinstance(out, MaskedArray):
            out.__setmask__(self._mask)
        return out

    def argsort(self, axis=np._NoValue, kind=None, order=None,
                endwith=True, fill_value=None):
        """
        Return an ndarray of indices that sort the array along the
        specified axis.  Masked values are filled beforehand to
        `fill_value`.

        Parameters
        ----------
        axis : int, optional
            Axis along which to sort. If None, the default, the flattened array
            is used.

            ..  versionchanged:: 1.13.0
                Previously, the default was documented to be -1, but that was
                in error. At some future date, the default will change to -1, as
                originally intended.
                Until then, the axis should be given explicitly when
                ``arr.ndim > 1``, to avoid a FutureWarning.
        kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional
            The sorting algorithm used.
        order : list, optional
            When `a` is an array with fields defined, this argument specifies
            which fields to compare first, second, etc.  Not all fields need be
            specified.
        endwith : {True, False}, optional
            Whether missing values (if any) should be treated as the largest values
            (True) or the smallest values (False)
            When the array contains unmasked values at the same extremes of the
            datatype, the ordering of these values and the masked values is
            undefined.
        fill_value : scalar or None, optional
            Value used internally for the masked values.
            If ``fill_value`` is not None, it supersedes ``endwith``.

        Returns
        -------
        index_array : ndarray, int
            Array of indices that sort `a` along the specified axis.
            In other words, ``a[index_array]`` yields a sorted `a`.

        See Also
        --------
        ma.MaskedArray.sort : Describes sorting algorithms used.
        lexsort : Indirect stable sort with multiple keys.
        numpy.ndarray.sort : Inplace sort.

        Notes
        -----
        See `sort` for notes on the different sorting algorithms.

        Examples
        --------
        >>> a = np.ma.array([3,2,1], mask=[False, False, True])
        >>> a
        masked_array(data=[3, 2, --],
                     mask=[False, False,  True],
               fill_value=999999)
        >>> a.argsort()
        array([1, 0, 2])

        """

        # 2017-04-11, Numpy 1.13.0, gh-8701: warn on axis default
        if axis is np._NoValue:
            axis = _deprecate_argsort_axis(self)

        if fill_value is None:
            if endwith:
                # nan > inf
                if np.issubdtype(self.dtype, np.floating):
                    fill_value = np.nan
                else:
                    fill_value = minimum_fill_value(self)
            else:
                fill_value = maximum_fill_value(self)

        filled = self.filled(fill_value)
        return filled.argsort(axis=axis, kind=kind, order=order)

    def argmin(self, axis=None, fill_value=None, out=None, *,
                keepdims=np._NoValue):
        """
        Return array of indices to the minimum values along the given axis.

        Parameters
        ----------
        axis : {None, integer}
            If None, the index is into the flattened array, otherwise along
            the specified axis
        fill_value : scalar or None, optional
            Value used to fill in the masked values.  If None, the output of
            minimum_fill_value(self._data) is used instead.
        out : {None, array}, optional
            Array into which the result can be placed. Its type is preserved
            and it must be of the right shape to hold the output.

        Returns
        -------
        ndarray or scalar
            If multi-dimension input, returns a new ndarray of indices to the
            minimum values along the given axis.  Otherwise, returns a scalar
            of index to the minimum values along the given axis.

        Examples
        --------
        >>> x = np.ma.array(np.arange(4), mask=[1,1,0,0])
        >>> x.shape = (2,2)
        >>> x
        masked_array(
          data=[[--, --],
                [2, 3]],
          mask=[[ True,  True],
                [False, False]],
          fill_value=999999)
        >>> x.argmin(axis=0, fill_value=-1)
        array([0, 0])
        >>> x.argmin(axis=0, fill_value=9)
        array([1, 1])

        """
        if fill_value is None:
            fill_value = minimum_fill_value(self)
        d = self.filled(fill_value).view(ndarray)
        keepdims = False if keepdims is np._NoValue else bool(keepdims)
        return d.argmin(axis, out=out, keepdims=keepdims)

    def argmax(self, axis=None, fill_value=None, out=None, *,
                keepdims=np._NoValue):
        """
        Returns array of indices of the maximum values along the given axis.
        Masked values are treated as if they had the value fill_value.

        Parameters
        ----------
        axis : {None, integer}
            If None, the index is into the flattened array, otherwise along
            the specified axis
        fill_value : scalar or None, optional
            Value used to fill in the masked values.  If None, the output of
            maximum_fill_value(self._data) is used instead.
        out : {None, array}, optional
            Array into which the result can be placed. Its type is preserved
            and it must be of the right shape to hold the output.

        Returns
        -------
        index_array : {integer_array}

        Examples
        --------
        >>> a = np.arange(6).reshape(2,3)
        >>> a.argmax()
        5
        >>> a.argmax(0)
        array([1, 1, 1])
        >>> a.argmax(1)
        array([2, 2])

        """
        if fill_value is None:
            fill_value = maximum_fill_value(self._data)
        d = self.filled(fill_value).view(ndarray)
        keepdims = False if keepdims is np._NoValue else bool(keepdims)
        return d.argmax(axis, out=out, keepdims=keepdims)

    def sort(self, axis=-1, kind=None, order=None,
             endwith=True, fill_value=None):
        """
        Sort the array, in-place

        Parameters
        ----------
        a : array_like
            Array to be sorted.
        axis : int, optional
            Axis along which to sort. If None, the array is flattened before
            sorting. The default is -1, which sorts along the last axis.
        kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, optional
            The sorting algorithm used.
        order : list, optional
            When `a` is a structured array, this argument specifies which fields
            to compare first, second, and so on.  This list does not need to
            include all of the fields.
        endwith : {True, False}, optional
            Whether missing values (if any) should be treated as the largest values
            (True) or the smallest values (False)
            When the array contains unmasked values sorting at the same extremes of the
            datatype, the ordering of these values and the masked values is
            undefined.
        fill_value : scalar or None, optional
            Value used internally for the masked values.
            If ``fill_value`` is not None, it supersedes ``endwith``.

        Returns
        -------
        sorted_array : ndarray
            Array of the same type and shape as `a`.

        See Also
        --------
        numpy.ndarray.sort : Method to sort an array in-place.
        argsort : Indirect sort.
        lexsort : Indirect stable sort on multiple keys.
        searchsorted : Find elements in a sorted array.

        Notes
        -----
        See ``sort`` for notes on the different sorting algorithms.

        Examples
        --------
        >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0])
        >>> # Default
        >>> a.sort()
        >>> a
        masked_array(data=[1, 3, 5, --, --],
                     mask=[False, False, False,  True,  True],
               fill_value=999999)

        >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0])
        >>> # Put missing values in the front
        >>> a.sort(endwith=False)
        >>> a
        masked_array(data=[--, --, 1, 3, 5],
                     mask=[ True,  True, False, False, False],
               fill_value=999999)

        >>> a = np.ma.array([1, 2, 5, 4, 3],mask=[0, 1, 0, 1, 0])
        >>> # fill_value takes over endwith
        >>> a.sort(endwith=False, fill_value=3)
        >>> a
        masked_array(data=[1, --, --, 3, 5],
                     mask=[False,  True,  True, False, False],
               fill_value=999999)

        """
        if self._mask is nomask:
            ndarray.sort(self, axis=axis, kind=kind, order=order)
            return

        if self is masked:
            return

        sidx = self.argsort(axis=axis, kind=kind, order=order,
                            fill_value=fill_value, endwith=endwith)

        self[...] = np.take_along_axis(self, sidx, axis=axis)

    def min(self, axis=None, out=None, fill_value=None, keepdims=np._NoValue):
        """
        Return the minimum along a given axis.

        Parameters
        ----------
        axis : None or int or tuple of ints, optional
            Axis along which to operate.  By default, ``axis`` is None and the
            flattened input is used.
            .. versionadded:: 1.7.0
            If this is a tuple of ints, the minimum is selected over multiple
            axes, instead of a single axis or all the axes as before.
        out : array_like, optional
            Alternative output array in which to place the result.  Must be of
            the same shape and buffer length as the expected output.
        fill_value : scalar or None, optional
            Value used to fill in the masked values.
            If None, use the output of `minimum_fill_value`.
        keepdims : bool, optional
            If this is set to True, the axes which are reduced are left
            in the result as dimensions with size one. With this option,
            the result will broadcast correctly against the array.

        Returns
        -------
        amin : array_like
            New array holding the result.
            If ``out`` was specified, ``out`` is returned.

        See Also
        --------
        ma.minimum_fill_value
            Returns the minimum filling value for a given datatype.

        Examples
        --------
        >>> import numpy.ma as ma
        >>> x = [[1., -2., 3.], [0.2, -0.7, 0.1]]
        >>> mask = [[1, 1, 0], [0, 0, 1]]
        >>> masked_x = ma.masked_array(x, mask)
        >>> masked_x
        masked_array(
          data=[[--, --, 3.0],
                [0.2, -0.7, --]],
          mask=[[ True,  True, False],
                [False, False,  True]],
          fill_value=1e+20)
        >>> ma.min(masked_x)
        -0.7
        >>> ma.min(masked_x, axis=-1)
        masked_array(data=[3.0, -0.7],
                     mask=[False, False],
                fill_value=1e+20)
        >>> ma.min(masked_x, axis=0, keepdims=True)
        masked_array(data=[[0.2, -0.7, 3.0]],
                     mask=[[False, False, False]],
                fill_value=1e+20)
        >>> mask = [[1, 1, 1,], [1, 1, 1]]
        >>> masked_x = ma.masked_array(x, mask)
        >>> ma.min(masked_x, axis=0)
        masked_array(data=[--, --, --],
                     mask=[ True,  True,  True],
                fill_value=1e+20,
                    dtype=float64)
        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        _mask = self._mask
        newmask = _check_mask_axis(_mask, axis, **kwargs)
        if fill_value is None:
            fill_value = minimum_fill_value(self)
        # No explicit output
        if out is None:
            result = self.filled(fill_value).min(
                axis=axis, out=out, **kwargs).view(type(self))
            if result.ndim:
                # Set the mask
                result.__setmask__(newmask)
                # Get rid of Infs
                if newmask.ndim:
                    np.copyto(result, result.fill_value, where=newmask)
            elif newmask:
                result = masked
            return result
        # Explicit output
        result = self.filled(fill_value).min(axis=axis, out=out, **kwargs)
        if isinstance(out, MaskedArray):
            outmask = getmask(out)
            if outmask is nomask:
                outmask = out._mask = make_mask_none(out.shape)
            outmask.flat = newmask
        else:
            if out.dtype.kind in 'biu':
                errmsg = "Masked data information would be lost in one or more"\
                         " location."
                raise MaskError(errmsg)
            np.copyto(out, np.nan, where=newmask)
        return out

    def max(self, axis=None, out=None, fill_value=None, keepdims=np._NoValue):
        """
        Return the maximum along a given axis.

        Parameters
        ----------
        axis : None or int or tuple of ints, optional
            Axis along which to operate.  By default, ``axis`` is None and the
            flattened input is used.
            .. versionadded:: 1.7.0
            If this is a tuple of ints, the maximum is selected over multiple
            axes, instead of a single axis or all the axes as before.
        out : array_like, optional
            Alternative output array in which to place the result.  Must
            be of the same shape and buffer length as the expected output.
        fill_value : scalar or None, optional
            Value used to fill in the masked values.
            If None, use the output of maximum_fill_value().
        keepdims : bool, optional
            If this is set to True, the axes which are reduced are left
            in the result as dimensions with size one. With this option,
            the result will broadcast correctly against the array.

        Returns
        -------
        amax : array_like
            New array holding the result.
            If ``out`` was specified, ``out`` is returned.

        See Also
        --------
        ma.maximum_fill_value
            Returns the maximum filling value for a given datatype.

        Examples
        --------
        >>> import numpy.ma as ma
        >>> x = [[-1., 2.5], [4., -2.], [3., 0.]]
        >>> mask = [[0, 0], [1, 0], [1, 0]]
        >>> masked_x = ma.masked_array(x, mask)
        >>> masked_x
        masked_array(
          data=[[-1.0, 2.5],
                [--, -2.0],
                [--, 0.0]],
          mask=[[False, False],
                [ True, False],
                [ True, False]],
          fill_value=1e+20)
        >>> ma.max(masked_x)
        2.5
        >>> ma.max(masked_x, axis=0)
        masked_array(data=[-1.0, 2.5],
                     mask=[False, False],
               fill_value=1e+20)
        >>> ma.max(masked_x, axis=1, keepdims=True)
        masked_array(
          data=[[2.5],
                [-2.0],
                [0.0]],
          mask=[[False],
                [False],
                [False]],
          fill_value=1e+20)
        >>> mask = [[1, 1], [1, 1], [1, 1]]
        >>> masked_x = ma.masked_array(x, mask)
        >>> ma.max(masked_x, axis=1)
        masked_array(data=[--, --, --],
                     mask=[ True,  True,  True],
               fill_value=1e+20,
                    dtype=float64)
        """
        kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

        _mask = self._mask
        newmask = _check_mask_axis(_mask, axis, **kwargs)
        if fill_value is None:
            fill_value = maximum_fill_value(self)
        # No explicit output
        if out is None:
            result = self.filled(fill_value).max(
                axis=axis, out=out, **kwargs).view(type(self))
            if result.ndim:
                # Set the mask
                result.__setmask__(newmask)
                # Get rid of Infs
                if newmask.ndim:
                    np.copyto(result, result.fill_value, where=newmask)
            elif newmask:
                result = masked
            return result
        # Explicit output
        result = self.filled(fill_value).max(axis=axis, out=out, **kwargs)
        if isinstance(out, MaskedArray):
            outmask = getmask(out)
            if outmask is nomask:
                outmask = out._mask = make_mask_none(out.shape)
            outmask.flat = newmask
        else:

            if out.dtype.kind in 'biu':
                errmsg = "Masked data information would be lost in one or more"\
                         " location."
                raise MaskError(errmsg)
            np.copyto(out, np.nan, where=newmask)
        return out

    def ptp(self, axis=None, out=None, fill_value=None, keepdims=False):
        """
        Return (maximum - minimum) along the given dimension
        (i.e. peak-to-peak value).

        .. warning::
            `ptp` preserves the data type of the array. This means the
            return value for an input of signed integers with n bits
            (e.g. `np.int8`, `np.int16`, etc) is also a signed integer
            with n bits.  In that case, peak-to-peak values greater than
            ``2**(n-1)-1`` will be returned as negative values. An example
            with a work-around is shown below.

        Parameters
        ----------
        axis : {None, int}, optional
            Axis along which to find the peaks.  If None (default) the
            flattened array is used.
        out : {None, array_like}, optional
            Alternative output array in which to place the result. It must
            have the same shape and buffer length as the expected output
            but the type will be cast if necessary.
        fill_value : scalar or None, optional
            Value used to fill in the masked values.
        keepdims : bool, optional
            If this is set to True, the axes which are reduced are left
            in the result as dimensions with size one. With this option,
            the result will broadcast correctly against the array.

        Returns
        -------
        ptp : ndarray.
            A new array holding the result, unless ``out`` was
            specified, in which case a reference to ``out`` is returned.

        Examples
        --------
        >>> x = np.ma.MaskedArray([[4, 9, 2, 10],
        ...                        [6, 9, 7, 12]])

        >>> x.ptp(axis=1)
        masked_array(data=[8, 6],
                     mask=False,
               fill_value=999999)

        >>> x.ptp(axis=0)
        masked_array(data=[2, 0, 5, 2],
                     mask=False,
               fill_value=999999)

        >>> x.ptp()
        10

        This example shows that a negative value can be returned when
        the input is an array of signed integers.

        >>> y = np.ma.MaskedArray([[1, 127],
        ...                        [0, 127],
        ...                        [-1, 127],
        ...                        [-2, 127]], dtype=np.int8)
        >>> y.ptp(axis=1)
        masked_array(data=[ 126,  127, -128, -127],
                     mask=False,
               fill_value=999999,
                    dtype=int8)

        A work-around is to use the `view()` method to view the result as
        unsigned integers with the same bit width:

        >>> y.ptp(axis=1).view(np.uint8)
        masked_array(data=[126, 127, 128, 129],
                     mask=False,
               fill_value=999999,
                    dtype=uint8)
        """
        if out is None:
            result = self.max(axis=axis, fill_value=fill_value,
                              keepdims=keepdims)
            result -= self.min(axis=axis, fill_value=fill_value,
                               keepdims=keepdims)
            return result
        out.flat = self.max(axis=axis, out=out, fill_value=fill_value,
                            keepdims=keepdims)
        min_value = self.min(axis=axis, fill_value=fill_value,
                             keepdims=keepdims)
        np.subtract(out, min_value, out=out, casting='unsafe')
        return out

    def partition(self, *args, **kwargs):
        warnings.warn("Warning: 'partition' will ignore the 'mask' "
                      f"of the {self.__class__.__name__}.",
                      stacklevel=2)
        return super().partition(*args, **kwargs)

    def argpartition(self, *args, **kwargs):
        warnings.warn("Warning: 'argpartition' will ignore the 'mask' "
                      f"of the {self.__class__.__name__}.",
                      stacklevel=2)
        return super().argpartition(*args, **kwargs)

    def take(self, indices, axis=None, out=None, mode='raise'):
        """
        """
        (_data, _mask) = (self._data, self._mask)
        cls = type(self)
        # Make sure the indices are not masked
        maskindices = getmask(indices)
        if maskindices is not nomask:
            indices = indices.filled(0)
        # Get the data, promoting scalars to 0d arrays with [...] so that
        # .view works correctly
        if out is None:
            out = _data.take(indices, axis=axis, mode=mode)[...].view(cls)
        else:
            np.take(_data, indices, axis=axis, mode=mode, out=out)
        # Get the mask
        if isinstance(out, MaskedArray):
            if _mask is nomask:
                outmask = maskindices
            else:
                outmask = _mask.take(indices, axis=axis, mode=mode)
                outmask |= maskindices
            out.__setmask__(outmask)
        # demote 0d arrays back to scalars, for consistency with ndarray.take
        return out[()]

    # Array methods
    copy = _arraymethod('copy')
    diagonal = _arraymethod('diagonal')
    flatten = _arraymethod('flatten')
    repeat = _arraymethod('repeat')
    squeeze = _arraymethod('squeeze')
    swapaxes = _arraymethod('swapaxes')
    T = property(fget=lambda self: self.transpose())
    transpose = _arraymethod('transpose')

    def tolist(self, fill_value=None):
        """
        Return the data portion of the masked array as a hierarchical Python list.

        Data items are converted to the nearest compatible Python type.
        Masked values are converted to `fill_value`. If `fill_value` is None,
        the corresponding entries in the output list will be ``None``.

        Parameters
        ----------
        fill_value : scalar, optional
            The value to use for invalid entries. Default is None.

        Returns
        -------
        result : list
            The Python list representation of the masked array.

        Examples
        --------
        >>> x = np.ma.array([[1,2,3], [4,5,6], [7,8,9]], mask=[0] + [1,0]*4)
        >>> x.tolist()
        [[1, None, 3], [None, 5, None], [7, None, 9]]
        >>> x.tolist(-999)
        [[1, -999, 3], [-999, 5, -999], [7, -999, 9]]

        """
        _mask = self._mask
        # No mask ? Just return .data.tolist ?
        if _mask is nomask:
            return self._data.tolist()
        # Explicit fill_value: fill the array and get the list
        if fill_value is not None:
            return self.filled(fill_value).tolist()
        # Structured array.
        names = self.dtype.names
        if names:
            result = self._data.astype([(_, object) for _ in names])
            for n in names:
                result[n][_mask[n]] = None
            return result.tolist()
        # Standard arrays.
        if _mask is nomask:
            return [None]
        # Set temps to save time when dealing w/ marrays.
        inishape = self.shape
        result = np.array(self._data.ravel(), dtype=object)
        result[_mask.ravel()] = None
        result.shape = inishape
        return result.tolist()

    def tostring(self, fill_value=None, order='C'):
        r"""
        A compatibility alias for `tobytes`, with exactly the same behavior.

        Despite its name, it returns `bytes` not `str`\ s.

        .. deprecated:: 1.19.0
        """
        # 2020-03-30, Numpy 1.19.0
        warnings.warn(
            "tostring() is deprecated. Use tobytes() instead.",
            DeprecationWarning, stacklevel=2)

        return self.tobytes(fill_value, order=order)

    def tobytes(self, fill_value=None, order='C'):
        """
        Return the array data as a string containing the raw bytes in the array.

        The array is filled with a fill value before the string conversion.

        .. versionadded:: 1.9.0

        Parameters
        ----------
        fill_value : scalar, optional
            Value used to fill in the masked values. Default is None, in which
            case `MaskedArray.fill_value` is used.
        order : {'C','F','A'}, optional
            Order of the data item in the copy. Default is 'C'.

            - 'C'   -- C order (row major).
            - 'F'   -- Fortran order (column major).
            - 'A'   -- Any, current order of array.
            - None  -- Same as 'A'.

        See Also
        --------
        numpy.ndarray.tobytes
        tolist, tofile

        Notes
        -----
        As for `ndarray.tobytes`, information about the shape, dtype, etc.,
        but also about `fill_value`, will be lost.

        Examples
        --------
        >>> x = np.ma.array(np.array([[1, 2], [3, 4]]), mask=[[0, 1], [1, 0]])
        >>> x.tobytes()
        b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00?B\\x0f\\x00\\x00\\x00\\x00\\x00?B\\x0f\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00'

        """
        return self.filled(fill_value).tobytes(order=order)

    def tofile(self, fid, sep="", format="%s"):
        """
        Save a masked array to a file in binary format.

        .. warning::
          This function is not implemented yet.

        Raises
        ------
        NotImplementedError
            When `tofile` is called.

        """
        raise NotImplementedError("MaskedArray.tofile() not implemented yet.")

    def toflex(self):
        """
        Transforms a masked array into a flexible-type array.

        The flexible type array that is returned will have two fields:

        * the ``_data`` field stores the ``_data`` part of the array.
        * the ``_mask`` field stores the ``_mask`` part of the array.

        Parameters
        ----------
        None

        Returns
        -------
        record : ndarray
            A new flexible-type `ndarray` with two fields: the first element
            containing a value, the second element containing the corresponding
            mask boolean. The returned record shape matches self.shape.

        Notes
        -----
        A side-effect of transforming a masked array into a flexible `ndarray` is
        that meta information (``fill_value``, ...) will be lost.

        Examples
        --------
        >>> x = np.ma.array([[1,2,3],[4,5,6],[7,8,9]], mask=[0] + [1,0]*4)
        >>> x
        masked_array(
          data=[[1, --, 3],
                [--, 5, --],
                [7, --, 9]],
          mask=[[False,  True, False],
                [ True, False,  True],
                [False,  True, False]],
          fill_value=999999)
        >>> x.toflex()
        array([[(1, False), (2,  True), (3, False)],
               [(4,  True), (5, False), (6,  True)],
               [(7, False), (8,  True), (9, False)]],
              dtype=[('_data', '<i8'), ('_mask', '?')])

        """
        # Get the basic dtype.
        ddtype = self.dtype
        # Make sure we have a mask
        _mask = self._mask
        if _mask is None:
            _mask = make_mask_none(self.shape, ddtype)
        # And get its dtype
        mdtype = self._mask.dtype

        record = np.ndarray(shape=self.shape,
                            dtype=[('_data', ddtype), ('_mask', mdtype)])
        record['_data'] = self._data
        record['_mask'] = self._mask
        return record
    torecords = toflex

    # Pickling
    def __getstate__(self):
        """Return the internal state of the masked array, for pickling
        purposes.

        """
        cf = 'CF'[self.flags.fnc]
        data_state = super().__reduce__()[2]
        return data_state + (getmaskarray(self).tobytes(cf), self._fill_value)

    def __setstate__(self, state):
        """Restore the internal state of the masked array, for
        pickling purposes.  ``state`` is typically the output of the
        ``__getstate__`` output, and is a 5-tuple:

        - class name
        - a tuple giving the shape of the data
        - a typecode for the data
        - a binary string for the data
        - a binary string for the mask.

        """
        (_, shp, typ, isf, raw, msk, flv) = state
        super().__setstate__((shp, typ, isf, raw))
        self._mask.__setstate__((shp, make_mask_descr(typ), isf, msk))
        self.fill_value = flv

    def __reduce__(self):
        """Return a 3-tuple for pickling a MaskedArray.

        """
        return (_mareconstruct,
                (self.__class__, self._baseclass, (0,), 'b',),
                self.__getstate__())

    def __deepcopy__(self, memo=None):
        from copy import deepcopy
        copied = MaskedArray.__new__(type(self), self, copy=True)
        if memo is None:
            memo = {}
        memo[id(self)] = copied
        for (k, v) in self.__dict__.items():
            copied.__dict__[k] = deepcopy(v, memo)
        return copied


def _mareconstruct(subtype, baseclass, baseshape, basetype,):
    """Internal function that builds a new MaskedArray from the
    information stored in a pickle.

    """
    _data = ndarray.__new__(baseclass, baseshape, basetype)
    _mask = ndarray.__new__(ndarray, baseshape, make_mask_descr(basetype))
    return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,)


class mvoid(MaskedArray):
    """
    Fake a 'void' object to use for masked array with structured dtypes.
    """

    def __new__(self, data, mask=nomask, dtype=None, fill_value=None,
                hardmask=False, copy=False, subok=True):
        _data = np.array(data, copy=copy, subok=subok, dtype=dtype)
        _data = _data.view(self)
        _data._hardmask = hardmask
        if mask is not nomask:
            if isinstance(mask, np.void):
                _data._mask = mask
            else:
                try:
                    # Mask is already a 0D array
                    _data._mask = np.void(mask)
                except TypeError:
                    # Transform the mask to a void
                    mdtype = make_mask_descr(dtype)
                    _data._mask = np.array(mask, dtype=mdtype)[()]
        if fill_value is not None:
            _data.fill_value = fill_value
        return _data

    @property
    def _data(self):
        # Make sure that the _data part is a np.void
        return super()._data[()]

    def __getitem__(self, indx):
        """
        Get the index.

        """
        m = self._mask
        if isinstance(m[indx], ndarray):
            # Can happen when indx is a multi-dimensional field:
            # A = ma.masked_array(data=[([0,1],)], mask=[([True,
            #                     False],)], dtype=[("A", ">i2", (2,))])
            # x = A[0]; y = x["A"]; then y.mask["A"].size==2
            # and we can not say masked/unmasked.
            # The result is no longer mvoid!
            # See also issue #6724.
            return masked_array(
                data=self._data[indx], mask=m[indx],
                fill_value=self._fill_value[indx],
                hard_mask=self._hardmask)
        if m is not nomask and m[indx]:
            return masked
        return self._data[indx]

    def __setitem__(self, indx, value):
        self._data[indx] = value
        if self._hardmask:
            self._mask[indx] |= getattr(value, "_mask", False)
        else:
            self._mask[indx] = getattr(value, "_mask", False)

    def __str__(self):
        m = self._mask
        if m is nomask:
            return str(self._data)

        rdtype = _replace_dtype_fields(self._data.dtype, "O")
        data_arr = super()._data
        res = data_arr.astype(rdtype)
        _recursive_printoption(res, self._mask, masked_print_option)
        return str(res)

    __repr__ = __str__

    def __iter__(self):
        "Defines an iterator for mvoid"
        (_data, _mask) = (self._data, self._mask)
        if _mask is nomask:
            yield from _data
        else:
            for (d, m) in zip(_data, _mask):
                if m:
                    yield masked
                else:
                    yield d

    def __len__(self):
        return self._data.__len__()

    def filled(self, fill_value=None):
        """
        Return a copy with masked fields filled with a given value.

        Parameters
        ----------
        fill_value : array_like, optional
            The value to use for invalid entries. Can be scalar or
            non-scalar. If latter is the case, the filled array should
            be broadcastable over input array. Default is None, in
            which case the `fill_value` attribute is used instead.

        Returns
        -------
        filled_void
            A `np.void` object

        See Also
        --------
        MaskedArray.filled

        """
        return asarray(self).filled(fill_value)[()]

    def tolist(self):
        """
    Transforms the mvoid object into a tuple.

    Masked fields are replaced by None.

    Returns
    -------
    returned_tuple
        Tuple of fields
        """
        _mask = self._mask
        if _mask is nomask:
            return self._data.tolist()
        result = []
        for (d, m) in zip(self._data, self._mask):
            if m:
                result.append(None)
            else:
                # .item() makes sure we return a standard Python object
                result.append(d.item())
        return tuple(result)


##############################################################################
#                                Shortcuts                                   #
##############################################################################


def isMaskedArray(x):
    """
    Test whether input is an instance of MaskedArray.

    This function returns True if `x` is an instance of MaskedArray
    and returns False otherwise.  Any object is accepted as input.

    Parameters
    ----------
    x : object
        Object to test.

    Returns
    -------
    result : bool
        True if `x` is a MaskedArray.

    See Also
    --------
    isMA : Alias to isMaskedArray.
    isarray : Alias to isMaskedArray.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.eye(3, 3)
    >>> a
    array([[ 1.,  0.,  0.],
           [ 0.,  1.,  0.],
           [ 0.,  0.,  1.]])
    >>> m = ma.masked_values(a, 0)
    >>> m
    masked_array(
      data=[[1.0, --, --],
            [--, 1.0, --],
            [--, --, 1.0]],
      mask=[[False,  True,  True],
            [ True, False,  True],
            [ True,  True, False]],
      fill_value=0.0)
    >>> ma.isMaskedArray(a)
    False
    >>> ma.isMaskedArray(m)
    True
    >>> ma.isMaskedArray([0, 1, 2])
    False

    """
    return isinstance(x, MaskedArray)


isarray = isMaskedArray
isMA = isMaskedArray  # backward compatibility


class MaskedConstant(MaskedArray):
    # the lone np.ma.masked instance
    __singleton = None

    @classmethod
    def __has_singleton(cls):
        # second case ensures `cls.__singleton` is not just a view on the
        # superclass singleton
        return cls.__singleton is not None and type(cls.__singleton) is cls

    def __new__(cls):
        if not cls.__has_singleton():
            # We define the masked singleton as a float for higher precedence.
            # Note that it can be tricky sometimes w/ type comparison
            data = np.array(0.)
            mask = np.array(True)

            # prevent any modifications
            data.flags.writeable = False
            mask.flags.writeable = False

            # don't fall back on MaskedArray.__new__(MaskedConstant), since
            # that might confuse it - this way, the construction is entirely
            # within our control
            cls.__singleton = MaskedArray(data, mask=mask).view(cls)

        return cls.__singleton

    def __array_finalize__(self, obj):
        if not self.__has_singleton():
            # this handles the `.view` in __new__, which we want to copy across
            # properties normally
            return super().__array_finalize__(obj)
        elif self is self.__singleton:
            # not clear how this can happen, play it safe
            pass
        else:
            # everywhere else, we want to downcast to MaskedArray, to prevent a
            # duplicate maskedconstant.
            self.__class__ = MaskedArray
            MaskedArray.__array_finalize__(self, obj)

    def __array_prepare__(self, obj, context=None):
        return self.view(MaskedArray).__array_prepare__(obj, context)

    def __array_wrap__(self, obj, context=None):
        return self.view(MaskedArray).__array_wrap__(obj, context)

    def __str__(self):
        return str(masked_print_option._display)

    def __repr__(self):
        if self is MaskedConstant.__singleton:
            return 'masked'
        else:
            # it's a subclass, or something is wrong, make it obvious
            return object.__repr__(self)

    def __format__(self, format_spec):
        # Replace ndarray.__format__ with the default, which supports no format characters.
        # Supporting format characters is unwise here, because we do not know what type
        # the user was expecting - better to not guess.
        try:
            return object.__format__(self, format_spec)
        except TypeError:
            # 2020-03-23, NumPy 1.19.0
            warnings.warn(
                "Format strings passed to MaskedConstant are ignored, but in future may "
                "error or produce different behavior",
                FutureWarning, stacklevel=2
            )
            return object.__format__(self, "")

    def __reduce__(self):
        """Override of MaskedArray's __reduce__.
        """
        return (self.__class__, ())

    # inplace operations have no effect. We have to override them to avoid
    # trying to modify the readonly data and mask arrays
    def __iop__(self, other):
        return self
    __iadd__ = \
    __isub__ = \
    __imul__ = \
    __ifloordiv__ = \
    __itruediv__ = \
    __ipow__ = \
        __iop__
    del __iop__  # don't leave this around

    def copy(self, *args, **kwargs):
        """ Copy is a no-op on the maskedconstant, as it is a scalar """
        # maskedconstant is a scalar, so copy doesn't need to copy. There's
        # precedent for this with `np.bool_` scalars.
        return self

    def __copy__(self):
        return self

    def __deepcopy__(self, memo):
        return self

    def __setattr__(self, attr, value):
        if not self.__has_singleton():
            # allow the singleton to be initialized
            return super().__setattr__(attr, value)
        elif self is self.__singleton:
            raise AttributeError(
                f"attributes of {self!r} are not writeable")
        else:
            # duplicate instance - we can end up here from __array_finalize__,
            # where we set the __class__ attribute
            return super().__setattr__(attr, value)


masked = masked_singleton = MaskedConstant()
masked_array = MaskedArray


def array(data, dtype=None, copy=False, order=None,
          mask=nomask, fill_value=None, keep_mask=True,
          hard_mask=False, shrink=True, subok=True, ndmin=0):
    """
    Shortcut to MaskedArray.

    The options are in a different order for convenience and backwards
    compatibility.

    """
    return MaskedArray(data, mask=mask, dtype=dtype, copy=copy,
                       subok=subok, keep_mask=keep_mask,
                       hard_mask=hard_mask, fill_value=fill_value,
                       ndmin=ndmin, shrink=shrink, order=order)
array.__doc__ = masked_array.__doc__


def is_masked(x):
    """
    Determine whether input has masked values.

    Accepts any object as input, but always returns False unless the
    input is a MaskedArray containing masked values.

    Parameters
    ----------
    x : array_like
        Array to check for masked values.

    Returns
    -------
    result : bool
        True if `x` is a MaskedArray with masked values, False otherwise.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = ma.masked_equal([0, 1, 0, 2, 3], 0)
    >>> x
    masked_array(data=[--, 1, --, 2, 3],
                 mask=[ True, False,  True, False, False],
           fill_value=0)
    >>> ma.is_masked(x)
    True
    >>> x = ma.masked_equal([0, 1, 0, 2, 3], 42)
    >>> x
    masked_array(data=[0, 1, 0, 2, 3],
                 mask=False,
           fill_value=42)
    >>> ma.is_masked(x)
    False

    Always returns False if `x` isn't a MaskedArray.

    >>> x = [False, True, False]
    >>> ma.is_masked(x)
    False
    >>> x = 'a string'
    >>> ma.is_masked(x)
    False

    """
    m = getmask(x)
    if m is nomask:
        return False
    elif m.any():
        return True
    return False


##############################################################################
#                             Extrema functions                              #
##############################################################################


class _extrema_operation(_MaskedUFunc):
    """
    Generic class for maximum/minimum functions.

    .. note::
      This is the base class for `_maximum_operation` and
      `_minimum_operation`.

    """
    def __init__(self, ufunc, compare, fill_value):
        super().__init__(ufunc)
        self.compare = compare
        self.fill_value_func = fill_value

    def __call__(self, a, b):
        "Executes the call behavior."

        return where(self.compare(a, b), a, b)

    def reduce(self, target, axis=np._NoValue):
        "Reduce target along the given axis."
        target = narray(target, copy=False, subok=True)
        m = getmask(target)

        if axis is np._NoValue and target.ndim > 1:
            # 2017-05-06, Numpy 1.13.0: warn on axis default
            warnings.warn(
                f"In the future the default for ma.{self.__name__}.reduce will be axis=0, "
                f"not the current None, to match np.{self.__name__}.reduce. "
                "Explicitly pass 0 or None to silence this warning.",
                MaskedArrayFutureWarning, stacklevel=2)
            axis = None

        if axis is not np._NoValue:
            kwargs = dict(axis=axis)
        else:
            kwargs = dict()

        if m is nomask:
            t = self.f.reduce(target, **kwargs)
        else:
            target = target.filled(
                self.fill_value_func(target)).view(type(target))
            t = self.f.reduce(target, **kwargs)
            m = umath.logical_and.reduce(m, **kwargs)
            if hasattr(t, '_mask'):
                t._mask = m
            elif m:
                t = masked
        return t

    def outer(self, a, b):
        "Return the function applied to the outer product of a and b."
        ma = getmask(a)
        mb = getmask(b)
        if ma is nomask and mb is nomask:
            m = nomask
        else:
            ma = getmaskarray(a)
            mb = getmaskarray(b)
            m = logical_or.outer(ma, mb)
        result = self.f.outer(filled(a), filled(b))
        if not isinstance(result, MaskedArray):
            result = result.view(MaskedArray)
        result._mask = m
        return result

def min(obj, axis=None, out=None, fill_value=None, keepdims=np._NoValue):
    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

    try:
        return obj.min(axis=axis, fill_value=fill_value, out=out, **kwargs)
    except (AttributeError, TypeError):
        # If obj doesn't have a min method, or if the method doesn't accept a
        # fill_value argument
        return asanyarray(obj).min(axis=axis, fill_value=fill_value,
                                   out=out, **kwargs)
min.__doc__ = MaskedArray.min.__doc__

def max(obj, axis=None, out=None, fill_value=None, keepdims=np._NoValue):
    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}

    try:
        return obj.max(axis=axis, fill_value=fill_value, out=out, **kwargs)
    except (AttributeError, TypeError):
        # If obj doesn't have a max method, or if the method doesn't accept a
        # fill_value argument
        return asanyarray(obj).max(axis=axis, fill_value=fill_value,
                                   out=out, **kwargs)
max.__doc__ = MaskedArray.max.__doc__


def ptp(obj, axis=None, out=None, fill_value=None, keepdims=np._NoValue):
    kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims}
    try:
        return obj.ptp(axis, out=out, fill_value=fill_value, **kwargs)
    except (AttributeError, TypeError):
        # If obj doesn't have a ptp method or if the method doesn't accept
        # a fill_value argument
        return asanyarray(obj).ptp(axis=axis, fill_value=fill_value,
                                   out=out, **kwargs)
ptp.__doc__ = MaskedArray.ptp.__doc__


##############################################################################
#           Definition of functions from the corresponding methods           #
##############################################################################


class _frommethod:
    """
    Define functions from existing MaskedArray methods.

    Parameters
    ----------
    methodname : str
        Name of the method to transform.

    """

    def __init__(self, methodname, reversed=False):
        self.__name__ = methodname
        self.__doc__ = self.getdoc()
        self.reversed = reversed

    def getdoc(self):
        "Return the doc of the function (from the doc of the method)."
        meth = getattr(MaskedArray, self.__name__, None) or\
            getattr(np, self.__name__, None)
        signature = self.__name__ + get_object_signature(meth)
        if meth is not None:
            doc = """    %s\n%s""" % (
                signature, getattr(meth, '__doc__', None))
            return doc

    def __call__(self, a, *args, **params):
        if self.reversed:
            args = list(args)
            a, args[0] = args[0], a

        marr = asanyarray(a)
        method_name = self.__name__
        method = getattr(type(marr), method_name, None)
        if method is None:
            # use the corresponding np function
            method = getattr(np, method_name)

        return method(marr, *args, **params)


all = _frommethod('all')
anomalies = anom = _frommethod('anom')
any = _frommethod('any')
compress = _frommethod('compress', reversed=True)
cumprod = _frommethod('cumprod')
cumsum = _frommethod('cumsum')
copy = _frommethod('copy')
diagonal = _frommethod('diagonal')
harden_mask = _frommethod('harden_mask')
ids = _frommethod('ids')
maximum = _extrema_operation(umath.maximum, greater, maximum_fill_value)
mean = _frommethod('mean')
minimum = _extrema_operation(umath.minimum, less, minimum_fill_value)
nonzero = _frommethod('nonzero')
prod = _frommethod('prod')
product = _frommethod('prod')
ravel = _frommethod('ravel')
repeat = _frommethod('repeat')
shrink_mask = _frommethod('shrink_mask')
soften_mask = _frommethod('soften_mask')
std = _frommethod('std')
sum = _frommethod('sum')
swapaxes = _frommethod('swapaxes')
#take = _frommethod('take')
trace = _frommethod('trace')
var = _frommethod('var')

count = _frommethod('count')

def take(a, indices, axis=None, out=None, mode='raise'):
    """
    """
    a = masked_array(a)
    return a.take(indices, axis=axis, out=out, mode=mode)


def power(a, b, third=None):
    """
    Returns element-wise base array raised to power from second array.

    This is the masked array version of `numpy.power`. For details see
    `numpy.power`.

    See Also
    --------
    numpy.power

    Notes
    -----
    The *out* argument to `numpy.power` is not supported, `third` has to be
    None.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = [11.2, -3.973, 0.801, -1.41]
    >>> mask = [0, 0, 0, 1]
    >>> masked_x = ma.masked_array(x, mask)
    >>> masked_x
    masked_array(data=[11.2, -3.973, 0.801, --],
             mask=[False, False, False,  True],
       fill_value=1e+20)
    >>> ma.power(masked_x, 2)
    masked_array(data=[125.43999999999998, 15.784728999999999,
                   0.6416010000000001, --],
             mask=[False, False, False,  True],
       fill_value=1e+20)
    >>> y = [-0.5, 2, 0, 17]
    >>> masked_y = ma.masked_array(y, mask)
    >>> masked_y
    masked_array(data=[-0.5, 2.0, 0.0, --],
             mask=[False, False, False,  True],
       fill_value=1e+20)
    >>> ma.power(masked_x, masked_y)
    masked_array(data=[0.29880715233359845, 15.784728999999999, 1.0, --],
             mask=[False, False, False,  True],
       fill_value=1e+20)

    """
    if third is not None:
        raise MaskError("3-argument power not supported.")
    # Get the masks
    ma = getmask(a)
    mb = getmask(b)
    m = mask_or(ma, mb)
    # Get the rawdata
    fa = getdata(a)
    fb = getdata(b)
    # Get the type of the result (so that we preserve subclasses)
    if isinstance(a, MaskedArray):
        basetype = type(a)
    else:
        basetype = MaskedArray
    # Get the result and view it as a (subclass of) MaskedArray
    with np.errstate(divide='ignore', invalid='ignore'):
        result = np.where(m, fa, umath.power(fa, fb)).view(basetype)
    result._update_from(a)
    # Find where we're in trouble w/ NaNs and Infs
    invalid = np.logical_not(np.isfinite(result.view(ndarray)))
    # Add the initial mask
    if m is not nomask:
        if not result.ndim:
            return masked
        result._mask = np.logical_or(m, invalid)
    # Fix the invalid parts
    if invalid.any():
        if not result.ndim:
            return masked
        elif result._mask is nomask:
            result._mask = invalid
        result._data[invalid] = result.fill_value
    return result

argmin = _frommethod('argmin')
argmax = _frommethod('argmax')

def argsort(a, axis=np._NoValue, kind=None, order=None, endwith=True, fill_value=None):
    "Function version of the eponymous method."
    a = np.asanyarray(a)

    # 2017-04-11, Numpy 1.13.0, gh-8701: warn on axis default
    if axis is np._NoValue:
        axis = _deprecate_argsort_axis(a)

    if isinstance(a, MaskedArray):
        return a.argsort(axis=axis, kind=kind, order=order,
                         endwith=endwith, fill_value=fill_value)
    else:
        return a.argsort(axis=axis, kind=kind, order=order)
argsort.__doc__ = MaskedArray.argsort.__doc__

def sort(a, axis=-1, kind=None, order=None, endwith=True, fill_value=None):
    """
    Return a sorted copy of the masked array.

    Equivalent to creating a copy of the array
    and applying the  MaskedArray ``sort()`` method.

    Refer to ``MaskedArray.sort`` for the full documentation

    See Also
    --------
    MaskedArray.sort : equivalent method
    """
    a = np.array(a, copy=True, subok=True)
    if axis is None:
        a = a.flatten()
        axis = 0

    if isinstance(a, MaskedArray):
        a.sort(axis=axis, kind=kind, order=order,
               endwith=endwith, fill_value=fill_value)
    else:
        a.sort(axis=axis, kind=kind, order=order)
    return a


def compressed(x):
    """
    Return all the non-masked data as a 1-D array.

    This function is equivalent to calling the "compressed" method of a
    `ma.MaskedArray`, see `ma.MaskedArray.compressed` for details.

    See Also
    --------
    ma.MaskedArray.compressed : Equivalent method.

    """
    return asanyarray(x).compressed()


def concatenate(arrays, axis=0):
    """
    Concatenate a sequence of arrays along the given axis.

    Parameters
    ----------
    arrays : sequence of array_like
        The arrays must have the same shape, except in the dimension
        corresponding to `axis` (the first, by default).
    axis : int, optional
        The axis along which the arrays will be joined. Default is 0.

    Returns
    -------
    result : MaskedArray
        The concatenated array with any masked entries preserved.

    See Also
    --------
    numpy.concatenate : Equivalent function in the top-level NumPy module.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = ma.arange(3)
    >>> a[1] = ma.masked
    >>> b = ma.arange(2, 5)
    >>> a
    masked_array(data=[0, --, 2],
                 mask=[False,  True, False],
           fill_value=999999)
    >>> b
    masked_array(data=[2, 3, 4],
                 mask=False,
           fill_value=999999)
    >>> ma.concatenate([a, b])
    masked_array(data=[0, --, 2, 2, 3, 4],
                 mask=[False,  True, False, False, False, False],
           fill_value=999999)

    """
    d = np.concatenate([getdata(a) for a in arrays], axis)
    rcls = get_masked_subclass(*arrays)
    data = d.view(rcls)
    # Check whether one of the arrays has a non-empty mask.
    for x in arrays:
        if getmask(x) is not nomask:
            break
    else:
        return data
    # OK, so we have to concatenate the masks
    dm = np.concatenate([getmaskarray(a) for a in arrays], axis)
    dm = dm.reshape(d.shape)

    # If we decide to keep a '_shrinkmask' option, we want to check that
    # all of them are True, and then check for dm.any()
    data._mask = _shrink_mask(dm)
    return data


def diag(v, k=0):
    """
    Extract a diagonal or construct a diagonal array.

    This function is the equivalent of `numpy.diag` that takes masked
    values into account, see `numpy.diag` for details.

    See Also
    --------
    numpy.diag : Equivalent function for ndarrays.

    """
    output = np.diag(v, k).view(MaskedArray)
    if getmask(v) is not nomask:
        output._mask = np.diag(v._mask, k)
    return output


def left_shift(a, n):
    """
    Shift the bits of an integer to the left.

    This is the masked array version of `numpy.left_shift`, for details
    see that function.

    See Also
    --------
    numpy.left_shift

    """
    m = getmask(a)
    if m is nomask:
        d = umath.left_shift(filled(a), n)
        return masked_array(d)
    else:
        d = umath.left_shift(filled(a, 0), n)
        return masked_array(d, mask=m)


def right_shift(a, n):
    """
    Shift the bits of an integer to the right.

    This is the masked array version of `numpy.right_shift`, for details
    see that function.

    See Also
    --------
    numpy.right_shift

    """
    m = getmask(a)
    if m is nomask:
        d = umath.right_shift(filled(a), n)
        return masked_array(d)
    else:
        d = umath.right_shift(filled(a, 0), n)
        return masked_array(d, mask=m)


def put(a, indices, values, mode='raise'):
    """
    Set storage-indexed locations to corresponding values.

    This function is equivalent to `MaskedArray.put`, see that method
    for details.

    See Also
    --------
    MaskedArray.put

    """
    # We can't use 'frommethod', the order of arguments is different
    try:
        return a.put(indices, values, mode=mode)
    except AttributeError:
        return narray(a, copy=False).put(indices, values, mode=mode)


def putmask(a, mask, values):  # , mode='raise'):
    """
    Changes elements of an array based on conditional and input values.

    This is the masked array version of `numpy.putmask`, for details see
    `numpy.putmask`.

    See Also
    --------
    numpy.putmask

    Notes
    -----
    Using a masked array as `values` will **not** transform a `ndarray` into
    a `MaskedArray`.

    """
    # We can't use 'frommethod', the order of arguments is different
    if not isinstance(a, MaskedArray):
        a = a.view(MaskedArray)
    (valdata, valmask) = (getdata(values), getmask(values))
    if getmask(a) is nomask:
        if valmask is not nomask:
            a._sharedmask = True
            a._mask = make_mask_none(a.shape, a.dtype)
            np.copyto(a._mask, valmask, where=mask)
    elif a._hardmask:
        if valmask is not nomask:
            m = a._mask.copy()
            np.copyto(m, valmask, where=mask)
            a.mask |= m
    else:
        if valmask is nomask:
            valmask = getmaskarray(values)
        np.copyto(a._mask, valmask, where=mask)
    np.copyto(a._data, valdata, where=mask)
    return


def transpose(a, axes=None):
    """
    Permute the dimensions of an array.

    This function is exactly equivalent to `numpy.transpose`.

    See Also
    --------
    numpy.transpose : Equivalent function in top-level NumPy module.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = ma.arange(4).reshape((2,2))
    >>> x[1, 1] = ma.masked
    >>> x
    masked_array(
      data=[[0, 1],
            [2, --]],
      mask=[[False, False],
            [False,  True]],
      fill_value=999999)

    >>> ma.transpose(x)
    masked_array(
      data=[[0, 2],
            [1, --]],
      mask=[[False, False],
            [False,  True]],
      fill_value=999999)
    """
    # We can't use 'frommethod', as 'transpose' doesn't take keywords
    try:
        return a.transpose(axes)
    except AttributeError:
        return narray(a, copy=False).transpose(axes).view(MaskedArray)


def reshape(a, new_shape, order='C'):
    """
    Returns an array containing the same data with a new shape.

    Refer to `MaskedArray.reshape` for full documentation.

    See Also
    --------
    MaskedArray.reshape : equivalent function

    """
    # We can't use 'frommethod', it whine about some parameters. Dmmit.
    try:
        return a.reshape(new_shape, order=order)
    except AttributeError:
        _tmp = narray(a, copy=False).reshape(new_shape, order=order)
        return _tmp.view(MaskedArray)


def resize(x, new_shape):
    """
    Return a new masked array with the specified size and shape.

    This is the masked equivalent of the `numpy.resize` function. The new
    array is filled with repeated copies of `x` (in the order that the
    data are stored in memory). If `x` is masked, the new array will be
    masked, and the new mask will be a repetition of the old one.

    See Also
    --------
    numpy.resize : Equivalent function in the top level NumPy module.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = ma.array([[1, 2] ,[3, 4]])
    >>> a[0, 1] = ma.masked
    >>> a
    masked_array(
      data=[[1, --],
            [3, 4]],
      mask=[[False,  True],
            [False, False]],
      fill_value=999999)
    >>> np.resize(a, (3, 3))
    masked_array(
      data=[[1, 2, 3],
            [4, 1, 2],
            [3, 4, 1]],
      mask=False,
      fill_value=999999)
    >>> ma.resize(a, (3, 3))
    masked_array(
      data=[[1, --, 3],
            [4, 1, --],
            [3, 4, 1]],
      mask=[[False,  True, False],
            [False, False,  True],
            [False, False, False]],
      fill_value=999999)

    A MaskedArray is always returned, regardless of the input type.

    >>> a = np.array([[1, 2] ,[3, 4]])
    >>> ma.resize(a, (3, 3))
    masked_array(
      data=[[1, 2, 3],
            [4, 1, 2],
            [3, 4, 1]],
      mask=False,
      fill_value=999999)

    """
    # We can't use _frommethods here, as N.resize is notoriously whiny.
    m = getmask(x)
    if m is not nomask:
        m = np.resize(m, new_shape)
    result = np.resize(x, new_shape).view(get_masked_subclass(x))
    if result.ndim:
        result._mask = m
    return result


def ndim(obj):
    """
    maskedarray version of the numpy function.

    """
    return np.ndim(getdata(obj))

ndim.__doc__ = np.ndim.__doc__


def shape(obj):
    "maskedarray version of the numpy function."
    return np.shape(getdata(obj))
shape.__doc__ = np.shape.__doc__


def size(obj, axis=None):
    "maskedarray version of the numpy function."
    return np.size(getdata(obj), axis)
size.__doc__ = np.size.__doc__


##############################################################################
#                            Extra functions                                 #
##############################################################################


def where(condition, x=_NoValue, y=_NoValue):
    """
    Return a masked array with elements from `x` or `y`, depending on condition.

    .. note::
        When only `condition` is provided, this function is identical to
        `nonzero`. The rest of this documentation covers only the case where
        all three arguments are provided.

    Parameters
    ----------
    condition : array_like, bool
        Where True, yield `x`, otherwise yield `y`.
    x, y : array_like, optional
        Values from which to choose. `x`, `y` and `condition` need to be
        broadcastable to some shape.

    Returns
    -------
    out : MaskedArray
        An masked array with `masked` elements where the condition is masked,
        elements from `x` where `condition` is True, and elements from `y`
        elsewhere.

    See Also
    --------
    numpy.where : Equivalent function in the top-level NumPy module.
    nonzero : The function that is called when x and y are omitted

    Examples
    --------
    >>> x = np.ma.array(np.arange(9.).reshape(3, 3), mask=[[0, 1, 0],
    ...                                                    [1, 0, 1],
    ...                                                    [0, 1, 0]])
    >>> x
    masked_array(
      data=[[0.0, --, 2.0],
            [--, 4.0, --],
            [6.0, --, 8.0]],
      mask=[[False,  True, False],
            [ True, False,  True],
            [False,  True, False]],
      fill_value=1e+20)
    >>> np.ma.where(x > 5, x, -3.1416)
    masked_array(
      data=[[-3.1416, --, -3.1416],
            [--, -3.1416, --],
            [6.0, --, 8.0]],
      mask=[[False,  True, False],
            [ True, False,  True],
            [False,  True, False]],
      fill_value=1e+20)

    """

    # handle the single-argument case
    missing = (x is _NoValue, y is _NoValue).count(True)
    if missing == 1:
        raise ValueError("Must provide both 'x' and 'y' or neither.")
    if missing == 2:
        return nonzero(condition)

    # we only care if the condition is true - false or masked pick y
    cf = filled(condition, False)
    xd = getdata(x)
    yd = getdata(y)

    # we need the full arrays here for correct final dimensions
    cm = getmaskarray(condition)
    xm = getmaskarray(x)
    ym = getmaskarray(y)

    # deal with the fact that masked.dtype == float64, but we don't actually
    # want to treat it as that.
    if x is masked and y is not masked:
        xd = np.zeros((), dtype=yd.dtype)
        xm = np.ones((),  dtype=ym.dtype)
    elif y is masked and x is not masked:
        yd = np.zeros((), dtype=xd.dtype)
        ym = np.ones((),  dtype=xm.dtype)

    data = np.where(cf, xd, yd)
    mask = np.where(cf, xm, ym)
    mask = np.where(cm, np.ones((), dtype=mask.dtype), mask)

    # collapse the mask, for backwards compatibility
    mask = _shrink_mask(mask)

    return masked_array(data, mask=mask)


def choose(indices, choices, out=None, mode='raise'):
    """
    Use an index array to construct a new array from a list of choices.

    Given an array of integers and a list of n choice arrays, this method
    will create a new array that merges each of the choice arrays.  Where a
    value in `index` is i, the new array will have the value that choices[i]
    contains in the same place.

    Parameters
    ----------
    indices : ndarray of ints
        This array must contain integers in ``[0, n-1]``, where n is the
        number of choices.
    choices : sequence of arrays
        Choice arrays. The index array and all of the choices should be
        broadcastable to the same shape.
    out : array, optional
        If provided, the result will be inserted into this array. It should
        be of the appropriate shape and `dtype`.
    mode : {'raise', 'wrap', 'clip'}, optional
        Specifies how out-of-bounds indices will behave.

        * 'raise' : raise an error
        * 'wrap' : wrap around
        * 'clip' : clip to the range

    Returns
    -------
    merged_array : array

    See Also
    --------
    choose : equivalent function

    Examples
    --------
    >>> choice = np.array([[1,1,1], [2,2,2], [3,3,3]])
    >>> a = np.array([2, 1, 0])
    >>> np.ma.choose(a, choice)
    masked_array(data=[3, 2, 1],
                 mask=False,
           fill_value=999999)

    """
    def fmask(x):
        "Returns the filled array, or True if masked."
        if x is masked:
            return True
        return filled(x)

    def nmask(x):
        "Returns the mask, True if ``masked``, False if ``nomask``."
        if x is masked:
            return True
        return getmask(x)
    # Get the indices.
    c = filled(indices, 0)
    # Get the masks.
    masks = [nmask(x) for x in choices]
    data = [fmask(x) for x in choices]
    # Construct the mask
    outputmask = np.choose(c, masks, mode=mode)
    outputmask = make_mask(mask_or(outputmask, getmask(indices)),
                           copy=False, shrink=True)
    # Get the choices.
    d = np.choose(c, data, mode=mode, out=out).view(MaskedArray)
    if out is not None:
        if isinstance(out, MaskedArray):
            out.__setmask__(outputmask)
        return out
    d.__setmask__(outputmask)
    return d


def round_(a, decimals=0, out=None):
    """
    Return a copy of a, rounded to 'decimals' places.

    When 'decimals' is negative, it specifies the number of positions
    to the left of the decimal point.  The real and imaginary parts of
    complex numbers are rounded separately. Nothing is done if the
    array is not of float type and 'decimals' is greater than or equal
    to 0.

    Parameters
    ----------
    decimals : int
        Number of decimals to round to. May be negative.
    out : array_like
        Existing array to use for output.
        If not given, returns a default copy of a.

    Notes
    -----
    If out is given and does not have a mask attribute, the mask of a
    is lost!

    Examples
    --------
    >>> import numpy.ma as ma
    >>> x = [11.2, -3.973, 0.801, -1.41]
    >>> mask = [0, 0, 0, 1]
    >>> masked_x = ma.masked_array(x, mask)
    >>> masked_x
    masked_array(data=[11.2, -3.973, 0.801, --],
                 mask=[False, False, False, True],
        fill_value=1e+20)
    >>> ma.round_(masked_x)
    masked_array(data=[11.0, -4.0, 1.0, --],
                 mask=[False, False, False, True],
        fill_value=1e+20)
    >>> ma.round(masked_x, decimals=1)
    masked_array(data=[11.2, -4.0, 0.8, --],
                 mask=[False, False, False, True],
        fill_value=1e+20)
    >>> ma.round_(masked_x, decimals=-1)
    masked_array(data=[10.0, -0.0, 0.0, --],
                 mask=[False, False, False, True],
        fill_value=1e+20)
    """
    if out is None:
        return np.round_(a, decimals, out)
    else:
        np.round_(getdata(a), decimals, out)
        if hasattr(out, '_mask'):
            out._mask = getmask(a)
        return out
round = round_


# Needed by dot, so move here from extras.py. It will still be exported
# from extras.py for compatibility.
def mask_rowcols(a, axis=None):
    """
    Mask rows and/or columns of a 2D array that contain masked values.

    Mask whole rows and/or columns of a 2D array that contain
    masked values.  The masking behavior is selected using the
    `axis` parameter.

      - If `axis` is None, rows *and* columns are masked.
      - If `axis` is 0, only rows are masked.
      - If `axis` is 1 or -1, only columns are masked.

    Parameters
    ----------
    a : array_like, MaskedArray
        The array to mask.  If not a MaskedArray instance (or if no array
        elements are masked).  The result is a MaskedArray with `mask` set
        to `nomask` (False). Must be a 2D array.
    axis : int, optional
        Axis along which to perform the operation. If None, applies to a
        flattened version of the array.

    Returns
    -------
    a : MaskedArray
        A modified version of the input array, masked depending on the value
        of the `axis` parameter.

    Raises
    ------
    NotImplementedError
        If input array `a` is not 2D.

    See Also
    --------
    mask_rows : Mask rows of a 2D array that contain masked values.
    mask_cols : Mask cols of a 2D array that contain masked values.
    masked_where : Mask where a condition is met.

    Notes
    -----
    The input array's mask is modified by this function.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.zeros((3, 3), dtype=int)
    >>> a[1, 1] = 1
    >>> a
    array([[0, 0, 0],
           [0, 1, 0],
           [0, 0, 0]])
    >>> a = ma.masked_equal(a, 1)
    >>> a
    masked_array(
      data=[[0, 0, 0],
            [0, --, 0],
            [0, 0, 0]],
      mask=[[False, False, False],
            [False,  True, False],
            [False, False, False]],
      fill_value=1)
    >>> ma.mask_rowcols(a)
    masked_array(
      data=[[0, --, 0],
            [--, --, --],
            [0, --, 0]],
      mask=[[False,  True, False],
            [ True,  True,  True],
            [False,  True, False]],
      fill_value=1)

    """
    a = array(a, subok=False)
    if a.ndim != 2:
        raise NotImplementedError("mask_rowcols works for 2D arrays only.")
    m = getmask(a)
    # Nothing is masked: return a
    if m is nomask or not m.any():
        return a
    maskedval = m.nonzero()
    a._mask = a._mask.copy()
    if not axis:
        a[np.unique(maskedval[0])] = masked
    if axis in [None, 1, -1]:
        a[:, np.unique(maskedval[1])] = masked
    return a


# Include masked dot here to avoid import problems in getting it from
# extras.py. Note that it is not included in __all__, but rather exported
# from extras in order to avoid backward compatibility problems.
def dot(a, b, strict=False, out=None):
    """
    Return the dot product of two arrays.

    This function is the equivalent of `numpy.dot` that takes masked values
    into account. Note that `strict` and `out` are in different position
    than in the method version. In order to maintain compatibility with the
    corresponding method, it is recommended that the optional arguments be
    treated as keyword only.  At some point that may be mandatory.

    .. note::
      Works only with 2-D arrays at the moment.


    Parameters
    ----------
    a, b : masked_array_like
        Inputs arrays.
    strict : bool, optional
        Whether masked data are propagated (True) or set to 0 (False) for
        the computation. Default is False.  Propagating the mask means that
        if a masked value appears in a row or column, the whole row or
        column is considered masked.
    out : masked_array, optional
        Output argument. This must have the exact kind that would be returned
        if it was not used. In particular, it must have the right type, must be
        C-contiguous, and its dtype must be the dtype that would be returned
        for `dot(a,b)`. This is a performance feature. Therefore, if these
        conditions are not met, an exception is raised, instead of attempting
        to be flexible.

        .. versionadded:: 1.10.2

    See Also
    --------
    numpy.dot : Equivalent function for ndarrays.

    Examples
    --------
    >>> a = np.ma.array([[1, 2, 3], [4, 5, 6]], mask=[[1, 0, 0], [0, 0, 0]])
    >>> b = np.ma.array([[1, 2], [3, 4], [5, 6]], mask=[[1, 0], [0, 0], [0, 0]])
    >>> np.ma.dot(a, b)
    masked_array(
      data=[[21, 26],
            [45, 64]],
      mask=[[False, False],
            [False, False]],
      fill_value=999999)
    >>> np.ma.dot(a, b, strict=True)
    masked_array(
      data=[[--, --],
            [--, 64]],
      mask=[[ True,  True],
            [ True, False]],
      fill_value=999999)

    """
    # !!!: Works only with 2D arrays. There should be a way to get it to run
    # with higher dimension
    if strict and (a.ndim == 2) and (b.ndim == 2):
        a = mask_rowcols(a, 0)
        b = mask_rowcols(b, 1)
    am = ~getmaskarray(a)
    bm = ~getmaskarray(b)

    if out is None:
        d = np.dot(filled(a, 0), filled(b, 0))
        m = ~np.dot(am, bm)
        if d.ndim == 0:
            d = np.asarray(d)
        r = d.view(get_masked_subclass(a, b))
        r.__setmask__(m)
        return r
    else:
        d = np.dot(filled(a, 0), filled(b, 0), out._data)
        if out.mask.shape != d.shape:
            out._mask = np.empty(d.shape, MaskType)
        np.dot(am, bm, out._mask)
        np.logical_not(out._mask, out._mask)
        return out


def inner(a, b):
    """
    Returns the inner product of a and b for arrays of floating point types.

    Like the generic NumPy equivalent the product sum is over the last dimension
    of a and b. The first argument is not conjugated.

    """
    fa = filled(a, 0)
    fb = filled(b, 0)
    if fa.ndim == 0:
        fa.shape = (1,)
    if fb.ndim == 0:
        fb.shape = (1,)
    return np.inner(fa, fb).view(MaskedArray)
inner.__doc__ = doc_note(np.inner.__doc__,
                         "Masked values are replaced by 0.")
innerproduct = inner


def outer(a, b):
    "maskedarray version of the numpy function."
    fa = filled(a, 0).ravel()
    fb = filled(b, 0).ravel()
    d = np.outer(fa, fb)
    ma = getmask(a)
    mb = getmask(b)
    if ma is nomask and mb is nomask:
        return masked_array(d)
    ma = getmaskarray(a)
    mb = getmaskarray(b)
    m = make_mask(1 - np.outer(1 - ma, 1 - mb), copy=False)
    return masked_array(d, mask=m)
outer.__doc__ = doc_note(np.outer.__doc__,
                         "Masked values are replaced by 0.")
outerproduct = outer


def _convolve_or_correlate(f, a, v, mode, propagate_mask):
    """
    Helper function for ma.correlate and ma.convolve
    """
    if propagate_mask:
        # results which are contributed to by either item in any pair being invalid
        mask = (
            f(getmaskarray(a), np.ones(np.shape(v), dtype=bool), mode=mode)
          | f(np.ones(np.shape(a), dtype=bool), getmaskarray(v), mode=mode)
        )
        data = f(getdata(a), getdata(v), mode=mode)
    else:
        # results which are not contributed to by any pair of valid elements
        mask = ~f(~getmaskarray(a), ~getmaskarray(v))
        data = f(filled(a, 0), filled(v, 0), mode=mode)

    return masked_array(data, mask=mask)


def correlate(a, v, mode='valid', propagate_mask=True):
    """
    Cross-correlation of two 1-dimensional sequences.

    Parameters
    ----------
    a, v : array_like
        Input sequences.
    mode : {'valid', 'same', 'full'}, optional
        Refer to the `np.convolve` docstring.  Note that the default
        is 'valid', unlike `convolve`, which uses 'full'.
    propagate_mask : bool
        If True, then a result element is masked if any masked element contributes towards it.
        If False, then a result element is only masked if no non-masked element
        contribute towards it

    Returns
    -------
    out : MaskedArray
        Discrete cross-correlation of `a` and `v`.

    See Also
    --------
    numpy.correlate : Equivalent function in the top-level NumPy module.
    """
    return _convolve_or_correlate(np.correlate, a, v, mode, propagate_mask)


def convolve(a, v, mode='full', propagate_mask=True):
    """
    Returns the discrete, linear convolution of two one-dimensional sequences.

    Parameters
    ----------
    a, v : array_like
        Input sequences.
    mode : {'valid', 'same', 'full'}, optional
        Refer to the `np.convolve` docstring.
    propagate_mask : bool
        If True, then if any masked element is included in the sum for a result
        element, then the result is masked.
        If False, then the result element is only masked if no non-masked cells
        contribute towards it

    Returns
    -------
    out : MaskedArray
        Discrete, linear convolution of `a` and `v`.

    See Also
    --------
    numpy.convolve : Equivalent function in the top-level NumPy module.
    """
    return _convolve_or_correlate(np.convolve, a, v, mode, propagate_mask)


def allequal(a, b, fill_value=True):
    """
    Return True if all entries of a and b are equal, using
    fill_value as a truth value where either or both are masked.

    Parameters
    ----------
    a, b : array_like
        Input arrays to compare.
    fill_value : bool, optional
        Whether masked values in a or b are considered equal (True) or not
        (False).

    Returns
    -------
    y : bool
        Returns True if the two arrays are equal within the given
        tolerance, False otherwise. If either array contains NaN,
        then False is returned.

    See Also
    --------
    all, any
    numpy.ma.allclose

    Examples
    --------
    >>> a = np.ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1])
    >>> a
    masked_array(data=[10000000000.0, 1e-07, --],
                 mask=[False, False,  True],
           fill_value=1e+20)

    >>> b = np.array([1e10, 1e-7, -42.0])
    >>> b
    array([  1.00000000e+10,   1.00000000e-07,  -4.20000000e+01])
    >>> np.ma.allequal(a, b, fill_value=False)
    False
    >>> np.ma.allequal(a, b)
    True

    """
    m = mask_or(getmask(a), getmask(b))
    if m is nomask:
        x = getdata(a)
        y = getdata(b)
        d = umath.equal(x, y)
        return d.all()
    elif fill_value:
        x = getdata(a)
        y = getdata(b)
        d = umath.equal(x, y)
        dm = array(d, mask=m, copy=False)
        return dm.filled(True).all(None)
    else:
        return False


def allclose(a, b, masked_equal=True, rtol=1e-5, atol=1e-8):
    """
    Returns True if two arrays are element-wise equal within a tolerance.

    This function is equivalent to `allclose` except that masked values
    are treated as equal (default) or unequal, depending on the `masked_equal`
    argument.

    Parameters
    ----------
    a, b : array_like
        Input arrays to compare.
    masked_equal : bool, optional
        Whether masked values in `a` and `b` are considered equal (True) or not
        (False). They are considered equal by default.
    rtol : float, optional
        Relative tolerance. The relative difference is equal to ``rtol * b``.
        Default is 1e-5.
    atol : float, optional
        Absolute tolerance. The absolute difference is equal to `atol`.
        Default is 1e-8.

    Returns
    -------
    y : bool
        Returns True if the two arrays are equal within the given
        tolerance, False otherwise. If either array contains NaN, then
        False is returned.

    See Also
    --------
    all, any
    numpy.allclose : the non-masked `allclose`.

    Notes
    -----
    If the following equation is element-wise True, then `allclose` returns
    True::

      absolute(`a` - `b`) <= (`atol` + `rtol` * absolute(`b`))

    Return True if all elements of `a` and `b` are equal subject to
    given tolerances.

    Examples
    --------
    >>> a = np.ma.array([1e10, 1e-7, 42.0], mask=[0, 0, 1])
    >>> a
    masked_array(data=[10000000000.0, 1e-07, --],
                 mask=[False, False,  True],
           fill_value=1e+20)
    >>> b = np.ma.array([1e10, 1e-8, -42.0], mask=[0, 0, 1])
    >>> np.ma.allclose(a, b)
    False

    >>> a = np.ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1])
    >>> b = np.ma.array([1.00001e10, 1e-9, -42.0], mask=[0, 0, 1])
    >>> np.ma.allclose(a, b)
    True
    >>> np.ma.allclose(a, b, masked_equal=False)
    False

    Masked values are not compared directly.

    >>> a = np.ma.array([1e10, 1e-8, 42.0], mask=[0, 0, 1])
    >>> b = np.ma.array([1.00001e10, 1e-9, 42.0], mask=[0, 0, 1])
    >>> np.ma.allclose(a, b)
    True
    >>> np.ma.allclose(a, b, masked_equal=False)
    False

    """
    x = masked_array(a, copy=False)
    y = masked_array(b, copy=False)

    # make sure y is an inexact type to avoid abs(MIN_INT); will cause
    # casting of x later.
    # NOTE: We explicitly allow timedelta, which used to work. This could
    #       possibly be deprecated. See also gh-18286.
    #       timedelta works if `atol` is an integer or also a timedelta.
    #       Although, the default tolerances are unlikely to be useful
    if y.dtype.kind != "m":
        dtype = np.result_type(y, 1.)
        if y.dtype != dtype:
            y = masked_array(y, dtype=dtype, copy=False)

    m = mask_or(getmask(x), getmask(y))
    xinf = np.isinf(masked_array(x, copy=False, mask=m)).filled(False)
    # If we have some infs, they should fall at the same place.
    if not np.all(xinf == filled(np.isinf(y), False)):
        return False
    # No infs at all
    if not np.any(xinf):
        d = filled(less_equal(absolute(x - y), atol + rtol * absolute(y)),
                   masked_equal)
        return np.all(d)

    if not np.all(filled(x[xinf] == y[xinf], masked_equal)):
        return False
    x = x[~xinf]
    y = y[~xinf]

    d = filled(less_equal(absolute(x - y), atol + rtol * absolute(y)),
               masked_equal)

    return np.all(d)


def asarray(a, dtype=None, order=None):
    """
    Convert the input to a masked array of the given data-type.

    No copy is performed if the input is already an `ndarray`. If `a` is
    a subclass of `MaskedArray`, a base class `MaskedArray` is returned.

    Parameters
    ----------
    a : array_like
        Input data, in any form that can be converted to a masked array. This
        includes lists, lists of tuples, tuples, tuples of tuples, tuples
        of lists, ndarrays and masked arrays.
    dtype : dtype, optional
        By default, the data-type is inferred from the input data.
    order : {'C', 'F'}, optional
        Whether to use row-major ('C') or column-major ('FORTRAN') memory
        representation.  Default is 'C'.

    Returns
    -------
    out : MaskedArray
        Masked array interpretation of `a`.

    See Also
    --------
    asanyarray : Similar to `asarray`, but conserves subclasses.

    Examples
    --------
    >>> x = np.arange(10.).reshape(2, 5)
    >>> x
    array([[0., 1., 2., 3., 4.],
           [5., 6., 7., 8., 9.]])
    >>> np.ma.asarray(x)
    masked_array(
      data=[[0., 1., 2., 3., 4.],
            [5., 6., 7., 8., 9.]],
      mask=False,
      fill_value=1e+20)
    >>> type(np.ma.asarray(x))
    <class 'numpy.ma.core.MaskedArray'>

    """
    order = order or 'C'
    return masked_array(a, dtype=dtype, copy=False, keep_mask=True,
                        subok=False, order=order)


def asanyarray(a, dtype=None):
    """
    Convert the input to a masked array, conserving subclasses.

    If `a` is a subclass of `MaskedArray`, its class is conserved.
    No copy is performed if the input is already an `ndarray`.

    Parameters
    ----------
    a : array_like
        Input data, in any form that can be converted to an array.
    dtype : dtype, optional
        By default, the data-type is inferred from the input data.
    order : {'C', 'F'}, optional
        Whether to use row-major ('C') or column-major ('FORTRAN') memory
        representation.  Default is 'C'.

    Returns
    -------
    out : MaskedArray
        MaskedArray interpretation of `a`.

    See Also
    --------
    asarray : Similar to `asanyarray`, but does not conserve subclass.

    Examples
    --------
    >>> x = np.arange(10.).reshape(2, 5)
    >>> x
    array([[0., 1., 2., 3., 4.],
           [5., 6., 7., 8., 9.]])
    >>> np.ma.asanyarray(x)
    masked_array(
      data=[[0., 1., 2., 3., 4.],
            [5., 6., 7., 8., 9.]],
      mask=False,
      fill_value=1e+20)
    >>> type(np.ma.asanyarray(x))
    <class 'numpy.ma.core.MaskedArray'>

    """
    # workaround for #8666, to preserve identity. Ideally the bottom line
    # would handle this for us.
    if isinstance(a, MaskedArray) and (dtype is None or dtype == a.dtype):
        return a
    return masked_array(a, dtype=dtype, copy=False, keep_mask=True, subok=True)


##############################################################################
#                               Pickling                                     #
##############################################################################


def fromfile(file, dtype=float, count=-1, sep=''):
    raise NotImplementedError(
        "fromfile() not yet implemented for a MaskedArray.")


def fromflex(fxarray):
    """
    Build a masked array from a suitable flexible-type array.

    The input array has to have a data-type with ``_data`` and ``_mask``
    fields. This type of array is output by `MaskedArray.toflex`.

    Parameters
    ----------
    fxarray : ndarray
        The structured input array, containing ``_data`` and ``_mask``
        fields. If present, other fields are discarded.

    Returns
    -------
    result : MaskedArray
        The constructed masked array.

    See Also
    --------
    MaskedArray.toflex : Build a flexible-type array from a masked array.

    Examples
    --------
    >>> x = np.ma.array(np.arange(9).reshape(3, 3), mask=[0] + [1, 0] * 4)
    >>> rec = x.toflex()
    >>> rec
    array([[(0, False), (1,  True), (2, False)],
           [(3,  True), (4, False), (5,  True)],
           [(6, False), (7,  True), (8, False)]],
          dtype=[('_data', '<i8'), ('_mask', '?')])
    >>> x2 = np.ma.fromflex(rec)
    >>> x2
    masked_array(
      data=[[0, --, 2],
            [--, 4, --],
            [6, --, 8]],
      mask=[[False,  True, False],
            [ True, False,  True],
            [False,  True, False]],
      fill_value=999999)

    Extra fields can be present in the structured array but are discarded:

    >>> dt = [('_data', '<i4'), ('_mask', '|b1'), ('field3', '<f4')]
    >>> rec2 = np.zeros((2, 2), dtype=dt)
    >>> rec2
    array([[(0, False, 0.), (0, False, 0.)],
           [(0, False, 0.), (0, False, 0.)]],
          dtype=[('_data', '<i4'), ('_mask', '?'), ('field3', '<f4')])
    >>> y = np.ma.fromflex(rec2)
    >>> y
    masked_array(
      data=[[0, 0],
            [0, 0]],
      mask=[[False, False],
            [False, False]],
      fill_value=999999,
      dtype=int32)

    """
    return masked_array(fxarray['_data'], mask=fxarray['_mask'])


class _convert2ma:

    """
    Convert functions from numpy to numpy.ma.

    Parameters
    ----------
        _methodname : string
            Name of the method to transform.

    """
    __doc__ = None

    def __init__(self, funcname, np_ret, np_ma_ret, params=None):
        self._func = getattr(np, funcname)
        self.__doc__ = self.getdoc(np_ret, np_ma_ret)
        self._extras = params or {}

    def getdoc(self, np_ret, np_ma_ret):
        "Return the doc of the function (from the doc of the method)."
        doc = getattr(self._func, '__doc__', None)
        sig = get_object_signature(self._func)
        if doc:
            doc = self._replace_return_type(doc, np_ret, np_ma_ret)
            # Add the signature of the function at the beginning of the doc
            if sig:
                sig = "%s%s\n" % (self._func.__name__, sig)
            doc = sig + doc
        return doc

    def _replace_return_type(self, doc, np_ret, np_ma_ret):
        """
        Replace documentation of ``np`` function's return type.

        Replaces it with the proper type for the ``np.ma`` function.

        Parameters
        ----------
        doc : str
            The documentation of the ``np`` method.
        np_ret : str
            The return type string of the ``np`` method that we want to
            replace. (e.g. "out : ndarray")
        np_ma_ret : str
            The return type string of the ``np.ma`` method.
            (e.g. "out : MaskedArray")
        """
        if np_ret not in doc:
            raise RuntimeError(
                f"Failed to replace `{np_ret}` with `{np_ma_ret}`. "
                f"The documentation string for return type, {np_ret}, is not "
                f"found in the docstring for `np.{self._func.__name__}`. "
                f"Fix the docstring for `np.{self._func.__name__}` or "
                "update the expected string for return type."
            )

        return doc.replace(np_ret, np_ma_ret)

    def __call__(self, *args, **params):
        # Find the common parameters to the call and the definition
        _extras = self._extras
        common_params = set(params).intersection(_extras)
        # Drop the common parameters from the call
        for p in common_params:
            _extras[p] = params.pop(p)
        # Get the result
        result = self._func.__call__(*args, **params).view(MaskedArray)
        if "fill_value" in common_params:
            result.fill_value = _extras.get("fill_value", None)
        if "hardmask" in common_params:
            result._hardmask = bool(_extras.get("hard_mask", False))
        return result


arange = _convert2ma(
    'arange',
    params=dict(fill_value=None, hardmask=False),
    np_ret='arange : ndarray',
    np_ma_ret='arange : MaskedArray',
)
clip = _convert2ma(
    'clip',
    params=dict(fill_value=None, hardmask=False),
    np_ret='clipped_array : ndarray',
    np_ma_ret='clipped_array : MaskedArray',
)
diff = _convert2ma(
    'diff',
    params=dict(fill_value=None, hardmask=False),
    np_ret='diff : ndarray',
    np_ma_ret='diff : MaskedArray',
)
empty = _convert2ma(
    'empty',
    params=dict(fill_value=None, hardmask=False),
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)
empty_like = _convert2ma(
    'empty_like',
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)
frombuffer = _convert2ma(
    'frombuffer',
    np_ret='out : ndarray',
    np_ma_ret='out: MaskedArray',
)
fromfunction = _convert2ma(
   'fromfunction',
   np_ret='fromfunction : any',
   np_ma_ret='fromfunction: MaskedArray',
)
identity = _convert2ma(
    'identity',
    params=dict(fill_value=None, hardmask=False),
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)
indices = _convert2ma(
    'indices',
    params=dict(fill_value=None, hardmask=False),
    np_ret='grid : one ndarray or tuple of ndarrays',
    np_ma_ret='grid : one MaskedArray or tuple of MaskedArrays',
)
ones = _convert2ma(
    'ones',
    params=dict(fill_value=None, hardmask=False),
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)
ones_like = _convert2ma(
    'ones_like',
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)
squeeze = _convert2ma(
    'squeeze',
    params=dict(fill_value=None, hardmask=False),
    np_ret='squeezed : ndarray',
    np_ma_ret='squeezed : MaskedArray',
)
zeros = _convert2ma(
    'zeros',
    params=dict(fill_value=None, hardmask=False),
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)
zeros_like = _convert2ma(
    'zeros_like',
    np_ret='out : ndarray',
    np_ma_ret='out : MaskedArray',
)


def append(a, b, axis=None):
    """Append values to the end of an array.

    .. versionadded:: 1.9.0

    Parameters
    ----------
    a : array_like
        Values are appended to a copy of this array.
    b : array_like
        These values are appended to a copy of `a`.  It must be of the
        correct shape (the same shape as `a`, excluding `axis`).  If `axis`
        is not specified, `b` can be any shape and will be flattened
        before use.
    axis : int, optional
        The axis along which `v` are appended.  If `axis` is not given,
        both `a` and `b` are flattened before use.

    Returns
    -------
    append : MaskedArray
        A copy of `a` with `b` appended to `axis`.  Note that `append`
        does not occur in-place: a new array is allocated and filled.  If
        `axis` is None, the result is a flattened array.

    See Also
    --------
    numpy.append : Equivalent function in the top-level NumPy module.

    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = ma.masked_values([1, 2, 3], 2)
    >>> b = ma.masked_values([[4, 5, 6], [7, 8, 9]], 7)
    >>> ma.append(a, b)
    masked_array(data=[1, --, 3, 4, 5, 6, --, 8, 9],
                 mask=[False,  True, False, False, False, False,  True, False,
                       False],
           fill_value=999999)
    """
    return concatenate([a, b], axis)

env_pyrecodes/lib/python3.9/site-packages/numpy/ma/core.py
NikolaBlagojevic/pyrecodes
# pylint: disable-msg=W0400,W0511,W0611,W0612,W0614,R0201,E1102
"""Tests suite for MaskedArray & subclassing.

:author: Pierre Gerard-Marchant
:contact: pierregm_at_uga_dot_edu
"""
__author__ = "Pierre GF Gerard-Marchant"

import sys
import warnings
import operator
import itertools
import textwrap
import pytest

from functools import reduce


import numpy as np
import numpy.ma.core
import numpy.core.fromnumeric as fromnumeric
import numpy.core.umath as umath
from numpy.testing import (
    assert_raises, assert_warns, suppress_warnings, IS_WASM
    )
from numpy.testing._private.utils import requires_memory
from numpy import ndarray
from numpy.compat import asbytes
from numpy.ma.testutils import (
    assert_, assert_array_equal, assert_equal, assert_almost_equal,
    assert_equal_records, fail_if_equal, assert_not_equal,
    assert_mask_equal
    )
from numpy.ma.core import (
    MAError, MaskError, MaskType, MaskedArray, abs, absolute, add, all,
    allclose, allequal, alltrue, angle, anom, arange, arccos, arccosh, arctan2,
    arcsin, arctan, argsort, array, asarray, choose, concatenate,
    conjugate, cos, cosh, count, default_fill_value, diag, divide, doc_note,
    empty, empty_like, equal, exp, flatten_mask, filled, fix_invalid,
    flatten_structured_array, fromflex, getmask, getmaskarray, greater,
    greater_equal, identity, inner, isMaskedArray, less, less_equal, log,
    log10, make_mask, make_mask_descr, mask_or, masked, masked_array,
    masked_equal, masked_greater, masked_greater_equal, masked_inside,
    masked_less, masked_less_equal, masked_not_equal, masked_outside,
    masked_print_option, masked_values, masked_where, max, maximum,
    maximum_fill_value, min, minimum, minimum_fill_value, mod, multiply,
    mvoid, nomask, not_equal, ones, ones_like, outer, power, product, put,
    putmask, ravel, repeat, reshape, resize, shape, sin, sinh, sometrue, sort,
    sqrt, subtract, sum, take, tan, tanh, transpose, where, zeros, zeros_like,
    )
from numpy.compat import pickle

pi = np.pi


suppress_copy_mask_on_assignment = suppress_warnings()
suppress_copy_mask_on_assignment.filter(
    numpy.ma.core.MaskedArrayFutureWarning,
    "setting an item on a masked array which has a shared mask will not copy")


# For parametrized numeric testing
num_dts = [np.dtype(dt_) for dt_ in '?bhilqBHILQefdgFD']
num_ids = [dt_.char for dt_ in num_dts]


class TestMaskedArray:
    # Base test class for MaskedArrays.

    def setup_method(self):
        # Base data definition.
        x = np.array([1., 1., 1., -2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.])
        y = np.array([5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.])
        a10 = 10.
        m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]
        m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1]
        xm = masked_array(x, mask=m1)
        ym = masked_array(y, mask=m2)
        z = np.array([-.5, 0., .5, .8])
        zm = masked_array(z, mask=[0, 1, 0, 0])
        xf = np.where(m1, 1e+20, x)
        xm.set_fill_value(1e+20)
        self.d = (x, y, a10, m1, m2, xm, ym, z, zm, xf)

    def test_basicattributes(self):
        # Tests some basic array attributes.
        a = array([1, 3, 2])
        b = array([1, 3, 2], mask=[1, 0, 1])
        assert_equal(a.ndim, 1)
        assert_equal(b.ndim, 1)
        assert_equal(a.size, 3)
        assert_equal(b.size, 3)
        assert_equal(a.shape, (3,))
        assert_equal(b.shape, (3,))

    def test_basic0d(self):
        # Checks masking a scalar
        x = masked_array(0)
        assert_equal(str(x), '0')
        x = masked_array(0, mask=True)
        assert_equal(str(x), str(masked_print_option))
        x = masked_array(0, mask=False)
        assert_equal(str(x), '0')
        x = array(0, mask=1)
        assert_(x.filled().dtype is x._data.dtype)

    def test_basic1d(self):
        # Test of basic array creation and properties in 1 dimension.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        assert_(not isMaskedArray(x))
        assert_(isMaskedArray(xm))
        assert_((xm - ym).filled(0).any())
        fail_if_equal(xm.mask.astype(int), ym.mask.astype(int))
        s = x.shape
        assert_equal(np.shape(xm), s)
        assert_equal(xm.shape, s)
        assert_equal(xm.dtype, x.dtype)
        assert_equal(zm.dtype, z.dtype)
        assert_equal(xm.size, reduce(lambda x, y:x * y, s))
        assert_equal(count(xm), len(m1) - reduce(lambda x, y:x + y, m1))
        assert_array_equal(xm, xf)
        assert_array_equal(filled(xm, 1.e20), xf)
        assert_array_equal(x, xm)

    def test_basic2d(self):
        # Test of basic array creation and properties in 2 dimensions.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        for s in [(4, 3), (6, 2)]:
            x.shape = s
            y.shape = s
            xm.shape = s
            ym.shape = s
            xf.shape = s

            assert_(not isMaskedArray(x))
            assert_(isMaskedArray(xm))
            assert_equal(shape(xm), s)
            assert_equal(xm.shape, s)
            assert_equal(xm.size, reduce(lambda x, y:x * y, s))
            assert_equal(count(xm), len(m1) - reduce(lambda x, y:x + y, m1))
            assert_equal(xm, xf)
            assert_equal(filled(xm, 1.e20), xf)
            assert_equal(x, xm)

    def test_concatenate_basic(self):
        # Tests concatenations.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        # basic concatenation
        assert_equal(np.concatenate((x, y)), concatenate((xm, ym)))
        assert_equal(np.concatenate((x, y)), concatenate((x, y)))
        assert_equal(np.concatenate((x, y)), concatenate((xm, y)))
        assert_equal(np.concatenate((x, y, x)), concatenate((x, ym, x)))

    def test_concatenate_alongaxis(self):
        # Tests concatenations.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        # Concatenation along an axis
        s = (3, 4)
        x.shape = y.shape = xm.shape = ym.shape = s
        assert_equal(xm.mask, np.reshape(m1, s))
        assert_equal(ym.mask, np.reshape(m2, s))
        xmym = concatenate((xm, ym), 1)
        assert_equal(np.concatenate((x, y), 1), xmym)
        assert_equal(np.concatenate((xm.mask, ym.mask), 1), xmym._mask)

        x = zeros(2)
        y = array(ones(2), mask=[False, True])
        z = concatenate((x, y))
        assert_array_equal(z, [0, 0, 1, 1])
        assert_array_equal(z.mask, [False, False, False, True])
        z = concatenate((y, x))
        assert_array_equal(z, [1, 1, 0, 0])
        assert_array_equal(z.mask, [False, True, False, False])

    def test_concatenate_flexible(self):
        # Tests the concatenation on flexible arrays.
        data = masked_array(list(zip(np.random.rand(10),
                                     np.arange(10))),
                            dtype=[('a', float), ('b', int)])

        test = concatenate([data[:5], data[5:]])
        assert_equal_records(test, data)

    def test_creation_ndmin(self):
        # Check the use of ndmin
        x = array([1, 2, 3], mask=[1, 0, 0], ndmin=2)
        assert_equal(x.shape, (1, 3))
        assert_equal(x._data, [[1, 2, 3]])
        assert_equal(x._mask, [[1, 0, 0]])

    def test_creation_ndmin_from_maskedarray(self):
        # Make sure we're not losing the original mask w/ ndmin
        x = array([1, 2, 3])
        x[-1] = masked
        xx = array(x, ndmin=2, dtype=float)
        assert_equal(x.shape, x._mask.shape)
        assert_equal(xx.shape, xx._mask.shape)

    def test_creation_maskcreation(self):
        # Tests how masks are initialized at the creation of Maskedarrays.
        data = arange(24, dtype=float)
        data[[3, 6, 15]] = masked
        dma_1 = MaskedArray(data)
        assert_equal(dma_1.mask, data.mask)
        dma_2 = MaskedArray(dma_1)
        assert_equal(dma_2.mask, dma_1.mask)
        dma_3 = MaskedArray(dma_1, mask=[1, 0, 0, 0] * 6)
        fail_if_equal(dma_3.mask, dma_1.mask)

        x = array([1, 2, 3], mask=True)
        assert_equal(x._mask, [True, True, True])
        x = array([1, 2, 3], mask=False)
        assert_equal(x._mask, [False, False, False])
        y = array([1, 2, 3], mask=x._mask, copy=False)
        assert_(np.may_share_memory(x.mask, y.mask))
        y = array([1, 2, 3], mask=x._mask, copy=True)
        assert_(not np.may_share_memory(x.mask, y.mask))

    def test_masked_singleton_array_creation_warns(self):
        # The first works, but should not (ideally), there may be no way
        # to solve this, however, as long as `np.ma.masked` is an ndarray.
        np.array(np.ma.masked)
        with pytest.warns(UserWarning):
            # Tries to create a float array, using `float(np.ma.masked)`.
            # We may want to define this is invalid behaviour in the future!
            # (requiring np.ma.masked to be a known NumPy scalar probably
            # with a DType.)
            np.array([3., np.ma.masked])

    def test_creation_with_list_of_maskedarrays(self):
        # Tests creating a masked array from a list of masked arrays.
        x = array(np.arange(5), mask=[1, 0, 0, 0, 0])
        data = array((x, x[::-1]))
        assert_equal(data, [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0]])
        assert_equal(data._mask, [[1, 0, 0, 0, 0], [0, 0, 0, 0, 1]])

        x.mask = nomask
        data = array((x, x[::-1]))
        assert_equal(data, [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0]])
        assert_(data.mask is nomask)

    def test_creation_with_list_of_maskedarrays_no_bool_cast(self):
        # Tests the regression in gh-18551
        masked_str = np.ma.masked_array(['a', 'b'], mask=[True, False])
        normal_int = np.arange(2)
        res = np.ma.asarray([masked_str, normal_int], dtype="U21")
        assert_array_equal(res.mask, [[True, False], [False, False]])

        # The above only failed due a long chain of oddity, try also with
        # an object array that cannot be converted to bool always:
        class NotBool():
            def __bool__(self):
                raise ValueError("not a bool!")
        masked_obj = np.ma.masked_array([NotBool(), 'b'], mask=[True, False])
        # Check that the NotBool actually fails like we would expect:
        with pytest.raises(ValueError, match="not a bool!"):
            np.asarray([masked_obj], dtype=bool)

        res = np.ma.asarray([masked_obj, normal_int])
        assert_array_equal(res.mask, [[True, False], [False, False]])

    def test_creation_from_ndarray_with_padding(self):
        x = np.array([('A', 0)], dtype={'names':['f0','f1'],
                                        'formats':['S4','i8'],
                                        'offsets':[0,8]})
        array(x)  # used to fail due to 'V' padding field in x.dtype.descr

    def test_unknown_keyword_parameter(self):
        with pytest.raises(TypeError, match="unexpected keyword argument"):
            MaskedArray([1, 2, 3], maks=[0, 1, 0])  # `mask` is misspelled.

    def test_asarray(self):
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        xm.fill_value = -9999
        xm._hardmask = True
        xmm = asarray(xm)
        assert_equal(xmm._data, xm._data)
        assert_equal(xmm._mask, xm._mask)
        assert_equal(xmm.fill_value, xm.fill_value)
        assert_equal(xmm._hardmask, xm._hardmask)

    def test_asarray_default_order(self):
        # See Issue #6646
        m = np.eye(3).T
        assert_(not m.flags.c_contiguous)

        new_m = asarray(m)
        assert_(new_m.flags.c_contiguous)

    def test_asarray_enforce_order(self):
        # See Issue #6646
        m = np.eye(3).T
        assert_(not m.flags.c_contiguous)

        new_m = asarray(m, order='C')
        assert_(new_m.flags.c_contiguous)

    def test_fix_invalid(self):
        # Checks fix_invalid.
        with np.errstate(invalid='ignore'):
            data = masked_array([np.nan, 0., 1.], mask=[0, 0, 1])
            data_fixed = fix_invalid(data)
            assert_equal(data_fixed._data, [data.fill_value, 0., 1.])
            assert_equal(data_fixed._mask, [1., 0., 1.])

    def test_maskedelement(self):
        # Test of masked element
        x = arange(6)
        x[1] = masked
        assert_(str(masked) == '--')
        assert_(x[1] is masked)
        assert_equal(filled(x[1], 0), 0)

    def test_set_element_as_object(self):
        # Tests setting elements with object
        a = empty(1, dtype=object)
        x = (1, 2, 3, 4, 5)
        a[0] = x
        assert_equal(a[0], x)
        assert_(a[0] is x)

        import datetime
        dt = datetime.datetime.now()
        a[0] = dt
        assert_(a[0] is dt)

    def test_indexing(self):
        # Tests conversions and indexing
        x1 = np.array([1, 2, 4, 3])
        x2 = array(x1, mask=[1, 0, 0, 0])
        x3 = array(x1, mask=[0, 1, 0, 1])
        x4 = array(x1)
        # test conversion to strings
        str(x2)  # raises?
        repr(x2)  # raises?
        assert_equal(np.sort(x1), sort(x2, endwith=False))
        # tests of indexing
        assert_(type(x2[1]) is type(x1[1]))
        assert_(x1[1] == x2[1])
        assert_(x2[0] is masked)
        assert_equal(x1[2], x2[2])
        assert_equal(x1[2:5], x2[2:5])
        assert_equal(x1[:], x2[:])
        assert_equal(x1[1:], x3[1:])
        x1[2] = 9
        x2[2] = 9
        assert_equal(x1, x2)
        x1[1:3] = 99
        x2[1:3] = 99
        assert_equal(x1, x2)
        x2[1] = masked
        assert_equal(x1, x2)
        x2[1:3] = masked
        assert_equal(x1, x2)
        x2[:] = x1
        x2[1] = masked
        assert_(allequal(getmask(x2), array([0, 1, 0, 0])))
        x3[:] = masked_array([1, 2, 3, 4], [0, 1, 1, 0])
        assert_(allequal(getmask(x3), array([0, 1, 1, 0])))
        x4[:] = masked_array([1, 2, 3, 4], [0, 1, 1, 0])
        assert_(allequal(getmask(x4), array([0, 1, 1, 0])))
        assert_(allequal(x4, array([1, 2, 3, 4])))
        x1 = np.arange(5) * 1.0
        x2 = masked_values(x1, 3.0)
        assert_equal(x1, x2)
        assert_(allequal(array([0, 0, 0, 1, 0], MaskType), x2.mask))
        assert_equal(3.0, x2.fill_value)
        x1 = array([1, 'hello', 2, 3], object)
        x2 = np.array([1, 'hello', 2, 3], object)
        s1 = x1[1]
        s2 = x2[1]
        assert_equal(type(s2), str)
        assert_equal(type(s1), str)
        assert_equal(s1, s2)
        assert_(x1[1:1].shape == (0,))

    @suppress_copy_mask_on_assignment
    def test_copy(self):
        # Tests of some subtle points of copying and sizing.
        n = [0, 0, 1, 0, 0]
        m = make_mask(n)
        m2 = make_mask(m)
        assert_(m is m2)
        m3 = make_mask(m, copy=True)
        assert_(m is not m3)

        x1 = np.arange(5)
        y1 = array(x1, mask=m)
        assert_equal(y1._data.__array_interface__, x1.__array_interface__)
        assert_(allequal(x1, y1.data))
        assert_equal(y1._mask.__array_interface__, m.__array_interface__)

        y1a = array(y1)
        # Default for masked array is not to copy; see gh-10318.
        assert_(y1a._data.__array_interface__ ==
                        y1._data.__array_interface__)
        assert_(y1a._mask.__array_interface__ ==
                        y1._mask.__array_interface__)

        y2 = array(x1, mask=m3)
        assert_(y2._data.__array_interface__ == x1.__array_interface__)
        assert_(y2._mask.__array_interface__ == m3.__array_interface__)
        assert_(y2[2] is masked)
        y2[2] = 9
        assert_(y2[2] is not masked)
        assert_(y2._mask.__array_interface__ == m3.__array_interface__)
        assert_(allequal(y2.mask, 0))

        y2a = array(x1, mask=m, copy=1)
        assert_(y2a._data.__array_interface__ != x1.__array_interface__)
        #assert_( y2a._mask is not m)
        assert_(y2a._mask.__array_interface__ != m.__array_interface__)
        assert_(y2a[2] is masked)
        y2a[2] = 9
        assert_(y2a[2] is not masked)
        #assert_( y2a._mask is not m)
        assert_(y2a._mask.__array_interface__ != m.__array_interface__)
        assert_(allequal(y2a.mask, 0))

        y3 = array(x1 * 1.0, mask=m)
        assert_(filled(y3).dtype is (x1 * 1.0).dtype)

        x4 = arange(4)
        x4[2] = masked
        y4 = resize(x4, (8,))
        assert_equal(concatenate([x4, x4]), y4)
        assert_equal(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0])
        y5 = repeat(x4, (2, 2, 2, 2), axis=0)
        assert_equal(y5, [0, 0, 1, 1, 2, 2, 3, 3])
        y6 = repeat(x4, 2, axis=0)
        assert_equal(y5, y6)
        y7 = x4.repeat((2, 2, 2, 2), axis=0)
        assert_equal(y5, y7)
        y8 = x4.repeat(2, 0)
        assert_equal(y5, y8)

        y9 = x4.copy()
        assert_equal(y9._data, x4._data)
        assert_equal(y9._mask, x4._mask)

        x = masked_array([1, 2, 3], mask=[0, 1, 0])
        # Copy is False by default
        y = masked_array(x)
        assert_equal(y._data.ctypes.data, x._data.ctypes.data)
        assert_equal(y._mask.ctypes.data, x._mask.ctypes.data)
        y = masked_array(x, copy=True)
        assert_not_equal(y._data.ctypes.data, x._data.ctypes.data)
        assert_not_equal(y._mask.ctypes.data, x._mask.ctypes.data)

    def test_copy_0d(self):
        # gh-9430
        x = np.ma.array(43, mask=True)
        xc = x.copy()
        assert_equal(xc.mask, True)

    def test_copy_on_python_builtins(self):
        # Tests copy works on python builtins (issue#8019)
        assert_(isMaskedArray(np.ma.copy([1,2,3])))
        assert_(isMaskedArray(np.ma.copy((1,2,3))))

    def test_copy_immutable(self):
        # Tests that the copy method is immutable, GitHub issue #5247
        a = np.ma.array([1, 2, 3])
        b = np.ma.array([4, 5, 6])
        a_copy_method = a.copy
        b.copy
        assert_equal(a_copy_method(), [1, 2, 3])

    def test_deepcopy(self):
        from copy import deepcopy
        a = array([0, 1, 2], mask=[False, True, False])
        copied = deepcopy(a)
        assert_equal(copied.mask, a.mask)
        assert_not_equal(id(a._mask), id(copied._mask))

        copied[1] = 1
        assert_equal(copied.mask, [0, 0, 0])
        assert_equal(a.mask, [0, 1, 0])

        copied = deepcopy(a)
        assert_equal(copied.mask, a.mask)
        copied.mask[1] = False
        assert_equal(copied.mask, [0, 0, 0])
        assert_equal(a.mask, [0, 1, 0])

    def test_format(self):
        a = array([0, 1, 2], mask=[False, True, False])
        assert_equal(format(a), "[0 -- 2]")
        assert_equal(format(masked), "--")
        assert_equal(format(masked, ""), "--")

        # Postponed from PR #15410, perhaps address in the future.
        # assert_equal(format(masked, " >5"), "   --")
        # assert_equal(format(masked, " <5"), "--   ")

        # Expect a FutureWarning for using format_spec with MaskedElement
        with assert_warns(FutureWarning):
            with_format_string = format(masked, " >5")
        assert_equal(with_format_string, "--")

    def test_str_repr(self):
        a = array([0, 1, 2], mask=[False, True, False])
        assert_equal(str(a), '[0 -- 2]')
        assert_equal(
            repr(a),
            textwrap.dedent('''\
            masked_array(data=[0, --, 2],
                         mask=[False,  True, False],
                   fill_value=999999)''')
        )

        # arrays with a continuation
        a = np.ma.arange(2000)
        a[1:50] = np.ma.masked
        assert_equal(
            repr(a),
            textwrap.dedent('''\
            masked_array(data=[0, --, --, ..., 1997, 1998, 1999],
                         mask=[False,  True,  True, ..., False, False, False],
                   fill_value=999999)''')
        )

        # line-wrapped 1d arrays are correctly aligned
        a = np.ma.arange(20)
        assert_equal(
            repr(a),
            textwrap.dedent('''\
            masked_array(data=[ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13,
                               14, 15, 16, 17, 18, 19],
                         mask=False,
                   fill_value=999999)''')
        )

        # 2d arrays cause wrapping
        a = array([[1, 2, 3], [4, 5, 6]], dtype=np.int8)
        a[1,1] = np.ma.masked
        assert_equal(
            repr(a),
            textwrap.dedent('''\
            masked_array(
              data=[[1, 2, 3],
                    [4, --, 6]],
              mask=[[False, False, False],
                    [False,  True, False]],
              fill_value=999999,
              dtype=int8)''')
        )

        # but not it they're a row vector
        assert_equal(
            repr(a[:1]),
            textwrap.dedent('''\
            masked_array(data=[[1, 2, 3]],
                         mask=[[False, False, False]],
                   fill_value=999999,
                        dtype=int8)''')
        )

        # dtype=int is implied, so not shown
        assert_equal(
            repr(a.astype(int)),
            textwrap.dedent('''\
            masked_array(
              data=[[1, 2, 3],
                    [4, --, 6]],
              mask=[[False, False, False],
                    [False,  True, False]],
              fill_value=999999)''')
        )

    def test_str_repr_legacy(self):
        oldopts = np.get_printoptions()
        np.set_printoptions(legacy='1.13')
        try:
            a = array([0, 1, 2], mask=[False, True, False])
            assert_equal(str(a), '[0 -- 2]')
            assert_equal(repr(a), 'masked_array(data = [0 -- 2],\n'
                                  '             mask = [False  True False],\n'
                                  '       fill_value = 999999)\n')

            a = np.ma.arange(2000)
            a[1:50] = np.ma.masked
            assert_equal(
                repr(a),
                'masked_array(data = [0 -- -- ..., 1997 1998 1999],\n'
                '             mask = [False  True  True ..., False False False],\n'
                '       fill_value = 999999)\n'
            )
        finally:
            np.set_printoptions(**oldopts)

    def test_0d_unicode(self):
        u = 'caf\xe9'
        utype = type(u)

        arr_nomask = np.ma.array(u)
        arr_masked = np.ma.array(u, mask=True)

        assert_equal(utype(arr_nomask), u)
        assert_equal(utype(arr_masked), '--')

    def test_pickling(self):
        # Tests pickling
        for dtype in (int, float, str, object):
            a = arange(10).astype(dtype)
            a.fill_value = 999

            masks = ([0, 0, 0, 1, 0, 1, 0, 1, 0, 1],  # partially masked
                     True,                            # Fully masked
                     False)                           # Fully unmasked

            for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
                for mask in masks:
                    a.mask = mask
                    a_pickled = pickle.loads(pickle.dumps(a, protocol=proto))
                    assert_equal(a_pickled._mask, a._mask)
                    assert_equal(a_pickled._data, a._data)
                    if dtype in (object, int):
                        assert_equal(a_pickled.fill_value, 999)
                    else:
                        assert_equal(a_pickled.fill_value, dtype(999))
                    assert_array_equal(a_pickled.mask, mask)

    def test_pickling_subbaseclass(self):
        # Test pickling w/ a subclass of ndarray
        x = np.array([(1.0, 2), (3.0, 4)],
                     dtype=[('x', float), ('y', int)]).view(np.recarray)
        a = masked_array(x, mask=[(True, False), (False, True)])
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            a_pickled = pickle.loads(pickle.dumps(a, protocol=proto))
            assert_equal(a_pickled._mask, a._mask)
            assert_equal(a_pickled, a)
            assert_(isinstance(a_pickled._data, np.recarray))

    def test_pickling_maskedconstant(self):
        # Test pickling MaskedConstant
        mc = np.ma.masked
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            mc_pickled = pickle.loads(pickle.dumps(mc, protocol=proto))
            assert_equal(mc_pickled._baseclass, mc._baseclass)
            assert_equal(mc_pickled._mask, mc._mask)
            assert_equal(mc_pickled._data, mc._data)

    def test_pickling_wstructured(self):
        # Tests pickling w/ structured array
        a = array([(1, 1.), (2, 2.)], mask=[(0, 0), (0, 1)],
                  dtype=[('a', int), ('b', float)])
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            a_pickled = pickle.loads(pickle.dumps(a, protocol=proto))
            assert_equal(a_pickled._mask, a._mask)
            assert_equal(a_pickled, a)

    def test_pickling_keepalignment(self):
        # Tests pickling w/ F_CONTIGUOUS arrays
        a = arange(10)
        a.shape = (-1, 2)
        b = a.T
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            test = pickle.loads(pickle.dumps(b, protocol=proto))
            assert_equal(test, b)

    def test_single_element_subscript(self):
        # Tests single element subscripts of Maskedarrays.
        a = array([1, 3, 2])
        b = array([1, 3, 2], mask=[1, 0, 1])
        assert_equal(a[0].shape, ())
        assert_equal(b[0].shape, ())
        assert_equal(b[1].shape, ())

    def test_topython(self):
        # Tests some communication issues with Python.
        assert_equal(1, int(array(1)))
        assert_equal(1.0, float(array(1)))
        assert_equal(1, int(array([[[1]]])))
        assert_equal(1.0, float(array([[1]])))
        assert_raises(TypeError, float, array([1, 1]))

        with suppress_warnings() as sup:
            sup.filter(UserWarning, 'Warning: converting a masked element')
            assert_(np.isnan(float(array([1], mask=[1]))))

            a = array([1, 2, 3], mask=[1, 0, 0])
            assert_raises(TypeError, lambda: float(a))
            assert_equal(float(a[-1]), 3.)
            assert_(np.isnan(float(a[0])))
        assert_raises(TypeError, int, a)
        assert_equal(int(a[-1]), 3)
        assert_raises(MAError, lambda:int(a[0]))

    def test_oddfeatures_1(self):
        # Test of other odd features
        x = arange(20)
        x = x.reshape(4, 5)
        x.flat[5] = 12
        assert_(x[1, 0] == 12)
        z = x + 10j * x
        assert_equal(z.real, x)
        assert_equal(z.imag, 10 * x)
        assert_equal((z * conjugate(z)).real, 101 * x * x)
        z.imag[...] = 0.0

        x = arange(10)
        x[3] = masked
        assert_(str(x[3]) == str(masked))
        c = x >= 8
        assert_(count(where(c, masked, masked)) == 0)
        assert_(shape(where(c, masked, masked)) == c.shape)

        z = masked_where(c, x)
        assert_(z.dtype is x.dtype)
        assert_(z[3] is masked)
        assert_(z[4] is not masked)
        assert_(z[7] is not masked)
        assert_(z[8] is masked)
        assert_(z[9] is masked)
        assert_equal(x, z)

    def test_oddfeatures_2(self):
        # Tests some more features.
        x = array([1., 2., 3., 4., 5.])
        c = array([1, 1, 1, 0, 0])
        x[2] = masked
        z = where(c, x, -x)
        assert_equal(z, [1., 2., 0., -4., -5])
        c[0] = masked
        z = where(c, x, -x)
        assert_equal(z, [1., 2., 0., -4., -5])
        assert_(z[0] is masked)
        assert_(z[1] is not masked)
        assert_(z[2] is masked)

    @suppress_copy_mask_on_assignment
    def test_oddfeatures_3(self):
        # Tests some generic features
        atest = array([10], mask=True)
        btest = array([20])
        idx = atest.mask
        atest[idx] = btest[idx]
        assert_equal(atest, [20])

    def test_filled_with_object_dtype(self):
        a = np.ma.masked_all(1, dtype='O')
        assert_equal(a.filled('x')[0], 'x')

    def test_filled_with_flexible_dtype(self):
        # Test filled w/ flexible dtype
        flexi = array([(1, 1, 1)],
                      dtype=[('i', int), ('s', '|S8'), ('f', float)])
        flexi[0] = masked
        assert_equal(flexi.filled(),
                     np.array([(default_fill_value(0),
                                default_fill_value('0'),
                                default_fill_value(0.),)], dtype=flexi.dtype))
        flexi[0] = masked
        assert_equal(flexi.filled(1),
                     np.array([(1, '1', 1.)], dtype=flexi.dtype))

    def test_filled_with_mvoid(self):
        # Test filled w/ mvoid
        ndtype = [('a', int), ('b', float)]
        a = mvoid((1, 2.), mask=[(0, 1)], dtype=ndtype)
        # Filled using default
        test = a.filled()
        assert_equal(tuple(test), (1, default_fill_value(1.)))
        # Explicit fill_value
        test = a.filled((-1, -1))
        assert_equal(tuple(test), (1, -1))
        # Using predefined filling values
        a.fill_value = (-999, -999)
        assert_equal(tuple(a.filled()), (1, -999))

    def test_filled_with_nested_dtype(self):
        # Test filled w/ nested dtype
        ndtype = [('A', int), ('B', [('BA', int), ('BB', int)])]
        a = array([(1, (1, 1)), (2, (2, 2))],
                  mask=[(0, (1, 0)), (0, (0, 1))], dtype=ndtype)
        test = a.filled(0)
        control = np.array([(1, (0, 1)), (2, (2, 0))], dtype=ndtype)
        assert_equal(test, control)

        test = a['B'].filled(0)
        control = np.array([(0, 1), (2, 0)], dtype=a['B'].dtype)
        assert_equal(test, control)

        # test if mask gets set correctly (see #6760)
        Z = numpy.ma.zeros(2, numpy.dtype([("A", "(2,2)i1,(2,2)i1", (2,2))]))
        assert_equal(Z.data.dtype, numpy.dtype([('A', [('f0', 'i1', (2, 2)),
                                          ('f1', 'i1', (2, 2))], (2, 2))]))
        assert_equal(Z.mask.dtype, numpy.dtype([('A', [('f0', '?', (2, 2)),
                                          ('f1', '?', (2, 2))], (2, 2))]))

    def test_filled_with_f_order(self):
        # Test filled w/ F-contiguous array
        a = array(np.array([(0, 1, 2), (4, 5, 6)], order='F'),
                  mask=np.array([(0, 0, 1), (1, 0, 0)], order='F'),
                  order='F')  # this is currently ignored
        assert_(a.flags['F_CONTIGUOUS'])
        assert_(a.filled(0).flags['F_CONTIGUOUS'])

    def test_optinfo_propagation(self):
        # Checks that _optinfo dictionary isn't back-propagated
        x = array([1, 2, 3, ], dtype=float)
        x._optinfo['info'] = '???'
        y = x.copy()
        assert_equal(y._optinfo['info'], '???')
        y._optinfo['info'] = '!!!'
        assert_equal(x._optinfo['info'], '???')

    def test_optinfo_forward_propagation(self):
        a = array([1,2,2,4])
        a._optinfo["key"] = "value"
        assert_equal(a._optinfo["key"], (a == 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a != 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a > 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a >= 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a <= 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a + 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a - 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a * 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], (a / 2)._optinfo["key"])
        assert_equal(a._optinfo["key"], a[:2]._optinfo["key"])
        assert_equal(a._optinfo["key"], a[[0,0,2]]._optinfo["key"])
        assert_equal(a._optinfo["key"], np.exp(a)._optinfo["key"])
        assert_equal(a._optinfo["key"], np.abs(a)._optinfo["key"])
        assert_equal(a._optinfo["key"], array(a, copy=True)._optinfo["key"])
        assert_equal(a._optinfo["key"], np.zeros_like(a)._optinfo["key"])

    def test_fancy_printoptions(self):
        # Test printing a masked array w/ fancy dtype.
        fancydtype = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
        test = array([(1, (2, 3.0)), (4, (5, 6.0))],
                     mask=[(1, (0, 1)), (0, (1, 0))],
                     dtype=fancydtype)
        control = "[(--, (2, --)) (4, (--, 6.0))]"
        assert_equal(str(test), control)

        # Test 0-d array with multi-dimensional dtype
        t_2d0 = masked_array(data = (0, [[0.0, 0.0, 0.0],
                                        [0.0, 0.0, 0.0]],
                                    0.0),
                             mask = (False, [[True, False, True],
                                             [False, False, True]],
                                     False),
                             dtype = "int, (2,3)float, float")
        control = "(0, [[--, 0.0, --], [0.0, 0.0, --]], 0.0)"
        assert_equal(str(t_2d0), control)

    def test_flatten_structured_array(self):
        # Test flatten_structured_array on arrays
        # On ndarray
        ndtype = [('a', int), ('b', float)]
        a = np.array([(1, 1), (2, 2)], dtype=ndtype)
        test = flatten_structured_array(a)
        control = np.array([[1., 1.], [2., 2.]], dtype=float)
        assert_equal(test, control)
        assert_equal(test.dtype, control.dtype)
        # On masked_array
        a = array([(1, 1), (2, 2)], mask=[(0, 1), (1, 0)], dtype=ndtype)
        test = flatten_structured_array(a)
        control = array([[1., 1.], [2., 2.]],
                        mask=[[0, 1], [1, 0]], dtype=float)
        assert_equal(test, control)
        assert_equal(test.dtype, control.dtype)
        assert_equal(test.mask, control.mask)
        # On masked array with nested structure
        ndtype = [('a', int), ('b', [('ba', int), ('bb', float)])]
        a = array([(1, (1, 1.1)), (2, (2, 2.2))],
                  mask=[(0, (1, 0)), (1, (0, 1))], dtype=ndtype)
        test = flatten_structured_array(a)
        control = array([[1., 1., 1.1], [2., 2., 2.2]],
                        mask=[[0, 1, 0], [1, 0, 1]], dtype=float)
        assert_equal(test, control)
        assert_equal(test.dtype, control.dtype)
        assert_equal(test.mask, control.mask)
        # Keeping the initial shape
        ndtype = [('a', int), ('b', float)]
        a = np.array([[(1, 1), ], [(2, 2), ]], dtype=ndtype)
        test = flatten_structured_array(a)
        control = np.array([[[1., 1.], ], [[2., 2.], ]], dtype=float)
        assert_equal(test, control)
        assert_equal(test.dtype, control.dtype)

    def test_void0d(self):
        # Test creating a mvoid object
        ndtype = [('a', int), ('b', int)]
        a = np.array([(1, 2,)], dtype=ndtype)[0]
        f = mvoid(a)
        assert_(isinstance(f, mvoid))

        a = masked_array([(1, 2)], mask=[(1, 0)], dtype=ndtype)[0]
        assert_(isinstance(a, mvoid))

        a = masked_array([(1, 2), (1, 2)], mask=[(1, 0), (0, 0)], dtype=ndtype)
        f = mvoid(a._data[0], a._mask[0])
        assert_(isinstance(f, mvoid))

    def test_mvoid_getitem(self):
        # Test mvoid.__getitem__
        ndtype = [('a', int), ('b', int)]
        a = masked_array([(1, 2,), (3, 4)], mask=[(0, 0), (1, 0)],
                         dtype=ndtype)
        # w/o mask
        f = a[0]
        assert_(isinstance(f, mvoid))
        assert_equal((f[0], f['a']), (1, 1))
        assert_equal(f['b'], 2)
        # w/ mask
        f = a[1]
        assert_(isinstance(f, mvoid))
        assert_(f[0] is masked)
        assert_(f['a'] is masked)
        assert_equal(f[1], 4)

        # exotic dtype
        A = masked_array(data=[([0,1],)],
                         mask=[([True, False],)],
                         dtype=[("A", ">i2", (2,))])
        assert_equal(A[0]["A"], A["A"][0])
        assert_equal(A[0]["A"], masked_array(data=[0, 1],
                         mask=[True, False], dtype=">i2"))

    def test_mvoid_iter(self):
        # Test iteration on __getitem__
        ndtype = [('a', int), ('b', int)]
        a = masked_array([(1, 2,), (3, 4)], mask=[(0, 0), (1, 0)],
                         dtype=ndtype)
        # w/o mask
        assert_equal(list(a[0]), [1, 2])
        # w/ mask
        assert_equal(list(a[1]), [masked, 4])

    def test_mvoid_print(self):
        # Test printing a mvoid
        mx = array([(1, 1), (2, 2)], dtype=[('a', int), ('b', int)])
        assert_equal(str(mx[0]), "(1, 1)")
        mx['b'][0] = masked
        ini_display = masked_print_option._display
        masked_print_option.set_display("-X-")
        try:
            assert_equal(str(mx[0]), "(1, -X-)")
            assert_equal(repr(mx[0]), "(1, -X-)")
        finally:
            masked_print_option.set_display(ini_display)

        # also check if there are object datatypes (see gh-7493)
        mx = array([(1,), (2,)], dtype=[('a', 'O')])
        assert_equal(str(mx[0]), "(1,)")

    def test_mvoid_multidim_print(self):

        # regression test for gh-6019
        t_ma = masked_array(data = [([1, 2, 3],)],
                            mask = [([False, True, False],)],
                            fill_value = ([999999, 999999, 999999],),
                            dtype = [('a', '<i4', (3,))])
        assert_(str(t_ma[0]) == "([1, --, 3],)")
        assert_(repr(t_ma[0]) == "([1, --, 3],)")

        # additional tests with structured arrays

        t_2d = masked_array(data = [([[1, 2], [3,4]],)],
                            mask = [([[False, True], [True, False]],)],
                            dtype = [('a', '<i4', (2,2))])
        assert_(str(t_2d[0]) == "([[1, --], [--, 4]],)")
        assert_(repr(t_2d[0]) == "([[1, --], [--, 4]],)")

        t_0d = masked_array(data = [(1,2)],
                            mask = [(True,False)],
                            dtype = [('a', '<i4'), ('b', '<i4')])
        assert_(str(t_0d[0]) == "(--, 2)")
        assert_(repr(t_0d[0]) == "(--, 2)")

        t_2d = masked_array(data = [([[1, 2], [3,4]], 1)],
                            mask = [([[False, True], [True, False]], False)],
                            dtype = [('a', '<i4', (2,2)), ('b', float)])
        assert_(str(t_2d[0]) == "([[1, --], [--, 4]], 1.0)")
        assert_(repr(t_2d[0]) == "([[1, --], [--, 4]], 1.0)")

        t_ne = masked_array(data=[(1, (1, 1))],
                            mask=[(True, (True, False))],
                            dtype = [('a', '<i4'), ('b', 'i4,i4')])
        assert_(str(t_ne[0]) == "(--, (--, 1))")
        assert_(repr(t_ne[0]) == "(--, (--, 1))")

    def test_object_with_array(self):
        mx1 = masked_array([1.], mask=[True])
        mx2 = masked_array([1., 2.])
        mx = masked_array([mx1, mx2], mask=[False, True], dtype=object)
        assert_(mx[0] is mx1)
        assert_(mx[1] is not mx2)
        assert_(np.all(mx[1].data == mx2.data))
        assert_(np.all(mx[1].mask))
        # check that we return a view.
        mx[1].data[0] = 0.
        assert_(mx2[0] == 0.)


class TestMaskedArrayArithmetic:
    # Base test class for MaskedArrays.

    def setup_method(self):
        # Base data definition.
        x = np.array([1., 1., 1., -2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.])
        y = np.array([5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.])
        a10 = 10.
        m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]
        m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1]
        xm = masked_array(x, mask=m1)
        ym = masked_array(y, mask=m2)
        z = np.array([-.5, 0., .5, .8])
        zm = masked_array(z, mask=[0, 1, 0, 0])
        xf = np.where(m1, 1e+20, x)
        xm.set_fill_value(1e+20)
        self.d = (x, y, a10, m1, m2, xm, ym, z, zm, xf)
        self.err_status = np.geterr()
        np.seterr(divide='ignore', invalid='ignore')

    def teardown_method(self):
        np.seterr(**self.err_status)

    def test_basic_arithmetic(self):
        # Test of basic arithmetic.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        a2d = array([[1, 2], [0, 4]])
        a2dm = masked_array(a2d, [[0, 0], [1, 0]])
        assert_equal(a2d * a2d, a2d * a2dm)
        assert_equal(a2d + a2d, a2d + a2dm)
        assert_equal(a2d - a2d, a2d - a2dm)
        for s in [(12,), (4, 3), (2, 6)]:
            x = x.reshape(s)
            y = y.reshape(s)
            xm = xm.reshape(s)
            ym = ym.reshape(s)
            xf = xf.reshape(s)
            assert_equal(-x, -xm)
            assert_equal(x + y, xm + ym)
            assert_equal(x - y, xm - ym)
            assert_equal(x * y, xm * ym)
            assert_equal(x / y, xm / ym)
            assert_equal(a10 + y, a10 + ym)
            assert_equal(a10 - y, a10 - ym)
            assert_equal(a10 * y, a10 * ym)
            assert_equal(a10 / y, a10 / ym)
            assert_equal(x + a10, xm + a10)
            assert_equal(x - a10, xm - a10)
            assert_equal(x * a10, xm * a10)
            assert_equal(x / a10, xm / a10)
            assert_equal(x ** 2, xm ** 2)
            assert_equal(abs(x) ** 2.5, abs(xm) ** 2.5)
            assert_equal(x ** y, xm ** ym)
            assert_equal(np.add(x, y), add(xm, ym))
            assert_equal(np.subtract(x, y), subtract(xm, ym))
            assert_equal(np.multiply(x, y), multiply(xm, ym))
            assert_equal(np.divide(x, y), divide(xm, ym))

    def test_divide_on_different_shapes(self):
        x = arange(6, dtype=float)
        x.shape = (2, 3)
        y = arange(3, dtype=float)

        z = x / y
        assert_equal(z, [[-1., 1., 1.], [-1., 4., 2.5]])
        assert_equal(z.mask, [[1, 0, 0], [1, 0, 0]])

        z = x / y[None,:]
        assert_equal(z, [[-1., 1., 1.], [-1., 4., 2.5]])
        assert_equal(z.mask, [[1, 0, 0], [1, 0, 0]])

        y = arange(2, dtype=float)
        z = x / y[:, None]
        assert_equal(z, [[-1., -1., -1.], [3., 4., 5.]])
        assert_equal(z.mask, [[1, 1, 1], [0, 0, 0]])

    def test_mixed_arithmetic(self):
        # Tests mixed arithmetic.
        na = np.array([1])
        ma = array([1])
        assert_(isinstance(na + ma, MaskedArray))
        assert_(isinstance(ma + na, MaskedArray))

    def test_limits_arithmetic(self):
        tiny = np.finfo(float).tiny
        a = array([tiny, 1. / tiny, 0.])
        assert_equal(getmaskarray(a / 2), [0, 0, 0])
        assert_equal(getmaskarray(2 / a), [1, 0, 1])

    def test_masked_singleton_arithmetic(self):
        # Tests some scalar arithmetic on MaskedArrays.
        # Masked singleton should remain masked no matter what
        xm = array(0, mask=1)
        assert_((1 / array(0)).mask)
        assert_((1 + xm).mask)
        assert_((-xm).mask)
        assert_(maximum(xm, xm).mask)
        assert_(minimum(xm, xm).mask)

    def test_masked_singleton_equality(self):
        # Tests (in)equality on masked singleton
        a = array([1, 2, 3], mask=[1, 1, 0])
        assert_((a[0] == 0) is masked)
        assert_((a[0] != 0) is masked)
        assert_equal((a[-1] == 0), False)
        assert_equal((a[-1] != 0), True)

    def test_arithmetic_with_masked_singleton(self):
        # Checks that there's no collapsing to masked
        x = masked_array([1, 2])
        y = x * masked
        assert_equal(y.shape, x.shape)
        assert_equal(y._mask, [True, True])
        y = x[0] * masked
        assert_(y is masked)
        y = x + masked
        assert_equal(y.shape, x.shape)
        assert_equal(y._mask, [True, True])

    def test_arithmetic_with_masked_singleton_on_1d_singleton(self):
        # Check that we're not losing the shape of a singleton
        x = masked_array([1, ])
        y = x + masked
        assert_equal(y.shape, x.shape)
        assert_equal(y.mask, [True, ])

    def test_scalar_arithmetic(self):
        x = array(0, mask=0)
        assert_equal(x.filled().ctypes.data, x.ctypes.data)
        # Make sure we don't lose the shape in some circumstances
        xm = array((0, 0)) / 0.
        assert_equal(xm.shape, (2,))
        assert_equal(xm.mask, [1, 1])

    def test_basic_ufuncs(self):
        # Test various functions such as sin, cos.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        assert_equal(np.cos(x), cos(xm))
        assert_equal(np.cosh(x), cosh(xm))
        assert_equal(np.sin(x), sin(xm))
        assert_equal(np.sinh(x), sinh(xm))
        assert_equal(np.tan(x), tan(xm))
        assert_equal(np.tanh(x), tanh(xm))
        assert_equal(np.sqrt(abs(x)), sqrt(xm))
        assert_equal(np.log(abs(x)), log(xm))
        assert_equal(np.log10(abs(x)), log10(xm))
        assert_equal(np.exp(x), exp(xm))
        assert_equal(np.arcsin(z), arcsin(zm))
        assert_equal(np.arccos(z), arccos(zm))
        assert_equal(np.arctan(z), arctan(zm))
        assert_equal(np.arctan2(x, y), arctan2(xm, ym))
        assert_equal(np.absolute(x), absolute(xm))
        assert_equal(np.angle(x + 1j*y), angle(xm + 1j*ym))
        assert_equal(np.angle(x + 1j*y, deg=True), angle(xm + 1j*ym, deg=True))
        assert_equal(np.equal(x, y), equal(xm, ym))
        assert_equal(np.not_equal(x, y), not_equal(xm, ym))
        assert_equal(np.less(x, y), less(xm, ym))
        assert_equal(np.greater(x, y), greater(xm, ym))
        assert_equal(np.less_equal(x, y), less_equal(xm, ym))
        assert_equal(np.greater_equal(x, y), greater_equal(xm, ym))
        assert_equal(np.conjugate(x), conjugate(xm))

    def test_count_func(self):
        # Tests count
        assert_equal(1, count(1))
        assert_equal(0, array(1, mask=[1]))

        ott = array([0., 1., 2., 3.], mask=[1, 0, 0, 0])
        res = count(ott)
        assert_(res.dtype.type is np.intp)
        assert_equal(3, res)

        ott = ott.reshape((2, 2))
        res = count(ott)
        assert_(res.dtype.type is np.intp)
        assert_equal(3, res)
        res = count(ott, 0)
        assert_(isinstance(res, ndarray))
        assert_equal([1, 2], res)
        assert_(getmask(res) is nomask)

        ott = array([0., 1., 2., 3.])
        res = count(ott, 0)
        assert_(isinstance(res, ndarray))
        assert_(res.dtype.type is np.intp)
        assert_raises(np.AxisError, ott.count, axis=1)

    def test_count_on_python_builtins(self):
        # Tests count works on python builtins (issue#8019)
        assert_equal(3, count([1,2,3]))
        assert_equal(2, count((1,2)))

    def test_minmax_func(self):
        # Tests minimum and maximum.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        # max doesn't work if shaped
        xr = np.ravel(x)
        xmr = ravel(xm)
        # following are true because of careful selection of data
        assert_equal(max(xr), maximum.reduce(xmr))
        assert_equal(min(xr), minimum.reduce(xmr))

        assert_equal(minimum([1, 2, 3], [4, 0, 9]), [1, 0, 3])
        assert_equal(maximum([1, 2, 3], [4, 0, 9]), [4, 2, 9])
        x = arange(5)
        y = arange(5) - 2
        x[3] = masked
        y[0] = masked
        assert_equal(minimum(x, y), where(less(x, y), x, y))
        assert_equal(maximum(x, y), where(greater(x, y), x, y))
        assert_(minimum.reduce(x) == 0)
        assert_(maximum.reduce(x) == 4)

        x = arange(4).reshape(2, 2)
        x[-1, -1] = masked
        assert_equal(maximum.reduce(x, axis=None), 2)

    def test_minimummaximum_func(self):
        a = np.ones((2, 2))
        aminimum = minimum(a, a)
        assert_(isinstance(aminimum, MaskedArray))
        assert_equal(aminimum, np.minimum(a, a))

        aminimum = minimum.outer(a, a)
        assert_(isinstance(aminimum, MaskedArray))
        assert_equal(aminimum, np.minimum.outer(a, a))

        amaximum = maximum(a, a)
        assert_(isinstance(amaximum, MaskedArray))
        assert_equal(amaximum, np.maximum(a, a))

        amaximum = maximum.outer(a, a)
        assert_(isinstance(amaximum, MaskedArray))
        assert_equal(amaximum, np.maximum.outer(a, a))

    def test_minmax_reduce(self):
        # Test np.min/maximum.reduce on array w/ full False mask
        a = array([1, 2, 3], mask=[False, False, False])
        b = np.maximum.reduce(a)
        assert_equal(b, 3)

    def test_minmax_funcs_with_output(self):
        # Tests the min/max functions with explicit outputs
        mask = np.random.rand(12).round()
        xm = array(np.random.uniform(0, 10, 12), mask=mask)
        xm.shape = (3, 4)
        for funcname in ('min', 'max'):
            # Initialize
            npfunc = getattr(np, funcname)
            mafunc = getattr(numpy.ma.core, funcname)
            # Use the np version
            nout = np.empty((4,), dtype=int)
            try:
                result = npfunc(xm, axis=0, out=nout)
            except MaskError:
                pass
            nout = np.empty((4,), dtype=float)
            result = npfunc(xm, axis=0, out=nout)
            assert_(result is nout)
            # Use the ma version
            nout.fill(-999)
            result = mafunc(xm, axis=0, out=nout)
            assert_(result is nout)

    def test_minmax_methods(self):
        # Additional tests on max/min
        (_, _, _, _, _, xm, _, _, _, _) = self.d
        xm.shape = (xm.size,)
        assert_equal(xm.max(), 10)
        assert_(xm[0].max() is masked)
        assert_(xm[0].max(0) is masked)
        assert_(xm[0].max(-1) is masked)
        assert_equal(xm.min(), -10.)
        assert_(xm[0].min() is masked)
        assert_(xm[0].min(0) is masked)
        assert_(xm[0].min(-1) is masked)
        assert_equal(xm.ptp(), 20.)
        assert_(xm[0].ptp() is masked)
        assert_(xm[0].ptp(0) is masked)
        assert_(xm[0].ptp(-1) is masked)

        x = array([1, 2, 3], mask=True)
        assert_(x.min() is masked)
        assert_(x.max() is masked)
        assert_(x.ptp() is masked)

    def test_minmax_dtypes(self):
        # Additional tests on max/min for non-standard float and complex dtypes
        x = np.array([1., 1., 1., -2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.])
        a10 = 10.
        an10 = -10.0
        m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]
        xm = masked_array(x, mask=m1)
        xm.set_fill_value(1e+20)
        float_dtypes = [np.half, np.single, np.double,
                        np.longdouble, np.cfloat, np.cdouble, np.clongdouble]
        for float_dtype in float_dtypes:
            assert_equal(masked_array(x, mask=m1, dtype=float_dtype).max(),
                         float_dtype(a10))
            assert_equal(masked_array(x, mask=m1, dtype=float_dtype).min(),
                         float_dtype(an10))

        assert_equal(xm.min(), an10)
        assert_equal(xm.max(), a10)

        # Non-complex type only test
        for float_dtype in float_dtypes[:4]:
            assert_equal(masked_array(x, mask=m1, dtype=float_dtype).max(),
                         float_dtype(a10))
            assert_equal(masked_array(x, mask=m1, dtype=float_dtype).min(),
                         float_dtype(an10))

        # Complex types only test
        for float_dtype in float_dtypes[-3:]:
            ym = masked_array([1e20+1j, 1e20-2j, 1e20-1j], mask=[0, 1, 0],
                          dtype=float_dtype)
            assert_equal(ym.min(), float_dtype(1e20-1j))
            assert_equal(ym.max(), float_dtype(1e20+1j))

            zm = masked_array([np.inf+2j, np.inf+3j, -np.inf-1j], mask=[0, 1, 0],
                              dtype=float_dtype)
            assert_equal(zm.min(), float_dtype(-np.inf-1j))
            assert_equal(zm.max(), float_dtype(np.inf+2j))

            cmax = np.inf - 1j * np.finfo(np.float64).max
            assert masked_array([-cmax, 0], mask=[0, 1]).max() == -cmax
            assert masked_array([cmax, 0], mask=[0, 1]).min() == cmax

    def test_addsumprod(self):
        # Tests add, sum, product.
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        assert_equal(np.add.reduce(x), add.reduce(x))
        assert_equal(np.add.accumulate(x), add.accumulate(x))
        assert_equal(4, sum(array(4), axis=0))
        assert_equal(4, sum(array(4), axis=0))
        assert_equal(np.sum(x, axis=0), sum(x, axis=0))
        assert_equal(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0))
        assert_equal(np.sum(x, 0), sum(x, 0))
        assert_equal(np.product(x, axis=0), product(x, axis=0))
        assert_equal(np.product(x, 0), product(x, 0))
        assert_equal(np.product(filled(xm, 1), axis=0), product(xm, axis=0))
        s = (3, 4)
        x.shape = y.shape = xm.shape = ym.shape = s
        if len(s) > 1:
            assert_equal(np.concatenate((x, y), 1), concatenate((xm, ym), 1))
            assert_equal(np.add.reduce(x, 1), add.reduce(x, 1))
            assert_equal(np.sum(x, 1), sum(x, 1))
            assert_equal(np.product(x, 1), product(x, 1))

    def test_binops_d2D(self):
        # Test binary operations on 2D data
        a = array([[1.], [2.], [3.]], mask=[[False], [True], [True]])
        b = array([[2., 3.], [4., 5.], [6., 7.]])

        test = a * b
        control = array([[2., 3.], [2., 2.], [3., 3.]],
                        mask=[[0, 0], [1, 1], [1, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

        test = b * a
        control = array([[2., 3.], [4., 5.], [6., 7.]],
                        mask=[[0, 0], [1, 1], [1, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

        a = array([[1.], [2.], [3.]])
        b = array([[2., 3.], [4., 5.], [6., 7.]],
                  mask=[[0, 0], [0, 0], [0, 1]])
        test = a * b
        control = array([[2, 3], [8, 10], [18, 3]],
                        mask=[[0, 0], [0, 0], [0, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

        test = b * a
        control = array([[2, 3], [8, 10], [18, 7]],
                        mask=[[0, 0], [0, 0], [0, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

    def test_domained_binops_d2D(self):
        # Test domained binary operations on 2D data
        a = array([[1.], [2.], [3.]], mask=[[False], [True], [True]])
        b = array([[2., 3.], [4., 5.], [6., 7.]])

        test = a / b
        control = array([[1. / 2., 1. / 3.], [2., 2.], [3., 3.]],
                        mask=[[0, 0], [1, 1], [1, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

        test = b / a
        control = array([[2. / 1., 3. / 1.], [4., 5.], [6., 7.]],
                        mask=[[0, 0], [1, 1], [1, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

        a = array([[1.], [2.], [3.]])
        b = array([[2., 3.], [4., 5.], [6., 7.]],
                  mask=[[0, 0], [0, 0], [0, 1]])
        test = a / b
        control = array([[1. / 2, 1. / 3], [2. / 4, 2. / 5], [3. / 6, 3]],
                        mask=[[0, 0], [0, 0], [0, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

        test = b / a
        control = array([[2 / 1., 3 / 1.], [4 / 2., 5 / 2.], [6 / 3., 7]],
                        mask=[[0, 0], [0, 0], [0, 1]])
        assert_equal(test, control)
        assert_equal(test.data, control.data)
        assert_equal(test.mask, control.mask)

    def test_noshrinking(self):
        # Check that we don't shrink a mask when not wanted
        # Binary operations
        a = masked_array([1., 2., 3.], mask=[False, False, False],
                         shrink=False)
        b = a + 1
        assert_equal(b.mask, [0, 0, 0])
        # In place binary operation
        a += 1
        assert_equal(a.mask, [0, 0, 0])
        # Domained binary operation
        b = a / 1.
        assert_equal(b.mask, [0, 0, 0])
        # In place binary operation
        a /= 1.
        assert_equal(a.mask, [0, 0, 0])

    def test_ufunc_nomask(self):
        # check the case ufuncs should set the mask to false
        m = np.ma.array([1])
        # check we don't get array([False], dtype=bool)
        assert_equal(np.true_divide(m, 5).mask.shape, ())

    def test_noshink_on_creation(self):
        # Check that the mask is not shrunk on array creation when not wanted
        a = np.ma.masked_values([1., 2.5, 3.1], 1.5, shrink=False)
        assert_equal(a.mask, [0, 0, 0])

    def test_mod(self):
        # Tests mod
        (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d
        assert_equal(mod(x, y), mod(xm, ym))
        test = mod(ym, xm)
        assert_equal(test, np.mod(ym, xm))
        assert_equal(test.mask, mask_or(xm.mask, ym.mask))
        test = mod(xm, ym)
        assert_equal(test, np.mod(xm, ym))
        assert_equal(test.mask, mask_or(mask_or(xm.mask, ym.mask), (ym == 0)))

    def test_TakeTransposeInnerOuter(self):
        # Test of take, transpose, inner, outer products
        x = arange(24)
        y = np.arange(24)
        x[5:6] = masked
        x = x.reshape(2, 3, 4)
        y = y.reshape(2, 3, 4)
        assert_equal(np.transpose(y, (2, 0, 1)), transpose(x, (2, 0, 1)))
        assert_equal(np.take(y, (2, 0, 1), 1), take(x, (2, 0, 1), 1))
        assert_equal(np.inner(filled(x, 0), filled(y, 0)),
                     inner(x, y))
        assert_equal(np.outer(filled(x, 0), filled(y, 0)),
                     outer(x, y))
        y = array(['abc', 1, 'def', 2, 3], object)
        y[2] = masked
        t = take(y, [0, 3, 4])
        assert_(t[0] == 'abc')
        assert_(t[1] == 2)
        assert_(t[2] == 3)

    def test_imag_real(self):
        # Check complex
        xx = array([1 + 10j, 20 + 2j], mask=[1, 0])
        assert_equal(xx.imag, [10, 2])
        assert_equal(xx.imag.filled(), [1e+20, 2])
        assert_equal(xx.imag.dtype, xx._data.imag.dtype)
        assert_equal(xx.real, [1, 20])
        assert_equal(xx.real.filled(), [1e+20, 20])
        assert_equal(xx.real.dtype, xx._data.real.dtype)

    def test_methods_with_output(self):
        xm = array(np.random.uniform(0, 10, 12)).reshape(3, 4)
        xm[:, 0] = xm[0] = xm[-1, -1] = masked

        funclist = ('sum', 'prod', 'var', 'std', 'max', 'min', 'ptp', 'mean',)

        for funcname in funclist:
            npfunc = getattr(np, funcname)
            xmmeth = getattr(xm, funcname)
            # A ndarray as explicit input
            output = np.empty(4, dtype=float)
            output.fill(-9999)
            result = npfunc(xm, axis=0, out=output)
            # ... the result should be the given output
            assert_(result is output)
            assert_equal(result, xmmeth(axis=0, out=output))

            output = empty(4, dtype=int)
            result = xmmeth(axis=0, out=output)
            assert_(result is output)
            assert_(output[0] is masked)

    def test_eq_on_structured(self):
        # Test the equality of structured arrays
        ndtype = [('A', int), ('B', int)]
        a = array([(1, 1), (2, 2)], mask=[(0, 1), (0, 0)], dtype=ndtype)

        test = (a == a)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [False, False])
        assert_(test.fill_value == True)

        test = (a == a[0])
        assert_equal(test.data, [True, False])
        assert_equal(test.mask, [False, False])
        assert_(test.fill_value == True)

        b = array([(1, 1), (2, 2)], mask=[(1, 0), (0, 0)], dtype=ndtype)
        test = (a == b)
        assert_equal(test.data, [False, True])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = (a[0] == b)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        b = array([(1, 1), (2, 2)], mask=[(0, 1), (1, 0)], dtype=ndtype)
        test = (a == b)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [False, False])
        assert_(test.fill_value == True)

        # complicated dtype, 2-dimensional array.
        ndtype = [('A', int), ('B', [('BA', int), ('BB', int)])]
        a = array([[(1, (1, 1)), (2, (2, 2))],
                   [(3, (3, 3)), (4, (4, 4))]],
                  mask=[[(0, (1, 0)), (0, (0, 1))],
                        [(1, (0, 0)), (1, (1, 1))]], dtype=ndtype)
        test = (a[0, 0] == a)
        assert_equal(test.data, [[True, False], [False, False]])
        assert_equal(test.mask, [[False, False], [False, True]])
        assert_(test.fill_value == True)

    def test_ne_on_structured(self):
        # Test the equality of structured arrays
        ndtype = [('A', int), ('B', int)]
        a = array([(1, 1), (2, 2)], mask=[(0, 1), (0, 0)], dtype=ndtype)

        test = (a != a)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [False, False])
        assert_(test.fill_value == True)

        test = (a != a[0])
        assert_equal(test.data, [False, True])
        assert_equal(test.mask, [False, False])
        assert_(test.fill_value == True)

        b = array([(1, 1), (2, 2)], mask=[(1, 0), (0, 0)], dtype=ndtype)
        test = (a != b)
        assert_equal(test.data, [True, False])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = (a[0] != b)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        b = array([(1, 1), (2, 2)], mask=[(0, 1), (1, 0)], dtype=ndtype)
        test = (a != b)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [False, False])
        assert_(test.fill_value == True)

        # complicated dtype, 2-dimensional array.
        ndtype = [('A', int), ('B', [('BA', int), ('BB', int)])]
        a = array([[(1, (1, 1)), (2, (2, 2))],
                   [(3, (3, 3)), (4, (4, 4))]],
                  mask=[[(0, (1, 0)), (0, (0, 1))],
                        [(1, (0, 0)), (1, (1, 1))]], dtype=ndtype)
        test = (a[0, 0] != a)
        assert_equal(test.data, [[False, True], [True, True]])
        assert_equal(test.mask, [[False, False], [False, True]])
        assert_(test.fill_value == True)

    def test_eq_ne_structured_extra(self):
        # ensure simple examples are symmetric and make sense.
        # from https://github.com/numpy/numpy/pull/8590#discussion_r101126465
        dt = np.dtype('i4,i4')
        for m1 in (mvoid((1, 2), mask=(0, 0), dtype=dt),
                   mvoid((1, 2), mask=(0, 1), dtype=dt),
                   mvoid((1, 2), mask=(1, 0), dtype=dt),
                   mvoid((1, 2), mask=(1, 1), dtype=dt)):
            ma1 = m1.view(MaskedArray)
            r1 = ma1.view('2i4')
            for m2 in (np.array((1, 1), dtype=dt),
                       mvoid((1, 1), dtype=dt),
                       mvoid((1, 0), mask=(0, 1), dtype=dt),
                       mvoid((3, 2), mask=(0, 1), dtype=dt)):
                ma2 = m2.view(MaskedArray)
                r2 = ma2.view('2i4')
                eq_expected = (r1 == r2).all()
                assert_equal(m1 == m2, eq_expected)
                assert_equal(m2 == m1, eq_expected)
                assert_equal(ma1 == m2, eq_expected)
                assert_equal(m1 == ma2, eq_expected)
                assert_equal(ma1 == ma2, eq_expected)
                # Also check it is the same if we do it element by element.
                el_by_el = [m1[name] == m2[name] for name in dt.names]
                assert_equal(array(el_by_el, dtype=bool).all(), eq_expected)
                ne_expected = (r1 != r2).any()
                assert_equal(m1 != m2, ne_expected)
                assert_equal(m2 != m1, ne_expected)
                assert_equal(ma1 != m2, ne_expected)
                assert_equal(m1 != ma2, ne_expected)
                assert_equal(ma1 != ma2, ne_expected)
                el_by_el = [m1[name] != m2[name] for name in dt.names]
                assert_equal(array(el_by_el, dtype=bool).any(), ne_expected)

    @pytest.mark.parametrize('dt', ['S', 'U'])
    @pytest.mark.parametrize('fill', [None, 'A'])
    def test_eq_for_strings(self, dt, fill):
        # Test the equality of structured arrays
        a = array(['a', 'b'], dtype=dt, mask=[0, 1], fill_value=fill)

        test = (a == a)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        test = (a == a[0])
        assert_equal(test.data, [True, False])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        b = array(['a', 'b'], dtype=dt, mask=[1, 0], fill_value=fill)
        test = (a == b)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, True])
        assert_(test.fill_value == True)

        test = (a[0] == b)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = (b == a[0])
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

    @pytest.mark.parametrize('dt', ['S', 'U'])
    @pytest.mark.parametrize('fill', [None, 'A'])
    def test_ne_for_strings(self, dt, fill):
        # Test the equality of structured arrays
        a = array(['a', 'b'], dtype=dt, mask=[0, 1], fill_value=fill)

        test = (a != a)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        test = (a != a[0])
        assert_equal(test.data, [False, True])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        b = array(['a', 'b'], dtype=dt, mask=[1, 0], fill_value=fill)
        test = (a != b)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, True])
        assert_(test.fill_value == True)

        test = (a[0] != b)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = (b != a[0])
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

    @pytest.mark.parametrize('dt1', num_dts, ids=num_ids)
    @pytest.mark.parametrize('dt2', num_dts, ids=num_ids)
    @pytest.mark.parametrize('fill', [None, 1])
    def test_eq_for_numeric(self, dt1, dt2, fill):
        # Test the equality of structured arrays
        a = array([0, 1], dtype=dt1, mask=[0, 1], fill_value=fill)

        test = (a == a)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        test = (a == a[0])
        assert_equal(test.data, [True, False])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        b = array([0, 1], dtype=dt2, mask=[1, 0], fill_value=fill)
        test = (a == b)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, True])
        assert_(test.fill_value == True)

        test = (a[0] == b)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = (b == a[0])
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

    @pytest.mark.parametrize('dt1', num_dts, ids=num_ids)
    @pytest.mark.parametrize('dt2', num_dts, ids=num_ids)
    @pytest.mark.parametrize('fill', [None, 1])
    def test_ne_for_numeric(self, dt1, dt2, fill):
        # Test the equality of structured arrays
        a = array([0, 1], dtype=dt1, mask=[0, 1], fill_value=fill)

        test = (a != a)
        assert_equal(test.data, [False, False])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        test = (a != a[0])
        assert_equal(test.data, [False, True])
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        b = array([0, 1], dtype=dt2, mask=[1, 0], fill_value=fill)
        test = (a != b)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, True])
        assert_(test.fill_value == True)

        test = (a[0] != b)
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = (b != a[0])
        assert_equal(test.data, [True, True])
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

    @pytest.mark.parametrize('dt1', num_dts, ids=num_ids)
    @pytest.mark.parametrize('dt2', num_dts, ids=num_ids)
    @pytest.mark.parametrize('fill', [None, 1])
    @pytest.mark.parametrize('op',
            [operator.le, operator.lt, operator.ge, operator.gt])
    def test_comparisons_for_numeric(self, op, dt1, dt2, fill):
        # Test the equality of structured arrays
        a = array([0, 1], dtype=dt1, mask=[0, 1], fill_value=fill)

        test = op(a, a)
        assert_equal(test.data, op(a._data, a._data))
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        test = op(a, a[0])
        assert_equal(test.data, op(a._data, a._data[0]))
        assert_equal(test.mask, [False, True])
        assert_(test.fill_value == True)

        b = array([0, 1], dtype=dt2, mask=[1, 0], fill_value=fill)
        test = op(a, b)
        assert_equal(test.data, op(a._data, b._data))
        assert_equal(test.mask, [True, True])
        assert_(test.fill_value == True)

        test = op(a[0], b)
        assert_equal(test.data, op(a._data[0], b._data))
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

        test = op(b, a[0])
        assert_equal(test.data, op(b._data, a._data[0]))
        assert_equal(test.mask, [True, False])
        assert_(test.fill_value == True)

    @pytest.mark.parametrize('op',
            [operator.le, operator.lt, operator.ge, operator.gt])
    @pytest.mark.parametrize('fill', [None, "N/A"])
    def test_comparisons_strings(self, op, fill):
        # See gh-21770, mask propagation is broken for strings (and some other
        # cases) so we explicitly test strings here.
        # In principle only == and != may need special handling...
        ma1 = masked_array(["a", "b", "cde"], mask=[0, 1, 0], fill_value=fill)
        ma2 = masked_array(["cde", "b", "a"], mask=[0, 1, 0], fill_value=fill)
        assert_equal(op(ma1, ma2)._data, op(ma1._data, ma2._data))

    def test_eq_with_None(self):
        # Really, comparisons with None should not be done, but check them
        # anyway. Note that pep8 will flag these tests.
        # Deprecation is in place for arrays, and when it happens this
        # test will fail (and have to be changed accordingly).

        # With partial mask
        with suppress_warnings() as sup:
            sup.filter(FutureWarning, "Comparison to `None`")
            a = array([None, 1], mask=[0, 1])
            assert_equal(a == None, array([True, False], mask=[0, 1]))
            assert_equal(a.data == None, [True, False])
            assert_equal(a != None, array([False, True], mask=[0, 1]))
            # With nomask
            a = array([None, 1], mask=False)
            assert_equal(a == None, [True, False])
            assert_equal(a != None, [False, True])
            # With complete mask
            a = array([None, 2], mask=True)
            assert_equal(a == None, array([False, True], mask=True))
            assert_equal(a != None, array([True, False], mask=True))
            # Fully masked, even comparison to None should return "masked"
            a = masked
            assert_equal(a == None, masked)

    def test_eq_with_scalar(self):
        a = array(1)
        assert_equal(a == 1, True)
        assert_equal(a == 0, False)
        assert_equal(a != 1, False)
        assert_equal(a != 0, True)
        b = array(1, mask=True)
        assert_equal(b == 0, masked)
        assert_equal(b == 1, masked)
        assert_equal(b != 0, masked)
        assert_equal(b != 1, masked)

    def test_eq_different_dimensions(self):
        m1 = array([1, 1], mask=[0, 1])
        # test comparison with both masked and regular arrays.
        for m2 in (array([[0, 1], [1, 2]]),
                   np.array([[0, 1], [1, 2]])):
            test = (m1 == m2)
            assert_equal(test.data, [[False, False],
                                     [True, False]])
            assert_equal(test.mask, [[False, True],
                                     [False, True]])

    def test_numpyarithmetic(self):
        # Check that the mask is not back-propagated when using numpy functions
        a = masked_array([-1, 0, 1, 2, 3], mask=[0, 0, 0, 0, 1])
        control = masked_array([np.nan, np.nan, 0, np.log(2), -1],
                               mask=[1, 1, 0, 0, 1])

        test = log(a)
        assert_equal(test, control)
        assert_equal(test.mask, control.mask)
        assert_equal(a.mask, [0, 0, 0, 0, 1])

        test = np.log(a)
        assert_equal(test, control)
        assert_equal(test.mask, control.mask)
        assert_equal(a.mask, [0, 0, 0, 0, 1])


class TestMaskedArrayAttributes:

    def test_keepmask(self):
        # Tests the keep mask flag
        x = masked_array([1, 2, 3], mask=[1, 0, 0])
        mx = masked_array(x)
        assert_equal(mx.mask, x.mask)
        mx = masked_array(x, mask=[0, 1, 0], keep_mask=False)
        assert_equal(mx.mask, [0, 1, 0])
        mx = masked_array(x, mask=[0, 1, 0], keep_mask=True)
        assert_equal(mx.mask, [1, 1, 0])
        # We default to true
        mx = masked_array(x, mask=[0, 1, 0])
        assert_equal(mx.mask, [1, 1, 0])

    def test_hardmask(self):
        # Test hard_mask
        d = arange(5)
        n = [0, 0, 0, 1, 1]
        m = make_mask(n)
        xh = array(d, mask=m, hard_mask=True)
        # We need to copy, to avoid updating d in xh !
        xs = array(d, mask=m, hard_mask=False, copy=True)
        xh[[1, 4]] = [10, 40]
        xs[[1, 4]] = [10, 40]
        assert_equal(xh._data, [0, 10, 2, 3, 4])
        assert_equal(xs._data, [0, 10, 2, 3, 40])
        assert_equal(xs.mask, [0, 0, 0, 1, 0])
        assert_(xh._hardmask)
        assert_(not xs._hardmask)
        xh[1:4] = [10, 20, 30]
        xs[1:4] = [10, 20, 30]
        assert_equal(xh._data, [0, 10, 20, 3, 4])
        assert_equal(xs._data, [0, 10, 20, 30, 40])
        assert_equal(xs.mask, nomask)
        xh[0] = masked
        xs[0] = masked
        assert_equal(xh.mask, [1, 0, 0, 1, 1])
        assert_equal(xs.mask, [1, 0, 0, 0, 0])
        xh[:] = 1
        xs[:] = 1
        assert_equal(xh._data, [0, 1, 1, 3, 4])
        assert_equal(xs._data, [1, 1, 1, 1, 1])
        assert_equal(xh.mask, [1, 0, 0, 1, 1])
        assert_equal(xs.mask, nomask)
        # Switch to soft mask
        xh.soften_mask()
        xh[:] = arange(5)
        assert_equal(xh._data, [0, 1, 2, 3, 4])
        assert_equal(xh.mask, nomask)
        # Switch back to hard mask
        xh.harden_mask()
        xh[xh < 3] = masked
        assert_equal(xh._data, [0, 1, 2, 3, 4])
        assert_equal(xh._mask, [1, 1, 1, 0, 0])
        xh[filled(xh > 1, False)] = 5
        assert_equal(xh._data, [0, 1, 2, 5, 5])
        assert_equal(xh._mask, [1, 1, 1, 0, 0])

        xh = array([[1, 2], [3, 4]], mask=[[1, 0], [0, 0]], hard_mask=True)
        xh[0] = 0
        assert_equal(xh._data, [[1, 0], [3, 4]])
        assert_equal(xh._mask, [[1, 0], [0, 0]])
        xh[-1, -1] = 5
        assert_equal(xh._data, [[1, 0], [3, 5]])
        assert_equal(xh._mask, [[1, 0], [0, 0]])
        xh[filled(xh < 5, False)] = 2
        assert_equal(xh._data, [[1, 2], [2, 5]])
        assert_equal(xh._mask, [[1, 0], [0, 0]])

    def test_hardmask_again(self):
        # Another test of hardmask
        d = arange(5)
        n = [0, 0, 0, 1, 1]
        m = make_mask(n)
        xh = array(d, mask=m, hard_mask=True)
        xh[4:5] = 999
        xh[0:1] = 999
        assert_equal(xh._data, [999, 1, 2, 3, 4])

    def test_hardmask_oncemore_yay(self):
        # OK, yet another test of hardmask
        # Make sure that harden_mask/soften_mask//unshare_mask returns self
        a = array([1, 2, 3], mask=[1, 0, 0])
        b = a.harden_mask()
        assert_equal(a, b)
        b[0] = 0
        assert_equal(a, b)
        assert_equal(b, array([1, 2, 3], mask=[1, 0, 0]))
        a = b.soften_mask()
        a[0] = 0
        assert_equal(a, b)
        assert_equal(b, array([0, 2, 3], mask=[0, 0, 0]))

    def test_smallmask(self):
        # Checks the behaviour of _smallmask
        a = arange(10)
        a[1] = masked
        a[1] = 1
        assert_equal(a._mask, nomask)
        a = arange(10)
        a._smallmask = False
        a[1] = masked
        a[1] = 1
        assert_equal(a._mask, zeros(10))

    def test_shrink_mask(self):
        # Tests .shrink_mask()
        a = array([1, 2, 3], mask=[0, 0, 0])
        b = a.shrink_mask()
        assert_equal(a, b)
        assert_equal(a.mask, nomask)

        # Mask cannot be shrunk on structured types, so is a no-op
        a = np.ma.array([(1, 2.0)], [('a', int), ('b', float)])
        b = a.copy()
        a.shrink_mask()
        assert_equal(a.mask, b.mask)

    def test_flat(self):
        # Test that flat can return all types of items [#4585, #4615]
        # test 2-D record array
        # ... on structured array w/ masked records
        x = array([[(1, 1.1, 'one'), (2, 2.2, 'two'), (3, 3.3, 'thr')],
                   [(4, 4.4, 'fou'), (5, 5.5, 'fiv'), (6, 6.6, 'six')]],
                  dtype=[('a', int), ('b', float), ('c', '|S8')])
        x['a'][0, 1] = masked
        x['b'][1, 0] = masked
        x['c'][0, 2] = masked
        x[-1, -1] = masked
        xflat = x.flat
        assert_equal(xflat[0], x[0, 0])
        assert_equal(xflat[1], x[0, 1])
        assert_equal(xflat[2], x[0, 2])
        assert_equal(xflat[:3], x[0])
        assert_equal(xflat[3], x[1, 0])
        assert_equal(xflat[4], x[1, 1])
        assert_equal(xflat[5], x[1, 2])
        assert_equal(xflat[3:], x[1])
        assert_equal(xflat[-1], x[-1, -1])
        i = 0
        j = 0
        for xf in xflat:
            assert_equal(xf, x[j, i])
            i += 1
            if i >= x.shape[-1]:
                i = 0
                j += 1

    def test_assign_dtype(self):
        # check that the mask's dtype is updated when dtype is changed
        a = np.zeros(4, dtype='f4,i4')

        m = np.ma.array(a)
        m.dtype = np.dtype('f4')
        repr(m)  # raises?
        assert_equal(m.dtype, np.dtype('f4'))

        # check that dtype changes that change shape of mask too much
        # are not allowed
        def assign():
            m = np.ma.array(a)
            m.dtype = np.dtype('f8')
        assert_raises(ValueError, assign)

        b = a.view(dtype='f4', type=np.ma.MaskedArray)  # raises?
        assert_equal(b.dtype, np.dtype('f4'))

        # check that nomask is preserved
        a = np.zeros(4, dtype='f4')
        m = np.ma.array(a)
        m.dtype = np.dtype('f4,i4')
        assert_equal(m.dtype, np.dtype('f4,i4'))
        assert_equal(m._mask, np.ma.nomask)


class TestFillingValues:

    def test_check_on_scalar(self):
        # Test _check_fill_value set to valid and invalid values
        _check_fill_value = np.ma.core._check_fill_value

        fval = _check_fill_value(0, int)
        assert_equal(fval, 0)
        fval = _check_fill_value(None, int)
        assert_equal(fval, default_fill_value(0))

        fval = _check_fill_value(0, "|S3")
        assert_equal(fval, b"0")
        fval = _check_fill_value(None, "|S3")
        assert_equal(fval, default_fill_value(b"camelot!"))
        assert_raises(TypeError, _check_fill_value, 1e+20, int)
        assert_raises(TypeError, _check_fill_value, 'stuff', int)

    def test_check_on_fields(self):
        # Tests _check_fill_value with records
        _check_fill_value = np.ma.core._check_fill_value
        ndtype = [('a', int), ('b', float), ('c', "|S3")]
        # A check on a list should return a single record
        fval = _check_fill_value([-999, -12345678.9, "???"], ndtype)
        assert_(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, b"???"])
        # A check on None should output the defaults
        fval = _check_fill_value(None, ndtype)
        assert_(isinstance(fval, ndarray))
        assert_equal(fval.item(), [default_fill_value(0),
                                   default_fill_value(0.),
                                   asbytes(default_fill_value("0"))])
        #.....Using a structured type as fill_value should work
        fill_val = np.array((-999, -12345678.9, "???"), dtype=ndtype)
        fval = _check_fill_value(fill_val, ndtype)
        assert_(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, b"???"])

        #.....Using a flexible type w/ a different type shouldn't matter
        # BEHAVIOR in 1.5 and earlier, and 1.13 and later: match structured
        # types by position
        fill_val = np.array((-999, -12345678.9, "???"),
                            dtype=[("A", int), ("B", float), ("C", "|S3")])
        fval = _check_fill_value(fill_val, ndtype)
        assert_(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, b"???"])

        #.....Using an object-array shouldn't matter either
        fill_val = np.ndarray(shape=(1,), dtype=object)
        fill_val[0] = (-999, -12345678.9, b"???")
        fval = _check_fill_value(fill_val, object)
        assert_(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, b"???"])
        # NOTE: This test was never run properly as "fill_value" rather than
        # "fill_val" was assigned.  Written properly, it fails.
        #fill_val = np.array((-999, -12345678.9, "???"))
        #fval = _check_fill_value(fill_val, ndtype)
        #assert_(isinstance(fval, ndarray))
        #assert_equal(fval.item(), [-999, -12345678.9, b"???"])
        #.....One-field-only flexible type should work as well
        ndtype = [("a", int)]
        fval = _check_fill_value(-999999999, ndtype)
        assert_(isinstance(fval, ndarray))
        assert_equal(fval.item(), (-999999999,))

    def test_fillvalue_conversion(self):
        # Tests the behavior of fill_value during conversion
        # We had a tailored comment to make sure special attributes are
        # properly dealt with
        a = array([b'3', b'4', b'5'])
        a._optinfo.update({'comment':"updated!"})

        b = array(a, dtype=int)
        assert_equal(b._data, [3, 4, 5])
        assert_equal(b.fill_value, default_fill_value(0))

        b = array(a, dtype=float)
        assert_equal(b._data, [3, 4, 5])
        assert_equal(b.fill_value, default_fill_value(0.))

        b = a.astype(int)
        assert_equal(b._data, [3, 4, 5])
        assert_equal(b.fill_value, default_fill_value(0))
        assert_equal(b._optinfo['comment'], "updated!")

        b = a.astype([('a', '|S3')])
        assert_equal(b['a']._data, a._data)
        assert_equal(b['a'].fill_value, a.fill_value)

    def test_default_fill_value(self):
        # check all calling conventions
        f1 = default_fill_value(1.)
        f2 = default_fill_value(np.array(1.))
        f3 = default_fill_value(np.array(1.).dtype)
        assert_equal(f1, f2)
        assert_equal(f1, f3)

    def test_default_fill_value_structured(self):
        fields = array([(1, 1, 1)],
                      dtype=[('i', int), ('s', '|S8'), ('f', float)])

        f1 = default_fill_value(fields)
        f2 = default_fill_value(fields.dtype)
        expected = np.array((default_fill_value(0),
                             default_fill_value('0'),
                             default_fill_value(0.)), dtype=fields.dtype)
        assert_equal(f1, expected)
        assert_equal(f2, expected)

    def test_default_fill_value_void(self):
        dt = np.dtype([('v', 'V7')])
        f = default_fill_value(dt)
        assert_equal(f['v'], np.array(default_fill_value(dt['v']), dt['v']))

    def test_fillvalue(self):
        # Yet more fun with the fill_value
        data = masked_array([1, 2, 3], fill_value=-999)
        series = data[[0, 2, 1]]
        assert_equal(series._fill_value, data._fill_value)

        mtype = [('f', float), ('s', '|S3')]
        x = array([(1, 'a'), (2, 'b'), (pi, 'pi')], dtype=mtype)
        x.fill_value = 999
        assert_equal(x.fill_value.item(), [999., b'999'])
        assert_equal(x['f'].fill_value, 999)
        assert_equal(x['s'].fill_value, b'999')

        x.fill_value = (9, '???')
        assert_equal(x.fill_value.item(), (9, b'???'))
        assert_equal(x['f'].fill_value, 9)
        assert_equal(x['s'].fill_value, b'???')

        x = array([1, 2, 3.1])
        x.fill_value = 999
        assert_equal(np.asarray(x.fill_value).dtype, float)
        assert_equal(x.fill_value, 999.)
        assert_equal(x._fill_value, np.array(999.))

    def test_subarray_fillvalue(self):
        # gh-10483   test multi-field index fill value
        fields = array([(1, 1, 1)],
                      dtype=[('i', int), ('s', '|S8'), ('f', float)])
        with suppress_warnings() as sup:
            sup.filter(FutureWarning, "Numpy has detected")
            subfields = fields[['i', 'f']]
            assert_equal(tuple(subfields.fill_value), (999999, 1.e+20))
            # test comparison does not raise:
            subfields[1:] == subfields[:-1]

    def test_fillvalue_exotic_dtype(self):
        # Tests yet more exotic flexible dtypes
        _check_fill_value = np.ma.core._check_fill_value
        ndtype = [('i', int), ('s', '|S8'), ('f', float)]
        control = np.array((default_fill_value(0),
                            default_fill_value('0'),
                            default_fill_value(0.),),
                           dtype=ndtype)
        assert_equal(_check_fill_value(None, ndtype), control)
        # The shape shouldn't matter
        ndtype = [('f0', float, (2, 2))]
        control = np.array((default_fill_value(0.),),
                           dtype=[('f0', float)]).astype(ndtype)
        assert_equal(_check_fill_value(None, ndtype), control)
        control = np.array((0,), dtype=[('f0', float)]).astype(ndtype)
        assert_equal(_check_fill_value(0, ndtype), control)

        ndtype = np.dtype("int, (2,3)float, float")
        control = np.array((default_fill_value(0),
                            default_fill_value(0.),
                            default_fill_value(0.),),
                           dtype="int, float, float").astype(ndtype)
        test = _check_fill_value(None, ndtype)
        assert_equal(test, control)
        control = np.array((0, 0, 0), dtype="int, float, float").astype(ndtype)
        assert_equal(_check_fill_value(0, ndtype), control)
        # but when indexing, fill value should become scalar not tuple
        # See issue #6723
        M = masked_array(control)
        assert_equal(M["f1"].fill_value.ndim, 0)

    def test_fillvalue_datetime_timedelta(self):
        # Test default fillvalue for datetime64 and timedelta64 types.
        # See issue #4476, this would return '?' which would cause errors
        # elsewhere

        for timecode in ("as", "fs", "ps", "ns", "us", "ms", "s", "m",
                         "h", "D", "W", "M", "Y"):
            control = numpy.datetime64("NaT", timecode)
            test = default_fill_value(numpy.dtype("<M8[" + timecode + "]"))
            np.testing.assert_equal(test, control)

            control = numpy.timedelta64("NaT", timecode)
            test = default_fill_value(numpy.dtype("<m8[" + timecode + "]"))
            np.testing.assert_equal(test, control)

    def test_extremum_fill_value(self):
        # Tests extremum fill values for flexible type.
        a = array([(1, (2, 3)), (4, (5, 6))],
                  dtype=[('A', int), ('B', [('BA', int), ('BB', int)])])
        test = a.fill_value
        assert_equal(test.dtype, a.dtype)
        assert_equal(test['A'], default_fill_value(a['A']))
        assert_equal(test['B']['BA'], default_fill_value(a['B']['BA']))
        assert_equal(test['B']['BB'], default_fill_value(a['B']['BB']))

        test = minimum_fill_value(a)
        assert_equal(test.dtype, a.dtype)
        assert_equal(test[0], minimum_fill_value(a['A']))
        assert_equal(test[1][0], minimum_fill_value(a['B']['BA']))
        assert_equal(test[1][1], minimum_fill_value(a['B']['BB']))
        assert_equal(test[1], minimum_fill_value(a['B']))

        test = maximum_fill_value(a)
        assert_equal(test.dtype, a.dtype)
        assert_equal(test[0], maximum_fill_value(a['A']))
        assert_equal(test[1][0], maximum_fill_value(a['B']['BA']))
        assert_equal(test[1][1], maximum_fill_value(a['B']['BB']))
        assert_equal(test[1], maximum_fill_value(a['B']))

    def test_extremum_fill_value_subdtype(self):
        a = array(([2, 3, 4],), dtype=[('value', np.int8, 3)])

        test = minimum_fill_value(a)
        assert_equal(test.dtype, a.dtype)
        assert_equal(test[0], np.full(3, minimum_fill_value(a['value'])))

        test = maximum_fill_value(a)
        assert_equal(test.dtype, a.dtype)
        assert_equal(test[0], np.full(3, maximum_fill_value(a['value'])))

    def test_fillvalue_individual_fields(self):
        # Test setting fill_value on individual fields
        ndtype = [('a', int), ('b', int)]
        # Explicit fill_value
        a = array(list(zip([1, 2, 3], [4, 5, 6])),
                  fill_value=(-999, -999), dtype=ndtype)
        aa = a['a']
        aa.set_fill_value(10)
        assert_equal(aa._fill_value, np.array(10))
        assert_equal(tuple(a.fill_value), (10, -999))
        a.fill_value['b'] = -10
        assert_equal(tuple(a.fill_value), (10, -10))
        # Implicit fill_value
        t = array(list(zip([1, 2, 3], [4, 5, 6])), dtype=ndtype)
        tt = t['a']
        tt.set_fill_value(10)
        assert_equal(tt._fill_value, np.array(10))
        assert_equal(tuple(t.fill_value), (10, default_fill_value(0)))

    def test_fillvalue_implicit_structured_array(self):
        # Check that fill_value is always defined for structured arrays
        ndtype = ('b', float)
        adtype = ('a', float)
        a = array([(1.,), (2.,)], mask=[(False,), (False,)],
                  fill_value=(np.nan,), dtype=np.dtype([adtype]))
        b = empty(a.shape, dtype=[adtype, ndtype])
        b['a'] = a['a']
        b['a'].set_fill_value(a['a'].fill_value)
        f = b._fill_value[()]
        assert_(np.isnan(f[0]))
        assert_equal(f[-1], default_fill_value(1.))

    def test_fillvalue_as_arguments(self):
        # Test adding a fill_value parameter to empty/ones/zeros
        a = empty(3, fill_value=999.)
        assert_equal(a.fill_value, 999.)

        a = ones(3, fill_value=999., dtype=float)
        assert_equal(a.fill_value, 999.)

        a = zeros(3, fill_value=0., dtype=complex)
        assert_equal(a.fill_value, 0.)

        a = identity(3, fill_value=0., dtype=complex)
        assert_equal(a.fill_value, 0.)

    def test_shape_argument(self):
        # Test that shape can be provides as an argument
        # GH issue 6106
        a = empty(shape=(3, ))
        assert_equal(a.shape, (3, ))

        a = ones(shape=(3, ), dtype=float)
        assert_equal(a.shape, (3, ))

        a = zeros(shape=(3, ), dtype=complex)
        assert_equal(a.shape, (3, ))

    def test_fillvalue_in_view(self):
        # Test the behavior of fill_value in view

        # Create initial masked array
        x = array([1, 2, 3], fill_value=1, dtype=np.int64)

        # Check that fill_value is preserved by default
        y = x.view()
        assert_(y.fill_value == 1)

        # Check that fill_value is preserved if dtype is specified and the
        # dtype is an ndarray sub-class and has a _fill_value attribute
        y = x.view(MaskedArray)
        assert_(y.fill_value == 1)

        # Check that fill_value is preserved if type is specified and the
        # dtype is an ndarray sub-class and has a _fill_value attribute (by
        # default, the first argument is dtype, not type)
        y = x.view(type=MaskedArray)
        assert_(y.fill_value == 1)

        # Check that code does not crash if passed an ndarray sub-class that
        # does not have a _fill_value attribute
        y = x.view(np.ndarray)
        y = x.view(type=np.ndarray)

        # Check that fill_value can be overridden with view
        y = x.view(MaskedArray, fill_value=2)
        assert_(y.fill_value == 2)

        # Check that fill_value can be overridden with view (using type=)
        y = x.view(type=MaskedArray, fill_value=2)
        assert_(y.fill_value == 2)

        # Check that fill_value gets reset if passed a dtype but not a
        # fill_value. This is because even though in some cases one can safely
        # cast the fill_value, e.g. if taking an int64 view of an int32 array,
        # in other cases, this cannot be done (e.g. int32 view of an int64
        # array with a large fill_value).
        y = x.view(dtype=np.int32)
        assert_(y.fill_value == 999999)

    def test_fillvalue_bytes_or_str(self):
        # Test whether fill values work as expected for structured dtypes
        # containing bytes or str.  See issue #7259.
        a = empty(shape=(3, ), dtype="(2)3S,(2)3U")
        assert_equal(a["f0"].fill_value, default_fill_value(b"spam"))
        assert_equal(a["f1"].fill_value, default_fill_value("eggs"))


class TestUfuncs:
    # Test class for the application of ufuncs on MaskedArrays.

    def setup_method(self):
        # Base data definition.
        self.d = (array([1.0, 0, -1, pi / 2] * 2, mask=[0, 1] + [0] * 6),
                  array([1.0, 0, -1, pi / 2] * 2, mask=[1, 0] + [0] * 6),)
        self.err_status = np.geterr()
        np.seterr(divide='ignore', invalid='ignore')

    def teardown_method(self):
        np.seterr(**self.err_status)

    def test_testUfuncRegression(self):
        # Tests new ufuncs on MaskedArrays.
        for f in ['sqrt', 'log', 'log10', 'exp', 'conjugate',
                  'sin', 'cos', 'tan',
                  'arcsin', 'arccos', 'arctan',
                  'sinh', 'cosh', 'tanh',
                  'arcsinh',
                  'arccosh',
                  'arctanh',
                  'absolute', 'fabs', 'negative',
                  'floor', 'ceil',
                  'logical_not',
                  'add', 'subtract', 'multiply',
                  'divide', 'true_divide', 'floor_divide',
                  'remainder', 'fmod', 'hypot', 'arctan2',
                  'equal', 'not_equal', 'less_equal', 'greater_equal',
                  'less', 'greater',
                  'logical_and', 'logical_or', 'logical_xor',
                  ]:
            try:
                uf = getattr(umath, f)
            except AttributeError:
                uf = getattr(fromnumeric, f)
            mf = getattr(numpy.ma.core, f)
            args = self.d[:uf.nin]
            ur = uf(*args)
            mr = mf(*args)
            assert_equal(ur.filled(0), mr.filled(0), f)
            assert_mask_equal(ur.mask, mr.mask, err_msg=f)

    def test_reduce(self):
        # Tests reduce on MaskedArrays.
        a = self.d[0]
        assert_(not alltrue(a, axis=0))
        assert_(sometrue(a, axis=0))
        assert_equal(sum(a[:3], axis=0), 0)
        assert_equal(product(a, axis=0), 0)
        assert_equal(add.reduce(a), pi)

    def test_minmax(self):
        # Tests extrema on MaskedArrays.
        a = arange(1, 13).reshape(3, 4)
        amask = masked_where(a < 5, a)
        assert_equal(amask.max(), a.max())
        assert_equal(amask.min(), 5)
        assert_equal(amask.max(0), a.max(0))
        assert_equal(amask.min(0), [5, 6, 7, 8])
        assert_(amask.max(1)[0].mask)
        assert_(amask.min(1)[0].mask)

    def test_ndarray_mask(self):
        # Check that the mask of the result is a ndarray (not a MaskedArray...)
        a = masked_array([-1, 0, 1, 2, 3], mask=[0, 0, 0, 0, 1])
        test = np.sqrt(a)
        control = masked_array([-1, 0, 1, np.sqrt(2), -1],
                               mask=[1, 0, 0, 0, 1])
        assert_equal(test, control)
        assert_equal(test.mask, control.mask)
        assert_(not isinstance(test.mask, MaskedArray))

    def test_treatment_of_NotImplemented(self):
        # Check that NotImplemented is returned at appropriate places

        a = masked_array([1., 2.], mask=[1, 0])
        assert_raises(TypeError, operator.mul, a, "abc")
        assert_raises(TypeError, operator.truediv, a, "abc")

        class MyClass:
            __array_priority__ = a.__array_priority__ + 1

            def __mul__(self, other):
                return "My mul"

            def __rmul__(self, other):
                return "My rmul"

        me = MyClass()
        assert_(me * a == "My mul")
        assert_(a * me == "My rmul")

        # and that __array_priority__ is respected
        class MyClass2:
            __array_priority__ = 100

            def __mul__(self, other):
                return "Me2mul"

            def __rmul__(self, other):
                return "Me2rmul"

            def __rdiv__(self, other):
                return "Me2rdiv"

            __rtruediv__ = __rdiv__

        me_too = MyClass2()
        assert_(a.__mul__(me_too) is NotImplemented)
        assert_(all(multiply.outer(a, me_too) == "Me2rmul"))
        assert_(a.__truediv__(me_too) is NotImplemented)
        assert_(me_too * a == "Me2mul")
        assert_(a * me_too == "Me2rmul")
        assert_(a / me_too == "Me2rdiv")

    def test_no_masked_nan_warnings(self):
        # check that a nan in masked position does not
        # cause ufunc warnings

        m = np.ma.array([0.5, np.nan], mask=[0,1])

        with warnings.catch_warnings():
            warnings.filterwarnings("error")

            # test unary and binary ufuncs
            exp(m)
            add(m, 1)
            m > 0

            # test different unary domains
            sqrt(m)
            log(m)
            tan(m)
            arcsin(m)
            arccos(m)
            arccosh(m)

            # test binary domains
            divide(m, 2)

            # also check that allclose uses ma ufuncs, to avoid warning
            allclose(m, 0.5)

class TestMaskedArrayInPlaceArithmetic:
    # Test MaskedArray Arithmetic

    def setup_method(self):
        x = arange(10)
        y = arange(10)
        xm = arange(10)
        xm[2] = masked
        self.intdata = (x, y, xm)
        self.floatdata = (x.astype(float), y.astype(float), xm.astype(float))
        self.othertypes = np.typecodes['AllInteger'] + np.typecodes['AllFloat']
        self.othertypes = [np.dtype(_).type for _ in self.othertypes]
        self.uint8data = (
            x.astype(np.uint8),
            y.astype(np.uint8),
            xm.astype(np.uint8)
        )

    def test_inplace_addition_scalar(self):
        # Test of inplace additions
        (x, y, xm) = self.intdata
        xm[2] = masked
        x += 1
        assert_equal(x, y + 1)
        xm += 1
        assert_equal(xm, y + 1)

        (x, _, xm) = self.floatdata
        id1 = x.data.ctypes.data
        x += 1.
        assert_(id1 == x.data.ctypes.data)
        assert_equal(x, y + 1.)

    def test_inplace_addition_array(self):
        # Test of inplace additions
        (x, y, xm) = self.intdata
        m = xm.mask
        a = arange(10, dtype=np.int16)
        a[-1] = masked
        x += a
        xm += a
        assert_equal(x, y + a)
        assert_equal(xm, y + a)
        assert_equal(xm.mask, mask_or(m, a.mask))

    def test_inplace_subtraction_scalar(self):
        # Test of inplace subtractions
        (x, y, xm) = self.intdata
        x -= 1
        assert_equal(x, y - 1)
        xm -= 1
        assert_equal(xm, y - 1)

    def test_inplace_subtraction_array(self):
        # Test of inplace subtractions
        (x, y, xm) = self.floatdata
        m = xm.mask
        a = arange(10, dtype=float)
        a[-1] = masked
        x -= a
        xm -= a
        assert_equal(x, y - a)
        assert_equal(xm, y - a)
        assert_equal(xm.mask, mask_or(m, a.mask))

    def test_inplace_multiplication_scalar(self):
        # Test of inplace multiplication
        (x, y, xm) = self.floatdata
        x *= 2.0
        assert_equal(x, y * 2)
        xm *= 2.0
        assert_equal(xm, y * 2)

    def test_inplace_multiplication_array(self):
        # Test of inplace multiplication
        (x, y, xm) = self.floatdata
        m = xm.mask
        a = arange(10, dtype=float)
        a[-1] = masked
        x *= a
        xm *= a
        assert_equal(x, y * a)
        assert_equal(xm, y * a)
        assert_equal(xm.mask, mask_or(m, a.mask))

    def test_inplace_division_scalar_int(self):
        # Test of inplace division
        (x, y, xm) = self.intdata
        x = arange(10) * 2
        xm = arange(10) * 2
        xm[2] = masked
        x //= 2
        assert_equal(x, y)
        xm //= 2
        assert_equal(xm, y)

    def test_inplace_division_scalar_float(self):
        # Test of inplace division
        (x, y, xm) = self.floatdata
        x /= 2.0
        assert_equal(x, y / 2.0)
        xm /= arange(10)
        assert_equal(xm, ones((10,)))

    def test_inplace_division_array_float(self):
        # Test of inplace division
        (x, y, xm) = self.floatdata
        m = xm.mask
        a = arange(10, dtype=float)
        a[-1] = masked
        x /= a
        xm /= a
        assert_equal(x, y / a)
        assert_equal(xm, y / a)
        assert_equal(xm.mask, mask_or(mask_or(m, a.mask), (a == 0)))

    def test_inplace_division_misc(self):

        x = [1., 1., 1., -2., pi / 2., 4., 5., -10., 10., 1., 2., 3.]
        y = [5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.]
        m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]
        m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1]
        xm = masked_array(x, mask=m1)
        ym = masked_array(y, mask=m2)

        z = xm / ym
        assert_equal(z._mask, [1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1])
        assert_equal(z._data,
                     [1., 1., 1., -1., -pi / 2., 4., 5., 1., 1., 1., 2., 3.])

        xm = xm.copy()
        xm /= ym
        assert_equal(xm._mask, [1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1])
        assert_equal(z._data,
                     [1., 1., 1., -1., -pi / 2., 4., 5., 1., 1., 1., 2., 3.])

    def test_datafriendly_add(self):
        # Test keeping data w/ (inplace) addition
        x = array([1, 2, 3], mask=[0, 0, 1])
        # Test add w/ scalar
        xx = x + 1
        assert_equal(xx.data, [2, 3, 3])
        assert_equal(xx.mask, [0, 0, 1])
        # Test iadd w/ scalar
        x += 1
        assert_equal(x.data, [2, 3, 3])
        assert_equal(x.mask, [0, 0, 1])
        # Test add w/ array
        x = array([1, 2, 3], mask=[0, 0, 1])
        xx = x + array([1, 2, 3], mask=[1, 0, 0])
        assert_equal(xx.data, [1, 4, 3])
        assert_equal(xx.mask, [1, 0, 1])
        # Test iadd w/ array
        x = array([1, 2, 3], mask=[0, 0, 1])
        x += array([1, 2, 3], mask=[1, 0, 0])
        assert_equal(x.data, [1, 4, 3])
        assert_equal(x.mask, [1, 0, 1])

    def test_datafriendly_sub(self):
        # Test keeping data w/ (inplace) subtraction
        # Test sub w/ scalar
        x = array([1, 2, 3], mask=[0, 0, 1])
        xx = x - 1
        assert_equal(xx.data, [0, 1, 3])
        assert_equal(xx.mask, [0, 0, 1])
        # Test isub w/ scalar
        x = array([1, 2, 3], mask=[0, 0, 1])
        x -= 1
        assert_equal(x.data, [0, 1, 3])
        assert_equal(x.mask, [0, 0, 1])
        # Test sub w/ array
        x = array([1, 2, 3], mask=[0, 0, 1])
        xx = x - array([1, 2, 3], mask=[1, 0, 0])
        assert_equal(xx.data, [1, 0, 3])
        assert_equal(xx.mask, [1, 0, 1])
        # Test isub w/ array
        x = array([1, 2, 3], mask=[0, 0, 1])
        x -= array([1, 2, 3], mask=[1, 0, 0])
        assert_equal(x.data, [1, 0, 3])
        assert_equal(x.mask, [1, 0, 1])

    def test_datafriendly_mul(self):
        # Test keeping data w/ (inplace) multiplication
        # Test mul w/ scalar
        x = array([1, 2, 3], mask=[0, 0, 1])
        xx = x * 2
        assert_equal(xx.data, [2, 4, 3])
        assert_equal(xx.mask, [0, 0, 1])
        # Test imul w/ scalar
        x = array([1, 2, 3], mask=[0, 0, 1])
        x *= 2
        assert_equal(x.data, [2, 4, 3])
        assert_equal(x.mask, [0, 0, 1])
        # Test mul w/ array
        x = array([1, 2, 3], mask=[0, 0, 1])
        xx = x * array([10, 20, 30], mask=[1, 0, 0])
        assert_equal(xx.data, [1, 40, 3])
        assert_equal(xx.mask, [1, 0, 1])
        # Test imul w/ array
        x = array([1, 2, 3], mask=[0, 0, 1])
        x *= array([10, 20, 30], mask=[1, 0, 0])
        assert_equal(x.data, [1, 40, 3])
        assert_equal(x.mask, [1, 0, 1])

    def test_datafriendly_div(self):
        # Test keeping data w/ (inplace) division
        # Test div on scalar
        x = array([1, 2, 3], mask=[0, 0, 1])
        xx = x / 2.
        assert_equal(xx.data, [1 / 2., 2 / 2., 3])
        assert_equal(xx.mask, [0, 0, 1])
        # Test idiv on scalar
        x = array([1., 2., 3.], mask=[0, 0, 1])
        x /= 2.
        assert_equal(x.data, [1 / 2., 2 / 2., 3])
        assert_equal(x.mask, [0, 0, 1])
        # Test div on array
        x = array([1., 2., 3.], mask=[0, 0, 1])
        xx = x / array([10., 20., 30.], mask=[1, 0, 0])
        assert_equal(xx.data, [1., 2. / 20., 3.])
        assert_equal(xx.mask, [1, 0, 1])
        # Test idiv on array
        x = array([1., 2., 3.], mask=[0, 0, 1])
        x /= array([10., 20., 30.], mask=[1, 0, 0])
        assert_equal(x.data, [1., 2 / 20., 3.])
        assert_equal(x.mask, [1, 0, 1])

    def test_datafriendly_pow(self):
        # Test keeping data w/ (inplace) power
        # Test pow on scalar
        x = array([1., 2., 3.], mask=[0, 0, 1])
        xx = x ** 2.5
        assert_equal(xx.data, [1., 2. ** 2.5, 3.])
        assert_equal(xx.mask, [0, 0, 1])
        # Test ipow on scalar
        x **= 2.5
        assert_equal(x.data, [1., 2. ** 2.5, 3])
        assert_equal(x.mask, [0, 0, 1])

    def test_datafriendly_add_arrays(self):
        a = array([[1, 1], [3, 3]])
        b = array([1, 1], mask=[0, 0])
        a += b
        assert_equal(a, [[2, 2], [4, 4]])
        if a.mask is not nomask:
            assert_equal(a.mask, [[0, 0], [0, 0]])

        a = array([[1, 1], [3, 3]])
        b = array([1, 1], mask=[0, 1])
        a += b
        assert_equal(a, [[2, 2], [4, 4]])
        assert_equal(a.mask, [[0, 1], [0, 1]])

    def test_datafriendly_sub_arrays(self):
        a = array([[1, 1], [3, 3]])
        b = array([1, 1], mask=[0, 0])
        a -= b
        assert_equal(a, [[0, 0], [2, 2]])
        if a.mask is not nomask:
            assert_equal(a.mask, [[0, 0], [0, 0]])

        a = array([[1, 1], [3, 3]])
        b = array([1, 1], mask=[0, 1])
        a -= b
        assert_equal(a, [[0, 0], [2, 2]])
        assert_equal(a.mask, [[0, 1], [0, 1]])

    def test_datafriendly_mul_arrays(self):
        a = array([[1, 1], [3, 3]])
        b = array([1, 1], mask=[0, 0])
        a *= b
        assert_equal(a, [[1, 1], [3, 3]])
        if a.mask is not nomask:
            assert_equal(a.mask, [[0, 0], [0, 0]])

        a = array([[1, 1], [3, 3]])
        b = array([1, 1], mask=[0, 1])
        a *= b
        assert_equal(a, [[1, 1], [3, 3]])
        assert_equal(a.mask, [[0, 1], [0, 1]])

    def test_inplace_addition_scalar_type(self):
        # Test of inplace additions
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                xm[2] = masked
                x += t(1)
                assert_equal(x, y + t(1))
                xm += t(1)
                assert_equal(xm, y + t(1))

    def test_inplace_addition_array_type(self):
        # Test of inplace additions
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                m = xm.mask
                a = arange(10, dtype=t)
                a[-1] = masked
                x += a
                xm += a
                assert_equal(x, y + a)
                assert_equal(xm, y + a)
                assert_equal(xm.mask, mask_or(m, a.mask))

    def test_inplace_subtraction_scalar_type(self):
        # Test of inplace subtractions
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                x -= t(1)
                assert_equal(x, y - t(1))
                xm -= t(1)
                assert_equal(xm, y - t(1))

    def test_inplace_subtraction_array_type(self):
        # Test of inplace subtractions
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                m = xm.mask
                a = arange(10, dtype=t)
                a[-1] = masked
                x -= a
                xm -= a
                assert_equal(x, y - a)
                assert_equal(xm, y - a)
                assert_equal(xm.mask, mask_or(m, a.mask))

    def test_inplace_multiplication_scalar_type(self):
        # Test of inplace multiplication
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                x *= t(2)
                assert_equal(x, y * t(2))
                xm *= t(2)
                assert_equal(xm, y * t(2))

    def test_inplace_multiplication_array_type(self):
        # Test of inplace multiplication
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                m = xm.mask
                a = arange(10, dtype=t)
                a[-1] = masked
                x *= a
                xm *= a
                assert_equal(x, y * a)
                assert_equal(xm, y * a)
                assert_equal(xm.mask, mask_or(m, a.mask))

    def test_inplace_floor_division_scalar_type(self):
        # Test of inplace division
        # Check for TypeError in case of unsupported types
        unsupported = {np.dtype(t).type for t in np.typecodes["Complex"]}
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                x = arange(10, dtype=t) * t(2)
                xm = arange(10, dtype=t) * t(2)
                xm[2] = masked
                try:
                    x //= t(2)
                    xm //= t(2)
                    assert_equal(x, y)
                    assert_equal(xm, y)
                except TypeError:
                    msg = f"Supported type {t} throwing TypeError"
                    assert t in unsupported, msg

    def test_inplace_floor_division_array_type(self):
        # Test of inplace division
        # Check for TypeError in case of unsupported types
        unsupported = {np.dtype(t).type for t in np.typecodes["Complex"]}
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                m = xm.mask
                a = arange(10, dtype=t)
                a[-1] = masked
                try:
                    x //= a
                    xm //= a
                    assert_equal(x, y // a)
                    assert_equal(xm, y // a)
                    assert_equal(
                        xm.mask,
                        mask_or(mask_or(m, a.mask), (a == t(0)))
                    )
                except TypeError:
                    msg = f"Supported type {t} throwing TypeError"
                    assert t in unsupported, msg

    def test_inplace_division_scalar_type(self):
        # Test of inplace division
        for t in self.othertypes:
            with suppress_warnings() as sup:
                sup.record(UserWarning)

                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                x = arange(10, dtype=t) * t(2)
                xm = arange(10, dtype=t) * t(2)
                xm[2] = masked

                # May get a DeprecationWarning or a TypeError.
                #
                # This is a consequence of the fact that this is true divide
                # and will require casting to float for calculation and
                # casting back to the original type. This will only be raised
                # with integers. Whether it is an error or warning is only
                # dependent on how stringent the casting rules are.
                #
                # Will handle the same way.
                try:
                    x /= t(2)
                    assert_equal(x, y)
                except (DeprecationWarning, TypeError) as e:
                    warnings.warn(str(e), stacklevel=1)
                try:
                    xm /= t(2)
                    assert_equal(xm, y)
                except (DeprecationWarning, TypeError) as e:
                    warnings.warn(str(e), stacklevel=1)

                if issubclass(t, np.integer):
                    assert_equal(len(sup.log), 2, f'Failed on type={t}.')
                else:
                    assert_equal(len(sup.log), 0, f'Failed on type={t}.')

    def test_inplace_division_array_type(self):
        # Test of inplace division
        for t in self.othertypes:
            with suppress_warnings() as sup:
                sup.record(UserWarning)
                (x, y, xm) = (_.astype(t) for _ in self.uint8data)
                m = xm.mask
                a = arange(10, dtype=t)
                a[-1] = masked

                # May get a DeprecationWarning or a TypeError.
                #
                # This is a consequence of the fact that this is true divide
                # and will require casting to float for calculation and
                # casting back to the original type. This will only be raised
                # with integers. Whether it is an error or warning is only
                # dependent on how stringent the casting rules are.
                #
                # Will handle the same way.
                try:
                    x /= a
                    assert_equal(x, y / a)
                except (DeprecationWarning, TypeError) as e:
                    warnings.warn(str(e), stacklevel=1)
                try:
                    xm /= a
                    assert_equal(xm, y / a)
                    assert_equal(
                        xm.mask,
                        mask_or(mask_or(m, a.mask), (a == t(0)))
                    )
                except (DeprecationWarning, TypeError) as e:
                    warnings.warn(str(e), stacklevel=1)

                if issubclass(t, np.integer):
                    assert_equal(len(sup.log), 2, f'Failed on type={t}.')
                else:
                    assert_equal(len(sup.log), 0, f'Failed on type={t}.')

    def test_inplace_pow_type(self):
        # Test keeping data w/ (inplace) power
        for t in self.othertypes:
            with warnings.catch_warnings():
                warnings.filterwarnings("error")
                # Test pow on scalar
                x = array([1, 2, 3], mask=[0, 0, 1], dtype=t)
                xx = x ** t(2)
                xx_r = array([1, 2 ** 2, 3], mask=[0, 0, 1], dtype=t)
                assert_equal(xx.data, xx_r.data)
                assert_equal(xx.mask, xx_r.mask)
                # Test ipow on scalar
                x **= t(2)
                assert_equal(x.data, xx_r.data)
                assert_equal(x.mask, xx_r.mask)


class TestMaskedArrayMethods:
    # Test class for miscellaneous MaskedArrays methods.
    def setup_method(self):
        # Base data definition.
        x = np.array([8.375, 7.545, 8.828, 8.5, 1.757, 5.928,
                      8.43, 7.78, 9.865, 5.878, 8.979, 4.732,
                      3.012, 6.022, 5.095, 3.116, 5.238, 3.957,
                      6.04, 9.63, 7.712, 3.382, 4.489, 6.479,
                      7.189, 9.645, 5.395, 4.961, 9.894, 2.893,
                      7.357, 9.828, 6.272, 3.758, 6.693, 0.993])
        X = x.reshape(6, 6)
        XX = x.reshape(3, 2, 2, 3)

        m = np.array([0, 1, 0, 1, 0, 0,
                     1, 0, 1, 1, 0, 1,
                     0, 0, 0, 1, 0, 1,
                     0, 0, 0, 1, 1, 1,
                     1, 0, 0, 1, 0, 0,
                     0, 0, 1, 0, 1, 0])
        mx = array(data=x, mask=m)
        mX = array(data=X, mask=m.reshape(X.shape))
        mXX = array(data=XX, mask=m.reshape(XX.shape))

        m2 = np.array([1, 1, 0, 1, 0, 0,
                      1, 1, 1, 1, 0, 1,
                      0, 0, 1, 1, 0, 1,
                      0, 0, 0, 1, 1, 1,
                      1, 0, 0, 1, 1, 0,
                      0, 0, 1, 0, 1, 1])
        m2x = array(data=x, mask=m2)
        m2X = array(data=X, mask=m2.reshape(X.shape))
        m2XX = array(data=XX, mask=m2.reshape(XX.shape))
        self.d = (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX)

    def test_generic_methods(self):
        # Tests some MaskedArray methods.
        a = array([1, 3, 2])
        assert_equal(a.any(), a._data.any())
        assert_equal(a.all(), a._data.all())
        assert_equal(a.argmax(), a._data.argmax())
        assert_equal(a.argmin(), a._data.argmin())
        assert_equal(a.choose(0, 1, 2, 3, 4), a._data.choose(0, 1, 2, 3, 4))
        assert_equal(a.compress([1, 0, 1]), a._data.compress([1, 0, 1]))
        assert_equal(a.conj(), a._data.conj())
        assert_equal(a.conjugate(), a._data.conjugate())

        m = array([[1, 2], [3, 4]])
        assert_equal(m.diagonal(), m._data.diagonal())
        assert_equal(a.sum(), a._data.sum())
        assert_equal(a.take([1, 2]), a._data.take([1, 2]))
        assert_equal(m.transpose(), m._data.transpose())

    def test_allclose(self):
        # Tests allclose on arrays
        a = np.random.rand(10)
        b = a + np.random.rand(10) * 1e-8
        assert_(allclose(a, b))
        # Test allclose w/ infs
        a[0] = np.inf
        assert_(not allclose(a, b))
        b[0] = np.inf
        assert_(allclose(a, b))
        # Test allclose w/ masked
        a = masked_array(a)
        a[-1] = masked
        assert_(allclose(a, b, masked_equal=True))
        assert_(not allclose(a, b, masked_equal=False))
        # Test comparison w/ scalar
        a *= 1e-8
        a[0] = 0
        assert_(allclose(a, 0, masked_equal=True))

        # Test that the function works for MIN_INT integer typed arrays
        a = masked_array([np.iinfo(np.int_).min], dtype=np.int_)
        assert_(allclose(a, a))

    def test_allclose_timedelta(self):
        # Allclose currently works for timedelta64 as long as `atol` is
        # an integer or also a timedelta64
        a = np.array([[1, 2, 3, 4]], dtype="m8[ns]")
        assert allclose(a, a, atol=0)
        assert allclose(a, a, atol=np.timedelta64(1, "ns"))

    def test_allany(self):
        # Checks the any/all methods/functions.
        x = np.array([[0.13, 0.26, 0.90],
                      [0.28, 0.33, 0.63],
                      [0.31, 0.87, 0.70]])
        m = np.array([[True, False, False],
                      [False, False, False],
                      [True, True, False]], dtype=np.bool_)
        mx = masked_array(x, mask=m)
        mxbig = (mx > 0.5)
        mxsmall = (mx < 0.5)

        assert_(not mxbig.all())
        assert_(mxbig.any())
        assert_equal(mxbig.all(0), [False, False, True])
        assert_equal(mxbig.all(1), [False, False, True])
        assert_equal(mxbig.any(0), [False, False, True])
        assert_equal(mxbig.any(1), [True, True, True])

        assert_(not mxsmall.all())
        assert_(mxsmall.any())
        assert_equal(mxsmall.all(0), [True, True, False])
        assert_equal(mxsmall.all(1), [False, False, False])
        assert_equal(mxsmall.any(0), [True, True, False])
        assert_equal(mxsmall.any(1), [True, True, False])

    def test_allany_oddities(self):
        # Some fun with all and any
        store = empty((), dtype=bool)
        full = array([1, 2, 3], mask=True)

        assert_(full.all() is masked)
        full.all(out=store)
        assert_(store)
        assert_(store._mask, True)
        assert_(store is not masked)

        store = empty((), dtype=bool)
        assert_(full.any() is masked)
        full.any(out=store)
        assert_(not store)
        assert_(store._mask, True)
        assert_(store is not masked)

    def test_argmax_argmin(self):
        # Tests argmin & argmax on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d

        assert_equal(mx.argmin(), 35)
        assert_equal(mX.argmin(), 35)
        assert_equal(m2x.argmin(), 4)
        assert_equal(m2X.argmin(), 4)
        assert_equal(mx.argmax(), 28)
        assert_equal(mX.argmax(), 28)
        assert_equal(m2x.argmax(), 31)
        assert_equal(m2X.argmax(), 31)

        assert_equal(mX.argmin(0), [2, 2, 2, 5, 0, 5])
        assert_equal(m2X.argmin(0), [2, 2, 4, 5, 0, 4])
        assert_equal(mX.argmax(0), [0, 5, 0, 5, 4, 0])
        assert_equal(m2X.argmax(0), [5, 5, 0, 5, 1, 0])

        assert_equal(mX.argmin(1), [4, 1, 0, 0, 5, 5, ])
        assert_equal(m2X.argmin(1), [4, 4, 0, 0, 5, 3])
        assert_equal(mX.argmax(1), [2, 4, 1, 1, 4, 1])
        assert_equal(m2X.argmax(1), [2, 4, 1, 1, 1, 1])

    def test_clip(self):
        # Tests clip on MaskedArrays.
        x = np.array([8.375, 7.545, 8.828, 8.5, 1.757, 5.928,
                      8.43, 7.78, 9.865, 5.878, 8.979, 4.732,
                      3.012, 6.022, 5.095, 3.116, 5.238, 3.957,
                      6.04, 9.63, 7.712, 3.382, 4.489, 6.479,
                      7.189, 9.645, 5.395, 4.961, 9.894, 2.893,
                      7.357, 9.828, 6.272, 3.758, 6.693, 0.993])
        m = np.array([0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1,
                      0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1,
                      1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0])
        mx = array(x, mask=m)
        clipped = mx.clip(2, 8)
        assert_equal(clipped.mask, mx.mask)
        assert_equal(clipped._data, x.clip(2, 8))
        assert_equal(clipped._data, mx._data.clip(2, 8))

    def test_clip_out(self):
        # gh-14140
        a = np.arange(10)
        m = np.ma.MaskedArray(a, mask=[0, 1] * 5)
        m.clip(0, 5, out=m)
        assert_equal(m.mask, [0, 1] * 5)

    def test_compress(self):
        # test compress
        a = masked_array([1., 2., 3., 4., 5.], fill_value=9999)
        condition = (a > 1.5) & (a < 3.5)
        assert_equal(a.compress(condition), [2., 3.])

        a[[2, 3]] = masked
        b = a.compress(condition)
        assert_equal(b._data, [2., 3.])
        assert_equal(b._mask, [0, 1])
        assert_equal(b.fill_value, 9999)
        assert_equal(b, a[condition])

        condition = (a < 4.)
        b = a.compress(condition)
        assert_equal(b._data, [1., 2., 3.])
        assert_equal(b._mask, [0, 0, 1])
        assert_equal(b.fill_value, 9999)
        assert_equal(b, a[condition])

        a = masked_array([[10, 20, 30], [40, 50, 60]],
                         mask=[[0, 0, 1], [1, 0, 0]])
        b = a.compress(a.ravel() >= 22)
        assert_equal(b._data, [30, 40, 50, 60])
        assert_equal(b._mask, [1, 1, 0, 0])

        x = np.array([3, 1, 2])
        b = a.compress(x >= 2, axis=1)
        assert_equal(b._data, [[10, 30], [40, 60]])
        assert_equal(b._mask, [[0, 1], [1, 0]])

    def test_compressed(self):
        # Tests compressed
        a = array([1, 2, 3, 4], mask=[0, 0, 0, 0])
        b = a.compressed()
        assert_equal(b, a)
        a[0] = masked
        b = a.compressed()
        assert_equal(b, [2, 3, 4])

    def test_empty(self):
        # Tests empty/like
        datatype = [('a', int), ('b', float), ('c', '|S8')]
        a = masked_array([(1, 1.1, '1.1'), (2, 2.2, '2.2'), (3, 3.3, '3.3')],
                         dtype=datatype)
        assert_equal(len(a.fill_value.item()), len(datatype))

        b = empty_like(a)
        assert_equal(b.shape, a.shape)
        assert_equal(b.fill_value, a.fill_value)

        b = empty(len(a), dtype=datatype)
        assert_equal(b.shape, a.shape)
        assert_equal(b.fill_value, a.fill_value)

        # check empty_like mask handling
        a = masked_array([1, 2, 3], mask=[False, True, False])
        b = empty_like(a)
        assert_(not np.may_share_memory(a.mask, b.mask))
        b = a.view(masked_array)
        assert_(np.may_share_memory(a.mask, b.mask))

    def test_zeros(self):
        # Tests zeros/like
        datatype = [('a', int), ('b', float), ('c', '|S8')]
        a = masked_array([(1, 1.1, '1.1'), (2, 2.2, '2.2'), (3, 3.3, '3.3')],
                         dtype=datatype)
        assert_equal(len(a.fill_value.item()), len(datatype))

        b = zeros(len(a), dtype=datatype)
        assert_equal(b.shape, a.shape)
        assert_equal(b.fill_value, a.fill_value)

        b = zeros_like(a)
        assert_equal(b.shape, a.shape)
        assert_equal(b.fill_value, a.fill_value)

        # check zeros_like mask handling
        a = masked_array([1, 2, 3], mask=[False, True, False])
        b = zeros_like(a)
        assert_(not np.may_share_memory(a.mask, b.mask))
        b = a.view()
        assert_(np.may_share_memory(a.mask, b.mask))

    def test_ones(self):
        # Tests ones/like
        datatype = [('a', int), ('b', float), ('c', '|S8')]
        a = masked_array([(1, 1.1, '1.1'), (2, 2.2, '2.2'), (3, 3.3, '3.3')],
                         dtype=datatype)
        assert_equal(len(a.fill_value.item()), len(datatype))

        b = ones(len(a), dtype=datatype)
        assert_equal(b.shape, a.shape)
        assert_equal(b.fill_value, a.fill_value)

        b = ones_like(a)
        assert_equal(b.shape, a.shape)
        assert_equal(b.fill_value, a.fill_value)

        # check ones_like mask handling
        a = masked_array([1, 2, 3], mask=[False, True, False])
        b = ones_like(a)
        assert_(not np.may_share_memory(a.mask, b.mask))
        b = a.view()
        assert_(np.may_share_memory(a.mask, b.mask))

    @suppress_copy_mask_on_assignment
    def test_put(self):
        # Tests put.
        d = arange(5)
        n = [0, 0, 0, 1, 1]
        m = make_mask(n)
        x = array(d, mask=m)
        assert_(x[3] is masked)
        assert_(x[4] is masked)
        x[[1, 4]] = [10, 40]
        assert_(x[3] is masked)
        assert_(x[4] is not masked)
        assert_equal(x, [0, 10, 2, -1, 40])

        x = masked_array(arange(10), mask=[1, 0, 0, 0, 0] * 2)
        i = [0, 2, 4, 6]
        x.put(i, [6, 4, 2, 0])
        assert_equal(x, asarray([6, 1, 4, 3, 2, 5, 0, 7, 8, 9, ]))
        assert_equal(x.mask, [0, 0, 0, 0, 0, 1, 0, 0, 0, 0])
        x.put(i, masked_array([0, 2, 4, 6], [1, 0, 1, 0]))
        assert_array_equal(x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ])
        assert_equal(x.mask, [1, 0, 0, 0, 1, 1, 0, 0, 0, 0])

        x = masked_array(arange(10), mask=[1, 0, 0, 0, 0] * 2)
        put(x, i, [6, 4, 2, 0])
        assert_equal(x, asarray([6, 1, 4, 3, 2, 5, 0, 7, 8, 9, ]))
        assert_equal(x.mask, [0, 0, 0, 0, 0, 1, 0, 0, 0, 0])
        put(x, i, masked_array([0, 2, 4, 6], [1, 0, 1, 0]))
        assert_array_equal(x, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ])
        assert_equal(x.mask, [1, 0, 0, 0, 1, 1, 0, 0, 0, 0])

    def test_put_nomask(self):
        # GitHub issue 6425
        x = zeros(10)
        z = array([3., -1.], mask=[False, True])

        x.put([1, 2], z)
        assert_(x[0] is not masked)
        assert_equal(x[0], 0)
        assert_(x[1] is not masked)
        assert_equal(x[1], 3)
        assert_(x[2] is masked)
        assert_(x[3] is not masked)
        assert_equal(x[3], 0)

    def test_put_hardmask(self):
        # Tests put on hardmask
        d = arange(5)
        n = [0, 0, 0, 1, 1]
        m = make_mask(n)
        xh = array(d + 1, mask=m, hard_mask=True, copy=True)
        xh.put([4, 2, 0, 1, 3], [1, 2, 3, 4, 5])
        assert_equal(xh._data, [3, 4, 2, 4, 5])

    def test_putmask(self):
        x = arange(6) + 1
        mx = array(x, mask=[0, 0, 0, 1, 1, 1])
        mask = [0, 0, 1, 0, 0, 1]
        # w/o mask, w/o masked values
        xx = x.copy()
        putmask(xx, mask, 99)
        assert_equal(xx, [1, 2, 99, 4, 5, 99])
        # w/ mask, w/o masked values
        mxx = mx.copy()
        putmask(mxx, mask, 99)
        assert_equal(mxx._data, [1, 2, 99, 4, 5, 99])
        assert_equal(mxx._mask, [0, 0, 0, 1, 1, 0])
        # w/o mask, w/ masked values
        values = array([10, 20, 30, 40, 50, 60], mask=[1, 1, 1, 0, 0, 0])
        xx = x.copy()
        putmask(xx, mask, values)
        assert_equal(xx._data, [1, 2, 30, 4, 5, 60])
        assert_equal(xx._mask, [0, 0, 1, 0, 0, 0])
        # w/ mask, w/ masked values
        mxx = mx.copy()
        putmask(mxx, mask, values)
        assert_equal(mxx._data, [1, 2, 30, 4, 5, 60])
        assert_equal(mxx._mask, [0, 0, 1, 1, 1, 0])
        # w/ mask, w/ masked values + hardmask
        mxx = mx.copy()
        mxx.harden_mask()
        putmask(mxx, mask, values)
        assert_equal(mxx, [1, 2, 30, 4, 5, 60])

    def test_ravel(self):
        # Tests ravel
        a = array([[1, 2, 3, 4, 5]], mask=[[0, 1, 0, 0, 0]])
        aravel = a.ravel()
        assert_equal(aravel._mask.shape, aravel.shape)
        a = array([0, 0], mask=[1, 1])
        aravel = a.ravel()
        assert_equal(aravel._mask.shape, a.shape)
        # Checks that small_mask is preserved
        a = array([1, 2, 3, 4], mask=[0, 0, 0, 0], shrink=False)
        assert_equal(a.ravel()._mask, [0, 0, 0, 0])
        # Test that the fill_value is preserved
        a.fill_value = -99
        a.shape = (2, 2)
        ar = a.ravel()
        assert_equal(ar._mask, [0, 0, 0, 0])
        assert_equal(ar._data, [1, 2, 3, 4])
        assert_equal(ar.fill_value, -99)
        # Test index ordering
        assert_equal(a.ravel(order='C'), [1, 2, 3, 4])
        assert_equal(a.ravel(order='F'), [1, 3, 2, 4])

    def test_reshape(self):
        # Tests reshape
        x = arange(4)
        x[0] = masked
        y = x.reshape(2, 2)
        assert_equal(y.shape, (2, 2,))
        assert_equal(y._mask.shape, (2, 2,))
        assert_equal(x.shape, (4,))
        assert_equal(x._mask.shape, (4,))

    def test_sort(self):
        # Test sort
        x = array([1, 4, 2, 3], mask=[0, 1, 0, 0], dtype=np.uint8)

        sortedx = sort(x)
        assert_equal(sortedx._data, [1, 2, 3, 4])
        assert_equal(sortedx._mask, [0, 0, 0, 1])

        sortedx = sort(x, endwith=False)
        assert_equal(sortedx._data, [4, 1, 2, 3])
        assert_equal(sortedx._mask, [1, 0, 0, 0])

        x.sort()
        assert_equal(x._data, [1, 2, 3, 4])
        assert_equal(x._mask, [0, 0, 0, 1])

        x = array([1, 4, 2, 3], mask=[0, 1, 0, 0], dtype=np.uint8)
        x.sort(endwith=False)
        assert_equal(x._data, [4, 1, 2, 3])
        assert_equal(x._mask, [1, 0, 0, 0])

        x = [1, 4, 2, 3]
        sortedx = sort(x)
        assert_(not isinstance(sorted, MaskedArray))

        x = array([0, 1, -1, -2, 2], mask=nomask, dtype=np.int8)
        sortedx = sort(x, endwith=False)
        assert_equal(sortedx._data, [-2, -1, 0, 1, 2])
        x = array([0, 1, -1, -2, 2], mask=[0, 1, 0, 0, 1], dtype=np.int8)
        sortedx = sort(x, endwith=False)
        assert_equal(sortedx._data, [1, 2, -2, -1, 0])
        assert_equal(sortedx._mask, [1, 1, 0, 0, 0])

        x = array([0, -1], dtype=np.int8)
        sortedx = sort(x, kind="stable")
        assert_equal(sortedx, array([-1, 0], dtype=np.int8))

    def test_stable_sort(self):
        x = array([1, 2, 3, 1, 2, 3], dtype=np.uint8)
        expected = array([0, 3, 1, 4, 2, 5])
        computed = argsort(x, kind='stable')
        assert_equal(computed, expected)

    def test_argsort_matches_sort(self):
        x = array([1, 4, 2, 3], mask=[0, 1, 0, 0], dtype=np.uint8)

        for kwargs in [dict(),
                       dict(endwith=True),
                       dict(endwith=False),
                       dict(fill_value=2),
                       dict(fill_value=2, endwith=True),
                       dict(fill_value=2, endwith=False)]:
            sortedx = sort(x, **kwargs)
            argsortedx = x[argsort(x, **kwargs)]
            assert_equal(sortedx._data, argsortedx._data)
            assert_equal(sortedx._mask, argsortedx._mask)

    def test_sort_2d(self):
        # Check sort of 2D array.
        # 2D array w/o mask
        a = masked_array([[8, 4, 1], [2, 0, 9]])
        a.sort(0)
        assert_equal(a, [[2, 0, 1], [8, 4, 9]])
        a = masked_array([[8, 4, 1], [2, 0, 9]])
        a.sort(1)
        assert_equal(a, [[1, 4, 8], [0, 2, 9]])
        # 2D array w/mask
        a = masked_array([[8, 4, 1], [2, 0, 9]], mask=[[1, 0, 0], [0, 0, 1]])
        a.sort(0)
        assert_equal(a, [[2, 0, 1], [8, 4, 9]])
        assert_equal(a._mask, [[0, 0, 0], [1, 0, 1]])
        a = masked_array([[8, 4, 1], [2, 0, 9]], mask=[[1, 0, 0], [0, 0, 1]])
        a.sort(1)
        assert_equal(a, [[1, 4, 8], [0, 2, 9]])
        assert_equal(a._mask, [[0, 0, 1], [0, 0, 1]])
        # 3D
        a = masked_array([[[7, 8, 9], [4, 5, 6], [1, 2, 3]],
                          [[1, 2, 3], [7, 8, 9], [4, 5, 6]],
                          [[7, 8, 9], [1, 2, 3], [4, 5, 6]],
                          [[4, 5, 6], [1, 2, 3], [7, 8, 9]]])
        a[a % 4 == 0] = masked
        am = a.copy()
        an = a.filled(99)
        am.sort(0)
        an.sort(0)
        assert_equal(am, an)
        am = a.copy()
        an = a.filled(99)
        am.sort(1)
        an.sort(1)
        assert_equal(am, an)
        am = a.copy()
        an = a.filled(99)
        am.sort(2)
        an.sort(2)
        assert_equal(am, an)

    def test_sort_flexible(self):
        # Test sort on structured dtype.
        a = array(
            data=[(3, 3), (3, 2), (2, 2), (2, 1), (1, 0), (1, 1), (1, 2)],
            mask=[(0, 0), (0, 1), (0, 0), (0, 0), (1, 0), (0, 0), (0, 0)],
            dtype=[('A', int), ('B', int)])
        mask_last = array(
            data=[(1, 1), (1, 2), (2, 1), (2, 2), (3, 3), (3, 2), (1, 0)],
            mask=[(0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 1), (1, 0)],
            dtype=[('A', int), ('B', int)])
        mask_first = array(
            data=[(1, 0), (1, 1), (1, 2), (2, 1), (2, 2), (3, 2), (3, 3)],
            mask=[(1, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 1), (0, 0)],
            dtype=[('A', int), ('B', int)])

        test = sort(a)
        assert_equal(test, mask_last)
        assert_equal(test.mask, mask_last.mask)

        test = sort(a, endwith=False)
        assert_equal(test, mask_first)
        assert_equal(test.mask, mask_first.mask)

        # Test sort on dtype with subarray (gh-8069)
        # Just check that the sort does not error, structured array subarrays
        # are treated as byte strings and that leads to differing behavior
        # depending on endianness and `endwith`.
        dt = np.dtype([('v', int, 2)])
        a = a.view(dt)
        test = sort(a)
        test = sort(a, endwith=False)

    def test_argsort(self):
        # Test argsort
        a = array([1, 5, 2, 4, 3], mask=[1, 0, 0, 1, 0])
        assert_equal(np.argsort(a), argsort(a))

    def test_squeeze(self):
        # Check squeeze
        data = masked_array([[1, 2, 3]])
        assert_equal(data.squeeze(), [1, 2, 3])
        data = masked_array([[1, 2, 3]], mask=[[1, 1, 1]])
        assert_equal(data.squeeze(), [1, 2, 3])
        assert_equal(data.squeeze()._mask, [1, 1, 1])

        # normal ndarrays return a view
        arr = np.array([[1]])
        arr_sq = arr.squeeze()
        assert_equal(arr_sq, 1)
        arr_sq[...] = 2
        assert_equal(arr[0,0], 2)

        # so maskedarrays should too
        m_arr = masked_array([[1]], mask=True)
        m_arr_sq = m_arr.squeeze()
        assert_(m_arr_sq is not np.ma.masked)
        assert_equal(m_arr_sq.mask, True)
        m_arr_sq[...] = 2
        assert_equal(m_arr[0,0], 2)

    def test_swapaxes(self):
        # Tests swapaxes on MaskedArrays.
        x = np.array([8.375, 7.545, 8.828, 8.5, 1.757, 5.928,
                      8.43, 7.78, 9.865, 5.878, 8.979, 4.732,
                      3.012, 6.022, 5.095, 3.116, 5.238, 3.957,
                      6.04, 9.63, 7.712, 3.382, 4.489, 6.479,
                      7.189, 9.645, 5.395, 4.961, 9.894, 2.893,
                      7.357, 9.828, 6.272, 3.758, 6.693, 0.993])
        m = np.array([0, 1, 0, 1, 0, 0,
                      1, 0, 1, 1, 0, 1,
                      0, 0, 0, 1, 0, 1,
                      0, 0, 0, 1, 1, 1,
                      1, 0, 0, 1, 0, 0,
                      0, 0, 1, 0, 1, 0])
        mX = array(x, mask=m).reshape(6, 6)
        mXX = mX.reshape(3, 2, 2, 3)

        mXswapped = mX.swapaxes(0, 1)
        assert_equal(mXswapped[-1], mX[:, -1])

        mXXswapped = mXX.swapaxes(0, 2)
        assert_equal(mXXswapped.shape, (2, 2, 3, 3))

    def test_take(self):
        # Tests take
        x = masked_array([10, 20, 30, 40], [0, 1, 0, 1])
        assert_equal(x.take([0, 0, 3]), masked_array([10, 10, 40], [0, 0, 1]))
        assert_equal(x.take([0, 0, 3]), x[[0, 0, 3]])
        assert_equal(x.take([[0, 1], [0, 1]]),
                     masked_array([[10, 20], [10, 20]], [[0, 1], [0, 1]]))

        # assert_equal crashes when passed np.ma.mask
        assert_(x[1] is np.ma.masked)
        assert_(x.take(1) is np.ma.masked)

        x = array([[10, 20, 30], [40, 50, 60]], mask=[[0, 0, 1], [1, 0, 0, ]])
        assert_equal(x.take([0, 2], axis=1),
                     array([[10, 30], [40, 60]], mask=[[0, 1], [1, 0]]))
        assert_equal(take(x, [0, 2], axis=1),
                     array([[10, 30], [40, 60]], mask=[[0, 1], [1, 0]]))

    def test_take_masked_indices(self):
        # Test take w/ masked indices
        a = np.array((40, 18, 37, 9, 22))
        indices = np.arange(3)[None,:] + np.arange(5)[:, None]
        mindices = array(indices, mask=(indices >= len(a)))
        # No mask
        test = take(a, mindices, mode='clip')
        ctrl = array([[40, 18, 37],
                      [18, 37, 9],
                      [37, 9, 22],
                      [9, 22, 22],
                      [22, 22, 22]])
        assert_equal(test, ctrl)
        # Masked indices
        test = take(a, mindices)
        ctrl = array([[40, 18, 37],
                      [18, 37, 9],
                      [37, 9, 22],
                      [9, 22, 40],
                      [22, 40, 40]])
        ctrl[3, 2] = ctrl[4, 1] = ctrl[4, 2] = masked
        assert_equal(test, ctrl)
        assert_equal(test.mask, ctrl.mask)
        # Masked input + masked indices
        a = array((40, 18, 37, 9, 22), mask=(0, 1, 0, 0, 0))
        test = take(a, mindices)
        ctrl[0, 1] = ctrl[1, 0] = masked
        assert_equal(test, ctrl)
        assert_equal(test.mask, ctrl.mask)

    def test_tolist(self):
        # Tests to list
        # ... on 1D
        x = array(np.arange(12))
        x[[1, -2]] = masked
        xlist = x.tolist()
        assert_(xlist[1] is None)
        assert_(xlist[-2] is None)
        # ... on 2D
        x.shape = (3, 4)
        xlist = x.tolist()
        ctrl = [[0, None, 2, 3], [4, 5, 6, 7], [8, 9, None, 11]]
        assert_equal(xlist[0], [0, None, 2, 3])
        assert_equal(xlist[1], [4, 5, 6, 7])
        assert_equal(xlist[2], [8, 9, None, 11])
        assert_equal(xlist, ctrl)
        # ... on structured array w/ masked records
        x = array(list(zip([1, 2, 3],
                           [1.1, 2.2, 3.3],
                           ['one', 'two', 'thr'])),
                  dtype=[('a', int), ('b', float), ('c', '|S8')])
        x[-1] = masked
        assert_equal(x.tolist(),
                     [(1, 1.1, b'one'),
                      (2, 2.2, b'two'),
                      (None, None, None)])
        # ... on structured array w/ masked fields
        a = array([(1, 2,), (3, 4)], mask=[(0, 1), (0, 0)],
                  dtype=[('a', int), ('b', int)])
        test = a.tolist()
        assert_equal(test, [[1, None], [3, 4]])
        # ... on mvoid
        a = a[0]
        test = a.tolist()
        assert_equal(test, [1, None])

    def test_tolist_specialcase(self):
        # Test mvoid.tolist: make sure we return a standard Python object
        a = array([(0, 1), (2, 3)], dtype=[('a', int), ('b', int)])
        # w/o mask: each entry is a np.void whose elements are standard Python
        for entry in a:
            for item in entry.tolist():
                assert_(not isinstance(item, np.generic))
        # w/ mask: each entry is a ma.void whose elements should be
        # standard Python
        a.mask[0] = (0, 1)
        for entry in a:
            for item in entry.tolist():
                assert_(not isinstance(item, np.generic))

    def test_toflex(self):
        # Test the conversion to records
        data = arange(10)
        record = data.toflex()
        assert_equal(record['_data'], data._data)
        assert_equal(record['_mask'], data._mask)

        data[[0, 1, 2, -1]] = masked
        record = data.toflex()
        assert_equal(record['_data'], data._data)
        assert_equal(record['_mask'], data._mask)

        ndtype = [('i', int), ('s', '|S3'), ('f', float)]
        data = array([(i, s, f) for (i, s, f) in zip(np.arange(10),
                                                     'ABCDEFGHIJKLM',
                                                     np.random.rand(10))],
                     dtype=ndtype)
        data[[0, 1, 2, -1]] = masked
        record = data.toflex()
        assert_equal(record['_data'], data._data)
        assert_equal(record['_mask'], data._mask)

        ndtype = np.dtype("int, (2,3)float, float")
        data = array([(i, f, ff) for (i, f, ff) in zip(np.arange(10),
                                                       np.random.rand(10),
                                                       np.random.rand(10))],
                     dtype=ndtype)
        data[[0, 1, 2, -1]] = masked
        record = data.toflex()
        assert_equal_records(record['_data'], data._data)
        assert_equal_records(record['_mask'], data._mask)

    def test_fromflex(self):
        # Test the reconstruction of a masked_array from a record
        a = array([1, 2, 3])
        test = fromflex(a.toflex())
        assert_equal(test, a)
        assert_equal(test.mask, a.mask)

        a = array([1, 2, 3], mask=[0, 0, 1])
        test = fromflex(a.toflex())
        assert_equal(test, a)
        assert_equal(test.mask, a.mask)

        a = array([(1, 1.), (2, 2.), (3, 3.)], mask=[(1, 0), (0, 0), (0, 1)],
                  dtype=[('A', int), ('B', float)])
        test = fromflex(a.toflex())
        assert_equal(test, a)
        assert_equal(test.data, a.data)

    def test_arraymethod(self):
        # Test a _arraymethod w/ n argument
        marray = masked_array([[1, 2, 3, 4, 5]], mask=[0, 0, 1, 0, 0])
        control = masked_array([[1], [2], [3], [4], [5]],
                               mask=[0, 0, 1, 0, 0])
        assert_equal(marray.T, control)
        assert_equal(marray.transpose(), control)

        assert_equal(MaskedArray.cumsum(marray.T, 0), control.cumsum(0))

    def test_arraymethod_0d(self):
        # gh-9430
        x = np.ma.array(42, mask=True)
        assert_equal(x.T.mask, x.mask)
        assert_equal(x.T.data, x.data)

    def test_transpose_view(self):
        x = np.ma.array([[1, 2, 3], [4, 5, 6]])
        x[0,1] = np.ma.masked
        xt = x.T

        xt[1,0] = 10
        xt[0,1] = np.ma.masked

        assert_equal(x.data, xt.T.data)
        assert_equal(x.mask, xt.T.mask)

    def test_diagonal_view(self):
        x = np.ma.zeros((3,3))
        x[0,0] = 10
        x[1,1] = np.ma.masked
        x[2,2] = 20
        xd = x.diagonal()
        x[1,1] = 15
        assert_equal(xd.mask, x.diagonal().mask)
        assert_equal(xd.data, x.diagonal().data)


class TestMaskedArrayMathMethods:

    def setup_method(self):
        # Base data definition.
        x = np.array([8.375, 7.545, 8.828, 8.5, 1.757, 5.928,
                      8.43, 7.78, 9.865, 5.878, 8.979, 4.732,
                      3.012, 6.022, 5.095, 3.116, 5.238, 3.957,
                      6.04, 9.63, 7.712, 3.382, 4.489, 6.479,
                      7.189, 9.645, 5.395, 4.961, 9.894, 2.893,
                      7.357, 9.828, 6.272, 3.758, 6.693, 0.993])
        X = x.reshape(6, 6)
        XX = x.reshape(3, 2, 2, 3)

        m = np.array([0, 1, 0, 1, 0, 0,
                     1, 0, 1, 1, 0, 1,
                     0, 0, 0, 1, 0, 1,
                     0, 0, 0, 1, 1, 1,
                     1, 0, 0, 1, 0, 0,
                     0, 0, 1, 0, 1, 0])
        mx = array(data=x, mask=m)
        mX = array(data=X, mask=m.reshape(X.shape))
        mXX = array(data=XX, mask=m.reshape(XX.shape))

        m2 = np.array([1, 1, 0, 1, 0, 0,
                      1, 1, 1, 1, 0, 1,
                      0, 0, 1, 1, 0, 1,
                      0, 0, 0, 1, 1, 1,
                      1, 0, 0, 1, 1, 0,
                      0, 0, 1, 0, 1, 1])
        m2x = array(data=x, mask=m2)
        m2X = array(data=X, mask=m2.reshape(X.shape))
        m2XX = array(data=XX, mask=m2.reshape(XX.shape))
        self.d = (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX)

    def test_cumsumprod(self):
        # Tests cumsum & cumprod on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d
        mXcp = mX.cumsum(0)
        assert_equal(mXcp._data, mX.filled(0).cumsum(0))
        mXcp = mX.cumsum(1)
        assert_equal(mXcp._data, mX.filled(0).cumsum(1))

        mXcp = mX.cumprod(0)
        assert_equal(mXcp._data, mX.filled(1).cumprod(0))
        mXcp = mX.cumprod(1)
        assert_equal(mXcp._data, mX.filled(1).cumprod(1))

    def test_cumsumprod_with_output(self):
        # Tests cumsum/cumprod w/ output
        xm = array(np.random.uniform(0, 10, 12)).reshape(3, 4)
        xm[:, 0] = xm[0] = xm[-1, -1] = masked

        for funcname in ('cumsum', 'cumprod'):
            npfunc = getattr(np, funcname)
            xmmeth = getattr(xm, funcname)

            # A ndarray as explicit input
            output = np.empty((3, 4), dtype=float)
            output.fill(-9999)
            result = npfunc(xm, axis=0, out=output)
            # ... the result should be the given output
            assert_(result is output)
            assert_equal(result, xmmeth(axis=0, out=output))

            output = empty((3, 4), dtype=int)
            result = xmmeth(axis=0, out=output)
            assert_(result is output)

    def test_ptp(self):
        # Tests ptp on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d
        (n, m) = X.shape
        assert_equal(mx.ptp(), mx.compressed().ptp())
        rows = np.zeros(n, float)
        cols = np.zeros(m, float)
        for k in range(m):
            cols[k] = mX[:, k].compressed().ptp()
        for k in range(n):
            rows[k] = mX[k].compressed().ptp()
        assert_equal(mX.ptp(0), cols)
        assert_equal(mX.ptp(1), rows)

    def test_add_object(self):
        x = masked_array(['a', 'b'], mask=[1, 0], dtype=object)
        y = x + 'x'
        assert_equal(y[1], 'bx')
        assert_(y.mask[0])

    def test_sum_object(self):
        # Test sum on object dtype
        a = masked_array([1, 2, 3], mask=[1, 0, 0], dtype=object)
        assert_equal(a.sum(), 5)
        a = masked_array([[1, 2, 3], [4, 5, 6]], dtype=object)
        assert_equal(a.sum(axis=0), [5, 7, 9])

    def test_prod_object(self):
        # Test prod on object dtype
        a = masked_array([1, 2, 3], mask=[1, 0, 0], dtype=object)
        assert_equal(a.prod(), 2 * 3)
        a = masked_array([[1, 2, 3], [4, 5, 6]], dtype=object)
        assert_equal(a.prod(axis=0), [4, 10, 18])

    def test_meananom_object(self):
        # Test mean/anom on object dtype
        a = masked_array([1, 2, 3], dtype=object)
        assert_equal(a.mean(), 2)
        assert_equal(a.anom(), [-1, 0, 1])

    def test_anom_shape(self):
        a = masked_array([1, 2, 3])
        assert_equal(a.anom().shape, a.shape)
        a.mask = True
        assert_equal(a.anom().shape, a.shape)
        assert_(np.ma.is_masked(a.anom()))

    def test_anom(self):
        a = masked_array(np.arange(1, 7).reshape(2, 3))
        assert_almost_equal(a.anom(),
                            [[-2.5, -1.5, -0.5], [0.5, 1.5, 2.5]])
        assert_almost_equal(a.anom(axis=0),
                            [[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]])
        assert_almost_equal(a.anom(axis=1),
                            [[-1., 0., 1.], [-1., 0., 1.]])
        a.mask = [[0, 0, 1], [0, 1, 0]]
        mval = -99
        assert_almost_equal(a.anom().filled(mval),
                            [[-2.25, -1.25, mval], [0.75, mval, 2.75]])
        assert_almost_equal(a.anom(axis=0).filled(mval),
                            [[-1.5, 0.0, mval], [1.5, mval, 0.0]])
        assert_almost_equal(a.anom(axis=1).filled(mval),
                            [[-0.5, 0.5, mval], [-1.0, mval, 1.0]])

    def test_trace(self):
        # Tests trace on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d
        mXdiag = mX.diagonal()
        assert_equal(mX.trace(), mX.diagonal().compressed().sum())
        assert_almost_equal(mX.trace(),
                            X.trace() - sum(mXdiag.mask * X.diagonal(),
                                            axis=0))
        assert_equal(np.trace(mX), mX.trace())

        # gh-5560
        arr = np.arange(2*4*4).reshape(2,4,4)
        m_arr = np.ma.masked_array(arr, False)
        assert_equal(arr.trace(axis1=1, axis2=2), m_arr.trace(axis1=1, axis2=2))

    def test_dot(self):
        # Tests dot on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d
        fx = mx.filled(0)
        r = mx.dot(mx)
        assert_almost_equal(r.filled(0), fx.dot(fx))
        assert_(r.mask is nomask)

        fX = mX.filled(0)
        r = mX.dot(mX)
        assert_almost_equal(r.filled(0), fX.dot(fX))
        assert_(r.mask[1,3])
        r1 = empty_like(r)
        mX.dot(mX, out=r1)
        assert_almost_equal(r, r1)

        mYY = mXX.swapaxes(-1, -2)
        fXX, fYY = mXX.filled(0), mYY.filled(0)
        r = mXX.dot(mYY)
        assert_almost_equal(r.filled(0), fXX.dot(fYY))
        r1 = empty_like(r)
        mXX.dot(mYY, out=r1)
        assert_almost_equal(r, r1)

    def test_dot_shape_mismatch(self):
        # regression test
        x = masked_array([[1,2],[3,4]], mask=[[0,1],[0,0]])
        y = masked_array([[1,2],[3,4]], mask=[[0,1],[0,0]])
        z = masked_array([[0,1],[3,3]])
        x.dot(y, out=z)
        assert_almost_equal(z.filled(0), [[1, 0], [15, 16]])
        assert_almost_equal(z.mask, [[0, 1], [0, 0]])

    def test_varmean_nomask(self):
        # gh-5769
        foo = array([1,2,3,4], dtype='f8')
        bar = array([1,2,3,4], dtype='f8')
        assert_equal(type(foo.mean()), np.float64)
        assert_equal(type(foo.var()), np.float64)
        assert((foo.mean() == bar.mean()) is np.bool_(True))

        # check array type is preserved and out works
        foo = array(np.arange(16).reshape((4,4)), dtype='f8')
        bar = empty(4, dtype='f4')
        assert_equal(type(foo.mean(axis=1)), MaskedArray)
        assert_equal(type(foo.var(axis=1)), MaskedArray)
        assert_(foo.mean(axis=1, out=bar) is bar)
        assert_(foo.var(axis=1, out=bar) is bar)

    def test_varstd(self):
        # Tests var & std on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d
        assert_almost_equal(mX.var(axis=None), mX.compressed().var())
        assert_almost_equal(mX.std(axis=None), mX.compressed().std())
        assert_almost_equal(mX.std(axis=None, ddof=1),
                            mX.compressed().std(ddof=1))
        assert_almost_equal(mX.var(axis=None, ddof=1),
                            mX.compressed().var(ddof=1))
        assert_equal(mXX.var(axis=3).shape, XX.var(axis=3).shape)
        assert_equal(mX.var().shape, X.var().shape)
        (mXvar0, mXvar1) = (mX.var(axis=0), mX.var(axis=1))
        assert_almost_equal(mX.var(axis=None, ddof=2),
                            mX.compressed().var(ddof=2))
        assert_almost_equal(mX.std(axis=None, ddof=2),
                            mX.compressed().std(ddof=2))
        for k in range(6):
            assert_almost_equal(mXvar1[k], mX[k].compressed().var())
            assert_almost_equal(mXvar0[k], mX[:, k].compressed().var())
            assert_almost_equal(np.sqrt(mXvar0[k]),
                                mX[:, k].compressed().std())

    @suppress_copy_mask_on_assignment
    def test_varstd_specialcases(self):
        # Test a special case for var
        nout = np.array(-1, dtype=float)
        mout = array(-1, dtype=float)

        x = array(arange(10), mask=True)
        for methodname in ('var', 'std'):
            method = getattr(x, methodname)
            assert_(method() is masked)
            assert_(method(0) is masked)
            assert_(method(-1) is masked)
            # Using a masked array as explicit output
            method(out=mout)
            assert_(mout is not masked)
            assert_equal(mout.mask, True)
            # Using a ndarray as explicit output
            method(out=nout)
            assert_(np.isnan(nout))

        x = array(arange(10), mask=True)
        x[-1] = 9
        for methodname in ('var', 'std'):
            method = getattr(x, methodname)
            assert_(method(ddof=1) is masked)
            assert_(method(0, ddof=1) is masked)
            assert_(method(-1, ddof=1) is masked)
            # Using a masked array as explicit output
            method(out=mout, ddof=1)
            assert_(mout is not masked)
            assert_equal(mout.mask, True)
            # Using a ndarray as explicit output
            method(out=nout, ddof=1)
            assert_(np.isnan(nout))

    def test_varstd_ddof(self):
        a = array([[1, 1, 0], [1, 1, 0]], mask=[[0, 0, 1], [0, 0, 1]])
        test = a.std(axis=0, ddof=0)
        assert_equal(test.filled(0), [0, 0, 0])
        assert_equal(test.mask, [0, 0, 1])
        test = a.std(axis=0, ddof=1)
        assert_equal(test.filled(0), [0, 0, 0])
        assert_equal(test.mask, [0, 0, 1])
        test = a.std(axis=0, ddof=2)
        assert_equal(test.filled(0), [0, 0, 0])
        assert_equal(test.mask, [1, 1, 1])

    def test_diag(self):
        # Test diag
        x = arange(9).reshape((3, 3))
        x[1, 1] = masked
        out = np.diag(x)
        assert_equal(out, [0, 4, 8])
        out = diag(x)
        assert_equal(out, [0, 4, 8])
        assert_equal(out.mask, [0, 1, 0])
        out = diag(out)
        control = array([[0, 0, 0], [0, 4, 0], [0, 0, 8]],
                        mask=[[0, 0, 0], [0, 1, 0], [0, 0, 0]])
        assert_equal(out, control)

    def test_axis_methods_nomask(self):
        # Test the combination nomask & methods w/ axis
        a = array([[1, 2, 3], [4, 5, 6]])

        assert_equal(a.sum(0), [5, 7, 9])
        assert_equal(a.sum(-1), [6, 15])
        assert_equal(a.sum(1), [6, 15])

        assert_equal(a.prod(0), [4, 10, 18])
        assert_equal(a.prod(-1), [6, 120])
        assert_equal(a.prod(1), [6, 120])

        assert_equal(a.min(0), [1, 2, 3])
        assert_equal(a.min(-1), [1, 4])
        assert_equal(a.min(1), [1, 4])

        assert_equal(a.max(0), [4, 5, 6])
        assert_equal(a.max(-1), [3, 6])
        assert_equal(a.max(1), [3, 6])

    @requires_memory(free_bytes=2 * 10000 * 1000 * 2)
    def test_mean_overflow(self):
        # Test overflow in masked arrays
        # gh-20272
        a = masked_array(np.full((10000, 10000), 65535, dtype=np.uint16),
                         mask=np.zeros((10000, 10000)))
        assert_equal(a.mean(), 65535.0)

class TestMaskedArrayMathMethodsComplex:
    # Test class for miscellaneous MaskedArrays methods.
    def setup_method(self):
        # Base data definition.
        x = np.array([8.375j, 7.545j, 8.828j, 8.5j, 1.757j, 5.928,
                      8.43, 7.78, 9.865, 5.878, 8.979, 4.732,
                      3.012, 6.022, 5.095, 3.116, 5.238, 3.957,
                      6.04, 9.63, 7.712, 3.382, 4.489, 6.479j,
                      7.189j, 9.645, 5.395, 4.961, 9.894, 2.893,
                      7.357, 9.828, 6.272, 3.758, 6.693, 0.993j])
        X = x.reshape(6, 6)
        XX = x.reshape(3, 2, 2, 3)

        m = np.array([0, 1, 0, 1, 0, 0,
                     1, 0, 1, 1, 0, 1,
                     0, 0, 0, 1, 0, 1,
                     0, 0, 0, 1, 1, 1,
                     1, 0, 0, 1, 0, 0,
                     0, 0, 1, 0, 1, 0])
        mx = array(data=x, mask=m)
        mX = array(data=X, mask=m.reshape(X.shape))
        mXX = array(data=XX, mask=m.reshape(XX.shape))

        m2 = np.array([1, 1, 0, 1, 0, 0,
                      1, 1, 1, 1, 0, 1,
                      0, 0, 1, 1, 0, 1,
                      0, 0, 0, 1, 1, 1,
                      1, 0, 0, 1, 1, 0,
                      0, 0, 1, 0, 1, 1])
        m2x = array(data=x, mask=m2)
        m2X = array(data=X, mask=m2.reshape(X.shape))
        m2XX = array(data=XX, mask=m2.reshape(XX.shape))
        self.d = (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX)

    def test_varstd(self):
        # Tests var & std on MaskedArrays.
        (x, X, XX, m, mx, mX, mXX, m2x, m2X, m2XX) = self.d
        assert_almost_equal(mX.var(axis=None), mX.compressed().var())
        assert_almost_equal(mX.std(axis=None), mX.compressed().std())
        assert_equal(mXX.var(axis=3).shape, XX.var(axis=3).shape)
        assert_equal(mX.var().shape, X.var().shape)
        (mXvar0, mXvar1) = (mX.var(axis=0), mX.var(axis=1))
        assert_almost_equal(mX.var(axis=None, ddof=2),
                            mX.compressed().var(ddof=2))
        assert_almost_equal(mX.std(axis=None, ddof=2),
                            mX.compressed().std(ddof=2))
        for k in range(6):
            assert_almost_equal(mXvar1[k], mX[k].compressed().var())
            assert_almost_equal(mXvar0[k], mX[:, k].compressed().var())
            assert_almost_equal(np.sqrt(mXvar0[k]),
                                mX[:, k].compressed().std())


class TestMaskedArrayFunctions:
    # Test class for miscellaneous functions.

    def setup_method(self):
        x = np.array([1., 1., 1., -2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.])
        y = np.array([5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.])
        m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]
        m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1]
        xm = masked_array(x, mask=m1)
        ym = masked_array(y, mask=m2)
        xm.set_fill_value(1e+20)
        self.info = (xm, ym)

    def test_masked_where_bool(self):
        x = [1, 2]
        y = masked_where(False, x)
        assert_equal(y, [1, 2])
        assert_equal(y[1], 2)

    def test_masked_equal_wlist(self):
        x = [1, 2, 3]
        mx = masked_equal(x, 3)
        assert_equal(mx, x)
        assert_equal(mx._mask, [0, 0, 1])
        mx = masked_not_equal(x, 3)
        assert_equal(mx, x)
        assert_equal(mx._mask, [1, 1, 0])

    def test_masked_equal_fill_value(self):
        x = [1, 2, 3]
        mx = masked_equal(x, 3)
        assert_equal(mx._mask, [0, 0, 1])
        assert_equal(mx.fill_value, 3)

    def test_masked_where_condition(self):
        # Tests masking functions.
        x = array([1., 2., 3., 4., 5.])
        x[2] = masked
        assert_equal(masked_where(greater(x, 2), x), masked_greater(x, 2))
        assert_equal(masked_where(greater_equal(x, 2), x),
                     masked_greater_equal(x, 2))
        assert_equal(masked_where(less(x, 2), x), masked_less(x, 2))
        assert_equal(masked_where(less_equal(x, 2), x),
                     masked_less_equal(x, 2))
        assert_equal(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2))
        assert_equal(masked_where(equal(x, 2), x), masked_equal(x, 2))
        assert_equal(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2))
        assert_equal(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
                     [99, 99, 3, 4, 5])

    def test_masked_where_oddities(self):
        # Tests some generic features.
        atest = ones((10, 10, 10), dtype=float)
        btest = zeros(atest.shape, MaskType)
        ctest = masked_where(btest, atest)
        assert_equal(atest, ctest)

    def test_masked_where_shape_constraint(self):
        a = arange(10)
        with assert_raises(IndexError):
            masked_equal(1, a)
        test = masked_equal(a, 1)
        assert_equal(test.mask, [0, 1, 0, 0, 0, 0, 0, 0, 0, 0])

    def test_masked_where_structured(self):
        # test that masked_where on a structured array sets a structured
        # mask (see issue #2972)
        a = np.zeros(10, dtype=[("A", "<f2"), ("B", "<f4")])
        with np.errstate(over="ignore"):
            # NOTE: The float16 "uses" 1e20 as mask, which overflows to inf
            #       and warns.  Unrelated to this test, but probably undesired.
            #       But NumPy previously did not warn for this overflow.
            am = np.ma.masked_where(a["A"] < 5, a)
        assert_equal(am.mask.dtype.names, am.dtype.names)
        assert_equal(am["A"],
                    np.ma.masked_array(np.zeros(10), np.ones(10)))

    def test_masked_where_mismatch(self):
        # gh-4520
        x = np.arange(10)
        y = np.arange(5)
        assert_raises(IndexError, np.ma.masked_where, y > 6, x)

    def test_masked_otherfunctions(self):
        assert_equal(masked_inside(list(range(5)), 1, 3),
                     [0, 199, 199, 199, 4])
        assert_equal(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199])
        assert_equal(masked_inside(array(list(range(5)),
                                         mask=[1, 0, 0, 0, 0]), 1, 3).mask,
                     [1, 1, 1, 1, 0])
        assert_equal(masked_outside(array(list(range(5)),
                                          mask=[0, 1, 0, 0, 0]), 1, 3).mask,
                     [1, 1, 0, 0, 1])
        assert_equal(masked_equal(array(list(range(5)),
                                        mask=[1, 0, 0, 0, 0]), 2).mask,
                     [1, 0, 1, 0, 0])
        assert_equal(masked_not_equal(array([2, 2, 1, 2, 1],
                                            mask=[1, 0, 0, 0, 0]), 2).mask,
                     [1, 0, 1, 0, 1])

    def test_round(self):
        a = array([1.23456, 2.34567, 3.45678, 4.56789, 5.67890],
                  mask=[0, 1, 0, 0, 0])
        assert_equal(a.round(), [1., 2., 3., 5., 6.])
        assert_equal(a.round(1), [1.2, 2.3, 3.5, 4.6, 5.7])
        assert_equal(a.round(3), [1.235, 2.346, 3.457, 4.568, 5.679])
        b = empty_like(a)
        a.round(out=b)
        assert_equal(b, [1., 2., 3., 5., 6.])

        x = array([1., 2., 3., 4., 5.])
        c = array([1, 1, 1, 0, 0])
        x[2] = masked
        z = where(c, x, -x)
        assert_equal(z, [1., 2., 0., -4., -5])
        c[0] = masked
        z = where(c, x, -x)
        assert_equal(z, [1., 2., 0., -4., -5])
        assert_(z[0] is masked)
        assert_(z[1] is not masked)
        assert_(z[2] is masked)

    def test_round_with_output(self):
        # Testing round with an explicit output

        xm = array(np.random.uniform(0, 10, 12)).reshape(3, 4)
        xm[:, 0] = xm[0] = xm[-1, -1] = masked

        # A ndarray as explicit input
        output = np.empty((3, 4), dtype=float)
        output.fill(-9999)
        result = np.round(xm, decimals=2, out=output)
        # ... the result should be the given output
        assert_(result is output)
        assert_equal(result, xm.round(decimals=2, out=output))

        output = empty((3, 4), dtype=float)
        result = xm.round(decimals=2, out=output)
        assert_(result is output)

    def test_round_with_scalar(self):
        # Testing round with scalar/zero dimension input
        # GH issue 2244
        a = array(1.1, mask=[False])
        assert_equal(a.round(), 1)

        a = array(1.1, mask=[True])
        assert_(a.round() is masked)

        a = array(1.1, mask=[False])
        output = np.empty(1, dtype=float)
        output.fill(-9999)
        a.round(out=output)
        assert_equal(output, 1)

        a = array(1.1, mask=[False])
        output = array(-9999., mask=[True])
        a.round(out=output)
        assert_equal(output[()], 1)

        a = array(1.1, mask=[True])
        output = array(-9999., mask=[False])
        a.round(out=output)
        assert_(output[()] is masked)

    def test_identity(self):
        a = identity(5)
        assert_(isinstance(a, MaskedArray))
        assert_equal(a, np.identity(5))

    def test_power(self):
        x = -1.1
        assert_almost_equal(power(x, 2.), 1.21)
        assert_(power(x, masked) is masked)
        x = array([-1.1, -1.1, 1.1, 1.1, 0.])
        b = array([0.5, 2., 0.5, 2., -1.], mask=[0, 0, 0, 0, 1])
        y = power(x, b)
        assert_almost_equal(y, [0, 1.21, 1.04880884817, 1.21, 0.])
        assert_equal(y._mask, [1, 0, 0, 0, 1])
        b.mask = nomask
        y = power(x, b)
        assert_equal(y._mask, [1, 0, 0, 0, 1])
        z = x ** b
        assert_equal(z._mask, y._mask)
        assert_almost_equal(z, y)
        assert_almost_equal(z._data, y._data)
        x **= b
        assert_equal(x._mask, y._mask)
        assert_almost_equal(x, y)
        assert_almost_equal(x._data, y._data)

    def test_power_with_broadcasting(self):
        # Test power w/ broadcasting
        a2 = np.array([[1., 2., 3.], [4., 5., 6.]])
        a2m = array(a2, mask=[[1, 0, 0], [0, 0, 1]])
        b1 = np.array([2, 4, 3])
        b2 = np.array([b1, b1])
        b2m = array(b2, mask=[[0, 1, 0], [0, 1, 0]])

        ctrl = array([[1 ** 2, 2 ** 4, 3 ** 3], [4 ** 2, 5 ** 4, 6 ** 3]],
                     mask=[[1, 1, 0], [0, 1, 1]])
        # No broadcasting, base & exp w/ mask
        test = a2m ** b2m
        assert_equal(test, ctrl)
        assert_equal(test.mask, ctrl.mask)
        # No broadcasting, base w/ mask, exp w/o mask
        test = a2m ** b2
        assert_equal(test, ctrl)
        assert_equal(test.mask, a2m.mask)
        # No broadcasting, base w/o mask, exp w/ mask
        test = a2 ** b2m
        assert_equal(test, ctrl)
        assert_equal(test.mask, b2m.mask)

        ctrl = array([[2 ** 2, 4 ** 4, 3 ** 3], [2 ** 2, 4 ** 4, 3 ** 3]],
                     mask=[[0, 1, 0], [0, 1, 0]])
        test = b1 ** b2m
        assert_equal(test, ctrl)
        assert_equal(test.mask, ctrl.mask)
        test = b2m ** b1
        assert_equal(test, ctrl)
        assert_equal(test.mask, ctrl.mask)

    @pytest.mark.skipif(IS_WASM, reason="fp errors don't work in wasm")
    def test_where(self):
        # Test the where function
        x = np.array([1., 1., 1., -2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.])
        y = np.array([5., 0., 3., 2., -1., -4., 0., -10., 10., 1., 0., 3.])
        m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]
        m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1]
        xm = masked_array(x, mask=m1)
        ym = masked_array(y, mask=m2)
        xm.set_fill_value(1e+20)

        d = where(xm > 2, xm, -9)
        assert_equal(d, [-9., -9., -9., -9., -9., 4.,
                         -9., -9., 10., -9., -9., 3.])
        assert_equal(d._mask, xm._mask)
        d = where(xm > 2, -9, ym)
        assert_equal(d, [5., 0., 3., 2., -1., -9.,
                         -9., -10., -9., 1., 0., -9.])
        assert_equal(d._mask, [1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0])
        d = where(xm > 2, xm, masked)
        assert_equal(d, [-9., -9., -9., -9., -9., 4.,
                         -9., -9., 10., -9., -9., 3.])
        tmp = xm._mask.copy()
        tmp[(xm <= 2).filled(True)] = True
        assert_equal(d._mask, tmp)

        with np.errstate(invalid="warn"):
            # The fill value is 1e20, it cannot be converted to `int`:
            with pytest.warns(RuntimeWarning, match="invalid value"):
                ixm = xm.astype(int)
        d = where(ixm > 2, ixm, masked)
        assert_equal(d, [-9, -9, -9, -9, -9, 4, -9, -9, 10, -9, -9, 3])
        assert_equal(d.dtype, ixm.dtype)

    def test_where_object(self):
        a = np.array(None)
        b = masked_array(None)
        r = b.copy()
        assert_equal(np.ma.where(True, a, a), r)
        assert_equal(np.ma.where(True, b, b), r)

    def test_where_with_masked_choice(self):
        x = arange(10)
        x[3] = masked
        c = x >= 8
        # Set False to masked
        z = where(c, x, masked)
        assert_(z.dtype is x.dtype)
        assert_(z[3] is masked)
        assert_(z[4] is masked)
        assert_(z[7] is masked)
        assert_(z[8] is not masked)
        assert_(z[9] is not masked)
        assert_equal(x, z)
        # Set True to masked
        z = where(c, masked, x)
        assert_(z.dtype is x.dtype)
        assert_(z[3] is masked)
        assert_(z[4] is not masked)
        assert_(z[7] is not masked)
        assert_(z[8] is masked)
        assert_(z[9] is masked)

    def test_where_with_masked_condition(self):
        x = array([1., 2., 3., 4., 5.])
        c = array([1, 1, 1, 0, 0])
        x[2] = masked
        z = where(c, x, -x)
        assert_equal(z, [1., 2., 0., -4., -5])
        c[0] = masked
        z = where(c, x, -x)
        assert_equal(z, [1., 2., 0., -4., -5])
        assert_(z[0] is masked)
        assert_(z[1] is not masked)
        assert_(z[2] is masked)

        x = arange(1, 6)
        x[-1] = masked
        y = arange(1, 6) * 10
        y[2] = masked
        c = array([1, 1, 1, 0, 0], mask=[1, 0, 0, 0, 0])
        cm = c.filled(1)
        z = where(c, x, y)
        zm = where(cm, x, y)
        assert_equal(z, zm)
        assert_(getmask(zm) is nomask)
        assert_equal(zm, [1, 2, 3, 40, 50])
        z = where(c, masked, 1)
        assert_equal(z, [99, 99, 99, 1, 1])
        z = where(c, 1, masked)
        assert_equal(z, [99, 1, 1, 99, 99])

    def test_where_type(self):
        # Test the type conservation with where
        x = np.arange(4, dtype=np.int32)
        y = np.arange(4, dtype=np.float32) * 2.2
        test = where(x > 1.5, y, x).dtype
        control = np.find_common_type([np.int32, np.float32], [])
        assert_equal(test, control)

    def test_where_broadcast(self):
        # Issue 8599
        x = np.arange(9).reshape(3, 3)
        y = np.zeros(3)
        core = np.where([1, 0, 1], x, y)
        ma = where([1, 0, 1], x, y)

        assert_equal(core, ma)
        assert_equal(core.dtype, ma.dtype)

    def test_where_structured(self):
        # Issue 8600
        dt = np.dtype([('a', int), ('b', int)])
        x = np.array([(1, 2), (3, 4), (5, 6)], dtype=dt)
        y = np.array((10, 20), dtype=dt)
        core = np.where([0, 1, 1], x, y)
        ma = np.where([0, 1, 1], x, y)

        assert_equal(core, ma)
        assert_equal(core.dtype, ma.dtype)

    def test_where_structured_masked(self):
        dt = np.dtype([('a', int), ('b', int)])
        x = np.array([(1, 2), (3, 4), (5, 6)], dtype=dt)

        ma = where([0, 1, 1], x, masked)
        expected = masked_where([1, 0, 0], x)

        assert_equal(ma.dtype, expected.dtype)
        assert_equal(ma, expected)
        assert_equal(ma.mask, expected.mask)

    def test_masked_invalid_error(self):
        a = np.arange(5, dtype=object)
        a[3] = np.PINF
        a[2] = np.NaN
        with pytest.raises(TypeError,
                           match="not supported for the input types"):
            np.ma.masked_invalid(a)

    def test_masked_invalid_pandas(self):
        # getdata() used to be bad for pandas series due to its _data
        # attribute.  This test is a regression test mainly and may be
        # removed if getdata() is adjusted.
        class Series():
            _data = "nonsense"

            def __array__(self):
                return np.array([5, np.nan, np.inf])

        arr = np.ma.masked_invalid(Series())
        assert_array_equal(arr._data, np.array(Series()))
        assert_array_equal(arr._mask, [False, True, True])

    @pytest.mark.parametrize("copy", [True, False])
    def test_masked_invalid_full_mask(self, copy):
        # Matplotlib relied on masked_invalid always returning a full mask
        # (Also astropy projects, but were ok with it gh-22720 and gh-22842)
        a = np.ma.array([1, 2, 3, 4])
        assert a._mask is nomask
        res = np.ma.masked_invalid(a, copy=copy)
        assert res.mask is not nomask
        # mask of a should not be mutated
        assert a.mask is nomask
        assert np.may_share_memory(a._data, res._data) != copy

    def test_choose(self):
        # Test choose
        choices = [[0, 1, 2, 3], [10, 11, 12, 13],
                   [20, 21, 22, 23], [30, 31, 32, 33]]
        chosen = choose([2, 3, 1, 0], choices)
        assert_equal(chosen, array([20, 31, 12, 3]))
        chosen = choose([2, 4, 1, 0], choices, mode='clip')
        assert_equal(chosen, array([20, 31, 12, 3]))
        chosen = choose([2, 4, 1, 0], choices, mode='wrap')
        assert_equal(chosen, array([20, 1, 12, 3]))
        # Check with some masked indices
        indices_ = array([2, 4, 1, 0], mask=[1, 0, 0, 1])
        chosen = choose(indices_, choices, mode='wrap')
        assert_equal(chosen, array([99, 1, 12, 99]))
        assert_equal(chosen.mask, [1, 0, 0, 1])
        # Check with some masked choices
        choices = array(choices, mask=[[0, 0, 0, 1], [1, 1, 0, 1],
                                       [1, 0, 0, 0], [0, 0, 0, 0]])
        indices_ = [2, 3, 1, 0]
        chosen = choose(indices_, choices, mode='wrap')
        assert_equal(chosen, array([20, 31, 12, 3]))
        assert_equal(chosen.mask, [1, 0, 0, 1])

    def test_choose_with_out(self):
        # Test choose with an explicit out keyword
        choices = [[0, 1, 2, 3], [10, 11, 12, 13],
                   [20, 21, 22, 23], [30, 31, 32, 33]]
        store = empty(4, dtype=int)
        chosen = choose([2, 3, 1, 0], choices, out=store)
        assert_equal(store, array([20, 31, 12, 3]))
        assert_(store is chosen)
        # Check with some masked indices + out
        store = empty(4, dtype=int)
        indices_ = array([2, 3, 1, 0], mask=[1, 0, 0, 1])
        chosen = choose(indices_, choices, mode='wrap', out=store)
        assert_equal(store, array([99, 31, 12, 99]))
        assert_equal(store.mask, [1, 0, 0, 1])
        # Check with some masked choices + out ina ndarray !
        choices = array(choices, mask=[[0, 0, 0, 1], [1, 1, 0, 1],
                                       [1, 0, 0, 0], [0, 0, 0, 0]])
        indices_ = [2, 3, 1, 0]
        store = empty(4, dtype=int).view(ndarray)
        chosen = choose(indices_, choices, mode='wrap', out=store)
        assert_equal(store, array([999999, 31, 12, 999999]))

    def test_reshape(self):
        a = arange(10)
        a[0] = masked
        # Try the default
        b = a.reshape((5, 2))
        assert_equal(b.shape, (5, 2))
        assert_(b.flags['C'])
        # Try w/ arguments as list instead of tuple
        b = a.reshape(5, 2)
        assert_equal(b.shape, (5, 2))
        assert_(b.flags['C'])
        # Try w/ order
        b = a.reshape((5, 2), order='F')
        assert_equal(b.shape, (5, 2))
        assert_(b.flags['F'])
        # Try w/ order
        b = a.reshape(5, 2, order='F')
        assert_equal(b.shape, (5, 2))
        assert_(b.flags['F'])

        c = np.reshape(a, (2, 5))
        assert_(isinstance(c, MaskedArray))
        assert_equal(c.shape, (2, 5))
        assert_(c[0, 0] is masked)
        assert_(c.flags['C'])

    def test_make_mask_descr(self):
        # Flexible
        ntype = [('a', float), ('b', float)]
        test = make_mask_descr(ntype)
        assert_equal(test, [('a', bool), ('b', bool)])
        assert_(test is make_mask_descr(test))

        # Standard w/ shape
        ntype = (float, 2)
        test = make_mask_descr(ntype)
        assert_equal(test, (bool, 2))
        assert_(test is make_mask_descr(test))

        # Standard standard
        ntype = float
        test = make_mask_descr(ntype)
        assert_equal(test, np.dtype(bool))
        assert_(test is make_mask_descr(test))

        # Nested
        ntype = [('a', float), ('b', [('ba', float), ('bb', float)])]
        test = make_mask_descr(ntype)
        control = np.dtype([('a', 'b1'), ('b', [('ba', 'b1'), ('bb', 'b1')])])
        assert_equal(test, control)
        assert_(test is make_mask_descr(test))

        # Named+ shape
        ntype = [('a', (float, 2))]
        test = make_mask_descr(ntype)
        assert_equal(test, np.dtype([('a', (bool, 2))]))
        assert_(test is make_mask_descr(test))

        # 2 names
        ntype = [(('A', 'a'), float)]
        test = make_mask_descr(ntype)
        assert_equal(test, np.dtype([(('A', 'a'), bool)]))
        assert_(test is make_mask_descr(test))

        # nested boolean types should preserve identity
        base_type = np.dtype([('a', int, 3)])
        base_mtype = make_mask_descr(base_type)
        sub_type = np.dtype([('a', int), ('b', base_mtype)])
        test = make_mask_descr(sub_type)
        assert_equal(test, np.dtype([('a', bool), ('b', [('a', bool, 3)])]))
        assert_(test.fields['b'][0] is base_mtype)

    def test_make_mask(self):
        # Test make_mask
        # w/ a list as an input
        mask = [0, 1]
        test = make_mask(mask)
        assert_equal(test.dtype, MaskType)
        assert_equal(test, [0, 1])
        # w/ a ndarray as an input
        mask = np.array([0, 1], dtype=bool)
        test = make_mask(mask)
        assert_equal(test.dtype, MaskType)
        assert_equal(test, [0, 1])
        # w/ a flexible-type ndarray as an input - use default
        mdtype = [('a', bool), ('b', bool)]
        mask = np.array([(0, 0), (0, 1)], dtype=mdtype)
        test = make_mask(mask)
        assert_equal(test.dtype, MaskType)
        assert_equal(test, [1, 1])
        # w/ a flexible-type ndarray as an input - use input dtype
        mdtype = [('a', bool), ('b', bool)]
        mask = np.array([(0, 0), (0, 1)], dtype=mdtype)
        test = make_mask(mask, dtype=mask.dtype)
        assert_equal(test.dtype, mdtype)
        assert_equal(test, mask)
        # w/ a flexible-type ndarray as an input - use input dtype
        mdtype = [('a', float), ('b', float)]
        bdtype = [('a', bool), ('b', bool)]
        mask = np.array([(0, 0), (0, 1)], dtype=mdtype)
        test = make_mask(mask, dtype=mask.dtype)
        assert_equal(test.dtype, bdtype)
        assert_equal(test, np.array([(0, 0), (0, 1)], dtype=bdtype))
        # Ensure this also works for void
        mask = np.array((False, True), dtype='?,?')[()]
        assert_(isinstance(mask, np.void))
        test = make_mask(mask, dtype=mask.dtype)
        assert_equal(test, mask)
        assert_(test is not mask)
        mask = np.array((0, 1), dtype='i4,i4')[()]
        test2 = make_mask(mask, dtype=mask.dtype)
        assert_equal(test2, test)
        # test that nomask is returned when m is nomask.
        bools = [True, False]
        dtypes = [MaskType, float]
        msgformat = 'copy=%s, shrink=%s, dtype=%s'
        for cpy, shr, dt in itertools.product(bools, bools, dtypes):
            res = make_mask(nomask, copy=cpy, shrink=shr, dtype=dt)
            assert_(res is nomask, msgformat % (cpy, shr, dt))

    def test_mask_or(self):
        # Initialize
        mtype = [('a', bool), ('b', bool)]
        mask = np.array([(0, 0), (0, 1), (1, 0), (0, 0)], dtype=mtype)
        # Test using nomask as input
        test = mask_or(mask, nomask)
        assert_equal(test, mask)
        test = mask_or(nomask, mask)
        assert_equal(test, mask)
        # Using False as input
        test = mask_or(mask, False)
        assert_equal(test, mask)
        # Using another array w / the same dtype
        other = np.array([(0, 1), (0, 1), (0, 1), (0, 1)], dtype=mtype)
        test = mask_or(mask, other)
        control = np.array([(0, 1), (0, 1), (1, 1), (0, 1)], dtype=mtype)
        assert_equal(test, control)
        # Using another array w / a different dtype
        othertype = [('A', bool), ('B', bool)]
        other = np.array([(0, 1), (0, 1), (0, 1), (0, 1)], dtype=othertype)
        try:
            test = mask_or(mask, other)
        except ValueError:
            pass
        # Using nested arrays
        dtype = [('a', bool), ('b', [('ba', bool), ('bb', bool)])]
        amask = np.array([(0, (1, 0)), (0, (1, 0))], dtype=dtype)
        bmask = np.array([(1, (0, 1)), (0, (0, 0))], dtype=dtype)
        cntrl = np.array([(1, (1, 1)), (0, (1, 0))], dtype=dtype)
        assert_equal(mask_or(amask, bmask), cntrl)

    def test_flatten_mask(self):
        # Tests flatten mask
        # Standard dtype
        mask = np.array([0, 0, 1], dtype=bool)
        assert_equal(flatten_mask(mask), mask)
        # Flexible dtype
        mask = np.array([(0, 0), (0, 1)], dtype=[('a', bool), ('b', bool)])
        test = flatten_mask(mask)
        control = np.array([0, 0, 0, 1], dtype=bool)
        assert_equal(test, control)

        mdtype = [('a', bool), ('b', [('ba', bool), ('bb', bool)])]
        data = [(0, (0, 0)), (0, (0, 1))]
        mask = np.array(data, dtype=mdtype)
        test = flatten_mask(mask)
        control = np.array([0, 0, 0, 0, 0, 1], dtype=bool)
        assert_equal(test, control)

    def test_on_ndarray(self):
        # Test functions on ndarrays
        a = np.array([1, 2, 3, 4])
        m = array(a, mask=False)
        test = anom(a)
        assert_equal(test, m.anom())
        test = reshape(a, (2, 2))
        assert_equal(test, m.reshape(2, 2))

    def test_compress(self):
        # Test compress function on ndarray and masked array
        # Address Github #2495.
        arr = np.arange(8)
        arr.shape = 4, 2
        cond = np.array([True, False, True, True])
        control = arr[[0, 2, 3]]
        test = np.ma.compress(cond, arr, axis=0)
        assert_equal(test, control)
        marr = np.ma.array(arr)
        test = np.ma.compress(cond, marr, axis=0)
        assert_equal(test, control)

    def test_compressed(self):
        # Test ma.compressed function.
        # Address gh-4026
        a = np.ma.array([1, 2])
        test = np.ma.compressed(a)
        assert_(type(test) is np.ndarray)

        # Test case when input data is ndarray subclass
        class A(np.ndarray):
            pass

        a = np.ma.array(A(shape=0))
        test = np.ma.compressed(a)
        assert_(type(test) is A)

        # Test that compress flattens
        test = np.ma.compressed([[1],[2]])
        assert_equal(test.ndim, 1)
        test = np.ma.compressed([[[[[1]]]]])
        assert_equal(test.ndim, 1)

        # Test case when input is MaskedArray subclass
        class M(MaskedArray):
            pass

        test = np.ma.compressed(M([[[]], [[]]]))
        assert_equal(test.ndim, 1)

        # with .compressed() overridden
        class M(MaskedArray):
            def compressed(self):
                return 42

        test = np.ma.compressed(M([[[]], [[]]]))
        assert_equal(test, 42)

    def test_convolve(self):
        a = masked_equal(np.arange(5), 2)
        b = np.array([1, 1])
        test = np.ma.convolve(a, b)
        assert_equal(test, masked_equal([0, 1, -1, -1, 7, 4], -1))

        test = np.ma.convolve(a, b, propagate_mask=False)
        assert_equal(test, masked_equal([0, 1, 1, 3, 7, 4], -1))

        test = np.ma.convolve([1, 1], [1, 1, 1])
        assert_equal(test, masked_equal([1, 2, 2, 1], -1))

        a = [1, 1]
        b = masked_equal([1, -1, -1, 1], -1)
        test = np.ma.convolve(a, b, propagate_mask=False)
        assert_equal(test, masked_equal([1, 1, -1, 1, 1], -1))
        test = np.ma.convolve(a, b, propagate_mask=True)
        assert_equal(test, masked_equal([-1, -1, -1, -1, -1], -1))


class TestMaskedFields:

    def setup_method(self):
        ilist = [1, 2, 3, 4, 5]
        flist = [1.1, 2.2, 3.3, 4.4, 5.5]
        slist = ['one', 'two', 'three', 'four', 'five']
        ddtype = [('a', int), ('b', float), ('c', '|S8')]
        mdtype = [('a', bool), ('b', bool), ('c', bool)]
        mask = [0, 1, 0, 0, 1]
        base = array(list(zip(ilist, flist, slist)), mask=mask, dtype=ddtype)
        self.data = dict(base=base, mask=mask, ddtype=ddtype, mdtype=mdtype)

    def test_set_records_masks(self):
        base = self.data['base']
        mdtype = self.data['mdtype']
        # Set w/ nomask or masked
        base.mask = nomask
        assert_equal_records(base._mask, np.zeros(base.shape, dtype=mdtype))
        base.mask = masked
        assert_equal_records(base._mask, np.ones(base.shape, dtype=mdtype))
        # Set w/ simple boolean
        base.mask = False
        assert_equal_records(base._mask, np.zeros(base.shape, dtype=mdtype))
        base.mask = True
        assert_equal_records(base._mask, np.ones(base.shape, dtype=mdtype))
        # Set w/ list
        base.mask = [0, 0, 0, 1, 1]
        assert_equal_records(base._mask,
                             np.array([(x, x, x) for x in [0, 0, 0, 1, 1]],
                                      dtype=mdtype))

    def test_set_record_element(self):
        # Check setting an element of a record)
        base = self.data['base']
        (base_a, base_b, base_c) = (base['a'], base['b'], base['c'])
        base[0] = (pi, pi, 'pi')

        assert_equal(base_a.dtype, int)
        assert_equal(base_a._data, [3, 2, 3, 4, 5])

        assert_equal(base_b.dtype, float)
        assert_equal(base_b._data, [pi, 2.2, 3.3, 4.4, 5.5])

        assert_equal(base_c.dtype, '|S8')
        assert_equal(base_c._data,
                     [b'pi', b'two', b'three', b'four', b'five'])

    def test_set_record_slice(self):
        base = self.data['base']
        (base_a, base_b, base_c) = (base['a'], base['b'], base['c'])
        base[:3] = (pi, pi, 'pi')

        assert_equal(base_a.dtype, int)
        assert_equal(base_a._data, [3, 3, 3, 4, 5])

        assert_equal(base_b.dtype, float)
        assert_equal(base_b._data, [pi, pi, pi, 4.4, 5.5])

        assert_equal(base_c.dtype, '|S8')
        assert_equal(base_c._data,
                     [b'pi', b'pi', b'pi', b'four', b'five'])

    def test_mask_element(self):
        "Check record access"
        base = self.data['base']
        base[0] = masked

        for n in ('a', 'b', 'c'):
            assert_equal(base[n].mask, [1, 1, 0, 0, 1])
            assert_equal(base[n]._data, base._data[n])

    def test_getmaskarray(self):
        # Test getmaskarray on flexible dtype
        ndtype = [('a', int), ('b', float)]
        test = empty(3, dtype=ndtype)
        assert_equal(getmaskarray(test),
                     np.array([(0, 0), (0, 0), (0, 0)],
                              dtype=[('a', '|b1'), ('b', '|b1')]))
        test[:] = masked
        assert_equal(getmaskarray(test),
                     np.array([(1, 1), (1, 1), (1, 1)],
                              dtype=[('a', '|b1'), ('b', '|b1')]))

    def test_view(self):
        # Test view w/ flexible dtype
        iterator = list(zip(np.arange(10), np.random.rand(10)))
        data = np.array(iterator)
        a = array(iterator, dtype=[('a', float), ('b', float)])
        a.mask[0] = (1, 0)
        controlmask = np.array([1] + 19 * [0], dtype=bool)
        # Transform globally to simple dtype
        test = a.view(float)
        assert_equal(test, data.ravel())
        assert_equal(test.mask, controlmask)
        # Transform globally to dty
        test = a.view((float, 2))
        assert_equal(test, data)
        assert_equal(test.mask, controlmask.reshape(-1, 2))

    def test_getitem(self):
        ndtype = [('a', float), ('b', float)]
        a = array(list(zip(np.random.rand(10), np.arange(10))), dtype=ndtype)
        a.mask = np.array(list(zip([0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
                                   [1, 0, 0, 0, 0, 0, 0, 0, 1, 0])),
                          dtype=[('a', bool), ('b', bool)])

        def _test_index(i):
            assert_equal(type(a[i]), mvoid)
            assert_equal_records(a[i]._data, a._data[i])
            assert_equal_records(a[i]._mask, a._mask[i])

            assert_equal(type(a[i, ...]), MaskedArray)
            assert_equal_records(a[i,...]._data, a._data[i,...])
            assert_equal_records(a[i,...]._mask, a._mask[i,...])

        _test_index(1)   # No mask
        _test_index(0)   # One element masked
        _test_index(-2)  # All element masked

    def test_setitem(self):
        # Issue 4866: check that one can set individual items in [record][col]
        # and [col][record] order
        ndtype = np.dtype([('a', float), ('b', int)])
        ma = np.ma.MaskedArray([(1.0, 1), (2.0, 2)], dtype=ndtype)
        ma['a'][1] = 3.0
        assert_equal(ma['a'], np.array([1.0, 3.0]))
        ma[1]['a'] = 4.0
        assert_equal(ma['a'], np.array([1.0, 4.0]))
        # Issue 2403
        mdtype = np.dtype([('a', bool), ('b', bool)])
        # soft mask
        control = np.array([(False, True), (True, True)], dtype=mdtype)
        a = np.ma.masked_all((2,), dtype=ndtype)
        a['a'][0] = 2
        assert_equal(a.mask, control)
        a = np.ma.masked_all((2,), dtype=ndtype)
        a[0]['a'] = 2
        assert_equal(a.mask, control)
        # hard mask
        control = np.array([(True, True), (True, True)], dtype=mdtype)
        a = np.ma.masked_all((2,), dtype=ndtype)
        a.harden_mask()
        a['a'][0] = 2
        assert_equal(a.mask, control)
        a = np.ma.masked_all((2,), dtype=ndtype)
        a.harden_mask()
        a[0]['a'] = 2
        assert_equal(a.mask, control)

    def test_setitem_scalar(self):
        # 8510
        mask_0d = np.ma.masked_array(1, mask=True)
        arr = np.ma.arange(3)
        arr[0] = mask_0d
        assert_array_equal(arr.mask, [True, False, False])

    def test_element_len(self):
        # check that len() works for mvoid (Github issue #576)
        for rec in self.data['base']:
            assert_equal(len(rec), len(self.data['ddtype']))


class TestMaskedObjectArray:

    def test_getitem(self):
        arr = np.ma.array([None, None])
        for dt in [float, object]:
            a0 = np.eye(2).astype(dt)
            a1 = np.eye(3).astype(dt)
            arr[0] = a0
            arr[1] = a1

            assert_(arr[0] is a0)
            assert_(arr[1] is a1)
            assert_(isinstance(arr[0,...], MaskedArray))
            assert_(isinstance(arr[1,...], MaskedArray))
            assert_(arr[0,...][()] is a0)
            assert_(arr[1,...][()] is a1)

            arr[0] = np.ma.masked

            assert_(arr[1] is a1)
            assert_(isinstance(arr[0,...], MaskedArray))
            assert_(isinstance(arr[1,...], MaskedArray))
            assert_equal(arr[0,...].mask, True)
            assert_(arr[1,...][()] is a1)

            # gh-5962 - object arrays of arrays do something special
            assert_equal(arr[0].data, a0)
            assert_equal(arr[0].mask, True)
            assert_equal(arr[0,...][()].data, a0)
            assert_equal(arr[0,...][()].mask, True)

    def test_nested_ma(self):

        arr = np.ma.array([None, None])
        # set the first object to be an unmasked masked constant. A little fiddly
        arr[0,...] = np.array([np.ma.masked], object)[0,...]

        # check the above line did what we were aiming for
        assert_(arr.data[0] is np.ma.masked)

        # test that getitem returned the value by identity
        assert_(arr[0] is np.ma.masked)

        # now mask the masked value!
        arr[0] = np.ma.masked
        assert_(arr[0] is np.ma.masked)


class TestMaskedView:

    def setup_method(self):
        iterator = list(zip(np.arange(10), np.random.rand(10)))
        data = np.array(iterator)
        a = array(iterator, dtype=[('a', float), ('b', float)])
        a.mask[0] = (1, 0)
        controlmask = np.array([1] + 19 * [0], dtype=bool)
        self.data = (data, a, controlmask)

    def test_view_to_nothing(self):
        (data, a, controlmask) = self.data
        test = a.view()
        assert_(isinstance(test, MaskedArray))
        assert_equal(test._data, a._data)
        assert_equal(test._mask, a._mask)

    def test_view_to_type(self):
        (data, a, controlmask) = self.data
        test = a.view(np.ndarray)
        assert_(not isinstance(test, MaskedArray))
        assert_equal(test, a._data)
        assert_equal_records(test, data.view(a.dtype).squeeze())

    def test_view_to_simple_dtype(self):
        (data, a, controlmask) = self.data
        # View globally
        test = a.view(float)
        assert_(isinstance(test, MaskedArray))
        assert_equal(test, data.ravel())
        assert_equal(test.mask, controlmask)

    def test_view_to_flexible_dtype(self):
        (data, a, controlmask) = self.data

        test = a.view([('A', float), ('B', float)])
        assert_equal(test.mask.dtype.names, ('A', 'B'))
        assert_equal(test['A'], a['a'])
        assert_equal(test['B'], a['b'])

        test = a[0].view([('A', float), ('B', float)])
        assert_(isinstance(test, MaskedArray))
        assert_equal(test.mask.dtype.names, ('A', 'B'))
        assert_equal(test['A'], a['a'][0])
        assert_equal(test['B'], a['b'][0])

        test = a[-1].view([('A', float), ('B', float)])
        assert_(isinstance(test, MaskedArray))
        assert_equal(test.dtype.names, ('A', 'B'))
        assert_equal(test['A'], a['a'][-1])
        assert_equal(test['B'], a['b'][-1])

    def test_view_to_subdtype(self):
        (data, a, controlmask) = self.data
        # View globally
        test = a.view((float, 2))
        assert_(isinstance(test, MaskedArray))
        assert_equal(test, data)
        assert_equal(test.mask, controlmask.reshape(-1, 2))
        # View on 1 masked element
        test = a[0].view((float, 2))
        assert_(isinstance(test, MaskedArray))
        assert_equal(test, data[0])
        assert_equal(test.mask, (1, 0))
        # View on 1 unmasked element
        test = a[-1].view((float, 2))
        assert_(isinstance(test, MaskedArray))
        assert_equal(test, data[-1])

    def test_view_to_dtype_and_type(self):
        (data, a, controlmask) = self.data

        test = a.view((float, 2), np.recarray)
        assert_equal(test, data)
        assert_(isinstance(test, np.recarray))
        assert_(not isinstance(test, MaskedArray))


class TestOptionalArgs:
    def test_ndarrayfuncs(self):
        # test axis arg behaves the same as ndarray (including multiple axes)

        d = np.arange(24.0).reshape((2,3,4))
        m = np.zeros(24, dtype=bool).reshape((2,3,4))
        # mask out last element of last dimension
        m[:,:,-1] = True
        a = np.ma.array(d, mask=m)

        def testaxis(f, a, d):
            numpy_f = numpy.__getattribute__(f)
            ma_f = np.ma.__getattribute__(f)

            # test axis arg
            assert_equal(ma_f(a, axis=1)[...,:-1], numpy_f(d[...,:-1], axis=1))
            assert_equal(ma_f(a, axis=(0,1))[...,:-1],
                         numpy_f(d[...,:-1], axis=(0,1)))

        def testkeepdims(f, a, d):
            numpy_f = numpy.__getattribute__(f)
            ma_f = np.ma.__getattribute__(f)

            # test keepdims arg
            assert_equal(ma_f(a, keepdims=True).shape,
                         numpy_f(d, keepdims=True).shape)
            assert_equal(ma_f(a, keepdims=False).shape,
                         numpy_f(d, keepdims=False).shape)

            # test both at once
            assert_equal(ma_f(a, axis=1, keepdims=True)[...,:-1],
                         numpy_f(d[...,:-1], axis=1, keepdims=True))
            assert_equal(ma_f(a, axis=(0,1), keepdims=True)[...,:-1],
                         numpy_f(d[...,:-1], axis=(0,1), keepdims=True))

        for f in ['sum', 'prod', 'mean', 'var', 'std']:
            testaxis(f, a, d)
            testkeepdims(f, a, d)

        for f in ['min', 'max']:
            testaxis(f, a, d)

        d = (np.arange(24).reshape((2,3,4))%2 == 0)
        a = np.ma.array(d, mask=m)
        for f in ['all', 'any']:
            testaxis(f, a, d)
            testkeepdims(f, a, d)

    def test_count(self):
        # test np.ma.count specially

        d = np.arange(24.0).reshape((2,3,4))
        m = np.zeros(24, dtype=bool).reshape((2,3,4))
        m[:,0,:] = True
        a = np.ma.array(d, mask=m)

        assert_equal(count(a), 16)
        assert_equal(count(a, axis=1), 2*ones((2,4)))
        assert_equal(count(a, axis=(0,1)), 4*ones((4,)))
        assert_equal(count(a, keepdims=True), 16*ones((1,1,1)))
        assert_equal(count(a, axis=1, keepdims=True), 2*ones((2,1,4)))
        assert_equal(count(a, axis=(0,1), keepdims=True), 4*ones((1,1,4)))
        assert_equal(count(a, axis=-2), 2*ones((2,4)))
        assert_raises(ValueError, count, a, axis=(1,1))
        assert_raises(np.AxisError, count, a, axis=3)

        # check the 'nomask' path
        a = np.ma.array(d, mask=nomask)

        assert_equal(count(a), 24)
        assert_equal(count(a, axis=1), 3*ones((2,4)))
        assert_equal(count(a, axis=(0,1)), 6*ones((4,)))
        assert_equal(count(a, keepdims=True), 24*ones((1,1,1)))
        assert_equal(np.ndim(count(a, keepdims=True)), 3)
        assert_equal(count(a, axis=1, keepdims=True), 3*ones((2,1,4)))
        assert_equal(count(a, axis=(0,1), keepdims=True), 6*ones((1,1,4)))
        assert_equal(count(a, axis=-2), 3*ones((2,4)))
        assert_raises(ValueError, count, a, axis=(1,1))
        assert_raises(np.AxisError, count, a, axis=3)

        # check the 'masked' singleton
        assert_equal(count(np.ma.masked), 0)

        # check 0-d arrays do not allow axis > 0
        assert_raises(np.AxisError, count, np.ma.array(1), axis=1)


class TestMaskedConstant:
    def _do_add_test(self, add):
        # sanity check
        assert_(add(np.ma.masked, 1) is np.ma.masked)

        # now try with a vector
        vector = np.array([1, 2, 3])
        result = add(np.ma.masked, vector)

        # lots of things could go wrong here
        assert_(result is not np.ma.masked)
        assert_(not isinstance(result, np.ma.core.MaskedConstant))
        assert_equal(result.shape, vector.shape)
        assert_equal(np.ma.getmask(result), np.ones(vector.shape, dtype=bool))

    def test_ufunc(self):
        self._do_add_test(np.add)

    def test_operator(self):
        self._do_add_test(lambda a, b: a + b)

    def test_ctor(self):
        m = np.ma.array(np.ma.masked)

        # most importantly, we do not want to create a new MaskedConstant
        # instance
        assert_(not isinstance(m, np.ma.core.MaskedConstant))
        assert_(m is not np.ma.masked)

    def test_repr(self):
        # copies should not exist, but if they do, it should be obvious that
        # something is wrong
        assert_equal(repr(np.ma.masked), 'masked')

        # create a new instance in a weird way
        masked2 = np.ma.MaskedArray.__new__(np.ma.core.MaskedConstant)
        assert_not_equal(repr(masked2), 'masked')

    def test_pickle(self):
        from io import BytesIO

        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            with BytesIO() as f:
                pickle.dump(np.ma.masked, f, protocol=proto)
                f.seek(0)
                res = pickle.load(f)
            assert_(res is np.ma.masked)

    def test_copy(self):
        # gh-9328
        # copy is a no-op, like it is with np.True_
        assert_equal(
            np.ma.masked.copy() is np.ma.masked,
            np.True_.copy() is np.True_)

    def test__copy(self):
        import copy
        assert_(
            copy.copy(np.ma.masked) is np.ma.masked)

    def test_deepcopy(self):
        import copy
        assert_(
            copy.deepcopy(np.ma.masked) is np.ma.masked)

    def test_immutable(self):
        orig = np.ma.masked
        assert_raises(np.ma.core.MaskError, operator.setitem, orig, (), 1)
        assert_raises(ValueError,operator.setitem, orig.data, (), 1)
        assert_raises(ValueError, operator.setitem, orig.mask, (), False)

        view = np.ma.masked.view(np.ma.MaskedArray)
        assert_raises(ValueError, operator.setitem, view, (), 1)
        assert_raises(ValueError, operator.setitem, view.data, (), 1)
        assert_raises(ValueError, operator.setitem, view.mask, (), False)

    def test_coercion_int(self):
        a_i = np.zeros((), int)
        assert_raises(MaskError, operator.setitem, a_i, (), np.ma.masked)
        assert_raises(MaskError, int, np.ma.masked)

    def test_coercion_float(self):
        a_f = np.zeros((), float)
        assert_warns(UserWarning, operator.setitem, a_f, (), np.ma.masked)
        assert_(np.isnan(a_f[()]))

    @pytest.mark.xfail(reason="See gh-9750")
    def test_coercion_unicode(self):
        a_u = np.zeros((), 'U10')
        a_u[()] = np.ma.masked
        assert_equal(a_u[()], '--')

    @pytest.mark.xfail(reason="See gh-9750")
    def test_coercion_bytes(self):
        a_b = np.zeros((), 'S10')
        a_b[()] = np.ma.masked
        assert_equal(a_b[()], b'--')

    def test_subclass(self):
        # https://github.com/astropy/astropy/issues/6645
        class Sub(type(np.ma.masked)): pass

        a = Sub()
        assert_(a is Sub())
        assert_(a is not np.ma.masked)
        assert_not_equal(repr(a), 'masked')

    def test_attributes_readonly(self):
        assert_raises(AttributeError, setattr, np.ma.masked, 'shape', (1,))
        assert_raises(AttributeError, setattr, np.ma.masked, 'dtype', np.int64)


class TestMaskedWhereAliases:

    # TODO: Test masked_object, masked_equal, ...

    def test_masked_values(self):
        res = masked_values(np.array([-32768.0]), np.int16(-32768))
        assert_equal(res.mask, [True])

        res = masked_values(np.inf, np.inf)
        assert_equal(res.mask, True)

        res = np.ma.masked_values(np.inf, -np.inf)
        assert_equal(res.mask, False)

        res = np.ma.masked_values([1, 2, 3, 4], 5, shrink=True)
        assert_(res.mask is np.ma.nomask)

        res = np.ma.masked_values([1, 2, 3, 4], 5, shrink=False)
        assert_equal(res.mask, [False] * 4)


def test_masked_array():
    a = np.ma.array([0, 1, 2, 3], mask=[0, 0, 1, 0])
    assert_equal(np.argwhere(a), [[1], [3]])

def test_masked_array_no_copy():
    # check nomask array is updated in place
    a = np.ma.array([1, 2, 3, 4])
    _ = np.ma.masked_where(a == 3, a, copy=False)
    assert_array_equal(a.mask, [False, False, True, False])
    # check masked array is updated in place
    a = np.ma.array([1, 2, 3, 4], mask=[1, 0, 0, 0])
    _ = np.ma.masked_where(a == 3, a, copy=False)
    assert_array_equal(a.mask, [True, False, True, False])
    # check masked array with masked_invalid is updated in place
    a = np.ma.array([np.inf, 1, 2, 3, 4])
    _ = np.ma.masked_invalid(a, copy=False)
    assert_array_equal(a.mask, [True, False, False, False, False])

def test_append_masked_array():
    a = np.ma.masked_equal([1,2,3], value=2)
    b = np.ma.masked_equal([4,3,2], value=2)

    result = np.ma.append(a, b)
    expected_data = [1, 2, 3, 4, 3, 2]
    expected_mask = [False, True, False, False, False, True]
    assert_array_equal(result.data, expected_data)
    assert_array_equal(result.mask, expected_mask)

    a = np.ma.masked_all((2,2))
    b = np.ma.ones((3,1))

    result = np.ma.append(a, b)
    expected_data = [1] * 3
    expected_mask = [True] * 4 + [False] * 3
    assert_array_equal(result.data[-3], expected_data)
    assert_array_equal(result.mask, expected_mask)

    result = np.ma.append(a, b, axis=None)
    assert_array_equal(result.data[-3], expected_data)
    assert_array_equal(result.mask, expected_mask)


def test_append_masked_array_along_axis():
    a = np.ma.masked_equal([1,2,3], value=2)
    b = np.ma.masked_values([[4, 5, 6], [7, 8, 9]], 7)

    # When `axis` is specified, `values` must have the correct shape.
    assert_raises(ValueError, np.ma.append, a, b, axis=0)

    result = np.ma.append(a[np.newaxis,:], b, axis=0)
    expected = np.ma.arange(1, 10)
    expected[[1, 6]] = np.ma.masked
    expected = expected.reshape((3,3))
    assert_array_equal(result.data, expected.data)
    assert_array_equal(result.mask, expected.mask)

def test_default_fill_value_complex():
    # regression test for Python 3, where 'unicode' was not defined
    assert_(default_fill_value(1 + 1j) == 1.e20 + 0.0j)


def test_ufunc_with_output():
    # check that giving an output argument always returns that output.
    # Regression test for gh-8416.
    x = array([1., 2., 3.], mask=[0, 0, 1])
    y = np.add(x, 1., out=x)
    assert_(y is x)


def test_ufunc_with_out_varied():
    """ Test that masked arrays are immune to gh-10459 """
    # the mask of the output should not affect the result, however it is passed
    a        = array([ 1,  2,  3], mask=[1, 0, 0])
    b        = array([10, 20, 30], mask=[1, 0, 0])
    out      = array([ 0,  0,  0], mask=[0, 0, 1])
    expected = array([11, 22, 33], mask=[1, 0, 0])

    out_pos = out.copy()
    res_pos = np.add(a, b, out_pos)

    out_kw = out.copy()
    res_kw = np.add(a, b, out=out_kw)

    out_tup = out.copy()
    res_tup = np.add(a, b, out=(out_tup,))

    assert_equal(res_kw.mask,  expected.mask)
    assert_equal(res_kw.data,  expected.data)
    assert_equal(res_tup.mask, expected.mask)
    assert_equal(res_tup.data, expected.data)
    assert_equal(res_pos.mask, expected.mask)
    assert_equal(res_pos.data, expected.data)


def test_astype_mask_ordering():
    descr = np.dtype([('v', int, 3), ('x', [('y', float)])])
    x = array([
        [([1, 2, 3], (1.0,)),  ([1, 2, 3], (2.0,))],
        [([1, 2, 3], (3.0,)),  ([1, 2, 3], (4.0,))]], dtype=descr)
    x[0]['v'][0] = np.ma.masked

    x_a = x.astype(descr)
    assert x_a.dtype.names == np.dtype(descr).names
    assert x_a.mask.dtype.names == np.dtype(descr).names
    assert_equal(x, x_a)

    assert_(x is x.astype(x.dtype, copy=False))
    assert_equal(type(x.astype(x.dtype, subok=False)), np.ndarray)

    x_f = x.astype(x.dtype, order='F')
    assert_(x_f.flags.f_contiguous)
    assert_(x_f.mask.flags.f_contiguous)

    # Also test the same indirectly, via np.array
    x_a2 = np.array(x, dtype=descr, subok=True)
    assert x_a2.dtype.names == np.dtype(descr).names
    assert x_a2.mask.dtype.names == np.dtype(descr).names
    assert_equal(x, x_a2)

    assert_(x is np.array(x, dtype=descr, copy=False, subok=True))

    x_f2 = np.array(x, dtype=x.dtype, order='F', subok=True)
    assert_(x_f2.flags.f_contiguous)
    assert_(x_f2.mask.flags.f_contiguous)


@pytest.mark.parametrize('dt1', num_dts, ids=num_ids)
@pytest.mark.parametrize('dt2', num_dts, ids=num_ids)
@pytest.mark.filterwarnings('ignore::numpy.ComplexWarning')
def test_astype_basic(dt1, dt2):
    # See gh-12070
    src = np.ma.array(ones(3, dt1), fill_value=1)
    dst = src.astype(dt2)

    assert_(src.fill_value == 1)
    assert_(src.dtype == dt1)
    assert_(src.fill_value.dtype == dt1)

    assert_(dst.fill_value == 1)
    assert_(dst.dtype == dt2)
    assert_(dst.fill_value.dtype == dt2)

    assert_equal(src, dst)


def test_fieldless_void():
    dt = np.dtype([])  # a void dtype with no fields
    x = np.empty(4, dt)

    # these arrays contain no values, so there's little to test - but this
    # shouldn't crash
    mx = np.ma.array(x)
    assert_equal(mx.dtype, x.dtype)
    assert_equal(mx.shape, x.shape)

    mx = np.ma.array(x, mask=x)
    assert_equal(mx.dtype, x.dtype)
    assert_equal(mx.shape, x.shape)


def test_mask_shape_assignment_does_not_break_masked():
    a = np.ma.masked
    b = np.ma.array(1, mask=a.mask)
    b.shape = (1,)
    assert_equal(a.mask.shape, ())

@pytest.mark.skipif(sys.flags.optimize > 1,
                    reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1")
def test_doc_note():
    def method(self):
        """This docstring

        Has multiple lines

        And notes

        Notes
        -----
        original note
        """
        pass

    expected_doc = """This docstring

Has multiple lines

And notes

Notes
-----
note

original note"""

    assert_equal(np.ma.core.doc_note(method.__doc__, "note"), expected_doc)

env_pyrecodes/lib/python3.9/site-packages/numpy/ma/tests/test_core.py
NikolaBlagojevic/pyrecodes
import sys
import sysconfig
import subprocess
import pkgutil
import types
import importlib
import warnings

import numpy as np
import numpy
import pytest
from numpy.testing import IS_WASM

try:
    import ctypes
except ImportError:
    ctypes = None


def check_dir(module, module_name=None):
    """Returns a mapping of all objects with the wrong __module__ attribute."""
    if module_name is None:
        module_name = module.__name__
    results = {}
    for name in dir(module):
        item = getattr(module, name)
        if (hasattr(item, '__module__') and hasattr(item, '__name__')
                and item.__module__ != module_name):
            results[name] = item.__module__ + '.' + item.__name__
    return results


def test_numpy_namespace():
    # None of these objects are publicly documented to be part of the main
    # NumPy namespace (some are useful though, others need to be cleaned up)
    undocumented = {
        'Tester': 'numpy.testing._private.nosetester.NoseTester',
        '_add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc',
        'add_docstring': 'numpy.core._multiarray_umath.add_docstring',
        'add_newdoc': 'numpy.core.function_base.add_newdoc',
        'add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc',
        'byte_bounds': 'numpy.lib.utils.byte_bounds',
        'compare_chararrays': 'numpy.core._multiarray_umath.compare_chararrays',
        'deprecate': 'numpy.lib.utils.deprecate',
        'deprecate_with_doc': 'numpy.lib.utils.deprecate_with_doc',
        'disp': 'numpy.lib.function_base.disp',
        'fastCopyAndTranspose': 'numpy.core._multiarray_umath.fastCopyAndTranspose',
        'get_array_wrap': 'numpy.lib.shape_base.get_array_wrap',
        'get_include': 'numpy.lib.utils.get_include',
        'recfromcsv': 'numpy.lib.npyio.recfromcsv',
        'recfromtxt': 'numpy.lib.npyio.recfromtxt',
        'safe_eval': 'numpy.lib.utils.safe_eval',
        'set_string_function': 'numpy.core.arrayprint.set_string_function',
        'show_config': 'numpy.__config__.show',
        'show_runtime': 'numpy.lib.utils.show_runtime',
        'who': 'numpy.lib.utils.who',
    }
    # We override dir to not show these members
    allowlist = undocumented
    bad_results = check_dir(np)
    # pytest gives better error messages with the builtin assert than with
    # assert_equal
    assert bad_results == allowlist


@pytest.mark.skipif(IS_WASM, reason="can't start subprocess")
@pytest.mark.parametrize('name', ['testing', 'Tester'])
def test_import_lazy_import(name):
    """Make sure we can actually use the modules we lazy load.

    While not exported as part of the public API, it was accessible.  With the
    use of __getattr__ and __dir__, this isn't always true It can happen that
    an infinite recursion may happen.

    This is the only way I found that would force the failure to appear on the
    badly implemented code.

    We also test for the presence of the lazily imported modules in dir

    """
    exe = (sys.executable, '-c', "import numpy; numpy." + name)
    result = subprocess.check_output(exe)
    assert not result

    # Make sure they are still in the __dir__
    assert name in dir(np)


def test_dir_testing():
    """Assert that output of dir has only one "testing/tester"
    attribute without duplicate"""
    assert len(dir(np)) == len(set(dir(np)))


def test_numpy_linalg():
    bad_results = check_dir(np.linalg)
    assert bad_results == {}


def test_numpy_fft():
    bad_results = check_dir(np.fft)
    assert bad_results == {}


@pytest.mark.skipif(ctypes is None,
                    reason="ctypes not available in this python")
def test_NPY_NO_EXPORT():
    cdll = ctypes.CDLL(np.core._multiarray_tests.__file__)
    # Make sure an arbitrary NPY_NO_EXPORT function is actually hidden
    f = getattr(cdll, 'test_not_exported', None)
    assert f is None, ("'test_not_exported' is mistakenly exported, "
                      "NPY_NO_EXPORT does not work")


# Historically NumPy has not used leading underscores for private submodules
# much.  This has resulted in lots of things that look like public modules
# (i.e. things that can be imported as `import numpy.somesubmodule.somefile`),
# but were never intended to be public.  The PUBLIC_MODULES list contains
# modules that are either public because they were meant to be, or because they
# contain public functions/objects that aren't present in any other namespace
# for whatever reason and therefore should be treated as public.
#
# The PRIVATE_BUT_PRESENT_MODULES list contains modules that look public (lack
# of underscores) but should not be used.  For many of those modules the
# current status is fine.  For others it may make sense to work on making them
# private, to clean up our public API and avoid confusion.
PUBLIC_MODULES = ['numpy.' + s for s in [
    "array_api",
    "array_api.linalg",
    "ctypeslib",
    "distutils",
    "distutils.cpuinfo",
    "distutils.exec_command",
    "distutils.misc_util",
    "distutils.log",
    "distutils.system_info",
    "doc",
    "doc.constants",
    "doc.ufuncs",
    "f2py",
    "fft",
    "lib",
    "lib.format",  # was this meant to be public?
    "lib.mixins",
    "lib.recfunctions",
    "lib.scimath",
    "lib.stride_tricks",
    "linalg",
    "ma",
    "ma.extras",
    "ma.mrecords",
    "matlib",
    "polynomial",
    "polynomial.chebyshev",
    "polynomial.hermite",
    "polynomial.hermite_e",
    "polynomial.laguerre",
    "polynomial.legendre",
    "polynomial.polynomial",
    "random",
    "testing",
    "typing",
    "typing.mypy_plugin",
    "version",
]]


PUBLIC_ALIASED_MODULES = [
    "numpy.char",
    "numpy.emath",
    "numpy.rec",
]


PRIVATE_BUT_PRESENT_MODULES = ['numpy.' + s for s in [
    "compat",
    "compat.py3k",
    "conftest",
    "core",
    "core.arrayprint",
    "core.defchararray",
    "core.einsumfunc",
    "core.fromnumeric",
    "core.function_base",
    "core.getlimits",
    "core.memmap",
    "core.multiarray",
    "core.numeric",
    "core.numerictypes",
    "core.overrides",
    "core.records",
    "core.shape_base",
    "core.umath",
    "core.umath_tests",
    "distutils.armccompiler",
    "distutils.ccompiler",
    'distutils.ccompiler_opt',
    "distutils.command",
    "distutils.command.autodist",
    "distutils.command.bdist_rpm",
    "distutils.command.build",
    "distutils.command.build_clib",
    "distutils.command.build_ext",
    "distutils.command.build_py",
    "distutils.command.build_scripts",
    "distutils.command.build_src",
    "distutils.command.config",
    "distutils.command.config_compiler",
    "distutils.command.develop",
    "distutils.command.egg_info",
    "distutils.command.install",
    "distutils.command.install_clib",
    "distutils.command.install_data",
    "distutils.command.install_headers",
    "distutils.command.sdist",
    "distutils.conv_template",
    "distutils.core",
    "distutils.extension",
    "distutils.fcompiler",
    "distutils.fcompiler.absoft",
    "distutils.fcompiler.arm",
    "distutils.fcompiler.compaq",
    "distutils.fcompiler.environment",
    "distutils.fcompiler.g95",
    "distutils.fcompiler.gnu",
    "distutils.fcompiler.hpux",
    "distutils.fcompiler.ibm",
    "distutils.fcompiler.intel",
    "distutils.fcompiler.lahey",
    "distutils.fcompiler.mips",
    "distutils.fcompiler.nag",
    "distutils.fcompiler.none",
    "distutils.fcompiler.pathf95",
    "distutils.fcompiler.pg",
    "distutils.fcompiler.nv",
    "distutils.fcompiler.sun",
    "distutils.fcompiler.vast",
    "distutils.fcompiler.fujitsu",
    "distutils.from_template",
    "distutils.intelccompiler",
    "distutils.lib2def",
    "distutils.line_endings",
    "distutils.mingw32ccompiler",
    "distutils.msvccompiler",
    "distutils.npy_pkg_config",
    "distutils.numpy_distribution",
    "distutils.pathccompiler",
    "distutils.unixccompiler",
    "dual",
    "f2py.auxfuncs",
    "f2py.capi_maps",
    "f2py.cb_rules",
    "f2py.cfuncs",
    "f2py.common_rules",
    "f2py.crackfortran",
    "f2py.diagnose",
    "f2py.f2py2e",
    "f2py.f90mod_rules",
    "f2py.func2subr",
    "f2py.rules",
    "f2py.symbolic",
    "f2py.use_rules",
    "fft.helper",
    "lib.arraypad",
    "lib.arraysetops",
    "lib.arrayterator",
    "lib.function_base",
    "lib.histograms",
    "lib.index_tricks",
    "lib.nanfunctions",
    "lib.npyio",
    "lib.polynomial",
    "lib.shape_base",
    "lib.twodim_base",
    "lib.type_check",
    "lib.ufunclike",
    "lib.user_array",  # note: not in np.lib, but probably should just be deleted
    "lib.utils",
    "linalg.lapack_lite",
    "linalg.linalg",
    "ma.bench",
    "ma.core",
    "ma.testutils",
    "ma.timer_comparison",
    "matrixlib",
    "matrixlib.defmatrix",
    "polynomial.polyutils",
    "random.mtrand",
    "random.bit_generator",
    "testing.print_coercion_tables",
    "testing.utils",
]]


def is_unexpected(name):
    """Check if this needs to be considered."""
    if '._' in name or '.tests' in name or '.setup' in name:
        return False

    if name in PUBLIC_MODULES:
        return False

    if name in PUBLIC_ALIASED_MODULES:
        return False

    if name in PRIVATE_BUT_PRESENT_MODULES:
        return False

    return True


# These are present in a directory with an __init__.py but cannot be imported
# code_generators/ isn't installed, but present for an inplace build
SKIP_LIST = [
    "numpy.core.code_generators",
    "numpy.core.code_generators.genapi",
    "numpy.core.code_generators.generate_umath",
    "numpy.core.code_generators.ufunc_docstrings",
    "numpy.core.code_generators.generate_numpy_api",
    "numpy.core.code_generators.generate_ufunc_api",
    "numpy.core.code_generators.numpy_api",
    "numpy.core.code_generators.generate_umath_doc",
    "numpy.core.cversions",
    "numpy.core.generate_numpy_api",
    "numpy.distutils.msvc9compiler",
]


def test_all_modules_are_expected():
    """
    Test that we don't add anything that looks like a new public module by
    accident.  Check is based on filenames.
    """

    modnames = []
    for _, modname, ispkg in pkgutil.walk_packages(path=np.__path__,
                                                   prefix=np.__name__ + '.',
                                                   onerror=None):
        if is_unexpected(modname) and modname not in SKIP_LIST:
            # We have a name that is new.  If that's on purpose, add it to
            # PUBLIC_MODULES.  We don't expect to have to add anything to
            # PRIVATE_BUT_PRESENT_MODULES.  Use an underscore in the name!
            modnames.append(modname)

    if modnames:
        raise AssertionError(f'Found unexpected modules: {modnames}')


# Stuff that clearly shouldn't be in the API and is detected by the next test
# below
SKIP_LIST_2 = [
    'numpy.math',
    'numpy.distutils.log.sys',
    'numpy.distutils.log.logging',
    'numpy.distutils.log.warnings',
    'numpy.doc.constants.re',
    'numpy.doc.constants.textwrap',
    'numpy.lib.emath',
    'numpy.lib.math',
    'numpy.matlib.char',
    'numpy.matlib.rec',
    'numpy.matlib.emath',
    'numpy.matlib.math',
    'numpy.matlib.linalg',
    'numpy.matlib.fft',
    'numpy.matlib.random',
    'numpy.matlib.ctypeslib',
    'numpy.matlib.ma',
]


def test_all_modules_are_expected_2():
    """
    Method checking all objects. The pkgutil-based method in
    `test_all_modules_are_expected` does not catch imports into a namespace,
    only filenames.  So this test is more thorough, and checks this like:

        import .lib.scimath as emath

    To check if something in a module is (effectively) public, one can check if
    there's anything in that namespace that's a public function/object but is
    not exposed in a higher-level namespace.  For example for a `numpy.lib`
    submodule::

        mod = np.lib.mixins
        for obj in mod.__all__:
            if obj in np.__all__:
                continue
            elif obj in np.lib.__all__:
                continue

            else:
                print(obj)

    """

    def find_unexpected_members(mod_name):
        members = []
        module = importlib.import_module(mod_name)
        if hasattr(module, '__all__'):
            objnames = module.__all__
        else:
            objnames = dir(module)

        for objname in objnames:
            if not objname.startswith('_'):
                fullobjname = mod_name + '.' + objname
                if isinstance(getattr(module, objname), types.ModuleType):
                    if is_unexpected(fullobjname):
                        if fullobjname not in SKIP_LIST_2:
                            members.append(fullobjname)

        return members

    unexpected_members = find_unexpected_members("numpy")
    for modname in PUBLIC_MODULES:
        unexpected_members.extend(find_unexpected_members(modname))

    if unexpected_members:
        raise AssertionError("Found unexpected object(s) that look like "
                             "modules: {}".format(unexpected_members))


def test_api_importable():
    """
    Check that all submodules listed higher up in this file can be imported

    Note that if a PRIVATE_BUT_PRESENT_MODULES entry goes missing, it may
    simply need to be removed from the list (deprecation may or may not be
    needed - apply common sense).
    """
    def check_importable(module_name):
        try:
            importlib.import_module(module_name)
        except (ImportError, AttributeError):
            return False

        return True

    module_names = []
    for module_name in PUBLIC_MODULES:
        if not check_importable(module_name):
            module_names.append(module_name)

    if module_names:
        raise AssertionError("Modules in the public API that cannot be "
                             "imported: {}".format(module_names))

    for module_name in PUBLIC_ALIASED_MODULES:
        try:
            eval(module_name)
        except AttributeError:
            module_names.append(module_name)

    if module_names:
        raise AssertionError("Modules in the public API that were not "
                             "found: {}".format(module_names))

    with warnings.catch_warnings(record=True) as w:
        warnings.filterwarnings('always', category=DeprecationWarning)
        warnings.filterwarnings('always', category=ImportWarning)
        for module_name in PRIVATE_BUT_PRESENT_MODULES:
            if not check_importable(module_name):
                module_names.append(module_name)

    if module_names:
        raise AssertionError("Modules that are not really public but looked "
                             "public and can not be imported: "
                             "{}".format(module_names))


@pytest.mark.xfail(
    sysconfig.get_config_var("Py_DEBUG") is not None,
    reason=(
        "NumPy possibly built with `USE_DEBUG=True ./tools/travis-test.sh`, "
        "which does not expose the `array_api` entry point. "
        "See https://github.com/numpy/numpy/pull/19800"
    ),
)
def test_array_api_entry_point():
    """
    Entry point for Array API implementation can be found with importlib and
    returns the numpy.array_api namespace.
    """
    eps = importlib.metadata.entry_points()
    try:
        xp_eps = eps.select(group="array_api")
    except AttributeError:
        # The select interface for entry_points was introduced in py3.10,
        # deprecating its dict interface. We fallback to dict keys for finding
        # Array API entry points so that running this test in <=3.9 will
        # still work - see https://github.com/numpy/numpy/pull/19800.
        xp_eps = eps.get("array_api", [])
    assert len(xp_eps) > 0, "No entry points for 'array_api' found"

    try:
        ep = next(ep for ep in xp_eps if ep.name == "numpy")
    except StopIteration:
        raise AssertionError("'numpy' not in array_api entry points") from None

    xp = ep.load()
    msg = (
        f"numpy entry point value '{ep.value}' "
        "does not point to our Array API implementation"
    )
    assert xp is numpy.array_api, msg

env_pyrecodes/lib/python3.9/site-packages/numpy/tests/test_public_api.py
NikolaBlagojevic/pyrecodes
import os
import sys
import stat
import select
import time
import errno

try:
    InterruptedError
except NameError:
    # Alias Python2 exception to Python3
    InterruptedError = select.error

if sys.version_info[0] >= 3:
    string_types = (str,)
else:
    string_types = (unicode, str)


def is_executable_file(path):
    """Checks that path is an executable regular file, or a symlink towards one.

    This is roughly ``os.path isfile(path) and os.access(path, os.X_OK)``.
    """
    # follow symlinks,
    fpath = os.path.realpath(path)

    if not os.path.isfile(fpath):
        # non-files (directories, fifo, etc.)
        return False

    mode = os.stat(fpath).st_mode

    if (sys.platform.startswith('sunos')
            and os.getuid() == 0):
        # When root on Solaris, os.X_OK is True for *all* files, irregardless
        # of their executability -- instead, any permission bit of any user,
        # group, or other is fine enough.
        #
        # (This may be true for other "Unix98" OS's such as HP-UX and AIX)
        return bool(mode & (stat.S_IXUSR |
                            stat.S_IXGRP |
                            stat.S_IXOTH))

    return os.access(fpath, os.X_OK)


def which(filename, env=None):
    '''This takes a given filename; tries to find it in the environment path;
    then checks if it is executable. This returns the full path to the filename
    if found and executable. Otherwise this returns None.'''

    # Special case where filename contains an explicit path.
    if os.path.dirname(filename) != '' and is_executable_file(filename):
        return filename
    if env is None:
        env = os.environ
    p = env.get('PATH')
    if not p:
        p = os.defpath
    pathlist = p.split(os.pathsep)
    for path in pathlist:
        ff = os.path.join(path, filename)
        if is_executable_file(ff):
            return ff
    return None


def split_command_line(command_line):

    '''This splits a command line into a list of arguments. It splits arguments
    on spaces, but handles embedded quotes, doublequotes, and escaped
    characters. It's impossible to do this with a regular expression, so I
    wrote a little state machine to parse the command line. '''

    arg_list = []
    arg = ''

    # Constants to name the states we can be in.
    state_basic = 0
    state_esc = 1
    state_singlequote = 2
    state_doublequote = 3
    # The state when consuming whitespace between commands.
    state_whitespace = 4
    state = state_basic

    for c in command_line:
        if state == state_basic or state == state_whitespace:
            if c == '\\':
                # Escape the next character
                state = state_esc
            elif c == r"'":
                # Handle single quote
                state = state_singlequote
            elif c == r'"':
                # Handle double quote
                state = state_doublequote
            elif c.isspace():
                # Add arg to arg_list if we aren't in the middle of whitespace.
                if state == state_whitespace:
                    # Do nothing.
                    None
                else:
                    arg_list.append(arg)
                    arg = ''
                    state = state_whitespace
            else:
                arg = arg + c
                state = state_basic
        elif state == state_esc:
            arg = arg + c
            state = state_basic
        elif state == state_singlequote:
            if c == r"'":
                state = state_basic
            else:
                arg = arg + c
        elif state == state_doublequote:
            if c == r'"':
                state = state_basic
            else:
                arg = arg + c

    if arg != '':
        arg_list.append(arg)
    return arg_list


def select_ignore_interrupts(iwtd, owtd, ewtd, timeout=None):

    '''This is a wrapper around select.select() that ignores signals. If
    select.select raises a select.error exception and errno is an EINTR
    error then it is ignored. Mainly this is used to ignore sigwinch
    (terminal resize). '''

    # if select() is interrupted by a signal (errno==EINTR) then
    # we loop back and enter the select() again.
    if timeout is not None:
        end_time = time.time() + timeout
    while True:
        try:
            return select.select(iwtd, owtd, ewtd, timeout)
        except InterruptedError:
            err = sys.exc_info()[1]
            if err.args[0] == errno.EINTR:
                # if we loop back we have to subtract the
                # amount of time we already waited.
                if timeout is not None:
                    timeout = end_time - time.time()
                    if timeout < 0:
                        return([], [], [])
            else:
                # something else caused the select.error, so
                # this actually is an exception.
                raise


def poll_ignore_interrupts(fds, timeout=None):
    '''Simple wrapper around poll to register file descriptors and
    ignore signals.'''

    if timeout is not None:
        end_time = time.time() + timeout

    poller = select.poll()
    for fd in fds:
        poller.register(fd, select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR)

    while True:
        try:
            timeout_ms = None if timeout is None else timeout * 1000
            results = poller.poll(timeout_ms)
            return [afd for afd, _ in results]
        except InterruptedError:
            err = sys.exc_info()[1]
            if err.args[0] == errno.EINTR:
                # if we loop back we have to subtract the
                # amount of time we already waited.
                if timeout is not None:
                    timeout = end_time - time.time()
                    if timeout < 0:
                        return []
            else:
                # something else caused the select.error, so
                # this actually is an exception.
                raise

env_pyrecodes/lib/python3.9/site-packages/pexpect/utils.py
NikolaBlagojevic/pyrecodes
from typing import Union

from .align import AlignMethod
from .cells import cell_len, set_cell_size
from .console import Console, ConsoleOptions, RenderResult
from .jupyter import JupyterMixin
from .measure import Measurement
from .style import Style
from .text import Text


class Rule(JupyterMixin):
    """A console renderable to draw a horizontal rule (line).

    Args:
        title (Union[str, Text], optional): Text to render in the rule. Defaults to "".
        characters (str, optional): Character(s) used to draw the line. Defaults to "─".
        style (StyleType, optional): Style of Rule. Defaults to "rule.line".
        end (str, optional): Character at end of Rule. defaults to "\\\\n"
        align (str, optional): How to align the title, one of "left", "center", or "right". Defaults to "center".
    """

    def __init__(
        self,
        title: Union[str, Text] = "",
        *,
        characters: str = "─",
        style: Union[str, Style] = "rule.line",
        end: str = "\n",
        align: AlignMethod = "center",
    ) -> None:
        if cell_len(characters) < 1:
            raise ValueError(
                "'characters' argument must have a cell width of at least 1"
            )
        if align not in ("left", "center", "right"):
            raise ValueError(
                f'invalid value for align, expected "left", "center", "right" (not {align!r})'
            )
        self.title = title
        self.characters = characters
        self.style = style
        self.end = end
        self.align = align

    def __repr__(self) -> str:
        return f"Rule({self.title!r}, {self.characters!r})"

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        width = options.max_width

        characters = (
            "-"
            if (options.ascii_only and not self.characters.isascii())
            else self.characters
        )

        chars_len = cell_len(characters)
        if not self.title:
            yield self._rule_line(chars_len, width)
            return

        if isinstance(self.title, Text):
            title_text = self.title
        else:
            title_text = console.render_str(self.title, style="rule.text")

        title_text.plain = title_text.plain.replace("\n", " ")
        title_text.expand_tabs()

        required_space = 4 if self.align == "center" else 2
        truncate_width = max(0, width - required_space)
        if not truncate_width:
            yield self._rule_line(chars_len, width)
            return

        rule_text = Text(end=self.end)
        if self.align == "center":
            title_text.truncate(truncate_width, overflow="ellipsis")
            side_width = (width - cell_len(title_text.plain)) // 2
            left = Text(characters * (side_width // chars_len + 1))
            left.truncate(side_width - 1)
            right_length = width - cell_len(left.plain) - cell_len(title_text.plain)
            right = Text(characters * (side_width // chars_len + 1))
            right.truncate(right_length)
            rule_text.append(left.plain + " ", self.style)
            rule_text.append(title_text)
            rule_text.append(" " + right.plain, self.style)
        elif self.align == "left":
            title_text.truncate(truncate_width, overflow="ellipsis")
            rule_text.append(title_text)
            rule_text.append(" ")
            rule_text.append(characters * (width - rule_text.cell_len), self.style)
        elif self.align == "right":
            title_text.truncate(truncate_width, overflow="ellipsis")
            rule_text.append(characters * (width - title_text.cell_len - 1), self.style)
            rule_text.append(" ")
            rule_text.append(title_text)

        rule_text.plain = set_cell_size(rule_text.plain, width)
        yield rule_text

    def _rule_line(self, chars_len: int, width: int) -> Text:
        rule_text = Text(self.characters * ((width // chars_len) + 1), self.style)
        rule_text.truncate(width)
        rule_text.plain = set_cell_size(rule_text.plain, width)
        return rule_text

    def __rich_measure__(
        self, console: Console, options: ConsoleOptions
    ) -> Measurement:
        return Measurement(1, 1)


if __name__ == "__main__":  # pragma: no cover
    import sys

    from pip._vendor.rich.console import Console

    try:
        text = sys.argv[1]
    except IndexError:
        text = "Hello, World"
    console = Console()
    console.print(Rule(title=text))

    console = Console()
    console.print(Rule("foo"), width=4)

env_pyrecodes/lib/python3.9/site-packages/pip/_vendor/rich/rule.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""POSIX specific tests."""

import datetime
import errno
import os
import re
import subprocess
import time
import unittest

import psutil
from psutil import AIX
from psutil import BSD
from psutil import LINUX
from psutil import MACOS
from psutil import OPENBSD
from psutil import POSIX
from psutil import SUNOS
from psutil.tests import HAS_NET_IO_COUNTERS
from psutil.tests import PYTHON_EXE
from psutil.tests import PsutilTestCase
from psutil.tests import mock
from psutil.tests import retry_on_failure
from psutil.tests import sh
from psutil.tests import skip_on_access_denied
from psutil.tests import spawn_testproc
from psutil.tests import terminate
from psutil.tests import which


if POSIX:
    import mmap
    import resource

    from psutil._psutil_posix import getpagesize


def ps(fmt, pid=None):
    """
    Wrapper for calling the ps command with a little bit of cross-platform
    support for a narrow range of features.
    """

    cmd = ['ps']

    if LINUX:
        cmd.append('--no-headers')

    if pid is not None:
        cmd.extend(['-p', str(pid)])
    else:
        if SUNOS or AIX:
            cmd.append('-A')
        else:
            cmd.append('ax')

    if SUNOS:
        # XXX: set() has not get() method so this cannot work; not sure
        # what I meant in here.
        fmt_map = set(('command', 'comm', 'start', 'stime'))
        fmt = fmt_map.get(fmt, fmt)

    cmd.extend(['-o', fmt])

    output = sh(cmd)

    if LINUX:
        output = output.splitlines()
    else:
        output = output.splitlines()[1:]

    all_output = []
    for line in output:
        line = line.strip()

        try:
            line = int(line)
        except ValueError:
            pass

        all_output.append(line)

    if pid is None:
        return all_output
    else:
        return all_output[0]

# ps "-o" field names differ wildly between platforms.
# "comm" means "only executable name" but is not available on BSD platforms.
# "args" means "command with all its arguments", and is also not available
# on BSD platforms.
# "command" is like "args" on most platforms, but like "comm" on AIX,
# and not available on SUNOS.
# so for the executable name we can use "comm" on Solaris and split "command"
# on other platforms.
# to get the cmdline (with args) we have to use "args" on AIX and
# Solaris, and can use "command" on all others.


def ps_name(pid):
    field = "command"
    if SUNOS:
        field = "comm"
    return ps(field, pid).split()[0]


def ps_args(pid):
    field = "command"
    if AIX or SUNOS:
        field = "args"
    out = ps(field, pid)
    # observed on BSD + Github CI: '/usr/local/bin/python3 -E -O (python3.9)'
    out = re.sub(r"\(python.*?\)$", "", out)
    return out.strip()


def ps_rss(pid):
    field = "rss"
    if AIX:
        field = "rssize"
    return ps(field, pid)


def ps_vsz(pid):
    field = "vsz"
    if AIX:
        field = "vsize"
    return ps(field, pid)


@unittest.skipIf(not POSIX, "POSIX only")
class TestProcess(PsutilTestCase):
    """Compare psutil results against 'ps' command line utility (mainly)."""

    @classmethod
    def setUpClass(cls):
        cls.pid = spawn_testproc([PYTHON_EXE, "-E", "-O"],
                                 stdin=subprocess.PIPE).pid

    @classmethod
    def tearDownClass(cls):
        terminate(cls.pid)

    def test_ppid(self):
        ppid_ps = ps('ppid', self.pid)
        ppid_psutil = psutil.Process(self.pid).ppid()
        self.assertEqual(ppid_ps, ppid_psutil)

    def test_uid(self):
        uid_ps = ps('uid', self.pid)
        uid_psutil = psutil.Process(self.pid).uids().real
        self.assertEqual(uid_ps, uid_psutil)

    def test_gid(self):
        gid_ps = ps('rgid', self.pid)
        gid_psutil = psutil.Process(self.pid).gids().real
        self.assertEqual(gid_ps, gid_psutil)

    def test_username(self):
        username_ps = ps('user', self.pid)
        username_psutil = psutil.Process(self.pid).username()
        self.assertEqual(username_ps, username_psutil)

    def test_username_no_resolution(self):
        # Emulate a case where the system can't resolve the uid to
        # a username in which case psutil is supposed to return
        # the stringified uid.
        p = psutil.Process()
        with mock.patch("psutil.pwd.getpwuid", side_effect=KeyError) as fun:
            self.assertEqual(p.username(), str(p.uids().real))
            assert fun.called

    @skip_on_access_denied()
    @retry_on_failure()
    def test_rss_memory(self):
        # give python interpreter some time to properly initialize
        # so that the results are the same
        time.sleep(0.1)
        rss_ps = ps_rss(self.pid)
        rss_psutil = psutil.Process(self.pid).memory_info()[0] / 1024
        self.assertEqual(rss_ps, rss_psutil)

    @skip_on_access_denied()
    @retry_on_failure()
    def test_vsz_memory(self):
        # give python interpreter some time to properly initialize
        # so that the results are the same
        time.sleep(0.1)
        vsz_ps = ps_vsz(self.pid)
        vsz_psutil = psutil.Process(self.pid).memory_info()[1] / 1024
        self.assertEqual(vsz_ps, vsz_psutil)

    def test_name(self):
        name_ps = ps_name(self.pid)
        # remove path if there is any, from the command
        name_ps = os.path.basename(name_ps).lower()
        name_psutil = psutil.Process(self.pid).name().lower()
        # ...because of how we calculate PYTHON_EXE; on MACOS this may
        # be "pythonX.Y".
        name_ps = re.sub(r"\d.\d", "", name_ps)
        name_psutil = re.sub(r"\d.\d", "", name_psutil)
        # ...may also be "python.X"
        name_ps = re.sub(r"\d", "", name_ps)
        name_psutil = re.sub(r"\d", "", name_psutil)
        self.assertEqual(name_ps, name_psutil)

    def test_name_long(self):
        # On UNIX the kernel truncates the name to the first 15
        # characters. In such a case psutil tries to determine the
        # full name from the cmdline.
        name = "long-program-name"
        cmdline = ["long-program-name-extended", "foo", "bar"]
        with mock.patch("psutil._psplatform.Process.name",
                        return_value=name):
            with mock.patch("psutil._psplatform.Process.cmdline",
                            return_value=cmdline):
                p = psutil.Process()
                self.assertEqual(p.name(), "long-program-name-extended")

    def test_name_long_cmdline_ad_exc(self):
        # Same as above but emulates a case where cmdline() raises
        # AccessDenied in which case psutil is supposed to return
        # the truncated name instead of crashing.
        name = "long-program-name"
        with mock.patch("psutil._psplatform.Process.name",
                        return_value=name):
            with mock.patch("psutil._psplatform.Process.cmdline",
                            side_effect=psutil.AccessDenied(0, "")):
                p = psutil.Process()
                self.assertEqual(p.name(), "long-program-name")

    def test_name_long_cmdline_nsp_exc(self):
        # Same as above but emulates a case where cmdline() raises NSP
        # which is supposed to propagate.
        name = "long-program-name"
        with mock.patch("psutil._psplatform.Process.name",
                        return_value=name):
            with mock.patch("psutil._psplatform.Process.cmdline",
                            side_effect=psutil.NoSuchProcess(0, "")):
                p = psutil.Process()
                self.assertRaises(psutil.NoSuchProcess, p.name)

    @unittest.skipIf(MACOS or BSD, 'ps -o start not available')
    def test_create_time(self):
        time_ps = ps('start', self.pid)
        time_psutil = psutil.Process(self.pid).create_time()
        time_psutil_tstamp = datetime.datetime.fromtimestamp(
            time_psutil).strftime("%H:%M:%S")
        # sometimes ps shows the time rounded up instead of down, so we check
        # for both possible values
        round_time_psutil = round(time_psutil)
        round_time_psutil_tstamp = datetime.datetime.fromtimestamp(
            round_time_psutil).strftime("%H:%M:%S")
        self.assertIn(time_ps, [time_psutil_tstamp, round_time_psutil_tstamp])

    def test_exe(self):
        ps_pathname = ps_name(self.pid)
        psutil_pathname = psutil.Process(self.pid).exe()
        try:
            self.assertEqual(ps_pathname, psutil_pathname)
        except AssertionError:
            # certain platforms such as BSD are more accurate returning:
            # "/usr/local/bin/python2.7"
            # ...instead of:
            # "/usr/local/bin/python"
            # We do not want to consider this difference in accuracy
            # an error.
            adjusted_ps_pathname = ps_pathname[:len(ps_pathname)]
            self.assertEqual(ps_pathname, adjusted_ps_pathname)

    # On macOS the official python installer exposes a python wrapper that
    # executes a python executable hidden inside an application bundle inside
    # the Python framework.
    # There's a race condition between the ps call & the psutil call below
    # depending on the completion of the execve call so let's retry on failure
    @retry_on_failure()
    def test_cmdline(self):
        ps_cmdline = ps_args(self.pid)
        psutil_cmdline = " ".join(psutil.Process(self.pid).cmdline())
        self.assertEqual(ps_cmdline, psutil_cmdline)

    # On SUNOS "ps" reads niceness /proc/pid/psinfo which returns an
    # incorrect value (20); the real deal is getpriority(2) which
    # returns 0; psutil relies on it, see:
    # https://github.com/giampaolo/psutil/issues/1082
    # AIX has the same issue
    @unittest.skipIf(SUNOS, "not reliable on SUNOS")
    @unittest.skipIf(AIX, "not reliable on AIX")
    def test_nice(self):
        ps_nice = ps('nice', self.pid)
        psutil_nice = psutil.Process().nice()
        self.assertEqual(ps_nice, psutil_nice)


@unittest.skipIf(not POSIX, "POSIX only")
class TestSystemAPIs(PsutilTestCase):
    """Test some system APIs."""

    @retry_on_failure()
    def test_pids(self):
        # Note: this test might fail if the OS is starting/killing
        # other processes in the meantime
        pids_ps = sorted(ps("pid"))
        pids_psutil = psutil.pids()

        # on MACOS and OPENBSD ps doesn't show pid 0
        if MACOS or OPENBSD and 0 not in pids_ps:
            pids_ps.insert(0, 0)

        # There will often be one more process in pids_ps for ps itself
        if len(pids_ps) - len(pids_psutil) > 1:
            difference = [x for x in pids_psutil if x not in pids_ps] + \
                         [x for x in pids_ps if x not in pids_psutil]
            raise self.fail("difference: " + str(difference))

    # for some reason ifconfig -a does not report all interfaces
    # returned by psutil
    @unittest.skipIf(SUNOS, "unreliable on SUNOS")
    @unittest.skipIf(not which('ifconfig'), "no ifconfig cmd")
    @unittest.skipIf(not HAS_NET_IO_COUNTERS, "not supported")
    def test_nic_names(self):
        output = sh("ifconfig -a")
        for nic in psutil.net_io_counters(pernic=True).keys():
            for line in output.split():
                if line.startswith(nic):
                    break
            else:
                raise self.fail(
                    "couldn't find %s nic in 'ifconfig -a' output\n%s" % (
                        nic, output))

    # @unittest.skipIf(CI_TESTING and not psutil.users(), "unreliable on CI")
    @retry_on_failure()
    def test_users(self):
        out = sh("who -u")
        if not out.strip():
            raise self.skipTest("no users on this system")
        lines = out.split('\n')
        users = [x.split()[0] for x in lines]
        terminals = [x.split()[1] for x in lines]
        self.assertEqual(len(users), len(psutil.users()))
        with self.subTest(psutil=psutil.users(), who=out):
            for idx, u in enumerate(psutil.users()):
                self.assertEqual(u.name, users[idx])
                self.assertEqual(u.terminal, terminals[idx])
                if u.pid is not None:  # None on OpenBSD
                    psutil.Process(u.pid)

    @retry_on_failure()
    def test_users_started(self):
        out = sh("who -u")
        if not out.strip():
            raise self.skipTest("no users on this system")
        # '2023-04-11 09:31' (Linux)
        started = re.findall(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d", out)
        if started:
            tstamp = "%Y-%m-%d %H:%M"
        else:
            # 'Apr 10 22:27' (macOS)
            started = re.findall(r"[A-Z][a-z][a-z] \d\d \d\d:\d\d", out)
            if started:
                tstamp = "%b %d %H:%M"
            else:
                raise ValueError(
                    "cannot interpret tstamp in who output\n%s" % (out))
        with self.subTest(psutil=psutil.users(), who=out):
            for idx, u in enumerate(psutil.users()):
                psutil_value = datetime.datetime.fromtimestamp(
                    u.started).strftime(tstamp)
                self.assertEqual(psutil_value, started[idx])

    def test_pid_exists_let_raise(self):
        # According to "man 2 kill" possible error values for kill
        # are (EINVAL, EPERM, ESRCH). Test that any other errno
        # results in an exception.
        with mock.patch("psutil._psposix.os.kill",
                        side_effect=OSError(errno.EBADF, "")) as m:
            self.assertRaises(OSError, psutil._psposix.pid_exists, os.getpid())
            assert m.called

    def test_os_waitpid_let_raise(self):
        # os.waitpid() is supposed to catch EINTR and ECHILD only.
        # Test that any other errno results in an exception.
        with mock.patch("psutil._psposix.os.waitpid",
                        side_effect=OSError(errno.EBADF, "")) as m:
            self.assertRaises(OSError, psutil._psposix.wait_pid, os.getpid())
            assert m.called

    def test_os_waitpid_eintr(self):
        # os.waitpid() is supposed to "retry" on EINTR.
        with mock.patch("psutil._psposix.os.waitpid",
                        side_effect=OSError(errno.EINTR, "")) as m:
            self.assertRaises(
                psutil._psposix.TimeoutExpired,
                psutil._psposix.wait_pid, os.getpid(), timeout=0.01)
            assert m.called

    def test_os_waitpid_bad_ret_status(self):
        # Simulate os.waitpid() returning a bad status.
        with mock.patch("psutil._psposix.os.waitpid",
                        return_value=(1, -1)) as m:
            self.assertRaises(ValueError,
                              psutil._psposix.wait_pid, os.getpid())
            assert m.called

    # AIX can return '-' in df output instead of numbers, e.g. for /proc
    @unittest.skipIf(AIX, "unreliable on AIX")
    @retry_on_failure()
    def test_disk_usage(self):
        def df(device):
            try:
                out = sh("df -k %s" % device).strip()
            except RuntimeError as err:
                if "device busy" in str(err).lower():
                    raise self.skipTest("df returned EBUSY")
                raise
            line = out.split('\n')[1]
            fields = line.split()
            total = int(fields[1]) * 1024
            used = int(fields[2]) * 1024
            free = int(fields[3]) * 1024
            percent = float(fields[4].replace('%', ''))
            return (total, used, free, percent)

        tolerance = 4 * 1024 * 1024  # 4MB
        for part in psutil.disk_partitions(all=False):
            usage = psutil.disk_usage(part.mountpoint)
            try:
                total, used, free, percent = df(part.device)
            except RuntimeError as err:
                # see:
                # https://travis-ci.org/giampaolo/psutil/jobs/138338464
                # https://travis-ci.org/giampaolo/psutil/jobs/138343361
                err = str(err).lower()
                if "no such file or directory" in err or \
                        "raw devices not supported" in err or \
                        "permission denied" in err:
                    continue
                raise
            else:
                self.assertAlmostEqual(usage.total, total, delta=tolerance)
                self.assertAlmostEqual(usage.used, used, delta=tolerance)
                self.assertAlmostEqual(usage.free, free, delta=tolerance)
                self.assertAlmostEqual(usage.percent, percent, delta=1)


@unittest.skipIf(not POSIX, "POSIX only")
class TestMisc(PsutilTestCase):

    def test_getpagesize(self):
        pagesize = getpagesize()
        self.assertGreater(pagesize, 0)
        self.assertEqual(pagesize, resource.getpagesize())
        self.assertEqual(pagesize, mmap.PAGESIZE)


if __name__ == '__main__':
    from psutil.tests.runner import run_from_name
    run_from_name(__file__)

env_pyrecodes/lib/python3.9/site-packages/psutil/tests/test_posix.py
NikolaBlagojevic/pyrecodes
"""
Python interface to PROJ (https://proj.org),
cartographic projections and coordinate transformations library.

Download: http://python.org/pypi/pyproj

Requirements: Python 3.8+.

Contact:  Jeffrey Whitaker <jeffrey.s.whitaker@noaa.gov>

Copyright (c) 2006-2018, Jeffrey Whitaker.
Copyright (c) 2019-2023, Open source contributors.

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import warnings

import pyproj.network
from pyproj._datadir import (  # noqa: F401 pylint: disable=unused-import
    _pyproj_global_context_initialize,
    set_use_global_context,
)
from pyproj._show_versions import (  # noqa: F401 pylint: disable=unused-import
    show_versions,
)
from pyproj.crs import CRS  # noqa: F401 pylint: disable=unused-import
from pyproj.database import (  # noqa: F401 pylint: disable=unused-import
    get_authorities,
    get_codes,
    get_units_map,
)
from pyproj.exceptions import (  # noqa: F401 pylint: disable=unused-import
    DataDirError,
    ProjError,
)
from pyproj.geod import (  # noqa: F401 pylint: disable=unused-import
    Geod,
    geodesic_version_str,
    pj_ellps,
)
from pyproj.list import (  # noqa: F401 pylint: disable=unused-import
    get_ellps_map,
    get_prime_meridians_map,
    get_proj_operations_map,
)
from pyproj.proj import Proj, pj_list  # noqa: F401 pylint: disable=unused-import
from pyproj.transformer import (  # noqa: F401 pylint: disable=unused-import
    Transformer,
    itransform,
    proj_version_str,
    transform,
)

__version__ = "3.5.0"
__all__ = [
    "Proj",
    "Geod",
    "CRS",
    "Transformer",
    "transform",
    "itransform",
    "pj_ellps",
    "pj_list",
    "get_ellps_map",
    "get_prime_meridians_map",
    "get_proj_operations_map",
    "get_units_map",
    "show_versions",
]
__proj_version__ = proj_version_str


try:
    _pyproj_global_context_initialize()
except DataDirError as err:
    warnings.warn(str(err))

pyproj.network.set_ca_bundle_path()

env_pyrecodes/lib/python3.9/site-packages/pyproj/__init__.py
NikolaBlagojevic/pyrecodes
#
# Copyright © 2009- The Spyder Development Team
# Copyright © 2014-2015 Colin Duquesnoy
#
# Licensed under the terms of the MIT License
# (see LICENSE.txt for details)

"""
**QtPy** is a shim over the various Python Qt bindings. It is used to write
Qt binding independent libraries or applications.

If one of the APIs has already been imported, then it will be used.

Otherwise, the shim will automatically select the first available API (PyQt5, PySide2,
PyQt6 and PySide6); in that case, you can force the use of one
specific bindings (e.g. if your application is using one specific bindings and
you need to use library that use QtPy) by setting up the ``QT_API`` environment
variable.

PyQt5
=====

For PyQt5, you don't have to set anything as it will be used automatically::

    >>> from qtpy import QtGui, QtWidgets, QtCore
    >>> print(QtWidgets.QWidget)

PySide2
======

Set the QT_API environment variable to 'pyside2' before importing other
packages::

    >>> import os
    >>> os.environ['QT_API'] = 'pyside2'
    >>> from qtpy import QtGui, QtWidgets, QtCore
    >>> print(QtWidgets.QWidget)

PyQt6
=====

    >>> import os
    >>> os.environ['QT_API'] = 'pyqt6'
    >>> from qtpy import QtGui, QtWidgets, QtCore
    >>> print(QtWidgets.QWidget)

PySide6
=======

    >>> import os
    >>> os.environ['QT_API'] = 'pyside6'
    >>> from qtpy import QtGui, QtWidgets, QtCore
    >>> print(QtWidgets.QWidget)

"""

import contextlib
import os
import platform
import sys
import warnings

from packaging.version import parse

# Version of QtPy
__version__ = "2.4.0"


class PythonQtError(RuntimeError):
    """Generic error superclass for QtPy."""


class PythonQtWarning(RuntimeWarning):
    """Warning class for QtPy."""


class PythonQtValueError(ValueError):
    """Error raised if an invalid QT_API is specified."""


class QtBindingsNotFoundError(PythonQtError, ImportError):
    """Error raised if no bindings could be selected."""

    _msg = "No Qt bindings could be found"

    def __init__(self):
        super().__init__(self._msg)


class QtModuleNotFoundError(ModuleNotFoundError, PythonQtError):
    """Raised when a Python Qt binding submodule is not installed/supported."""

    _msg = "The {name} module was not found."
    _msg_binding = "{binding}"
    _msg_extra = ""

    def __init__(self, *, name, msg=None, **msg_kwargs):
        global API_NAME
        binding = self._msg_binding.format(binding=API_NAME)
        msg = msg or f"{self._msg} {self._msg_extra}".strip()
        msg = msg.format(name=name, binding=binding, **msg_kwargs)
        super().__init__(msg, name=name)


class QtModuleNotInOSError(QtModuleNotFoundError):
    """Raised when a module is not supported on the current operating system."""

    _msg = "{name} does not exist on this operating system."


class QtModuleNotInQtVersionError(QtModuleNotFoundError):
    """Raised when a module is not implemented in the current Qt version."""

    _msg = "{name} does not exist in {version}."

    def __init__(self, *, name, msg=None, **msg_kwargs):
        global QT5, QT6
        version = "Qt5" if QT5 else "Qt6"
        super().__init__(name=name, version=version)


class QtBindingMissingModuleError(QtModuleNotFoundError):
    """Raised when a module is not supported by a given binding."""

    _msg_extra = "It is not currently implemented in {binding}."


class QtModuleNotInstalledError(QtModuleNotFoundError):
    """Raise when a module is supported by the binding, but not installed."""

    _msg_extra = "It must be installed separately"

    def __init__(self, *, missing_package=None, **superclass_kwargs):
        self.missing_package = missing_package
        if missing_package is not None:
            self._msg_extra += " as {missing_package}."
        super().__init__(missing_package=missing_package, **superclass_kwargs)


# Qt API environment variable name
QT_API = "QT_API"

# Names of the expected PyQt5 api
PYQT5_API = ["pyqt5"]

PYQT6_API = ["pyqt6"]

# Names of the expected PySide2 api
PYSIDE2_API = ["pyside2"]

# Names of the expected PySide6 api
PYSIDE6_API = ["pyside6"]

# Minimum supported versions of Qt and the bindings
QT5_VERSION_MIN = PYQT5_VERSION_MIN = "5.9.0"
PYSIDE2_VERSION_MIN = "5.12.0"
QT6_VERSION_MIN = PYQT6_VERSION_MIN = PYSIDE6_VERSION_MIN = "6.2.0"

QT_VERSION_MIN = QT5_VERSION_MIN
PYQT_VERSION_MIN = PYQT5_VERSION_MIN
PYSIDE_VERSION_MIN = PYSIDE2_VERSION_MIN

# Detecting if a binding was specified by the user
binding_specified = QT_API in os.environ

API_NAMES = {
    "pyqt5": "PyQt5",
    "pyside2": "PySide2",
    "pyqt6": "PyQt6",
    "pyside6": "PySide6",
}
API = os.environ.get(QT_API, "pyqt5").lower()
initial_api = API
if API not in API_NAMES:
    raise PythonQtValueError(
        f"Specified QT_API={QT_API.lower()!r} is not in valid options: "
        f"{API_NAMES}",
    )

is_old_pyqt = is_pyqt46 = False
QT5 = PYQT5 = True
QT4 = QT6 = PYQT4 = PYQT6 = PYSIDE = PYSIDE2 = PYSIDE6 = False

PYQT_VERSION = None
PYSIDE_VERSION = None
QT_VERSION = None

# Unless `FORCE_QT_API` is set, use previously imported Qt Python bindings
if not os.environ.get("FORCE_QT_API"):
    if "PyQt5" in sys.modules:
        API = initial_api if initial_api in PYQT5_API else "pyqt5"
    elif "PySide2" in sys.modules:
        API = initial_api if initial_api in PYSIDE2_API else "pyside2"
    elif "PyQt6" in sys.modules:
        API = initial_api if initial_api in PYQT6_API else "pyqt6"
    elif "PySide6" in sys.modules:
        API = initial_api if initial_api in PYSIDE6_API else "pyside6"

if API in PYQT5_API:
    try:
        from PyQt5.QtCore import (
            PYQT_VERSION_STR as PYQT_VERSION,
        )
        from PyQt5.QtCore import (
            QT_VERSION_STR as QT_VERSION,
        )

        QT5 = PYQT5 = True

        if sys.platform == "darwin":
            macos_version = parse(platform.mac_ver()[0])
            qt_ver = parse(QT_VERSION)
            if macos_version < parse("10.10") and qt_ver >= parse("5.9"):
                raise PythonQtError(
                    "Qt 5.9 or higher only works in "
                    "macOS 10.10 or higher. Your "
                    "program will fail in this "
                    "system.",
                )
            elif macos_version < parse("10.11") and qt_ver >= parse("5.11"):
                raise PythonQtError(
                    "Qt 5.11 or higher only works in "
                    "macOS 10.11 or higher. Your "
                    "program will fail in this "
                    "system.",
                )

            del macos_version
            del qt_ver
    except ImportError:
        API = "pyside2"
    else:
        os.environ[QT_API] = API

if API in PYSIDE2_API:
    try:
        from PySide2 import __version__ as PYSIDE_VERSION  # analysis:ignore
        from PySide2.QtCore import __version__ as QT_VERSION  # analysis:ignore

        PYQT5 = False
        QT5 = PYSIDE2 = True

        if sys.platform == "darwin":
            macos_version = parse(platform.mac_ver()[0])
            qt_ver = parse(QT_VERSION)
            if macos_version < parse("10.11") and qt_ver >= parse("5.11"):
                raise PythonQtError(
                    "Qt 5.11 or higher only works in "
                    "macOS 10.11 or higher. Your "
                    "program will fail in this "
                    "system.",
                )

            del macos_version
            del qt_ver
    except ImportError:
        API = "pyqt6"
    else:
        os.environ[QT_API] = API

if API in PYQT6_API:
    try:
        from PyQt6.QtCore import (
            PYQT_VERSION_STR as PYQT_VERSION,
        )
        from PyQt6.QtCore import (
            QT_VERSION_STR as QT_VERSION,
        )

        QT5 = PYQT5 = False
        QT6 = PYQT6 = True

    except ImportError:
        API = "pyside6"
    else:
        os.environ[QT_API] = API

if API in PYSIDE6_API:
    try:
        from PySide6 import __version__ as PYSIDE_VERSION  # analysis:ignore
        from PySide6.QtCore import __version__ as QT_VERSION  # analysis:ignore

        QT5 = PYQT5 = False
        QT6 = PYSIDE6 = True

    except ImportError:
        raise QtBindingsNotFoundError from None
    else:
        os.environ[QT_API] = API


# If a correct API name is passed to QT_API and it could not be found,
# switches to another and informs through the warning
if initial_api != API and binding_specified:
    warnings.warn(
        f"Selected binding {initial_api!r} could not be found; "
        f"falling back to {API!r}",
        PythonQtWarning,
        stacklevel=2,
    )


# Set display name of the Qt API
API_NAME = API_NAMES[API]

with contextlib.suppress(ImportError, PythonQtError):
    # QtDataVisualization backward compatibility (QtDataVisualization vs. QtDatavisualization)
    # Only available for Qt5 bindings > 5.9 on Windows
    from . import QtDataVisualization as QtDatavisualization  # analysis:ignore


def _warn_old_minor_version(name, old_version, min_version):
    """Warn if using a Qt or binding version no longer supported by QtPy."""
    warning_message = (
        f"{name} version {old_version} is not supported by QtPy. "
        "To ensure your application works correctly with QtPy, "
        f"please upgrade to {name} {min_version} or later."
    )
    warnings.warn(warning_message, PythonQtWarning, stacklevel=2)


# Warn if using an End of Life or unsupported Qt API/binding minor version
if QT_VERSION:
    if QT5 and (parse(QT_VERSION) < parse(QT5_VERSION_MIN)):
        _warn_old_minor_version("Qt5", QT_VERSION, QT5_VERSION_MIN)
    elif QT6 and (parse(QT_VERSION) < parse(QT6_VERSION_MIN)):
        _warn_old_minor_version("Qt6", QT_VERSION, QT6_VERSION_MIN)

if PYQT_VERSION:
    if PYQT5 and (parse(PYQT_VERSION) < parse(PYQT5_VERSION_MIN)):
        _warn_old_minor_version("PyQt5", PYQT_VERSION, PYQT5_VERSION_MIN)
    elif PYQT6 and (parse(PYQT_VERSION) < parse(PYQT6_VERSION_MIN)):
        _warn_old_minor_version("PyQt6", PYQT_VERSION, PYQT6_VERSION_MIN)
elif PYSIDE_VERSION:
    if PYSIDE2 and (parse(PYSIDE_VERSION) < parse(PYSIDE2_VERSION_MIN)):
        _warn_old_minor_version("PySide2", PYSIDE_VERSION, PYSIDE2_VERSION_MIN)
    elif PYSIDE6 and (parse(PYSIDE_VERSION) < parse(PYSIDE6_VERSION_MIN)):
        _warn_old_minor_version("PySide6", PYSIDE_VERSION, PYSIDE6_VERSION_MIN)

env_pyrecodes/lib/python3.9/site-packages/qtpy/__init__.py
NikolaBlagojevic/pyrecodes
"""Rasterio"""

from collections import namedtuple
import glob
import logging
from logging import NullHandler
import os
import platform
import sys
import warnings

# On Windows we must explicitly register the directories that contain
# the GDAL and supporting DLLs starting with Python 3.8. Presently, we
# support the rasterio-wheels location or directories on the system's
# executable path.
if platform.system() == "Windows":
    _whl_dir = os.path.join(os.path.dirname(__file__), ".libs")
    if os.path.exists(_whl_dir):
        os.add_dll_directory(_whl_dir)
    else:
        if "PATH" in os.environ:
            for p in os.environ["PATH"].split(os.pathsep):
                if p and glob.glob(os.path.join(p, "gdal*.dll")):
                    os.add_dll_directory(os.path.abspath(p))


from rasterio._show_versions import show_versions
from rasterio._version import gdal_version, get_geos_version, get_proj_version
from rasterio.crs import CRS
from rasterio.drivers import driver_from_extension, is_blacklisted
from rasterio.dtypes import (
    bool_,
    ubyte,
    sbyte,
    uint8,
    int8,
    uint16,
    int16,
    uint32,
    int32,
    int64,
    uint64,
    float32,
    float64,
    complex_,
    check_dtype,
    complex_int16,
)
from rasterio.env import ensure_env_with_credentials, Env
from rasterio.errors import (
    RasterioIOError,
    DriverCapabilityError,
    RasterioDeprecationWarning,
)
from rasterio.io import (
    DatasetReader,
    get_writer_for_path,
    get_writer_for_driver,
    MemoryFile,
)
from rasterio.profiles import default_gtiff_profile
from rasterio.transform import Affine, guard_transform
from rasterio._path import _parse_path

# These modules are imported from the Cython extensions, but are also import
# here to help tools like cx_Freeze find them automatically
import rasterio._err
import rasterio.coords
import rasterio.enums
import rasterio._path

try:
    from rasterio.io import FilePath

    have_vsi_plugin = True
except ImportError:

    class FilePath:
        pass

    have_vsi_plugin = False

__all__ = ['band', 'open', 'pad', 'Env', 'CRS']
__version__ = "1.3.6"
__gdal_version__ = gdal_version()
__proj_version__ = ".".join([str(version) for version in get_proj_version()])
__geos_version__ = ".".join([str(version) for version in get_geos_version()])

# Rasterio attaches NullHandler to the 'rasterio' logger and its
# descendents. See
# https://docs.python.org/2/howto/logging.html#configuring-logging-for-a-library
# Applications must attach their own handlers in order to see messages.
# See rasterio/rio/main.py for an example.
log = logging.getLogger(__name__)
log.addHandler(NullHandler())


# Remove this in 1.4.0 (see comment on gh-2423).
def parse_path(path):
    warnings.warn(
        "rasterio.parse_path will be removed in version 1.4.",
        RasterioDeprecationWarning,
        stacklevel=2,
    )
    return _parse_path(path)


@ensure_env_with_credentials
def open(fp, mode='r', driver=None, width=None, height=None, count=None,
         crs=None, transform=None, dtype=None, nodata=None, sharing=False,
         **kwargs):
    """Open a dataset for reading or writing.

    The dataset may be located in a local file, in a resource located by
    a URL, or contained within a stream of bytes.

    In read ('r') or read/write ('r+') mode, no keyword arguments are
    required: these attributes are supplied by the opened dataset.

    In write ('w' or 'w+') mode, the driver, width, height, count, and dtype
    keywords are strictly required.

    Parameters
    ----------
    fp : str, file object, PathLike object, FilePath, or MemoryFile
        A filename or URL, a file object opened in binary ('rb') mode, a
        Path object, or one of the rasterio classes that provides the
        dataset-opening interface (has an open method that returns a
        dataset).
    mode : str, optional
        'r' (read, the default), 'r+' (read/write), 'w' (write), or
        'w+' (write/read).
    driver : str, optional
        A short format driver name (e.g. "GTiff" or "JPEG") or a list of
        such names (see GDAL docs at
        https://gdal.org/drivers/raster/index.html). In 'w' or 'w+' modes
        a single name is required. In 'r' or 'r+' modes the driver can
        usually be omitted. Registered drivers will be tried
        sequentially until a match is found. When multiple drivers are
        available for a format such as JPEG2000, one of them can be
        selected by using this keyword argument.
    width : int, optional
        The number of columns of the raster dataset. Required in 'w' or
        'w+' modes, it is ignored in 'r' or 'r+' modes.
    height : int, optional
        The number of rows of the raster dataset. Required in 'w' or
        'w+' modes, it is ignored in 'r' or 'r+' modes.
    count : int, optional
        The count of dataset bands. Required in 'w' or 'w+' modes, it is
        ignored in 'r' or 'r+' modes.
    crs : str, dict, or CRS; optional
        The coordinate reference system. Required in 'w' or 'w+' modes,
        it is ignored in 'r' or 'r+' modes.
    transform : Affine instance, optional
        Affine transformation mapping the pixel space to geographic
        space. Required in 'w' or 'w+' modes, it is ignored in 'r' or
        'r+' modes.
    dtype : str or numpy dtype
        The data type for bands. For example: 'uint8' or
        ``rasterio.uint16``. Required in 'w' or 'w+' modes, it is
        ignored in 'r' or 'r+' modes.
    nodata : int, float, or nan; optional
        Defines the pixel value to be interpreted as not valid data.
        Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+'
        modes.
    sharing : bool; optional
        To reduce overhead and prevent programs from running out of file
        descriptors, rasterio maintains a pool of shared low level
        dataset handles. When `True` this function will use a shared
        handle if one is available. Multithreaded programs must avoid
        sharing and should set *sharing* to `False`.
    kwargs : optional
        These are passed to format drivers as directives for creating or
        interpreting datasets. For example: in 'w' or 'w+' modes
        a `tiled=True` keyword argument will direct the GeoTIFF format
        driver to create a tiled, rather than striped, TIFF.

    Returns
    -------
    A ``DatasetReader`` or ``DatasetWriter`` object.

    Examples
    --------

    To open a GeoTIFF for reading using standard driver discovery and
    no directives:

    >>> import rasterio
    >>> with rasterio.open('example.tif') as dataset:
    ...     print(dataset.profile)

    To open a JPEG2000 using only the JP2OpenJPEG driver:

    >>> with rasterio.open(
    ...         'example.jp2', driver='JP2OpenJPEG') as dataset:
    ...     print(dataset.profile)

    To create a new 8-band, 16-bit unsigned, tiled, and LZW-compressed
    GeoTIFF with a global extent and 0.5 degree resolution:

    >>> from rasterio.transform import from_origin
    >>> with rasterio.open(
    ...         'example.tif', 'w', driver='GTiff', dtype='uint16',
    ...         width=720, height=360, count=8, crs='EPSG:4326',
    ...         transform=from_origin(-180.0, 90.0, 0.5, 0.5),
    ...         nodata=0, tiled=True, compress='lzw') as dataset:
    ...     dataset.write(...)
    """

    if not isinstance(fp, str):
        if not (
            hasattr(fp, "read")
            or hasattr(fp, "write")
            or isinstance(fp, (os.PathLike, MemoryFile, FilePath))
        ):
            raise TypeError("invalid path or file: {0!r}".format(fp))
    if mode and not isinstance(mode, str):
        raise TypeError("invalid mode: {0!r}".format(mode))
    if driver and not isinstance(driver, str):
        raise TypeError("invalid driver: {0!r}".format(driver))
    if dtype and not check_dtype(dtype):
        raise TypeError("invalid dtype: {0!r}".format(dtype))
    if nodata is not None:
        nodata = float(nodata)
    if transform:
        transform = guard_transform(transform)

    # Check driver/mode blacklist.
    if driver and is_blacklisted(driver, mode):
        raise RasterioIOError(
            "Blacklisted: file cannot be opened by "
            "driver '{0}' in '{1}' mode".format(driver, mode))

    # If the fp argument is a file-like object and can be adapted by
    # rasterio's FilePath we do so. Otherwise, we use a MemoryFile to
    # hold fp's contents and store that in an ExitStack attached to the
    # dataset object that we will return. When a dataset's close method
    # is called, this ExitStack will be unwound and the MemoryFile's
    # storage will be cleaned up.
    if mode == 'r' and hasattr(fp, 'read'):
        if have_vsi_plugin:
            return FilePath(fp).open(driver=driver, sharing=sharing, **kwargs)
        else:
            memfile = MemoryFile(fp.read())
            dataset = memfile.open(driver=driver, sharing=sharing, **kwargs)
            dataset._env.enter_context(memfile)
            return dataset

    elif mode in ('w', 'w+') and hasattr(fp, 'write'):
        memfile = MemoryFile()
        dataset = memfile.open(
            driver=driver,
            width=width,
            height=height,
            count=count,
            crs=crs,
            transform=transform,
            dtype=dtype,
            nodata=nodata,
            sharing=sharing,
            **kwargs
        )
        dataset._env.enter_context(memfile)

        # For the writing case we push an extra callback onto the
        # ExitStack. It ensures that the MemoryFile's contents are
        # copied to the open file object.
        def func(*args, **kwds):
            memfile.seek(0)
            fp.write(memfile.read())

        dataset._env.callback(func)
        return dataset

    # TODO: test for a shared base class or abstract type.
    elif isinstance(fp, (FilePath, MemoryFile)):
        if mode.startswith("r"):
            dataset = fp.open(driver=driver, sharing=sharing, **kwargs)

        # Note: FilePath does not support writing and an exception will
        # result from this.
        elif mode.startswith("w"):
            dataset = fp.open(
                driver=driver,
                width=width,
                height=height,
                count=count,
                crs=crs,
                transform=transform,
                dtype=dtype,
                nodata=nodata,
                sharing=sharing,
                **kwargs
            )
        return dataset

    # At this point, the fp argument is a string or path-like object
    # which can be converted to a string.
    else:
        raw_dataset_path = os.fspath(fp)
        path = _parse_path(raw_dataset_path)

        if mode == "r":
            dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
        elif mode == "r+":
            dataset = get_writer_for_path(path, driver=driver)(
                path, mode, driver=driver, sharing=sharing, **kwargs
            )
        elif mode.startswith("w"):
            if not driver:
                driver = driver_from_extension(path)
            writer = get_writer_for_driver(driver)
            if writer is not None:
                dataset = writer(
                    path,
                    mode,
                    driver=driver,
                    width=width,
                    height=height,
                    count=count,
                    crs=crs,
                    transform=transform,
                    dtype=dtype,
                    nodata=nodata,
                    sharing=sharing,
                    **kwargs
                )
            else:
                raise DriverCapabilityError(
                    "Writer does not exist for driver: %s" % str(driver)
                )
        else:
            raise DriverCapabilityError(
                "mode must be one of 'r', 'r+', or 'w', not %s" % mode)

        return dataset


Band = namedtuple('Band', ['ds', 'bidx', 'dtype', 'shape'])


def band(ds, bidx):
    """A dataset and one or more of its bands

    Parameters
    ----------
    ds: dataset object
        An opened rasterio dataset object.
    bidx: int or sequence of ints
        Band number(s), index starting at 1.

    Returns
    -------
    rasterio.Band
    """
    return Band(ds, bidx, set(ds.dtypes).pop(), ds.shape)


def pad(array, transform, pad_width, mode=None, **kwargs):
    """pad array and adjust affine transform matrix.

    Parameters
    ----------
    array: ndarray
        Numpy ndarray, for best results a 2D array
    transform: Affine transform
        transform object mapping pixel space to coordinates
    pad_width: int
        number of pixels to pad array on all four
    mode: str or function
        define the method for determining padded values

    Returns
    -------
    (array, transform): tuple
        Tuple of new array and affine transform

    Notes
    -----
    See numpy docs for details on mode and other kwargs:
    http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.pad.html
    """
    import numpy as np
    transform = guard_transform(transform)
    padded_array = np.pad(array, pad_width, mode, **kwargs)
    padded_trans = list(transform)
    padded_trans[2] -= pad_width * padded_trans[0]
    padded_trans[5] -= pad_width * padded_trans[4]
    return padded_array, Affine(*padded_trans[:6])

env_pyrecodes/lib/python3.9/site-packages/rasterio/__init__.py
NikolaBlagojevic/pyrecodes
"""Dataset paths, identifiers, and filenames

Note: this module is not part of Rasterio's API. It is for internal use
only.

"""

import pathlib
import re
import sys
from urllib.parse import urlparse

import attr

from rasterio.errors import PathError

# Supported URI schemes and their mapping to GDAL's VSI suffix.
# TODO: extend for other cloud plaforms.
SCHEMES = {
    'ftp': 'curl',
    'gzip': 'gzip',
    'http': 'curl',
    'https': 'curl',
    's3': 's3',
    'tar': 'tar',
    'zip': 'zip',
    'file': 'file',
    'oss': 'oss',
    'gs': 'gs',
    'az': 'az',
}

CURLSCHEMES = set([k for k, v in SCHEMES.items() if v == 'curl'])

# TODO: extend for other cloud plaforms.
REMOTESCHEMES = set([k for k, v in SCHEMES.items() if v in ('curl', 's3', 'oss', 'gs', 'az',)])


class _Path:
    """Base class for dataset paths"""

    def as_vsi(self):
        return _vsi_path(self)


@attr.s(slots=True)
class _ParsedPath(_Path):
    """Result of parsing a dataset URI/Path

    Attributes
    ----------
    path : str
        Parsed path. Includes the hostname and query string in the case
        of a URI.
    archive : str
        Parsed archive path.
    scheme : str
        URI scheme such as "https" or "zip+s3".
    """
    path = attr.ib()
    archive = attr.ib()
    scheme = attr.ib()

    @classmethod
    def from_uri(cls, uri):
        parts = urlparse(uri)
        path = pathlib.Path(parts.path).as_posix() if parts.path else parts.path
        scheme = parts.scheme or None

        if parts.query:
            path += "?" + parts.query

        if parts.scheme and parts.netloc:
            path = parts.netloc + path

        parts = path.split('!')
        path = parts.pop() if parts else None
        archive = parts.pop() if parts else None
        return _ParsedPath(path, archive, scheme)

    @property
    def name(self):
        """The parsed path's original URI"""
        if not self.scheme:
            return self.path
        elif self.archive:
            return "{}://{}!{}".format(self.scheme, self.archive, self.path)
        else:
            return "{}://{}".format(self.scheme, self.path)

    @property
    def is_remote(self):
        """Test if the path is a remote, network URI"""
        return bool(self.scheme) and self.scheme.split("+")[-1] in REMOTESCHEMES

    @property
    def is_local(self):
        """Test if the path is a local URI"""
        return not self.scheme or (self.scheme and self.scheme.split('+')[-1] not in REMOTESCHEMES)


@attr.s(slots=True)
class _UnparsedPath(_Path):
    """Encapsulates legacy GDAL filenames

    Attributes
    ----------
    path : str
        The legacy GDAL filename.
    """
    path = attr.ib()

    @property
    def name(self):
        """The unparsed path's original path"""
        return self.path


def _parse_path(path):
    """Parse a dataset's identifier or path into its parts

    Parameters
    ----------
    path : str or path-like object
        The path to be parsed.

    Returns
    -------
    ParsedPath or UnparsedPath

    Notes
    -----
    When legacy GDAL filenames are encountered, they will be returned
    in a UnparsedPath.

    """
    if isinstance(path, _Path):
        return path

    elif pathlib and isinstance(path, pathlib.PurePath):
        return _ParsedPath(path.as_posix(), None, None)

    elif isinstance(path, str):

        if sys.platform == "win32" and re.match(r"^[a-zA-Z]\:", path):
            if pathlib:
                return _ParsedPath(pathlib.Path(path).as_posix(), None, None)
            else:
                return _UnparsedPath(path)

        elif path.startswith('/vsi'):
            return _UnparsedPath(path)

        else:
            parts = urlparse(path)

    else:
        raise PathError("invalid path '{!r}'".format(path))

    # if the scheme is not one of Rasterio's supported schemes, we
    # return an UnparsedPath.
    if parts.scheme:

        if all(p in SCHEMES for p in parts.scheme.split('+')):
            return _ParsedPath.from_uri(path)

    return _UnparsedPath(path)


def _vsi_path(path):
    """Convert a parsed path to a GDAL VSI path

    Parameters
    ----------
    path : Path
        A ParsedPath or UnparsedPath object.

    Returns
    -------
    str

    """
    if isinstance(path, _UnparsedPath):
        return path.path

    elif isinstance(path, _ParsedPath):

        if not path.scheme:
            return path.path

        else:
            if path.scheme.split('+')[-1] in CURLSCHEMES:
                suffix = '{}://'.format(path.scheme.split('+')[-1])
            else:
                suffix = ''

            prefix = '/'.join('vsi{0}'.format(SCHEMES[p]) for p in path.scheme.split('+') if p != 'file')

            if prefix:
                if path.archive:
                    result = '/{}/{}{}/{}'.format(prefix, suffix, path.archive, path.path.lstrip('/'))
                else:
                    result = '/{}/{}{}'.format(prefix, suffix, path.path)
            else:
                result = path.path
            return result

    else:
        raise ValueError("path must be a ParsedPath or UnparsedPath object")

env_pyrecodes/lib/python3.9/site-packages/rasterio/_path.py
NikolaBlagojevic/pyrecodes
"""
Utility methods to print system info for debugging

adapted from :func:`sklearn.utils._show_versions`
which was adapted from :func:`pandas.show_versions`
"""
import importlib
import os
import platform
import sys


def _get_sys_info():
    """System information

    Return
    ------
    dict:
        system and Python version information
    """
    blob = [
        ("python", sys.version.replace("\n", " ")),
        ("executable", sys.executable),
        ("machine", platform.platform()),
    ]

    return dict(blob)


def _get_gdal_info():
    """Information on system GDAL

    Returns
    -------
    dict:
        system GDAL information
    """
    # pylint: disable=import-outside-toplevel
    import rasterio

    blob = [
        ("rasterio", rasterio.__version__),
        ("GDAL", rasterio.__gdal_version__),
        ("PROJ", rasterio.__proj_version__),
        ("GEOS", rasterio.__geos_version__),
        ("PROJ DATA", os.pathsep.join(rasterio._env.get_proj_data_search_paths())),
        ("GDAL DATA", rasterio._env.get_gdal_data()),
    ]

    return dict(blob)


def _get_deps_info():
    """Overview of the installed version of main dependencies

    Returns
    -------
    dict:
        version information on relevant Python libraries
    """
    deps = [
        "affine",
        "attrs",
        "certifi",
        "click",
        "cligj",
        "cython",
        "numpy",
        "snuggs",
        "click-plugins",
        "setuptools",
    ]

    def get_version(module):
        try:
            return module.__version__
        except AttributeError:
            return module.version

    deps_info = {}

    for modname in deps:
        try:
            if modname in sys.modules:
                mod = sys.modules[modname]
            else:
                mod = importlib.import_module(modname)
            deps_info[modname] = get_version(mod)
        except ImportError:
            deps_info[modname] = None

    return deps_info


def _print_info_dict(info_dict):
    """Print the information dictionary"""
    for key, stat in info_dict.items():
        print(f"{key:>10}: {stat}")


def show_versions():
    """
    Print useful debugging information

    Example
    -------
    > python -c "import rasterio; rasterio.show_versions()"

    """
    print("rasterio info:")
    _print_info_dict(_get_gdal_info())
    print("\nSystem:")
    _print_info_dict(_get_sys_info())
    print("\nPython deps:")
    _print_info_dict(_get_deps_info())

env_pyrecodes/lib/python3.9/site-packages/rasterio/_show_versions.py
NikolaBlagojevic/pyrecodes
"""Ground control points"""

import uuid


class GroundControlPoint:
    """A mapping of row, col image coordinates to x, y, z."""

    def __init__(self, row=None, col=None, x=None, y=None, z=None,
                 id=None, info=None):
        """Create a new ground control point

        Parameters
        ----------
        row, col : float, required
            The row (or line) and column (or pixel) coordinates that
            map to spatial coordinate values ``y`` and ``x``,
            respectively.
        x, y : float, required
            Spatial coordinates of a ground control point.
        z : float, optional
            Optional ``z`` coordinate.
        id : str, optional
            A unique identifer for the ground control point.
        info : str, optional
            A short description for the ground control point.
        """
        if any(x is None for x in (row, col, x, y)):
            raise ValueError("row, col, x, and y are required parameters.")
        if id is None:
            id = str(uuid.uuid4())
        self.id = id
        self.info = info
        self.row = row
        self.col = col
        self.x = x
        self.y = y
        self.z = z

    def __repr__(self):
        args = ', '.join(['{}={}'.format(att, repr(getattr(self, att)))
                         for att in ('row', 'col', 'x', 'y', 'z', 'id', 'info')
                         if getattr(self, att) is not None])
        return "GroundControlPoint({})".format(args)

    def asdict(self):
        """A dict representation of the GCP"""
        return {'id': self.id, 'info': self.info, 'row': self.row,
                'col': self.col, 'x': self.x, 'y': self.y, 'z': self.z}

    @property
    def __geo_interface__(self):
        """A GeoJSON representation of the GCP"""
        coords = [self.x, self.y]
        if self.z is not None:
            coords.append(self.z)
        return {'id': self.id, 'type': 'Feature',
                'geometry': {'type': 'Point', 'coordinates': tuple(coords)},
                'properties': self.asdict()}

env_pyrecodes/lib/python3.9/site-packages/rasterio/control.py
NikolaBlagojevic/pyrecodes
"""Bounding box tuple, and disjoint operator."""

from collections import namedtuple

BoundingBox = namedtuple('BoundingBox', ('left', 'bottom', 'right', 'top'))
BoundingBox.__doc__ = \
    """Bounding box named tuple, defining extent in cartesian coordinates.

    .. code::

        BoundingBox(left, bottom, right, top)

    Attributes
    ----------
    left :
        Left coordinate
    bottom :
        Bottom coordinate
    right :
        Right coordinate
    top :
        Top coordinate
    """

def disjoint_bounds(bounds1, bounds2):
    """Compare two bounds and determine if they are disjoint.

    Parameters
    ----------
    bounds1: 4-tuple
        rasterio bounds tuple (left, bottom, right, top)
    bounds2: 4-tuple
        rasterio bounds tuple

    Returns
    -------
    boolean
    ``True`` if bounds are disjoint,
    ``False`` if bounds overlap

    """
    bounds1_north_up = bounds1[3] > bounds1[1]
    bounds2_north_up = bounds2[3] > bounds2[1]

    if not bounds1_north_up and bounds2_north_up:
        # or both south-up (also True)
        raise ValueError("Bounds must both have the same orientation")

    if bounds1_north_up:
        return (bounds1[0] > bounds2[2] or bounds2[0] > bounds1[2] or
                bounds1[1] > bounds2[3] or bounds2[1] > bounds1[3])
    else:
        return (bounds1[0] > bounds2[2] or bounds2[0] > bounds1[2] or
                bounds1[3] > bounds2[1] or bounds2[3] > bounds1[1])

env_pyrecodes/lib/python3.9/site-packages/rasterio/coords.py
NikolaBlagojevic/pyrecodes
"""Driver policies and utilities

GDAL has many standard and extension format drivers and completeness of
these drivers varies greatly. It's possible to succeed poorly with some
formats and drivers, meaning that easy problems can be solved but that
harder problems are blocked by limitations of the drivers and formats.

NetCDF writing, for example, is presently blacklisted. Rasterio users
should use netcdf4-python instead:
http://unidata.github.io/netcdf4-python/.
"""
import os

from rasterio._base import _raster_driver_extensions
from rasterio.env import GDALVersion, ensure_env

# Methods like `rasterio.open()` may use this blacklist to preempt
# combinations of drivers and file modes.
blacklist = {
    # See https://github.com/rasterio/rasterio/issues/638 for discussion
    # about writing NetCDF files.
    'netCDF': ('r+', 'w')}


@ensure_env
def raster_driver_extensions():
    """
    Returns
    -------
    dict:
        Map of extensions to the driver.
    """
    return _raster_driver_extensions()


def driver_from_extension(path):
    """
    Attempt to auto-detect driver based on the extension.

    Parameters
    ----------
    path: str or pathlike object
        The path to the dataset to write with.

    Returns
    -------
    str:
        The name of the driver for the extension.
    """
    try:
        # in case the path is a file handle
        # or a partsed path
        path = path.name
    except AttributeError:
        pass

    # dynamic driver extension lists added in GDAL 2
    if GDALVersion().runtime() < GDALVersion.parse('2.0'):
        # basic list for GDAL 1
        driver_extensions = {
            'tif': 'GTiff',
            'tiff': 'GTiff',
            'png': 'PNG',
            'jpg': 'JPEG',
            'jpeg': 'JPEG',
        }
    else:
        driver_extensions = raster_driver_extensions()

    try:
        return driver_extensions[os.path.splitext(path)[-1].lstrip(".").lower()]
    except KeyError:
        raise ValueError("Unable to detect driver. Please specify driver.")


def is_blacklisted(name, mode):
    """Returns True if driver `name` and `mode` are blacklisted."""
    return mode in blacklist.get(name, ())

env_pyrecodes/lib/python3.9/site-packages/rasterio/drivers.py
NikolaBlagojevic/pyrecodes
"""Mapping of GDAL to Numpy data types.

Since 0.13 we are not importing numpy here and data types are strings.
Happily strings can be used throughout Numpy and so existing code will
not break.

"""
import numpy

from rasterio.env import GDALVersion

_GDAL_AT_LEAST_35 = GDALVersion.runtime().at_least("3.5")

bool_ = 'bool'
ubyte = uint8 = 'uint8'
sbyte = int8 = 'int8'
uint16 = 'uint16'
int16 = 'int16'
uint32 = 'uint32'
int32 = 'int32'
uint64 = 'uint64'
int64 = 'int64'
float32 = 'float32'
float64 = 'float64'
complex_ = 'complex'
complex64 = 'complex64'
complex128 = 'complex128'

complex_int16 = "complex_int16"

dtype_fwd = {
    0: None,  # GDT_Unknown
    1: ubyte,  # GDT_Byte
    2: uint16,  # GDT_UInt16
    3: int16,  # GDT_Int16
    4: uint32,  # GDT_UInt32
    5: int32,  # GDT_Int32
    6: float32,  # GDT_Float32
    7: float64,  # GDT_Float64
    8: complex_int16,  # GDT_CInt16
    9: complex64,  # GDT_CInt32
    10: complex64,  # GDT_CFloat32
    11: complex128,  # GDT_CFloat64
}

if _GDAL_AT_LEAST_35:
    dtype_fwd[12] = int64 # GDT_Int64
    dtype_fwd[13] = uint64 # GDT_UInt64

dtype_rev = dict((v, k) for k, v in dtype_fwd.items())

dtype_rev["uint8"] = 1
dtype_rev["int8"] = 1
dtype_rev["complex"] = 11
dtype_rev["complex_int16"] = 8


def _get_gdal_dtype(type_name):
    try:
        return dtype_rev[type_name]
    except KeyError:
        raise TypeError(
            f"Unsupported data type {type_name}. "
            f"Allowed data types: {list(dtype_rev)}."
        )

typename_fwd = {
    0: 'Unknown',
    1: 'Byte',
    2: 'UInt16',
    3: 'Int16',
    4: 'UInt32',
    5: 'Int32',
    6: 'Float32',
    7: 'Float64',
    8: 'CInt16',
    9: 'CInt32',
    10: 'CFloat32',
    11: 'CFloat64'}

if _GDAL_AT_LEAST_35:
    typename_fwd[12] = 'Int64'
    typename_fwd[13] = 'UInt64'

typename_rev = dict((v, k) for k, v in typename_fwd.items())

dtype_ranges = {
    'int8': (-128, 127),
    'uint8': (0, 255),
    'uint16': (0, 65535),
    'int16': (-32768, 32767),
    'uint32': (0, 4294967295),
    'int32': (-2147483648, 2147483647),
    'float32': (-3.4028235e+38, 3.4028235e+38),
    'float64': (-1.7976931348623157e+308, 1.7976931348623157e+308)}

if _GDAL_AT_LEAST_35:
    dtype_ranges['int64'] = (-9223372036854775808, 9223372036854775807)
    dtype_ranges['uint64'] = (0, 18446744073709551615)


def in_dtype_range(value, dtype):
    """
    Check if the value is within the dtype range
    """
    if numpy.dtype(dtype).kind == "f" and (numpy.isinf(value) or numpy.isnan(value)):
        return True
    range_min, range_max = dtype_ranges[dtype]
    return range_min <= value <= range_max


def _gdal_typename(dt):
    try:
        return typename_fwd[dtype_rev[dt]]
    except KeyError:
        return typename_fwd[dtype_rev[dt().dtype.name]]


def check_dtype(dt):
    """Check if dtype is a known dtype."""
    if str(dt) in dtype_rev:
        return True
    elif callable(dt) and str(dt().dtype) in dtype_rev:
        return True
    return False


def get_minimum_dtype(values):
    """Determine minimum type to represent values.

    Uses range checking to determine the minimum integer or floating point
    data type required to represent values.

    Parameters
    ----------
    values: list-like


    Returns
    -------
    rasterio dtype string
    """
    import numpy as np

    if not is_ndarray(values):
        values = np.array(values)

    min_value = values.min()
    max_value = values.max()

    if values.dtype.kind in ('i', 'u'):
        if min_value >= 0:
            if max_value <= 255:
                return uint8
            elif max_value <= 65535:
                return uint16
            elif max_value <= 4294967295:
                return uint32
            if not _GDAL_AT_LEAST_35:
                raise ValueError("Values out of range for supported dtypes")
            return uint64
        elif min_value >= -32768 and max_value <= 32767:
            return int16
        elif min_value >= -2147483648 and max_value <= 2147483647:
            return int32
        if not _GDAL_AT_LEAST_35:
            raise ValueError("Values out of range for supported dtypes")
        return int64

    else:
        if min_value >= -3.4028235e+38 and max_value <= 3.4028235e+38:
            return float32
        return float64


def is_ndarray(array):
    """Check if array is a ndarray."""
    import numpy as np

    return isinstance(array, np.ndarray) or hasattr(array, '__array__')


def can_cast_dtype(values, dtype):
    """Test if values can be cast to dtype without loss of information.

    Parameters
    ----------
    values: list-like
    dtype: numpy dtype or string

    Returns
    -------
    boolean
        True if values can be cast to data type.
    """
    import numpy as np

    if not is_ndarray(values):
        values = np.array(values)

    if values.dtype.name == _getnpdtype(dtype).name:
        return True

    elif values.dtype.kind == 'f':
        return np.allclose(values, values.astype(dtype), equal_nan=True)

    else:
        return np.array_equal(values, values.astype(dtype))


def validate_dtype(values, valid_dtypes):
    """Test if dtype of values is one of valid_dtypes.

    Parameters
    ----------
    values: list-like
    valid_dtypes: list-like
        list of valid dtype strings, e.g., ('int16', 'int32')

    Returns
    -------
    boolean:
        True if dtype of values is one of valid_dtypes
    """
    import numpy as np

    if not is_ndarray(values):
        values = np.array(values)

    return (values.dtype.name in valid_dtypes or
            get_minimum_dtype(values) in valid_dtypes)


def _is_complex_int(dtype):
    return isinstance(dtype, str) and dtype.startswith("complex_int")


def _getnpdtype(dtype):
    import numpy as np
    if _is_complex_int(dtype):
        return np.dtype("complex64")
    else:
        return np.dtype(dtype)

env_pyrecodes/lib/python3.9/site-packages/rasterio/dtypes.py
NikolaBlagojevic/pyrecodes
"""Enumerations."""

from enum import Enum, IntEnum

class TransformDirection(IntEnum):
    """Coordinate transform direction
    
    Forward transform direction defined as image pixel (row, col) to 
    geographic/projected (x, y) coordinates. Reverse transform direction defined as
    geographic/projected (x, y) to image pixel (row, col) coordinates.

    Notes
    -----
    The convention for transform direction for RPC based coordinate transform is
    typically the opposite of what is previously described. For consistency
    all coordinate transforms methods use the same convention.
    """
    forward = 1
    reverse = 0

class TransformMethod(Enum):
    affine = 'transform'
    gcps = 'gcps'
    rpcs = 'rpcs'

class ColorInterp(IntEnum):
    """Raster band color interpretation."""
    undefined = 0
    gray = 1
    grey = 1
    palette = 2
    red = 3
    green = 4
    blue = 5
    alpha = 6
    hue = 7
    saturation = 8
    lightness = 9
    cyan = 10
    magenta = 11
    yellow = 12
    black = 13
    Y = 14
    Cb = 15
    Cr = 16


class Resampling(IntEnum):
    """Available warp resampling algorithms.
    
    Attributes
    ----------
    nearest
        Nearest neighbor resampling (default, fastest algorithm, worst interpolation quality).
    bilinear
        Bilinear resampling.
    cubic
        Cubic resampling.
    cubic_spline
        Cubic spline resampling.
    lanczos
        Lanczos windowed sinc resampling.
    average
        Average resampling, computes the weighted average of all non-NODATA contributing pixels.
    mode
        Mode resampling, selects the value which appears most often of all the sampled points.
    gauss
        Gaussian resampling, Note: not available to the functions in rio.warp.
    max
        Maximum resampling, selects the maximum value from all non-NODATA contributing pixels. (GDAL >= 2.0)
    min
        Minimum resampling, selects the minimum value from all non-NODATA contributing pixels. (GDAL >= 2.0)
    med
        Median resampling, selects the median value of all non-NODATA contributing pixels. (GDAL >= 2.0)
    q1
        Q1, first quartile resampling, selects the first quartile value of all non-NODATA contributing pixels. (GDAL >= 2.0)
    q3
        Q3, third quartile resampling, selects the third quartile value of all non-NODATA contributing pixels. (GDAL >= 2.0)
    sum
        Sum, compute the weighted sum of all non-NODATA contributing pixels. (GDAL >= 3.1)
    rms
        RMS, root mean square / quadratic mean of all non-NODATA contributing pixels. (GDAL >= 3.3)
    
    Notes
    ----------
    The first 8, 'nearest', 'bilinear', 'cubic', 'cubic_spline',
    'lanczos', 'average', 'mode', and 'gauss', are available for making
    dataset overviews.

    'max', 'min', 'med', 'q1', 'q3' are only supported in GDAL >= 2.0.0.

    'nearest', 'bilinear', 'cubic', 'cubic_spline', 'lanczos',
    'average', 'mode' are always available (GDAL >= 1.10).

    'sum' is only supported in GDAL >= 3.1.

    'rms' is only supported in GDAL >= 3.3.

    Note: 'gauss' is not available to the functions in rio.warp.
    """
    nearest = 0
    bilinear = 1
    cubic = 2
    cubic_spline = 3
    lanczos = 4
    average = 5
    mode = 6
    gauss = 7
    max = 8
    min = 9
    med = 10
    q1 = 11
    q3 = 12
    sum = 13
    rms = 14


class _OverviewResampling(IntEnum):
    """Available Overview resampling algorithms.

    The first 8, 'nearest', 'bilinear', 'cubic', 'cubic_spline',
    'lanczos', 'average', 'mode', and 'gauss', are available for making
    dataset overviews.

    'nearest', 'bilinear', 'cubic', 'cubic_spline', 'lanczos',
    'average', 'mode' are always available (GDAL >= 1.10).

    'rms' is only supported in GDAL >= 3.3.

    """
    nearest = 0
    bilinear = 1
    cubic = 2
    cubic_spline = 3
    lanczos = 4
    average = 5
    mode = 6
    gauss = 7
    rms = 14


class Compression(Enum):
    """Available compression algorithms."""
    jpeg = 'JPEG'
    lzw = 'LZW'
    packbits = 'PACKBITS'
    deflate = 'DEFLATE'
    ccittrle = 'CCITTRLE'
    ccittfax3 = 'CCITTFAX3'
    ccittfax4 = 'CCITTFAX4'
    lzma = 'LZMA'
    none = 'NONE'
    zstd = 'ZSTD'
    lerc = 'LERC'
    lerc_deflate = 'LERC_DEFLATE'
    lerc_zstd = 'LERC_ZSTD'
    webp = 'WEBP'
    jpeg2000 = 'JPEG2000'


class Interleaving(Enum):
    pixel = 'PIXEL'
    line = 'LINE'
    band = 'BAND'


class MaskFlags(IntEnum):
    all_valid = 1
    per_dataset = 2
    alpha = 4
    nodata = 8


class PhotometricInterp(Enum):
    black = 'MINISBLACK'
    white = 'MINISWHITE'
    rgb = 'RGB'
    cmyk = 'CMYK'
    ycbcr = 'YCbCr'
    cielab = 'CIELAB'
    icclab = 'ICCLAB'
    itulab = 'ITULAB'


class MergeAlg(Enum):
    """Available rasterization algorithms"""
    replace = 'REPLACE'
    add = 'ADD'


class WktVersion(Enum):
    """
     .. versionadded:: 1.3.0

    Supported CRS WKT string versions
    """

    #: WKT Version 2 from 2015
    WKT2_2015 = "WKT2_2015"
    #: Alias for latest WKT Version 2
    WKT2 = "WKT2"
    #: WKT Version 2 from 2019
    WKT2_2019 = "WKT2_2018"
    #: WKT Version 1 GDAL Style
    WKT1_GDAL = "WKT1_GDAL"
    #: Alias for WKT Version 1 GDAL Style
    WKT1 = "WKT1"
    #: WKT Version 1 ESRI Style
    WKT1_ESRI = "WKT1_ESRI"


    @classmethod
    def _missing_(cls, value):
        if value == "WKT2_2019":
            # WKT2_2019 alias added in GDAL 3.2, use WKT2_2018 for compatibility
            return WktVersion.WKT2_2019
        raise ValueError(f"Invalid value for WktVersion: {value}")

env_pyrecodes/lib/python3.9/site-packages/rasterio/enums.py
NikolaBlagojevic/pyrecodes
"""Rasterio's GDAL/AWS environment"""

from contextlib import ExitStack
from functools import wraps, total_ordering
from inspect import getfullargspec as getargspec
import logging
import os
import re
import threading
import warnings

import attr

from rasterio._env import (
    GDALEnv,
    get_gdal_config,
    set_gdal_config,
    GDALDataFinder,
    PROJDataFinder,
    set_proj_data_search_path,
)
from rasterio._version import gdal_version
from rasterio.errors import EnvError, GDALVersionError, RasterioDeprecationWarning
from rasterio.session import Session, DummySession


class ThreadEnv(threading.local):
    def __init__(self):
        self._env = None  # Initialises in each thread

        # When the outermost 'rasterio.Env()' executes '__enter__' it
        # probes the GDAL environment to see if any of the supplied
        # config options already exist, the assumption being that they
        # were set with 'osgeo.gdal.SetConfigOption()' or possibly
        # 'rasterio.env.set_gdal_config()'.  The discovered options are
        # reinstated when the outermost Rasterio environment exits.
        # Without this check any environment options that are present in
        # the GDAL environment and are also passed to 'rasterio.Env()'
        # will be unset when 'rasterio.Env()' tears down, regardless of
        # their value.  For example:
        #
        #   from osgeo import gdal import rasterio
        #
        #   gdal.SetConfigOption('key', 'value') with
        #   rasterio.Env(key='something'): pass
        #
        # The config option 'key' would be unset when 'Env()' exits.
        # A more comprehensive solution would also leverage
        # https://trac.osgeo.org/gdal/changeset/37273 but this gets
        # Rasterio + older versions of GDAL halfway there.  One major
        # assumption is that environment variables are not set directly
        # with 'osgeo.gdal.SetConfigOption()' OR
        # 'rasterio.env.set_gdal_config()' inside of a 'rasterio.Env()'.
        self._discovered_options = None


local = ThreadEnv()

log = logging.getLogger(__name__)


class Env:
    """Abstraction for GDAL and AWS configuration

    The GDAL library is stateful: it has a registry of format drivers,
    an error stack, and dozens of configuration options.

    Rasterio's approach to working with GDAL is to wrap all the state
    up using a Python context manager (see PEP 343,
    https://www.python.org/dev/peps/pep-0343/). When the context is
    entered GDAL drivers are registered, error handlers are
    configured, and configuration options are set. When the context
    is exited, drivers are removed from the registry and other
    configurations are removed.

    Example
    -------
    .. code-block:: python

        with rasterio.Env(GDAL_CACHEMAX=128000000) as env:
            # All drivers are registered, GDAL's raster block cache
            # size is set to 128 MB.
            # Commence processing...
            ...
            # End of processing.

        # At this point, configuration options are set to their
        # previous (possible unset) values.

    A boto3 session or boto3 session constructor arguments
    `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token`
    may be passed to Env's constructor. In the latter case, a session
    will be created as soon as needed. AWS credentials are configured
    for GDAL as needed.
    """

    @classmethod
    def default_options(cls):
        """Default configuration options

        Parameters
        ----------
        None

        Returns
        -------
        dict
        """
        return {
            'GTIFF_IMPLICIT_JPEG_OVR': False,
            "RASTERIO_ENV": True
        }

    def __init__(self, session=None, aws_unsigned=False, profile_name=None,
                 session_class=Session.aws_or_dummy, **options):
        """Create a new GDAL/AWS environment.

        Note: this class is a context manager. GDAL isn't configured
        until the context is entered via `with rasterio.Env():`

        Parameters
        ----------
        session : optional
            A Session object.
        aws_unsigned : bool, optional
            Do not sign cloud requests.
        profile_name : str, optional
            A shared credentials profile name, as per boto3.
        session_class : Session, optional
            A sub-class of Session.
        **options : optional
            A mapping of GDAL configuration options, e.g.,
            `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.

        Returns
        -------
        Env

        Notes
        -----
        We raise EnvError if the GDAL config options
        AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
        credentials are handled exclusively by boto3.

        Examples
        --------

        >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
        ...     with rasterio.open("https://example.com/a.tif") as src:
        ...         print(src.profile)

        For access to secured cloud resources, a Rasterio Session or a
        foreign session object may be passed to the constructor.

        >>> import boto3
        >>> from rasterio.session import AWSSession
        >>> boto3_session = boto3.Session(...)
        >>> with Env(AWSSession(boto3_session)):
        ...     with rasterio.open("s3://mybucket/a.tif") as src:
        ...         print(src.profile)

        """
        aws_access_key_id = options.pop('aws_access_key_id', None)
        # Before 1.0, Rasterio only supported AWS. We will special
        # case AWS in 1.0.x. TODO: warn deprecation in 1.1.
        if aws_access_key_id:
            warnings.warn(
                "Passing abstract session keyword arguments is deprecated. "
                "Pass a Rasterio AWSSession object instead.",
                RasterioDeprecationWarning
            )

        aws_secret_access_key = options.pop('aws_secret_access_key', None)
        aws_session_token = options.pop('aws_session_token', None)
        region_name = options.pop('region_name', None)

        if ('AWS_ACCESS_KEY_ID' in options or
                'AWS_SECRET_ACCESS_KEY' in options):
            raise EnvError(
                "GDAL's AWS config options can not be directly set. "
                "AWS credentials are handled exclusively by boto3.")

        if session:
            # Passing a session via keyword argument is the canonical
            # way to configure access to secured cloud resources.
            if not isinstance(session, Session):
                warnings.warn(
                    "Passing a boto3 session is deprecated. Pass a Rasterio "
                    "AWSSession object instead.",
                    RasterioDeprecationWarning
                )
                session = Session.aws_or_dummy(session=session)

            self.session = session

        elif aws_access_key_id or profile_name or aws_unsigned:
            self.session = Session.aws_or_dummy(
                aws_access_key_id=aws_access_key_id,
                aws_secret_access_key=aws_secret_access_key,
                aws_session_token=aws_session_token,
                region_name=region_name,
                profile_name=profile_name,
                aws_unsigned=aws_unsigned)

        elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ:
            self.session = Session.from_environ()

        else:
            self.session = DummySession()

        self.options = options.copy()
        self.context_options = {}

    @classmethod
    def from_defaults(cls, *args, **kwargs):
        """Create an environment with default config options

        Parameters
        ----------
        args : optional
            Positional arguments for Env()
        kwargs : optional
            Keyword arguments for Env()

        Returns
        -------
        Env

        Notes
        -----
        The items in kwargs will be overlaid on the default values.

        """
        options = Env.default_options()
        options.update(**kwargs)
        return Env(*args, **options)

    def credentialize(self):
        """Get credentials and configure GDAL

        Note well: this method is a no-op if the GDAL environment
        already has credentials, unless session is not None.

        Returns
        -------
        None

        """
        cred_opts = self.session.get_credential_options()
        self.options.update(**cred_opts)
        setenv(**cred_opts)

    def drivers(self):
        """Return a mapping of registered drivers."""
        return local._env.drivers()

    def _dump_open_datasets(self):
        """Writes descriptions of open datasets to stderr

        For debugging and testing purposes.
        """
        return local._env._dump_open_datasets()

    def _dump_vsimem(self):
        """Returns contents of /vsimem/.

        For debugging and testing purposes.
        """
        return local._env._dump_vsimem()

    def __enter__(self):
        log.debug("Entering env context: %r", self)
        if local._env is None:
            log.debug("Starting outermost env")
            self._has_parent_env = False

            # See note directly above where _discovered_options is globally
            # defined.  This MUST happen before calling 'defenv()'.
            local._discovered_options = {}
            # Don't want to reinstate the "RASTERIO_ENV" option.
            probe_env = {k for k in self.options.keys() if k != "RASTERIO_ENV"}
            for key in probe_env:
                val = get_gdal_config(key, normalize=False)
                if val is not None:
                    local._discovered_options[key] = val

            defenv(**self.options)
            self.context_options = {}
        else:
            self._has_parent_env = True
            self.context_options = getenv()
            setenv(**self.options)

        self.credentialize()

        log.debug("Entered env context: %r", self)
        return self

    def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
        log.debug("Exiting env context: %r", self)
        delenv()
        if self._has_parent_env:
            defenv()
            setenv(**self.context_options)
        else:
            log.debug("Exiting outermost env")
            # See note directly above where _discovered_options is globally
            # defined.
            while local._discovered_options:
                key, val = local._discovered_options.popitem()
                set_gdal_config(key, val, normalize=False)
            local._discovered_options = None
        log.debug("Exited env context: %r", self)


def defenv(**options):
    """Create a default environment if necessary."""
    if local._env:
        log.debug("GDAL environment exists: %r", local._env)
    else:
        log.debug("No GDAL environment exists")
        local._env = GDALEnv()
        local._env.update_config_options(**options)
        log.debug(
            "New GDAL environment %r created", local._env)
    local._env.start()


def getenv():
    """Get a mapping of current options."""
    if not local._env:
        raise EnvError("No GDAL environment exists")
    else:
        log.debug("Got a copy of environment %r options", local._env)
        return local._env.options.copy()


def hasenv():
    return bool(local._env)


def setenv(**options):
    """Set options in the existing environment."""
    if not local._env:
        raise EnvError("No GDAL environment exists")
    else:
        local._env.update_config_options(**options)


def hascreds():
    warnings.warn("Please use Env.session.hascreds() instead", RasterioDeprecationWarning)
    return local._env is not None and all(key in local._env.get_config_options() for key in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'])


def delenv():
    """Delete options in the existing environment."""
    if not local._env:
        raise EnvError("No GDAL environment exists")
    else:
        local._env.clear_config_options()
        log.debug("Cleared existing %r options", local._env)
    local._env.stop()
    local._env = None


class NullContextManager:

    def __init__(self):
        pass

    def __enter__(self):
        return self

    def __exit__(self, *args):
        pass


def env_ctx_if_needed():
    """Return an Env if one does not exist

    Returns
    -------
    Env or a do-nothing context manager

    """
    if local._env:
        return NullContextManager()
    else:
        return Env.from_defaults()


def ensure_env(f):
    """A decorator that ensures an env exists before a function
    calls any GDAL C functions."""
    @wraps(f)
    def wrapper(*args, **kwds):
        if local._env:
            return f(*args, **kwds)
        else:
            with Env.from_defaults():
                return f(*args, **kwds)
    return wrapper


def ensure_env_credentialled(f):
    """DEPRECATED alias for ensure_env_with_credentials"""
    warnings.warn("Please use ensure_env_with_credentials instead", RasterioDeprecationWarning)
    return ensure_env_with_credentials(f)


def ensure_env_with_credentials(f):
    """Ensures a config environment exists and is credentialized

    Parameters
    ----------
    f : function
        A function.

    Returns
    -------
    A function wrapper.

    Notes
    -----
    The function wrapper checks the first argument of f and
    credentializes the environment if the first argument is a URI with
    scheme "s3".

    """
    @wraps(f)
    def wrapper(*args, **kwds):
        if local._env:
            env_ctor = Env
        else:
            env_ctor = Env.from_defaults

        fp_arg = kwds.get("fp", None) or args[0]

        if isinstance(fp_arg, str):
            session_cls = Session.cls_from_path(fp_arg)

            if local._env and session_cls.hascreds(getenv()):
                session_cls = DummySession

            session = session_cls()

        else:
            session = DummySession()

        with env_ctor(session=session):
            return f(*args, **kwds)

    return wrapper


@attr.s(slots=True)
@total_ordering
class GDALVersion:
    """Convenience class for obtaining GDAL major and minor version components
    and comparing between versions.  This is highly simplistic and assumes a
    very normal numbering scheme for versions and ignores everything except
    the major and minor components."""

    major = attr.ib(default=0, validator=attr.validators.instance_of(int))
    minor = attr.ib(default=0, validator=attr.validators.instance_of(int))

    def __eq__(self, other):
        return (self.major, self.minor) == tuple(other.major, other.minor)

    def __lt__(self, other):
        return (self.major, self.minor) < tuple(other.major, other.minor)

    def __repr__(self):
        return "GDALVersion(major={0}, minor={1})".format(self.major, self.minor)

    def __str__(self):
        return "{0}.{1}".format(self.major, self.minor)

    @classmethod
    def parse(cls, input):
        """
        Parses input tuple or string to GDALVersion. If input is a GDALVersion
        instance, it is returned.

        Parameters
        ----------
        input: tuple of (major, minor), string, or instance of GDALVersion

        Returns
        -------
        GDALVersion instance
        """

        if isinstance(input, cls):
            return input
        if isinstance(input, tuple):
            return cls(*input)
        elif isinstance(input, str):
            # Extract major and minor version components.
            # alpha, beta, rc suffixes ignored
            match = re.search(r'^\d+\.\d+', input)
            if not match:
                raise ValueError(
                    "value does not appear to be a valid GDAL version "
                    "number: {}".format(input))
            major, minor = (int(c) for c in match.group().split('.'))
            return cls(major=major, minor=minor)

        raise TypeError("GDALVersion can only be parsed from a string or tuple")

    @classmethod
    def runtime(cls):
        """Return GDALVersion of current GDAL runtime"""
        return cls.parse(gdal_version())

    def at_least(self, other):
        other = self.__class__.parse(other)
        return self >= other


def require_gdal_version(version, param=None, values=None, is_max_version=False,
                         reason=''):
    """A decorator that ensures the called function or parameters are supported
    by the runtime version of GDAL.  Raises GDALVersionError if conditions
    are not met.

    Examples
    --------

    .. code-block:: python

        @require_gdal_version('2.2')
        def some_func():

    calling `some_func` with a runtime version of GDAL that is < 2.2 raises a
    GDALVersionErorr.

    .. code-block:: python

        @require_gdal_version('2.2', param='foo')
        def some_func(foo='bar'):

    calling `some_func` with parameter `foo` of any value on GDAL < 2.2 raises
    a GDALVersionError.

    .. code-block:: python

        @require_gdal_version('2.2', param='foo', values=('bar',))
        def some_func(foo=None):

    calling `some_func` with parameter `foo` and value `bar` on GDAL < 2.2
    raises a GDALVersionError.


    Parameters
    ------------
    version: tuple, string, or GDALVersion
    param: string (optional, default: None)
        If `values` are absent, then all use of this parameter with a value
        other than default value requires at least GDAL `version`.
    values: tuple, list, or set (optional, default: None)
        contains values that require at least GDAL `version`.  `param`
        is required for `values`.
    is_max_version: bool (optional, default: False)
        if `True` indicates that the version provided is the maximum version
        allowed, instead of requiring at least that version.
    reason: string (optional: default: '')
        custom error message presented to user in addition to message about
        GDAL version.  Use this to provide an explanation of what changed
        if necessary context to the user.

    Returns
    ---------
    wrapped function
    """

    if values is not None:
        if param is None:
            raise ValueError(
                'require_gdal_version: param must be provided with values')

        if not isinstance(values, (tuple, list, set)):
            raise ValueError(
                'require_gdal_version: values must be a tuple, list, or set')

    version = GDALVersion.parse(version)
    runtime = GDALVersion.runtime()
    inequality = '>=' if runtime < version else '<='
    reason = '\n{0}'.format(reason) if reason else reason

    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwds):
            if ((runtime < version and not is_max_version) or
                    (is_max_version and runtime > version)):

                if param is None:
                    raise GDALVersionError(
                        "GDAL version must be {0} {1}{2}".format(
                            inequality, str(version), reason))

                # normalize args and kwds to dict
                argspec = getargspec(f)
                full_kwds = kwds.copy()

                if argspec.args:
                    full_kwds.update(dict(zip(argspec.args[:len(args)], args)))

                if argspec.defaults:
                    defaults = dict(zip(
                        reversed(argspec.args), reversed(argspec.defaults)))
                else:
                    defaults = {}

                if param in full_kwds:
                    if values is None:
                        if param not in defaults or (
                                full_kwds[param] != defaults[param]):
                            raise GDALVersionError(
                                'usage of parameter "{0}" requires '
                                'GDAL {1} {2}{3}'.format(param, inequality,
                                                         version, reason))

                    elif full_kwds[param] in values:
                        raise GDALVersionError(
                            'parameter "{0}={1}" requires '
                            'GDAL {2} {3}{4}'.format(
                                param, full_kwds[param], inequality, version, reason))

            return f(*args, **kwds)

        return wrapper

    return decorator


# Patch the environment if needed, such as in the installed wheel case.

if 'GDAL_DATA' not in os.environ:

    path = GDALDataFinder().search_wheel()

    if path:
        log.debug("GDAL data found in package: path=%r.", path)
        set_gdal_config("GDAL_DATA", path)

    # See https://github.com/rasterio/rasterio/issues/1631.
    elif GDALDataFinder().find_file("header.dxf"):
        log.debug("GDAL data files are available at built-in paths.")

    else:
        path = GDALDataFinder().search()

        if path:
            set_gdal_config("GDAL_DATA", path)
            log.debug("GDAL data found in other locations: path=%r.", path)

if "PROJ_DATA" in os.environ:
    # PROJ 9.1+
    path = os.environ["PROJ_DATA"]
    set_proj_data_search_path(path)
elif "PROJ_LIB" in os.environ:
    # PROJ < 9.1
    path = os.environ["PROJ_LIB"]
    set_proj_data_search_path(path)

elif PROJDataFinder().search_wheel():
    path = PROJDataFinder().search_wheel()
    log.debug("PROJ data found in package: path=%r.", path)
    set_proj_data_search_path(path)

# See https://github.com/rasterio/rasterio/issues/1631.
elif PROJDataFinder().has_data():
    log.debug("PROJ data files are available at built-in paths.")

else:
    path = PROJDataFinder().search()

    if path:
        log.debug("PROJ data found in other locations: path=%r.", path)
        set_proj_data_search_path(path)

env_pyrecodes/lib/python3.9/site-packages/rasterio/env.py
NikolaBlagojevic/pyrecodes
"""Errors and Warnings."""

from click import FileError


class RasterioError(Exception):
    """Root exception class"""


class InvalidArrayError(RasterioError):
    """Raised when methods are passed invalid arrays"""


class WindowError(RasterioError):
    """Raised when errors occur during window operations"""


class CRSError(ValueError):
    """Raised when a CRS string or mapping is invalid or cannot serve
    to define a coordinate transformation."""


class EnvError(RasterioError):
    """Raised when the state of GDAL/AWS environment cannot be created
    or modified."""


class DriverCapabilityError(RasterioError, ValueError):
    """Raised when a format driver can't a feature such as writing."""


class DriverRegistrationError(ValueError):
    """Raised when a format driver is requested but is not registered."""


class FileOverwriteError(FileError):
    """Raised when Rasterio's CLI refuses to clobber output files."""

    def __init__(self, message):
        """Raise FileOverwriteError with message as hint."""
        super().__init__('', hint=message)


class RasterioIOError(OSError):
    """Raised when a dataset cannot be opened using one of the
    registered format drivers."""


class NodataShadowWarning(UserWarning):
    """Warn that a dataset's nodata attribute is shadowing its alpha band."""

    def __str__(self):
        return ("The dataset's nodata attribute is shadowing "
                "the alpha band. All masks will be determined "
                "by the nodata attribute")


class NotGeoreferencedWarning(UserWarning):
    """Warn that a dataset isn't georeferenced."""


class TransformWarning(UserWarning):
    """Warn that coordinate transformations may behave unexpectedly"""

class RPCError(ValueError):
    """Raised when RPC transformation is invalid"""


class ShapeSkipWarning(UserWarning):
    """Warn that an invalid or empty shape in a collection has been skipped"""


class GDALBehaviorChangeException(RuntimeError):
    """Raised when GDAL's behavior differs from the given arguments.  For
    example, antimeridian cutting is always on as of GDAL 2.2.0.  Users
    expecting it to be off will be presented with a MultiPolygon when the
    rest of their code expects a Polygon.

    Examples
    --------

    .. code-block:: python

        # Raises an exception on GDAL >= 2.2.0
        rasterio.warp.transform_geometry(
            src_crs, dst_crs, antimeridian_cutting=False)
    """


class GDALOptionNotImplementedError(RasterioError):
    """A dataset opening or dataset creation option can't be supported

    This will be raised from Rasterio's shim modules. For example, when
    a user passes arguments to open_dataset() that can't be evaluated
    by GDAL 1.x.
    """


class GDALVersionError(RasterioError):
    """Raised if the runtime version of GDAL does not meet the required version of GDAL."""


class WindowEvaluationError(ValueError):
    """Raised when window evaluation fails"""


class RasterioDeprecationWarning(FutureWarning):
    """Rasterio module deprecations

    Following https://www.python.org/dev/peps/pep-0565/#additional-use-case-for-futurewarning
    we base this on FutureWarning while continuing to support Python < 3.7.
    """


class RasterBlockError(RasterioError):
    """Raised when raster block access fails"""


class BandOverviewError(UserWarning):
    """Raised when a band overview access fails."""


class WarpOptionsError(RasterioError):
    """Raised when options for a warp operation are invalid"""


class UnsupportedOperation(RasterioError):
    """Raised when reading from a file opened in 'w' mode"""


class OverviewCreationError(RasterioError):
    """Raised when creation of an overview fails"""


class DatasetAttributeError(RasterioError, NotImplementedError):
    """Raised when dataset attributes are misused"""


class PathError(RasterioError):
    """Raised when a dataset path is malformed or invalid"""


class ResamplingAlgorithmError(RasterioError):
    """Raised when a resampling algorithm is invalid or inapplicable"""


class TransformError(RasterioError):
    """Raised when transform arguments are invalid"""


class WarpedVRTError(RasterioError):
    """Raised when WarpedVRT can't be initialized"""


class DatasetIOShapeError(RasterioError):
    """Raised when data buffer shape is a mismatch when reading and writing"""


class WarpOperationError(RasterioError):
    """Raised when a warp operation fails."""


class StatisticsError(RasterioError):
    """Raised when dataset statistics cannot be computed."""

env_pyrecodes/lib/python3.9/site-packages/rasterio/errors.py
NikolaBlagojevic/pyrecodes
"""Functions for working with features in a raster dataset."""

import logging
import math
import os
import warnings

import numpy as np

import rasterio
from rasterio.dtypes import (
    validate_dtype,
    can_cast_dtype,
    get_minimum_dtype,
    _getnpdtype,
)
from rasterio.enums import MergeAlg
from rasterio.env import ensure_env, GDALVersion
from rasterio.errors import ShapeSkipWarning
from rasterio._features import _shapes, _sieve, _rasterize, _bounds
from rasterio import warp
from rasterio.rio.helpers import coords
from rasterio.transform import Affine
from rasterio.transform import IDENTITY, guard_transform, rowcol
from rasterio.windows import Window

log = logging.getLogger(__name__)


@ensure_env
def geometry_mask(
        geometries,
        out_shape,
        transform,
        all_touched=False,
        invert=False):
    """Create a mask from shapes.

    By default, mask is intended for use as a
    numpy mask, where pixels that overlap shapes are False.

    Parameters
    ----------
    geometries : iterable over geometries (GeoJSON-like objects)
    out_shape : tuple or list
        Shape of output numpy ndarray.
    transform : Affine transformation object
        Transformation from pixel coordinates of `source` to the
        coordinate system of the input `shapes`. See the `transform`
        property of dataset objects.
    all_touched : boolean, optional
        If True, all pixels touched by geometries will be burned in.  If
        false, only pixels whose center is within the polygon or that
        are selected by Bresenham's line algorithm will be burned in.
    invert: boolean, optional
        If True, mask will be True for pixels that overlap shapes.
        False by default.

    Returns
    -------
    numpy ndarray of type 'bool'
        Result

    Notes
    -----
    See rasterize() for performance notes.

    """
    fill, mask_value = (0, 1) if invert else (1, 0)

    return rasterize(
        geometries,
        out_shape=out_shape,
        transform=transform,
        all_touched=all_touched,
        fill=fill,
        default_value=mask_value).astype('bool')


@ensure_env
def shapes(source, mask=None, connectivity=4, transform=IDENTITY):
    """Get shapes and values of connected regions in a dataset or array.

    Parameters
    ----------
    source : array, dataset object, Band, or tuple(dataset, bidx)
        Data type must be one of rasterio.int16, rasterio.int32,
        rasterio.uint8, rasterio.uint16, or rasterio.float32.
    mask : numpy ndarray or rasterio Band object, optional
        Must evaluate to bool (rasterio.bool_ or rasterio.uint8). Values
        of False or 0 will be excluded from feature generation.  Note
        well that this is the inverse sense from Numpy's, where a mask
        value of True indicates invalid data in an array. If `source` is
        a Numpy masked array and `mask` is None, the source's mask will
        be inverted and used in place of `mask`.
    connectivity : int, optional
        Use 4 or 8 pixel connectivity for grouping pixels into features
    transform : Affine transformation, optional
        If not provided, feature coordinates will be generated based on
        pixel coordinates

    Yields
    -------
    polygon, value
        A pair of (polygon, value) for each feature found in the image.
        Polygons are GeoJSON-like dicts and the values are the
        associated value from the image, in the data type of the image.
        Note: due to floating point precision issues, values returned
        from a floating point image may not exactly match the original
        values.

    Notes
    -----
    The amount of memory used by this algorithm is proportional to the
    number and complexity of polygons produced.  This algorithm is most
    appropriate for simple thematic data.  Data with high pixel-to-pixel
    variability, such as imagery, may produce one polygon per pixel and
    consume large amounts of memory.

    Because the low-level implementation uses either an int32 or float32
    buffer, uint32 and float64 data cannot be operated on without
    truncation issues.

    """
    if hasattr(source, 'mask') and mask is None:
        mask = ~source.mask
        source = source.data

    transform = guard_transform(transform)
    for s, v in _shapes(source, mask, connectivity, transform):
        yield s, v


@ensure_env
def sieve(source, size, out=None, mask=None, connectivity=4):
    """Replace small polygons in `source` with value of their largest neighbor.

    Polygons are found for each set of neighboring pixels of the same value.

    Parameters
    ----------
    source : array or dataset object opened in 'r' mode or Band or tuple(dataset, bidx)
        Must be of type rasterio.int16, rasterio.int32, rasterio.uint8,
        rasterio.uint16, or rasterio.float32
    size : int
        minimum polygon size (number of pixels) to retain.
    out : numpy ndarray, optional
        Array of same shape and data type as `source` in which to store results.
    mask : numpy ndarray or rasterio Band object, optional
        Values of False or 0 will be excluded from feature generation
        Must evaluate to bool (rasterio.bool_ or rasterio.uint8)
    connectivity : int, optional
        Use 4 or 8 pixel connectivity for grouping pixels into features

    Returns
    -------
    out : numpy ndarray
        Result

    Notes
    -----
    GDAL only supports values that can be cast to 32-bit integers for this
    operation.

    The amount of memory used by this algorithm is proportional to the number
    and complexity of polygons found in the image.  This algorithm is most
    appropriate for simple thematic data.  Data with high pixel-to-pixel
    variability, such as imagery, may produce one polygon per pixel and consume
    large amounts of memory.

    """

    if out is None:
        out = np.zeros(source.shape, source.dtype)
    _sieve(source, size, out, mask, connectivity)
    return out


@ensure_env
def rasterize(
        shapes,
        out_shape=None,
        fill=0,
        out=None,
        transform=IDENTITY,
        all_touched=False,
        merge_alg=MergeAlg.replace,
        default_value=1,
        dtype=None):
    """Return an image array with input geometries burned in.

    Warnings will be raised for any invalid or empty geometries, and
    an exception will be raised if there are no valid shapes
    to rasterize.

    Parameters
    ----------
    shapes : iterable of (`geometry`, `value`) pairs or geometries
        The `geometry` can either be an object that implements the geo
        interface or GeoJSON-like object. If no `value` is provided
        the `default_value` will be used. If `value` is `None` the
        `fill` value will be used.
    out_shape : tuple or list with 2 integers
        Shape of output numpy ndarray.
    fill : int or float, optional
        Used as fill value for all areas not covered by input
        geometries.
    out : numpy ndarray, optional
        Array of same shape and data type as `source` in which to store
        results.
    transform : Affine transformation object, optional
        Transformation from pixel coordinates of `source` to the
        coordinate system of the input `shapes`. See the `transform`
        property of dataset objects.
    all_touched : boolean, optional
        If True, all pixels touched by geometries will be burned in.  If
        false, only pixels whose center is within the polygon or that
        are selected by Bresenham's line algorithm will be burned in.
    merge_alg : MergeAlg, optional
        Merge algorithm to use. One of:
            MergeAlg.replace (default):
                the new value will overwrite the existing value.
            MergeAlg.add:
                the new value will be added to the existing raster.
    default_value : int or float, optional
        Used as value for all geometries, if not provided in `shapes`.
    dtype : rasterio or numpy data type, optional
        Used as data type for results, if `out` is not provided.

    Returns
    -------
    numpy ndarray
        If `out` was not None then `out` is returned, it will have been
        modified in-place. If `out` was None, this will be a new array.

    Notes
    -----
    Valid data types for `fill`, `default_value`, `out`, `dtype` and
    shape values are "int16", "int32", "uint8", "uint16", "uint32",
    "float32", and "float64".

    This function requires significant memory resources. The shapes
    iterator will be materialized to a Python list and another C copy of
    that list will be made. The `out` array will be copied and
    additional temporary raster memory equal to 2x the smaller of `out`
    data or GDAL's max cache size (controlled by GDAL_CACHEMAX, default
    is 5% of the computer's physical memory) is required.

    If GDAL max cache size is smaller than the output data, the array of
    shapes will be iterated multiple times. Performance is thus a linear
    function of buffer size. For maximum speed, ensure that
    GDAL_CACHEMAX is larger than the size of `out` or `out_shape`.

    """
    valid_dtypes = (
        'int16', 'int32', 'uint8', 'uint16', 'uint32', 'float32', 'float64'
    )
    if GDALVersion.runtime().at_least("3.5"):
        valid_dtypes = valid_dtypes + ("int64", "uint64")

    def format_invalid_dtype(param):
        return '{0} dtype must be one of: {1}'.format(
            param, ', '.join(valid_dtypes)
        )

    def format_cast_error(param, dtype):
        return '{0} cannot be cast to specified dtype: {1}'.format(param, dtype)

    if fill != 0:
        fill_array = np.array([fill])
        if not validate_dtype(fill_array, valid_dtypes):
            raise ValueError(format_invalid_dtype('fill'))

        if dtype is not None and not can_cast_dtype(fill_array, dtype):
            raise ValueError(format_cast_error('fill', dtype))

    if default_value != 1:
        default_value_array = np.array([default_value])
        if not validate_dtype(default_value_array, valid_dtypes):
            raise ValueError(format_invalid_dtype('default_value'))

        if dtype is not None and not can_cast_dtype(default_value_array, dtype):
            raise ValueError(format_cast_error('default_vaue', dtype))

    if dtype is not None and _getnpdtype(dtype).name not in valid_dtypes:
        raise ValueError(format_invalid_dtype('dtype'))

    valid_shapes = []
    shape_values = []
    for index, item in enumerate(shapes):
        if isinstance(item, (tuple, list)):
            geom, value = item
            if value is None:
                value = fill
        else:
            geom = item
            value = default_value
        geom = getattr(geom, '__geo_interface__', None) or geom

        if is_valid_geom(geom):
            shape_values.append(value)

            geom_type = geom['type']

            if geom_type == 'GeometryCollection':
                # GeometryCollections need to be handled as individual parts to
                # avoid holes in output:
                # https://github.com/rasterio/rasterio/issues/1253.
                # Only 1-level deep since GeoJSON spec discourages nested
                # GeometryCollections
                for part in geom['geometries']:
                    valid_shapes.append((part, value))

            elif geom_type == 'MultiPolygon':
                # Same issue as above
                for poly in geom['coordinates']:
                    valid_shapes.append(({'type': 'Polygon', 'coordinates': poly}, value))

            else:
                valid_shapes.append((geom, value))

        else:
            # invalid or empty geometries are skipped and raise a warning instead
            warnings.warn('Invalid or empty shape {} at index {} will not be rasterized.'.format(geom, index), ShapeSkipWarning)

    if not valid_shapes:
        raise ValueError('No valid geometry objects found for rasterize')

    shape_values = np.array(shape_values)

    if not validate_dtype(shape_values, valid_dtypes):
        raise ValueError(format_invalid_dtype('shape values'))

    if dtype is None:
        dtype = get_minimum_dtype(np.append(shape_values, fill))

    elif not can_cast_dtype(shape_values, dtype):
        raise ValueError(format_cast_error('shape values', dtype))

    if out is not None:
        if _getnpdtype(out.dtype).name not in valid_dtypes:
            raise ValueError(format_invalid_dtype('out'))

        if not can_cast_dtype(shape_values, out.dtype):
            raise ValueError(format_cast_error('shape values', out.dtype.name))

    elif out_shape is not None:

        if len(out_shape) != 2:
            raise ValueError('Invalid out_shape, must be 2D')

        out = np.empty(out_shape, dtype=dtype)
        out.fill(fill)

    else:
        raise ValueError('Either an out_shape or image must be provided')

    if min(out.shape) == 0:
        raise ValueError("width and height must be > 0")

    transform = guard_transform(transform)
    _rasterize(valid_shapes, out, transform, all_touched, merge_alg)
    return out


def bounds(geometry, north_up=True, transform=None):
    """Return a (left, bottom, right, top) bounding box.

    From Fiona 1.4.8. Modified to return bbox from geometry if available.

    Parameters
    ----------
    geometry: GeoJSON-like feature (implements __geo_interface__),
              feature collection, or geometry.

    Returns
    -------
    tuple
        Bounding box: (left, bottom, right, top)
    """

    geometry = getattr(geometry, '__geo_interface__', None) or geometry

    if 'bbox' in geometry:
        return tuple(geometry['bbox'])

    geom = geometry.get('geometry') or geometry

    # geometry must be a geometry, GeometryCollection, or FeatureCollection
    if not ('coordinates' in geom or 'geometries' in geom or 'features' in geom):
        raise ValueError(
            "geometry must be a GeoJSON-like geometry, GeometryCollection, "
            "or FeatureCollection"
        )

    return _bounds(geom, north_up=north_up, transform=transform)


def geometry_window(
    dataset,
    shapes,
    pad_x=0,
    pad_y=0,
    north_up=None,
    rotated=None,
    pixel_precision=None,
    boundless=False,
):
    """Calculate the window within the raster that fits the bounds of
    the geometry plus optional padding.  The window is the outermost
    pixel indices that contain the geometry (floor of offsets, ceiling
    of width and height).

    If shapes do not overlap raster, a WindowError is raised.

    Parameters
    ----------
    dataset : dataset object opened in 'r' mode
        Raster for which the mask will be created.
    shapes : iterable over geometries.
        A geometry is a GeoJSON-like object or implements the geo
        interface.  Must be in same coordinate system as dataset.
    pad_x : float
        Amount of padding (as fraction of raster's x pixel size) to add
        to left and right side of bounds.
    pad_y : float
        Amount of padding (as fraction of raster's y pixel size) to add
        to top and bottom of bounds.
    north_up : optional
        This parameter is ignored since version 1.2.1. A deprecation
        warning will be emitted in 1.3.0.
    rotated : optional
        This parameter is ignored since version 1.2.1. A deprecation
        warning will be emitted in 1.3.0.
    pixel_precision : int or float, optional
        Number of places of rounding precision or absolute precision for
        evaluating bounds of shapes.
    boundless : bool, optional
        Whether to allow a boundless window or not.

    Returns
    -------
    rasterio.windows.Window

    """

    all_bounds = [bounds(shape, transform=~dataset.transform) for shape in shapes]

    cols = [
        x
        for (left, bottom, right, top) in all_bounds
        for x in (left - pad_x, right + pad_x, right + pad_x, left - pad_x)
    ]
    rows = [
        y
        for (left, bottom, right, top) in all_bounds
        for y in (top - pad_y, top - pad_y, bottom + pad_y, bottom + pad_y)
    ]

    row_start, row_stop = int(math.floor(min(rows))), int(math.ceil(max(rows)))
    col_start, col_stop = int(math.floor(min(cols))), int(math.ceil(max(cols)))

    window = Window(
        col_off=col_start,
        row_off=row_start,
        width=max(col_stop - col_start, 0.0),
        height=max(row_stop - row_start, 0.0),
    )

    # Make sure that window overlaps raster
    raster_window = Window(0, 0, dataset.width, dataset.height)
    if not boundless:
        window = window.intersection(raster_window)

    return window


def is_valid_geom(geom):
    """
    Checks to see if geometry is a valid GeoJSON geometry type or
    GeometryCollection.  Geometry must be GeoJSON or implement the geo
    interface.

    Geometries must be non-empty, and have at least x, y coordinates.

    Note: only the first coordinate is checked for validity.

    Parameters
    ----------
    geom: an object that implements the geo interface or GeoJSON-like object

    Returns
    -------
    bool: True if object is a valid GeoJSON geometry type
    """

    geom_types = {'Point', 'MultiPoint', 'LineString', 'LinearRing',
                  'MultiLineString', 'Polygon', 'MultiPolygon'}

    geom = getattr(geom, '__geo_interface__', None) or geom

    try:
        geom_type = geom["type"]
        if geom_type not in geom_types.union({'GeometryCollection'}):
            return False

    except (KeyError, TypeError):
        return False

    if geom_type in geom_types:
        if 'coordinates' not in geom:
            return False

        coords = geom['coordinates']

        if geom_type == 'Point':
            # Points must have at least x, y
            return len(coords) >= 2

        if geom_type == 'MultiPoint':
            # Multi points must have at least one point with at least x, y
            return len(coords) > 0 and len(coords[0]) >= 2

        if geom_type == 'LineString':
            # Lines must have at least 2 coordinates and at least x, y for
            # a coordinate
            return len(coords) >= 2 and len(coords[0]) >= 2

        if geom_type == 'LinearRing':
            # Rings must have at least 4 coordinates and at least x, y for
            # a coordinate
            return len(coords) >= 4 and len(coords[0]) >= 2

        if geom_type == 'MultiLineString':
            # Multi lines must have at least one LineString
            return (len(coords) > 0 and len(coords[0]) >= 2 and
                    len(coords[0][0]) >= 2)

        if geom_type == 'Polygon':
            # Polygons must have at least 1 ring, with at least 4 coordinates,
            # with at least x, y for a coordinate
            return (len(coords) > 0 and len(coords[0]) >= 4 and
                    len(coords[0][0]) >= 2)

        if geom_type == 'MultiPolygon':
            # Muti polygons must have at least one Polygon
            return (len(coords) > 0 and len(coords[0]) > 0 and
                    len(coords[0][0]) >= 4 and len(coords[0][0][0]) >= 2)

    if geom_type == 'GeometryCollection':
        if 'geometries' not in geom:
            return False

        if not len(geom['geometries']) > 0:
            # While technically valid according to GeoJSON spec, an empty
            # GeometryCollection will cause issues if used in rasterio
            return False

        for g in geom['geometries']:
            if not is_valid_geom(g):
                return False  # short-circuit and fail early

    return True


def dataset_features(
        src,
        bidx=None,
        sampling=1,
        band=True,
        as_mask=False,
        with_nodata=False,
        geographic=True,
        precision=-1):
    """Yield GeoJSON features for the dataset

    The geometries are polygons bounding contiguous regions of the same raster value.

    Parameters
    ----------
    src: Rasterio Dataset

    bidx: int
        band index

    sampling: int (DEFAULT: 1)
        Inverse of the sampling fraction; a value of 10 decimates

    band: boolean (DEFAULT: True)
        extract features from a band (True) or a mask (False)

    as_mask: boolean (DEFAULT: False)
        Interpret band as a mask and output only one class of valid data shapes?

    with_nodata: boolean (DEFAULT: False)
        Include nodata regions?

    geographic: str (DEFAULT: True)
        Output shapes in EPSG:4326? Otherwise use the native CRS.

    precision: int (DEFAULT: -1)
        Decimal precision of coordinates. -1 for full float precision output

    Yields
    ------
    GeoJSON-like Feature dictionaries for shapes found in the given band
    """
    if bidx is not None and bidx > src.count:
        raise ValueError('bidx is out of range for raster')

    img = None
    msk = None

    # Adjust transforms.
    transform = src.transform
    if sampling > 1:
        # Determine the target shape (to decimate)
        shape = (int(math.ceil(src.height / sampling)),
                 int(math.ceil(src.width / sampling)))

        # Calculate independent sampling factors
        x_sampling = src.width / shape[1]
        y_sampling = src.height / shape[0]

        # Decimation of the raster produces a georeferencing
        # shift that we correct with a translation.
        transform *= Affine.translation(
            src.width % x_sampling, src.height % y_sampling)

        # And follow by scaling.
        transform *= Affine.scale(x_sampling, y_sampling)

    # Most of the time, we'll use the valid data mask.
    # We skip reading it if we're extracting every possible
    # feature (even invalid data features) from a band.
    if not band or (band and not as_mask and not with_nodata):
        if sampling == 1:
            msk = src.read_masks(bidx)
        else:
            msk_shape = shape
            if bidx is None:
                msk = np.zeros(
                    (src.count,) + msk_shape, 'uint8')
            else:
                msk = np.zeros(msk_shape, 'uint8')
            msk = src.read_masks(bidx, msk)

        if bidx is None:
            msk = np.logical_or.reduce(msk).astype('uint8')

        # Possibly overridden below.
        img = msk

    # Read the band data unless the --mask option is given.
    if band:
        if sampling == 1:
            img = src.read(bidx, masked=False)
        else:
            img = np.zeros(
                shape,
                dtype=src.dtypes[src.indexes.index(bidx)])
            img = src.read(bidx, img, masked=False)

    # If as_mask option was given, convert the image
    # to a binary image. This reduces the number of shape
    # categories to 2 and likely reduces the number of
    # shapes.
    if as_mask:
        tmp = np.ones_like(img, 'uint8') * 255
        tmp[img == 0] = 0
        img = tmp
        if not with_nodata:
            msk = tmp

    # Prepare keyword arguments for shapes().
    kwargs = {'transform': transform}
    if not with_nodata:
        kwargs['mask'] = msk

    src_basename = os.path.basename(src.name)

    # Yield GeoJSON features.
    for i, (g, val) in enumerate(
            rasterio.features.shapes(img, **kwargs)):
        if geographic:
            g = warp.transform_geom(
                src.crs, 'EPSG:4326', g,
                antimeridian_cutting=True, precision=precision)
        xs, ys = zip(*coords(g))
        yield {
            'type': 'Feature',
            'id': "{0}:{1}".format(src_basename, i),
            'properties': {
                'val': val,
                'filename': src_basename
            },
            'bbox': [min(xs), min(ys), max(xs), max(ys)],
            'geometry': g
        }

env_pyrecodes/lib/python3.9/site-packages/rasterio/features.py
NikolaBlagojevic/pyrecodes
"""Fill holes in raster dataset by interpolation from the edges."""

from numpy.ma import MaskedArray

import rasterio
from rasterio._fill import _fillnodata
from rasterio.env import ensure_env
from rasterio import dtypes


@ensure_env
def fillnodata(
        image,
        mask=None,
        max_search_distance=100.0,
        smoothing_iterations=0):
    """Fill holes in raster data by interpolation

    This algorithm will interpolate values for all designated nodata
    pixels (marked by zeros in `mask`). For each pixel a four direction
    conic search is done to find values to interpolate from (using
    inverse distance weighting). Once all values are interpolated, zero
    or more smoothing iterations (3x3 average filters on interpolated
    pixels) are applied to smooth out artifacts.

    This algorithm is generally suitable for interpolating missing
    regions of fairly continuously varying rasters (such as elevation
    models for instance). It is also suitable for filling small holes
    and cracks in more irregularly varying images (like aerial photos).
    It is generally not so great for interpolating a raster from sparse
    point data.

    Parameters
    ----------
    image : numpy ndarray
        The source image with holes to be filled. If a MaskedArray, the
        inverse of its mask will define the pixels to be filled --
        unless the ``mask`` argument is not None (see below).`
    mask : numpy ndarray or None
        A mask band indicating which pixels to interpolate. Pixels to
        interpolate into are indicated by the value 0. Values
        > 0 indicate areas to use during interpolation. Must be same
        shape as image. This array always takes precedence over the
        image's mask (see above). If None, the inverse of the image's
        mask will be used if available.
    max_search_distance : float, optional
        The maximum number of pixels to search in all directions to
        find values to interpolate from. The default is 100.
    smoothing_iterations : integer, optional
        The number of 3x3 smoothing filter passes to run. The default is
        0.

    Returns
    -------
    out : numpy ndarray
        The filled raster array.
    """
    if mask is None and isinstance(image, MaskedArray):
        mask = ~image.mask
    if not dtypes.is_ndarray(mask):
        raise ValueError("An mask array is required")

    if isinstance(image, MaskedArray):
        image = image.data
    if not dtypes.is_ndarray(image):
        raise ValueError("An image array is required")

    max_search_distance = float(max_search_distance)
    smoothing_iterations = int(smoothing_iterations)
    return _fillnodata(
        image, mask, max_search_distance, smoothing_iterations)

env_pyrecodes/lib/python3.9/site-packages/rasterio/fill.py
NikolaBlagojevic/pyrecodes
"""Classes capable of reading and writing datasets

Instances of these classes are called dataset objects.
"""

import logging

from rasterio._base import get_dataset_driver, driver_can_create, driver_can_create_copy
from rasterio._io import (
    DatasetReaderBase,
    DatasetWriterBase,
    BufferedDatasetWriterBase,
    MemoryFileBase,
)
from rasterio.windows import WindowMethodsMixin
from rasterio.env import ensure_env
from rasterio.transform import TransformMethodsMixin
from rasterio._path import _UnparsedPath

try:
    from rasterio._filepath import FilePathBase
except ImportError:
    FilePathBase = object


log = logging.getLogger(__name__)


class DatasetReader(DatasetReaderBase, WindowMethodsMixin, TransformMethodsMixin):
    """An unbuffered data and metadata reader"""

    def __repr__(self):
        return "<{} DatasetReader name='{}' mode='{}'>".format(
            self.closed and 'closed' or 'open', self.name, self.mode)


class DatasetWriter(DatasetWriterBase, WindowMethodsMixin, TransformMethodsMixin):
    """An unbuffered data and metadata writer. Its methods write data
    directly to disk.
    """

    def __repr__(self):
        return "<{} DatasetWriter name='{}' mode='{}'>".format(
            self.closed and 'closed' or 'open', self.name, self.mode)


class BufferedDatasetWriter(
    BufferedDatasetWriterBase, WindowMethodsMixin, TransformMethodsMixin
):
    """Maintains data and metadata in a buffer, writing to disk or
    network only when `close()` is called.

    This allows incremental updates to datasets using formats that don't
    otherwise support updates, such as JPEG.
    """

    def __repr__(self):
        return "<{} BufferedDatasetWriter name='{}' mode='{}'>".format(
            self.closed and 'closed' or 'open', self.name, self.mode)


class MemoryFile(MemoryFileBase):
    """A BytesIO-like object, backed by an in-memory file.

    This allows formatted files to be read and written without I/O.

    A MemoryFile created with initial bytes becomes immutable. A
    MemoryFile created without initial bytes may be written to using
    either file-like or dataset interfaces.

    Examples
    --------

    A GeoTIFF can be loaded in memory and accessed using the GeoTIFF
    format driver

    >>> with open('tests/data/RGB.byte.tif', 'rb') as f, MemoryFile(f) as memfile:
    ...     with memfile.open() as src:
    ...         pprint.pprint(src.profile)
    ...
    {'count': 3,
     'crs': CRS({'init': 'epsg:32618'}),
     'driver': 'GTiff',
     'dtype': 'uint8',
     'height': 718,
     'interleave': 'pixel',
     'nodata': 0.0,
     'tiled': False,
     'transform': Affine(300.0379266750948, 0.0, 101985.0,
           0.0, -300.041782729805, 2826915.0),
     'width': 791}

    """

    def __init__(self, file_or_bytes=None, dirname=None, filename=None, ext=".tif"):
        """Create a new file in memory

        Parameters
        ----------
        file_or_bytes : file-like object or bytes, optional
            File or bytes holding initial data.
        filename : str, optional
            An optional filename. A unique one will otherwise be generated.
        ext : str, optional
            An optional extension.

        Returns
        -------
        MemoryFile
        """
        super().__init__(
            file_or_bytes=file_or_bytes, dirname=dirname, filename=filename, ext=ext
        )

    @ensure_env
    def open(self, driver=None, width=None, height=None, count=None, crs=None,
             transform=None, dtype=None, nodata=None, sharing=False, **kwargs):
        """Open the file and return a Rasterio dataset object.

        If data has already been written, the file is opened in 'r'
        mode. Otherwise, the file is opened in 'w' mode.

        Parameters
        ----------
        Note well that there is no `path` parameter: a `MemoryFile`
        contains a single dataset and there is no need to specify a
        path.

        Other parameters are optional and have the same semantics as the
        parameters of `rasterio.open()`.
        """
        mempath = _UnparsedPath(self.name)

        if self.closed:
            raise OSError("I/O operation on closed file.")
        if len(self) > 0:
            log.debug("VSI path: {}".format(mempath.path))
            return DatasetReader(mempath, driver=driver, sharing=sharing, **kwargs)
        else:
            writer = get_writer_for_driver(driver)
            return writer(
                mempath,
                "w+",
                driver=driver,
                width=width,
                height=height,
                count=count,
                crs=crs,
                transform=transform,
                dtype=dtype,
                nodata=nodata,
                sharing=sharing,
                **kwargs
            )

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()


class _FilePath(FilePathBase):
    """A BytesIO-like object, backed by a Python file object.

    Examples
    --------

    A GeoTIFF can be loaded in memory and accessed using the GeoTIFF
    format driver

    >>> with open('tests/data/RGB.byte.tif', 'rb') as f, FilePath(f) as vsi_file:
    ...     with vsi_file.open() as src:
    ...         pprint.pprint(src.profile)
    ...
    {'count': 3,
     'crs': CRS({'init': 'epsg:32618'}),
     'driver': 'GTiff',
     'dtype': 'uint8',
     'height': 718,
     'interleave': 'pixel',
     'nodata': 0.0,
     'tiled': False,
     'transform': Affine(300.0379266750948, 0.0, 101985.0,
           0.0, -300.041782729805, 2826915.0),
     'width': 791}

    """

    def __init__(self, filelike_obj, dirname=None, filename=None):
        """Create a new wrapper around the provided file-like object.

        Parameters
        ----------
        filelike_obj : file-like object
            Open file-like object. Currently only reading is supported.
        filename : str, optional
            An optional filename. A unique one will otherwise be generated.

        Returns
        -------
        PythonVSIFile
        """
        super().__init__(
            filelike_obj, dirname=dirname, filename=filename
        )

    @ensure_env
    def open(self, driver=None, sharing=False, **kwargs):
        """Open the file and return a Rasterio dataset object.

        The provided file-like object is assumed to be readable.
        Writing is currently not supported.

        Parameters are optional and have the same semantics as the
        parameters of `rasterio.open()`.
        """
        mempath = _UnparsedPath(self.name)

        if self.closed:
            raise IOError("I/O operation on closed file.")
        # Assume we were given a non-empty file-like object
        log.debug("VSI path: {}".format(mempath.path))
        return DatasetReader(mempath, driver=driver, sharing=sharing, **kwargs)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()


if FilePathBase is not object:
    # only make this object available if the cython extension was compiled
    FilePath = _FilePath


class ZipMemoryFile(MemoryFile):
    """A read-only BytesIO-like object backed by an in-memory zip file.

    This allows a zip file containing formatted files to be read
    without I/O.
    """

    def __init__(self, file_or_bytes=None):
        super().__init__(file_or_bytes, ext="zip")

    @ensure_env
    def open(self, path, driver=None, sharing=False, **kwargs):
        """Open a dataset within the zipped stream.

        Parameters
        ----------
        path : str
            Path to a dataset in the zip file, relative to the root of the
            archive.

        Other parameters are optional and have the same semantics as the
        parameters of `rasterio.open()`.

        Returns
        -------
        A Rasterio dataset object
        """
        zippath = _UnparsedPath('/vsizip{0}/{1}'.format(self.name, path.lstrip('/')))

        if self.closed:
            raise OSError("I/O operation on closed file.")
        return DatasetReader(zippath, driver=driver, sharing=sharing, **kwargs)


def get_writer_for_driver(driver):
    """Return the writer class appropriate for the specified driver."""
    if not driver:
        raise ValueError("'driver' is required to read/write dataset.")
    cls = None
    if driver_can_create(driver):
        cls = DatasetWriter
    elif driver_can_create_copy(driver):  # pragma: no branch
        cls = BufferedDatasetWriter
    return cls


def get_writer_for_path(path, driver=None):
    """Return the writer class appropriate for the existing dataset."""
    if not driver:
        driver = get_dataset_driver(path)
    return get_writer_for_driver(driver)

env_pyrecodes/lib/python3.9/site-packages/rasterio/io.py
NikolaBlagojevic/pyrecodes
"""Copy valid pixels from input files to an output file."""

from contextlib import contextmanager
import logging
import os
import math
from pathlib import Path
import warnings

import numpy as np

import rasterio
from rasterio.coords import disjoint_bounds
from rasterio.enums import Resampling
from rasterio.errors import RasterioDeprecationWarning
from rasterio import windows
from rasterio.transform import Affine

logger = logging.getLogger(__name__)


def copy_first(merged_data, new_data, merged_mask, new_mask, **kwargs):
    """Returns the first available pixel."""
    mask = np.empty_like(merged_mask, dtype="bool")
    np.logical_not(new_mask, out=mask)
    np.logical_and(merged_mask, mask, out=mask)
    np.copyto(merged_data, new_data, where=mask, casting="unsafe")


def copy_last(merged_data, new_data, merged_mask, new_mask, **kwargs):
    """Returns the last available pixel."""
    mask = np.empty_like(merged_mask, dtype="bool")
    np.logical_not(new_mask, out=mask)
    np.copyto(merged_data, new_data, where=mask, casting="unsafe")


def copy_min(merged_data, new_data, merged_mask, new_mask, **kwargs):
    """Returns the minimum value pixel."""
    mask = np.empty_like(merged_mask, dtype="bool")
    np.logical_or(merged_mask, new_mask, out=mask)
    np.logical_not(mask, out=mask)
    np.minimum(merged_data, new_data, out=merged_data, where=mask, casting="unsafe")
    np.logical_not(new_mask, out=mask)
    np.logical_and(merged_mask, mask, out=mask)
    np.copyto(merged_data, new_data, where=mask, casting="unsafe")


def copy_max(merged_data, new_data, merged_mask, new_mask, **kwargs):
    """Returns the maximum value pixel."""
    mask = np.empty_like(merged_mask, dtype="bool")
    np.logical_or(merged_mask, new_mask, out=mask)
    np.logical_not(mask, out=mask)
    np.maximum(merged_data, new_data, out=merged_data, where=mask, casting="unsafe")
    np.logical_not(new_mask, out=mask)
    np.logical_and(merged_mask, mask, out=mask)
    np.copyto(merged_data, new_data, where=mask, casting="unsafe")


def copy_sum(merged_data, new_data, merged_mask, new_mask, **kwargs):
    """Returns the sum of all pixel values."""
    mask = np.empty_like(merged_mask, dtype="bool")
    np.logical_or(merged_mask, new_mask, out=mask)
    np.logical_not(mask, out=mask)
    np.add(merged_data, new_data, out=merged_data, where=mask, casting="unsafe")
    np.logical_not(new_mask, out=mask)
    np.logical_and(merged_mask, mask, out=mask)
    np.copyto(merged_data, new_data, where=mask, casting="unsafe")


def copy_count(merged_data, new_data, merged_mask, new_mask, **kwargs):
    """Returns the count of valid pixels."""
    mask = np.empty_like(merged_mask, dtype="bool")
    np.logical_or(merged_mask, new_mask, out=mask)
    np.logical_not(mask, out=mask)
    np.add(merged_data, mask, out=merged_data, where=mask, casting="unsafe")
    np.logical_not(new_mask, out=mask)
    np.logical_and(merged_mask, mask, out=mask)
    np.copyto(merged_data, mask, where=mask, casting="unsafe")


MERGE_METHODS = {
    "first": copy_first,
    "last": copy_last,
    "min": copy_min,
    "max": copy_max,
    "sum": copy_sum,
    "count": copy_count,
}


def merge(
    datasets,
    bounds=None,
    res=None,
    nodata=None,
    dtype=None,
    precision=None,
    indexes=None,
    output_count=None,
    resampling=Resampling.nearest,
    method="first",
    target_aligned_pixels=False,
    dst_path=None,
    dst_kwds=None,
):
    """Copy valid pixels from input files to an output file.

    All files must have the same number of bands, data type, and
    coordinate reference system.

    Input files are merged in their listed order using the reverse
    painter's algorithm (default) or another method. If the output file exists,
    its values will be overwritten by input values.

    Geospatial bounds and resolution of a new output file in the
    units of the input file coordinate reference system may be provided
    and are otherwise taken from the first input file.

    Parameters
    ----------
    datasets : list of dataset objects opened in 'r' mode, filenames or PathLike objects
        source datasets to be merged.
    bounds: tuple, optional
        Bounds of the output image (left, bottom, right, top).
        If not set, bounds are determined from bounds of input rasters.
    res: tuple, optional
        Output resolution in units of coordinate reference system. If not set,
        the resolution of the first raster is used. If a single value is passed,
        output pixels will be square.
    nodata: float, optional
        nodata value to use in output file. If not set, uses the nodata value
        in the first input raster.
    dtype: numpy dtype or string
        dtype to use in outputfile. If not set, uses the dtype value in the
        first input raster.
    precision: int, optional
        This parameters is unused, deprecated in rasterio 1.3.0, and
        will be removed in version 2.0.0.
    indexes : list of ints or a single int, optional
        bands to read and merge
    output_count: int, optional
        If using callable it may be useful to have additional bands in the output
        in addition to the indexes specified for read
    resampling : Resampling, optional
        Resampling algorithm used when reading input files.
        Default: `Resampling.nearest`.
    method : str or callable
        pre-defined method:
            first: reverse painting
            last: paint valid new on top of existing
            min: pixel-wise min of existing and new
            max: pixel-wise max of existing and new
        or custom callable with signature:
            merged_data : array_like
                array to update with new_data
            new_data : array_like
                data to merge
                same shape as merged_data
            merged_mask, new_mask : array_like
                boolean masks where merged/new data pixels are invalid
                same shape as merged_data
            index: int
                index of the current dataset within the merged dataset collection
            roff: int
                row offset in base array
            coff: int
                column offset in base array

    target_aligned_pixels : bool, optional
        Whether to adjust output image bounds so that pixel coordinates
        are integer multiples of pixel size, matching the ``-tap``
        options of GDAL utilities.  Default: False.
    dst_path : str or PathLike, optional
        Path of output dataset
    dst_kwds : dict, optional
        Dictionary of creation options and other paramters that will be
        overlaid on the profile of the output dataset.

    Returns
    -------
    tuple

        Two elements:

            dest: numpy ndarray
                Contents of all input rasters in single array

            out_transform: affine.Affine()
                Information for mapping pixel coordinates in `dest` to another
                coordinate system

    """
    if precision is not None:
        warnings.warn(
            "The precision parameter is unused, deprecated, and will be removed in 2.0.0.",
            RasterioDeprecationWarning,
        )

    if method in MERGE_METHODS:
        copyto = MERGE_METHODS[method]
    elif callable(method):
        copyto = method
    else:
        raise ValueError('Unknown method {0}, must be one of {1} or callable'
                         .format(method, list(MERGE_METHODS.keys())))

    # Create a dataset_opener object to use in several places in this function.
    if isinstance(datasets[0], (str, os.PathLike)):
        dataset_opener = rasterio.open
    else:

        @contextmanager
        def nullcontext(obj):
            try:
                yield obj
            finally:
                pass

        dataset_opener = nullcontext

    with dataset_opener(datasets[0]) as first:
        first_profile = first.profile
        first_res = first.res
        nodataval = first.nodatavals[0]
        dt = first.dtypes[0]

        if indexes is None:
            src_count = first.count
        elif isinstance(indexes, int):
            src_count = indexes
        else:
            src_count = len(indexes)

        try:
            first_colormap = first.colormap(1)
        except ValueError:
            first_colormap = None

    if not output_count:
        output_count = src_count

    # Extent from option or extent of all inputs
    if bounds:
        dst_w, dst_s, dst_e, dst_n = bounds
    else:
        # scan input files
        xs = []
        ys = []
        for dataset in datasets:
            with dataset_opener(dataset) as src:
                left, bottom, right, top = src.bounds
            xs.extend([left, right])
            ys.extend([bottom, top])
        dst_w, dst_s, dst_e, dst_n = min(xs), min(ys), max(xs), max(ys)

    # Resolution/pixel size
    if not res:
        res = first_res
    elif not np.iterable(res):
        res = (res, res)
    elif len(res) == 1:
        res = (res[0], res[0])

    if target_aligned_pixels:
        dst_w = math.floor(dst_w / res[0]) * res[0]
        dst_e = math.ceil(dst_e / res[0]) * res[0]
        dst_s = math.floor(dst_s / res[1]) * res[1]
        dst_n = math.ceil(dst_n / res[1]) * res[1]

    # Compute output array shape. We guarantee it will cover the output
    # bounds completely
    output_width = int(round((dst_e - dst_w) / res[0]))
    output_height = int(round((dst_n - dst_s) / res[1]))

    output_transform = Affine.translation(dst_w, dst_n) * Affine.scale(res[0], -res[1])

    if dtype is not None:
        dt = dtype
        logger.debug("Set dtype: %s", dt)

    out_profile = first_profile
    out_profile.update(**(dst_kwds or {}))

    out_profile["transform"] = output_transform
    out_profile["height"] = output_height
    out_profile["width"] = output_width
    out_profile["count"] = output_count
    out_profile["dtype"] = dt
    if nodata is not None:
        out_profile["nodata"] = nodata

    # create destination array
    dest = np.zeros((output_count, output_height, output_width), dtype=dt)

    if nodata is not None:
        nodataval = nodata
        logger.debug("Set nodataval: %r", nodataval)

    if nodataval is not None:
        # Only fill if the nodataval is within dtype's range
        inrange = False
        if np.issubdtype(dt, np.integer):
            info = np.iinfo(dt)
            inrange = (info.min <= nodataval <= info.max)
        elif np.issubdtype(dt, np.floating):
            if math.isnan(nodataval):
                inrange = True
            else:
                info = np.finfo(dt)
                inrange = (info.min <= nodataval <= info.max)
        if inrange:
            dest.fill(nodataval)
        else:
            warnings.warn(
                "The nodata value, %s, is beyond the valid "
                "range of the chosen data type, %s. Consider overriding it "
                "using the --nodata option for better results." % (
                    nodataval, dt))
    else:
        nodataval = 0

    for idx, dataset in enumerate(datasets):
        with dataset_opener(dataset) as src:
            # Real World (tm) use of boundless reads.
            # This approach uses the maximum amount of memory to solve the
            # problem. Making it more efficient is a TODO.

            if disjoint_bounds((dst_w, dst_s, dst_e, dst_n), src.bounds):
                logger.debug("Skipping source: src=%r, window=%r", src)
                continue

            # 1. Compute spatial intersection of destination and source
            src_w, src_s, src_e, src_n = src.bounds

            int_w = src_w if src_w > dst_w else dst_w
            int_s = src_s if src_s > dst_s else dst_s
            int_e = src_e if src_e < dst_e else dst_e
            int_n = src_n if src_n < dst_n else dst_n

            # 2. Compute the source window
            src_window = windows.from_bounds(int_w, int_s, int_e, int_n, src.transform)

            # 3. Compute the destination window
            dst_window = windows.from_bounds(
                int_w, int_s, int_e, int_n, output_transform
            )

            # 4. Read data in source window into temp
            src_window_rnd_shp = src_window.round_lengths()
            dst_window_rnd_shp = dst_window.round_lengths()
            dst_window_rnd_off = dst_window_rnd_shp.round_offsets()

            temp_height, temp_width = (
                dst_window_rnd_off.height,
                dst_window_rnd_off.width,
            )
            temp_shape = (src_count, temp_height, temp_width)

            temp_src = src.read(
                out_shape=temp_shape,
                window=src_window_rnd_shp,
                boundless=False,
                masked=True,
                indexes=indexes,
                resampling=resampling,
            )

        # 5. Copy elements of temp into dest
        roff, coff = (
            max(0, dst_window_rnd_off.row_off),
            max(0, dst_window_rnd_off.col_off),
        )
        region = dest[:, roff : roff + temp_height, coff : coff + temp_width]

        if math.isnan(nodataval):
            region_mask = np.isnan(region)
        elif np.issubdtype(region.dtype, np.floating):
            region_mask = np.isclose(region, nodataval)
        else:
            region_mask = region == nodataval

        # Ensure common shape, resolving issue #2202.
        temp = temp_src[:, : region.shape[1], : region.shape[2]]
        temp_mask = np.ma.getmask(temp)
        copyto(region, temp, region_mask, temp_mask, index=idx, roff=roff, coff=coff)

    if dst_path is None:
        return dest, output_transform

    else:
        with rasterio.open(dst_path, "w", **out_profile) as dst:
            dst.write(dest)
            if first_colormap:
                dst.write_colormap(1, first_colormap)

env_pyrecodes/lib/python3.9/site-packages/rasterio/merge.py
NikolaBlagojevic/pyrecodes
"""Dataset paths, identifiers, and filenames

Note well: this module is deprecated in 1.3.0 and will be removed in a
future version.
"""

import warnings

from rasterio._path import _ParsedPath as ParsedPath
from rasterio._path import _UnparsedPath as UnparsedPath
from rasterio._path import _parse_path as parse_path
from rasterio._path import _vsi_path as vsi_path
from rasterio.errors import RasterioDeprecationWarning

warnings.warn(
    "rasterio.path will be removed in version 1.4.", RasterioDeprecationWarning
)

env_pyrecodes/lib/python3.9/site-packages/rasterio/path.py
NikolaBlagojevic/pyrecodes
"""Implementations of various common operations.

Including `show()` for displaying an array or with matplotlib.
Most can handle a numpy array or `rasterio.Band()`.
Primarily supports `$ rio insp`.
"""

from collections import OrderedDict
from itertools import zip_longest
import logging
import warnings

import numpy as np

import rasterio
from rasterio.io import DatasetReader
from rasterio.transform import guard_transform

logger = logging.getLogger(__name__)


def get_plt():
    """import matplotlib.pyplot
    raise import error if matplotlib is not installed
    """
    try:
        import matplotlib.pyplot as plt
        return plt
    except (ImportError, RuntimeError):  # pragma: no cover
        msg = "Could not import matplotlib\n"
        msg += "matplotlib required for plotting functions"
        raise ImportError(msg)


def show(source, with_bounds=True, contour=False, contour_label_kws=None,
         ax=None, title=None, transform=None, adjust=False, **kwargs):
    """Display a raster or raster band using matplotlib.

    Parameters
    ----------
    source : array or dataset object opened in 'r' mode or Band or tuple(dataset, bidx)
        If Band or tuple (dataset, bidx), display the selected band.
        If raster dataset display the rgb image
        as defined in the colorinterp metadata, or default to first band.
    with_bounds : bool (opt)
        Whether to change the image extent to the spatial bounds of the image,
        rather than pixel coordinates. Only works when source is
        (raster dataset, bidx) or raster dataset.
    contour : bool (opt)
        Whether to plot the raster data as contours
    contour_label_kws : dictionary (opt)
        Keyword arguments for labeling the contours,
        empty dictionary for no labels.
    ax : matplotlib axes (opt)
        Axes to plot on, otherwise uses current axes.
    title : str, optional
        Title for the figure.
    transform : Affine, optional
        Defines the affine transform if source is an array
    adjust : bool
        If the plotted data is an RGB image, adjust the values of
        each band so that they fall between 0 and 1 before plotting. If
        True, values will be adjusted by the min / max of each band. If
        False, no adjustment will be applied.
    **kwargs : key, value pairings optional
        These will be passed to the matplotlib imshow or contour method
        depending on contour argument.
        See full lists at:
        https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.imshow.html
        or
        https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.contour.html

    Returns
    -------
    ax : matplotlib Axes
        Axes with plot.

    """
    plt = get_plt()

    if isinstance(source, tuple):
        arr = source[0].read(source[1])

        if len(arr.shape) >= 3:
            arr = reshape_as_image(arr)

        if with_bounds:
            kwargs['extent'] = plotting_extent(source[0])

    elif isinstance(source, DatasetReader):
        if with_bounds:
            kwargs['extent'] = plotting_extent(source)

        if source.count == 1:
            arr = source.read(1, masked=True)
        else:
            try:
                # Lookup table for the color space in the source file.
                # This will allow us to re-order it to RGB if needed
                source_colorinterp = OrderedDict(zip(source.colorinterp, source.indexes))
                colorinterp = rasterio.enums.ColorInterp

                # Gather the indexes of the RGB channels in that order
                rgb_indexes = [
                    source_colorinterp[ci]
                    for ci in (colorinterp.red, colorinterp.green, colorinterp.blue)
                ]

                # Read the image in the proper order so the numpy array
                # will have the colors in the order expected by
                # matplotlib (RGB)
                arr = source.read(rgb_indexes, masked=True)
                arr = reshape_as_image(arr)

            except KeyError:
                arr = source.read(1, masked=True)
    else:
        # The source is a numpy array reshape it to image if it has 3+ bands
        source = np.ma.squeeze(source)

        if len(source.shape) >= 3:
            arr = reshape_as_image(source)
        else:
            arr = source

        if transform and with_bounds:
            kwargs['extent'] = plotting_extent(arr, transform)

    if adjust and arr.ndim >= 3:
        # Adjust each band by the min/max so it will plot as RGB.
        arr = reshape_as_raster(arr).astype("float64")

        for ii, band in enumerate(arr):
            arr[ii] = adjust_band(band)

        arr = reshape_as_image(arr)

    show = False

    if not ax:
        show = True
        ax = plt.gca()

    if contour:
        if 'cmap' not in kwargs:
            kwargs['colors'] = kwargs.get('colors', 'red')

        kwargs['linewidths'] = kwargs.get('linewidths', 1.5)
        kwargs['alpha'] = kwargs.get('alpha', 0.8)

        C = ax.contour(arr, origin='upper', **kwargs)

        if contour_label_kws is None:
            # no explicit label kws passed use defaults
            contour_label_kws = dict(fontsize=8, inline=True)

        if contour_label_kws:
            ax.clabel(C, **contour_label_kws)
    else:
        ax.imshow(arr, **kwargs)

    if title:
        ax.set_title(title, fontweight='bold')

    if show:
        plt.show()

    return ax


def plotting_extent(source, transform=None):
    """Returns an extent in the format needed
     for matplotlib's imshow (left, right, bottom, top)
     instead of rasterio's bounds (left, bottom, right, top)

    Parameters
    ----------
    source : array or dataset object opened in 'r' mode
        If array, data in the order rows, columns and optionally bands. If array
        is band order (bands in the first dimension), use arr[0]
    transform: Affine, required if source is array
        Defines the affine transform if source is an array

    Returns
    -------
    tuple of float
        left, right, bottom, top
    """
    if hasattr(source, 'bounds'):
        extent = (source.bounds.left, source.bounds.right,
                  source.bounds.bottom, source.bounds.top)
    elif not transform:
        raise ValueError(
            "transform is required if source is an array")
    else:
        transform = guard_transform(transform)
        rows, cols = source.shape[0:2]
        left, top = transform * (0, 0)
        right, bottom = transform * (cols, rows)
        extent = (left, right, bottom, top)

    return extent


def reshape_as_image(arr):
    """Returns the source array reshaped into the order
    expected by image processing and visualization software
    (matplotlib, scikit-image, etc)
    by swapping the axes order from (bands, rows, columns)
    to (rows, columns, bands)

    Parameters
    ----------
    arr : array-like of shape (bands, rows, columns)
        image to reshape
    """
    # swap the axes order from (bands, rows, columns) to (rows, columns, bands)
    im = np.ma.transpose(arr, [1, 2, 0])
    return im


def reshape_as_raster(arr):
    """Returns the array in a raster order
    by swapping the axes order from (rows, columns, bands)
    to (bands, rows, columns)

    Parameters
    ----------
    arr : array-like in the image form of (rows, columns, bands)
        image to reshape
    """
    # swap the axes order from (rows, columns, bands) to (bands, rows, columns)
    im = np.transpose(arr, [2, 0, 1])
    return im


def show_hist(source, bins=10, masked=True, title='Histogram', ax=None, label=None, **kwargs):
    """Easily display a histogram with matplotlib.

    Parameters
    ----------
    source : array or dataset object opened in 'r' mode or Band or tuple(dataset, bidx)
        Input data to display.
        The first three arrays in multi-dimensional
        arrays are plotted as red, green, and blue.
    bins : int, optional
        Compute histogram across N bins.
    masked : bool, optional
        When working with a `rasterio.Band()` object, specifies if the data
        should be masked on read.
    title : str, optional
        Title for the figure.
    ax : matplotlib axes (opt)
        The raster will be added to this axes if passed.
    label : matplotlib labels (opt)
        If passed, matplotlib will use this label list.
        Otherwise, a default label list will be automatically created
    **kwargs : optional keyword arguments
        These will be passed to the matplotlib hist method. See full list at:
        http://matplotlib.org/api/axes_api.html?highlight=imshow#matplotlib.axes.Axes.hist
    """
    plt = get_plt()

    if isinstance(source, DatasetReader):
        arr = source.read(masked=masked)
    elif isinstance(source, (tuple, rasterio.Band)):
        arr = source[0].read(source[1], masked=masked)
    else:
        arr = source

    # The histogram is computed individually for each 'band' in the array
    # so we need the overall min/max to constrain the plot
    rng = np.nanmin(arr), np.nanmax(arr)

    if len(arr.shape) == 2:
        arr = np.expand_dims(arr.flatten(), 0).T
        colors = ['gold']
    else:
        arr = arr.reshape(arr.shape[0], -1).T
        colors = ['red', 'green', 'blue', 'violet', 'gold', 'saddlebrown']

    # The goal is to provide a curated set of colors for working with
    # smaller datasets and let matplotlib define additional colors when
    # working with larger datasets.
    if arr.shape[-1] > len(colors):
        n = arr.shape[-1] - len(colors)
        colors.extend(np.ndarray.tolist(plt.get_cmap('Accent')(np.linspace(0, 1, n))))
    else:
        colors = colors[:arr.shape[-1]]

    # if the user used the label argument, pass them drectly to matplotlib
    if label:
        labels = label
    # else, create default labels
    else:
        # If a rasterio.Band() is given make sure the proper index is displayed
        # in the legend.
        if isinstance(source, (tuple, rasterio.Band)):
            labels = [str(source[1])]
        else:
            labels = (str(i + 1) for i in range(len(arr)))

    if ax:
        show = False
    else:
        show = True
        ax = plt.gca()

    fig = ax.get_figure()

    ax.hist(arr,
            bins=bins,
            color=colors,
            label=labels,
            range=rng,
            **kwargs)

    ax.legend(loc="upper right")
    ax.set_title(title, fontweight='bold')
    ax.grid(True)
    ax.set_xlabel('DN')
    ax.set_ylabel('Frequency')
    if show:
        plt.show()


def adjust_band(band, kind=None):
    """Adjust a band to be between 0 and 1.

    Parameters
    ----------
    band : array, shape (height, width)
        A band of a raster object.
    kind : str
        An unused option. For now, there is only one option ('linear').

    Returns
    -------
    band_normed : array, shape (height, width)
        An adjusted version of the input band.

    """
    imin = np.float64(np.nanmin(band))
    imax = np.float64(np.nanmax(band))
    return (band - imin) / (imax - imin)

env_pyrecodes/lib/python3.9/site-packages/rasterio/plot.py
NikolaBlagojevic/pyrecodes
"""Raster dataset profiles."""

from collections import UserDict
import warnings

from rasterio.dtypes import uint8


class Profile(UserDict):
    """Base class for Rasterio dataset profiles.

    Subclasses will declare driver-specific creation options.
    """

    defaults = {}

    def __init__(self, data={}, **kwds):
        """Create a new profile based on the class defaults, which are
        overlaid with items from the `data` dict and keyword arguments."""
        UserDict.__init__(self)
        initdata = self.defaults.copy()
        initdata.update(data)
        initdata.update(**kwds)
        self.data.update(initdata)

    def __getitem__(self, key):
        """Like normal item access but with affine alias."""
        return self.data[key]

    def __setitem__(self, key, val):
        """Like normal item setter but forbidding affine item."""
        if key == 'affine':
            raise TypeError("affine key is prohibited")
        self.data[key] = val


class DefaultGTiffProfile(Profile):
    """Tiled, band-interleaved, LZW-compressed, 8-bit GTiff."""

    defaults = {
        'driver': 'GTiff',
        'interleave': 'band',
        'tiled': True,
        'blockxsize': 256,
        'blockysize': 256,
        'compress': 'lzw',
        'nodata': 0,
        'dtype': uint8
    }


default_gtiff_profile = DefaultGTiffProfile()

env_pyrecodes/lib/python3.9/site-packages/rasterio/profiles.py
NikolaBlagojevic/pyrecodes
"""$ rio blocks"""


import json
import logging
import os.path

import click
import cligj

import rasterio
from rasterio.rio import options
from rasterio.rio.helpers import write_features
from rasterio.warp import transform_bounds


logger = logging.getLogger(__name__)


class _Collection:

    """For use with `rasterio.rio.helpers.write_features()`."""

    def __init__(self, dataset, bidx, precision=6, geographic=True):

        """Export raster dataset windows to GeoJSON polygon features.

        Parameters
        ----------
        dataset : a dataset object opened in 'r' mode
            Source dataset
        bidx : int
            Extract windows from this band
        precision : int, optional
            Coordinate precision
        geographic : bool, optional
            Reproject geometries to ``EPSG:4326`` if ``True``

        Yields
        ------
        dict
            GeoJSON polygon feature
        """

        self._src = dataset
        self._bidx = bidx
        self._precision = precision
        self._geographic = geographic

    def _normalize_bounds(self, bounds):
        if self._geographic:
            bounds = transform_bounds(self._src.crs, 'EPSG:4326', *bounds)
        if self._precision >= 0:
            bounds = (round(v, self._precision) for v in bounds)
        return bounds

    @property
    def bbox(self):
        return tuple(self._normalize_bounds(self._src.bounds))

    def __call__(self):
        gen = self._src.block_windows(bidx=self._bidx)
        for idx, (block, window) in enumerate(gen):
            bounds = self._normalize_bounds(self._src.window_bounds(window))
            xmin, ymin, xmax, ymax = bounds
            yield {
                'type': 'Feature',
                'id': '{0}:{1}'.format(os.path.basename(self._src.name), idx),
                'properties': {
                    'block': json.dumps(block),
                    'window': window.todict(),
                },
                'geometry': {
                    'type': 'Polygon',
                    'coordinates': [[
                        (xmin, ymin),
                        (xmin, ymax),
                        (xmax, ymax),
                        (xmax, ymin)
                    ]]
                }
            }


@click.command()
@options.file_in_arg
@options.output_opt
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@cligj.projection_projected_opt
@cligj.sequence_opt
@cligj.use_rs_opt
@click.option(
    '--bidx', type=click.INT, default=0,
    help="Index of the band that is the source of shapes.")
@click.pass_context
def blocks(
    ctx, input, output, precision, indent, compact, projection, sequence, use_rs, bidx
):
    """Write dataset blocks as GeoJSON features.

    This command writes features describing a raster's internal blocks, which
    are used directly for raster I/O.  These features can be used to visualize
    how a windowed operation would operate using those blocks.

    Output features have two JSON encoded properties: block and window.  Block
    is a two element array like '[0, 0]' describing the window's position
    in the input band's window layout.  Window is a two element array
    containing two more two element arrays like '[[0, 256], [0, 256]' and
    describes the range of pixels the window covers in the input band.  Values
    are JSON encoded for better interoperability.

    Block windows are extracted from the dataset (all bands must have matching
    block windows) by default, or from the band specified using the '--bidx
    option:
    \b

        $ rio shapes --bidx 3 tests/data/RGB.byte.tif

    By default a GeoJSON 'FeatureCollection' is written, but the --sequence'
    option produces a GeoJSON feature stream instead.
    \b

        $ rio shapes tests/data/RGB.byte.tif --sequence

    Output features are reprojected to 'WGS84' unless the '--projected' flag is
    provided, which causes the output to be kept in the input datasource's
    coordinate reference system.

    For more information on exactly what blocks and windows represent, see
    'dataset.block_windows()'.

    """
    dump_kwds = {'sort_keys': True}

    if indent:
        dump_kwds['indent'] = indent
    if compact:
        dump_kwds['separators'] = (',', ':')

    stdout = click.open_file(
        output, 'w') if output else click.get_text_stream('stdout')

    with ctx.obj['env'], rasterio.open(input) as src:

        if bidx and bidx not in src.indexes:
            raise click.BadParameter("Not a valid band index")

        collection = _Collection(
            dataset=src,
            bidx=bidx,
            precision=precision,
            geographic=projection != "projected",
        )
        write_features(
            stdout,
            collection,
            sequence=sequence,
            geojson_type="feature" if sequence else "collection",
            use_rs=use_rs,
            **dump_kwds
        )

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/blocks.py
NikolaBlagojevic/pyrecodes
import logging
import os

import click
from cligj import (
    precision_opt, indent_opt, compact_opt, projection_geographic_opt,
    projection_mercator_opt, projection_projected_opt,
    use_rs_opt, geojson_type_feature_opt, geojson_type_bbox_opt,
    geojson_type_collection_opt)

from .helpers import write_features, to_lower
from rasterio.rio import options
from rasterio.warp import transform_bounds


logger = logging.getLogger(__name__)


# Bounds command.
@click.command(short_help="Write bounding boxes to stdout as GeoJSON.")
# One or more files, the bounds of each are a feature in the collection
# object or feature sequence.
@click.argument('INPUT', nargs=-1, type=click.Path(), required=True)
@precision_opt
@indent_opt
@compact_opt
@projection_geographic_opt
@projection_projected_opt
@projection_mercator_opt
@click.option(
    '--dst-crs', default='', metavar="EPSG:NNNN", callback=to_lower,
    help="Output in specified coordinates.")
@options.sequence_opt
@use_rs_opt
@geojson_type_feature_opt(True)
@geojson_type_bbox_opt(False)
@click.pass_context
def bounds(
    ctx,
    input,
    precision,
    indent,
    compact,
    projection,
    dst_crs,
    sequence,
    use_rs,
    geojson_type,
):
    """Write bounding boxes to stdout as GeoJSON for use with, e.g.,
    geojsonio

      $ rio bounds *.tif | geojsonio

    If a destination crs is passed via dst_crs, it takes precedence over
    the projection parameter.
    """
    import rasterio.warp
    dump_kwds = {'sort_keys': True}
    if indent:
        dump_kwds['indent'] = indent
    if compact:
        dump_kwds['separators'] = (',', ':')
    stdout = click.get_text_stream('stdout')

    # This is the generator for (feature, bbox) pairs.
    class Collection:

        def __init__(self, env):
            self._xs = []
            self._ys = []
            self.env = env

        @property
        def bbox(self):
            return min(self._xs), min(self._ys), max(self._xs), max(self._ys)

        def __call__(self):
            for i, path in enumerate(input):
                with rasterio.open(path) as src:
                    bounds = src.bounds
                    if dst_crs:
                        bbox = transform_bounds(src.crs,
                                                dst_crs, *bounds)
                    elif projection == 'mercator':
                        bbox = transform_bounds(src.crs,
                                                {'init': 'epsg:3857'}, *bounds)
                    elif projection == 'geographic':
                        bbox = transform_bounds(src.crs,
                                                {'init': 'epsg:4326'}, *bounds)
                    else:
                        bbox = bounds

                if precision >= 0:
                    bbox = [round(b, precision) for b in bbox]

                yield {
                    'type': 'Feature',
                    'bbox': bbox,
                    'geometry': {
                        'type': 'Polygon',
                        'coordinates': [[
                            [bbox[0], bbox[1]],
                            [bbox[2], bbox[1]],
                            [bbox[2], bbox[3]],
                            [bbox[0], bbox[3]],
                            [bbox[0], bbox[1]]]]},
                    'properties': {
                        'id': str(i),
                        'title': path,
                        'filename': os.path.basename(path)}}

                self._xs.extend(bbox[::2])
                self._ys.extend(bbox[1::2])

    with ctx.obj["env"] as env:
        write_features(
            stdout,
            Collection(env),
            sequence=sequence,
            geojson_type=geojson_type,
            use_rs=use_rs,
            **dump_kwds
        )

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/bounds.py
NikolaBlagojevic/pyrecodes
"""$ rio calc"""

from __future__ import division

from collections import OrderedDict
from contextlib import ExitStack
from distutils.version import LooseVersion
import math

import click
import snuggs

import rasterio
from rasterio.features import sieve
from rasterio.fill import fillnodata
from rasterio.windows import Window
from rasterio.rio import options
from rasterio.rio.helpers import resolve_inout


def _get_bands(inputs, sources, d, i=None):
    """Get a rasterio.Band object from calc's inputs"""
    idx = d if d in dict(inputs) else int(d) - 1
    src = sources[idx]
    return (rasterio.band(src, i) if i else
            [rasterio.band(src, j) for j in src.indexes])


def _read_array(ix, subix=None, dtype=None):
    """Change the type of a read array"""
    arr = snuggs._ctx.lookup(ix, subix)
    if dtype:
        arr = arr.astype(dtype)
    return arr


def _chunk_output(width, height, count, itemsize, mem_limit=1):
    """Divide the calculation output into chunks

    This function determines the chunk size such that an array of shape
    (chunk_size, chunk_size, count) with itemsize bytes per element
    requires no more than mem_limit megabytes of memory.

    Output chunks are described by rasterio Windows.

    Parameters
    ----------
    width : int
        Output width
    height : int
        Output height
    count : int
        Number of output bands
    itemsize : int
        Number of bytes per pixel
    mem_limit : int, default
        The maximum size in memory of a chunk array

    Returns
    -------
    sequence of Windows
    """
    max_pixels = mem_limit * 1.0e+6 / itemsize * count
    chunk_size = int(math.floor(math.sqrt(max_pixels)))
    ncols = int(math.ceil(width / chunk_size))
    nrows = int(math.ceil(height / chunk_size))
    chunk_windows = []

    for col in range(ncols):
        col_offset = col * chunk_size
        w = min(chunk_size, width - col_offset)
        for row in range(nrows):
            row_offset = row * chunk_size
            h = min(chunk_size, height - row_offset)
            chunk_windows.append(((row, col), Window(col_offset, row_offset, w, h)))

    return chunk_windows


@click.command(short_help="Raster data calculator.")
@click.argument('command')
@options.files_inout_arg
@options.output_opt
@options.format_opt
@click.option('--name', multiple=True,
              help='Specify an input file with a unique short (alphas only) '
                   'name for use in commands like '
                   '"a=tests/data/RGB.byte.tif".')
@options.dtype_opt
@options.masked_opt
@options.overwrite_opt
@click.option("--mem-limit", type=int, default=64, help="Limit on memory used to perform calculations, in MB.")
@options.creation_options
@click.pass_context
def calc(ctx, command, files, output, driver, name, dtype, masked, overwrite, mem_limit, creation_options):
    """A raster data calculator

    Evaluates an expression using input datasets and writes the result
    to a new dataset.

    Command syntax is lisp-like. An expression consists of an operator
    or function name and one or more strings, numbers, or expressions
    enclosed in parentheses. Functions include ``read`` (gets a raster
    array) and ``asarray`` (makes a 3-D array from 2-D arrays).

    \b
        * (read i) evaluates to the i-th input dataset (a 3-D array).
        * (read i j) evaluates to the j-th band of the i-th dataset (a
          2-D array).
        * (read i j 'float64') casts the array to, e.g. float64. This
          is critical if calculations will produces values that exceed
          the limits of the dataset's natural data type.
        * (take foo j) evaluates to the j-th band of a dataset named foo
          (see help on the --name option above).
        * Standard numpy array operators (+, -, *, /) are available.
        * When the final result is a list of arrays, a multiple band
          output file is written.
        * When the final result is a single array, a single band output
          file is written.

    Example:

    \b
         $ rio calc "(+ 2 (* 0.95 (read 1)))" tests/data/RGB.byte.tif \\
         > /tmp/out.tif

    The command above produces a 3-band GeoTIFF with all values scaled
    by 0.95 and incremented by 2.

    \b
        $ rio calc "(asarray (+ 125 (read 1)) (read 1) (read 1))" \\
        > tests/data/shade.tif /tmp/out.tif

    The command above produces a 3-band RGB GeoTIFF, with red levels
    incremented by 125, from the single-band input.

    The maximum amount of memory used to perform caculations defaults to
    64 MB. This number can be increased to improve speed of calculation.

    """
    import numpy as np

    dst = None
    sources = []

    try:
        with ctx.obj["env"], ExitStack() as stack:
            output, files = resolve_inout(
                files=files, output=output, overwrite=overwrite
            )
            inputs = [tuple(n.split("=")) for n in name] + [(None, n) for n in files]
            sources = [
                stack.enter_context(rasterio.open(path)) for name, path in inputs
            ]

            first = sources[0]
            kwargs = first.profile
            kwargs.update(**creation_options)
            dtype = dtype or first.meta['dtype']
            kwargs['dtype'] = dtype
            kwargs.pop("driver", None)
            if driver:
                kwargs['driver'] = driver

            snuggs.func_map['read'] = _read_array
            snuggs.func_map['band'] = lambda d, i: _get_bands(inputs, sources, d, i)
            snuggs.func_map['bands'] = lambda d: _get_bands(inputs, sources, d)
            snuggs.func_map['fillnodata'] = lambda *args: fillnodata(*args)
            snuggs.func_map['sieve'] = lambda *args: sieve(*args)

            # The windows iterator is initialized with a single sample.
            # The actual work windows will be added in the second
            # iteration of the loop.
            work_windows = [(None, Window(0, 0, 16, 16))]

            for ij, window in work_windows:
                ctxkwds = OrderedDict()

                for i, ((name, path), src) in enumerate(zip(inputs, sources)):
                    ctxkwds[name or '_i%d' % (i + 1)] = src.read(masked=masked, window=window)

                res = snuggs.eval(command, **ctxkwds)
                results = res.astype(dtype)

                if isinstance(results, np.ma.core.MaskedArray):
                    results = results.filled(float(kwargs['nodata']))
                    if len(results.shape) == 2:
                        results = np.ma.asanyarray([results])
                elif len(results.shape) == 2:
                    results = np.asanyarray([results])

                # The first iteration is only to get sample results and from them
                # compute some properties of the output dataset.
                if dst is None:
                    kwargs['count'] = results.shape[0]
                    dst = rasterio.open(output, 'w', **kwargs)
                    work_windows.extend(
                        _chunk_output(
                            dst.width,
                            dst.height,
                            dst.count,
                            np.dtype(dst.dtypes[0]).itemsize,
                            mem_limit=mem_limit,
                        )
                    )

                # In subsequent iterations we write results.
                else:
                    dst.write(results, window=window)

    except snuggs.ExpressionError as err:
        click.echo("Expression Error:")
        click.echo("  {}".format(err.text))
        click.echo(" {}^".format(" " * err.offset))
        click.echo(err)
        raise click.Abort()

    finally:
        if dst:
            dst.close()
        for src in sources:
            src.close()

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/calc.py
NikolaBlagojevic/pyrecodes
"""File translation command"""

import logging

import click

from .helpers import resolve_inout
from . import options
import rasterio
from rasterio.coords import disjoint_bounds
from rasterio.crs import CRS
from rasterio.enums import MaskFlags
from rasterio.windows import Window

logger = logging.getLogger(__name__)

# Geographic (default), projected, or Mercator switch.
projection_geographic_opt = click.option(
    '--geographic',
    'projection',
    flag_value='geographic',
    help="Bounds in geographic coordinates.")

projection_projected_opt = click.option(
    '--projected',
    'projection',
    flag_value='projected',
    default=True,
    help="Bounds in input's own projected coordinates (the default).")


# Clip command
@click.command(short_help='Clip a raster to given bounds.')
@click.argument(
    'files',
    nargs=-1,
    type=click.Path(),
    required=True,
    metavar="INPUT OUTPUT")
@options.output_opt
@options.bounds_opt
@click.option(
    '--like',
    type=click.Path(exists=True),
    help='Raster dataset to use as a template for bounds')
@options.format_opt
@options.nodata_opt
@projection_geographic_opt
@projection_projected_opt
@options.overwrite_opt
@options.creation_options
@click.option(
    "--with-complement/--without-complement",
    default=False,
    help="Include the relative complement of the raster in the given bounds (giving a larger result), else return results only from the intersection of the raster and the bounds (the default).",
)
@click.pass_context
def clip(
    ctx,
    files,
    output,
    bounds,
    like,
    driver,
    nodata,
    projection,
    overwrite,
    creation_options,
    with_complement,
):
    """Clips a raster using projected or geographic bounds.

    The values of --bounds are presumed to be from the coordinate
    reference system of the input dataset unless the --geographic option
    is used, in which case the values may be longitude and latitude
    bounds. Either JSON, for example "[west, south, east, north]", or
    plain text "west south east north" representations of a bounding box
    are acceptable.

    If using --like, bounds will automatically be transformed to match
    the coordinate reference system of the input.

    Datasets with non-rectilinear geo transforms (i.e. with rotation
    and/or shear) may not be cropped using this command. They must be
    processed with rio-warp.

    Examples
    --------
    $ rio clip input.tif output.tif --bounds xmin ymin xmax ymax

    $ rio clip input.tif output.tif --like template.tif

    """
    from rasterio.warp import transform_bounds

    with ctx.obj['env']:

        output, files = resolve_inout(files=files, output=output, overwrite=overwrite)
        input = files[0]

        with rasterio.open(input) as src:
            if not src.transform.is_rectilinear:
                raise click.BadParameter(
                    "Non-rectilinear rasters (i.e. with rotation or shear) cannot be clipped"
                )

            if bounds:
                if projection == 'geographic':
                    bounds = transform_bounds(CRS.from_epsg(4326), src.crs, *bounds)
                if disjoint_bounds(bounds, src.bounds):
                    raise click.BadParameter('must overlap the extent of '
                                             'the input raster',
                                             param='--bounds',
                                             param_hint='--bounds')
            elif like:
                with rasterio.open(like) as template_ds:
                    bounds = template_ds.bounds
                    if template_ds.crs != src.crs:
                        bounds = transform_bounds(template_ds.crs, src.crs,
                                                  *bounds)

                    if disjoint_bounds(bounds, src.bounds):
                        raise click.BadParameter('must overlap the extent of '
                                                 'the input raster',
                                                 param='--like',
                                                 param_hint='--like')

            else:
                raise click.UsageError('--bounds or --like required')

            bounds_window = src.window(*bounds)

            if not with_complement:
                bounds_window = bounds_window.intersection(
                    Window(0, 0, src.width, src.height)
                )

            # Align window, as in gdal_translate.
            out_window = bounds_window.round_lengths()
            out_window = out_window.round_offsets()

            height = int(out_window.height)
            width = int(out_window.width)

            out_kwargs = src.profile

            if driver:
                out_kwargs["driver"] = driver

            if nodata is not None:
                out_kwargs["nodata"] = nodata

            out_kwargs.update({
                'height': height,
                'width': width,
                'transform': src.window_transform(out_window)})

            out_kwargs.update(**creation_options)

            if "blockxsize" in out_kwargs and int(out_kwargs["blockxsize"]) > width:
                del out_kwargs["blockxsize"]
                logger.warning(
                    "Blockxsize removed from creation options to accomodate small output width"
                )
            if "blockysize" in out_kwargs and int(out_kwargs["blockysize"]) > height:
                del out_kwargs["blockysize"]
                logger.warning(
                    "Blockysize removed from creation options to accomodate small output height"
                )

            with rasterio.open(output, "w", **out_kwargs) as out:
                out.write(
                    src.read(
                        window=out_window,
                        out_shape=(src.count, height, width),
                        boundless=True,
                        masked=True,
                    )
                )

                if MaskFlags.per_dataset in src.mask_flag_enums[0]:
                    out.write_mask(
                        src.read_masks(
                            window=out_window,
                            out_shape=(src.count, height, width),
                            boundless=True,
                        )[0]
                    )

                # TODO: copy other properties (GCPs etc). Several other
                # programs need the same utility.

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/clip.py
NikolaBlagojevic/pyrecodes
"""File translation command"""

import click
import numpy as np

import rasterio
from rasterio.enums import MaskFlags
from rasterio.rio import options
from rasterio.rio.helpers import resolve_inout


@click.command(short_help="Copy and convert raster dataset.")
@options.files_inout_arg
@options.output_opt
@options.format_opt
@options.dtype_opt
@click.option('--scale-ratio', type=float, default=None,
              help="Source to destination scaling ratio.")
@click.option('--scale-offset', type=float, default=None,
              help="Source to destination scaling offset.")
@options.rgb_opt
@options.overwrite_opt
@options.creation_options
@click.pass_context
def convert(
        ctx, files, output, driver, dtype, scale_ratio, scale_offset,
        photometric, overwrite, creation_options):
    """Copy and convert raster datasets to other data types and formats.

    Data values may be linearly scaled when copying by using the
    --scale-ratio and --scale-offset options. Destination raster values
    are calculated as

      dst = scale_ratio * src + scale_offset

    For example, to scale uint16 data with an actual range of 0-4095 to
    0-255 as uint8:

      $ rio convert in16.tif out8.tif --dtype uint8 --scale-ratio 0.0625

    Format specific creation options may also be passed using --co. To
    tile a new GeoTIFF output file, do the following.

      --co tiled=true --co blockxsize=256 --co blockysize=256

    To compress it using the LZW method, add

      --co compress=LZW

    """
    with ctx.obj['env']:

        outputfile, files = resolve_inout(
            files=files, output=output, overwrite=overwrite, num_inputs=1
        )
        inputfile = files[0]

        with rasterio.open(inputfile) as src:

            # Use the input file's profile, updated by CLI
            # options, as the profile for the output file.
            profile = src.profile

            profile.pop('driver', None)
            if driver:
                profile['driver'] = driver
            if dtype:
                profile['dtype'] = dtype
            dst_dtype = profile['dtype']

            if photometric:
                creation_options['photometric'] = photometric

            profile.update(**creation_options)

            with rasterio.open(outputfile, 'w', **profile) as dst:

                data = src.read()

                if scale_ratio:
                    # Cast to float64 before multiplying.
                    data = data.astype('float64', casting='unsafe', copy=False)
                    np.multiply(
                        data, scale_ratio, out=data, casting='unsafe')

                if scale_offset:
                    # My understanding of copy=False is that this is a
                    # no-op if the array was cast for multiplication.
                    data = data.astype('float64', casting='unsafe', copy=False)
                    np.add(
                        data, scale_offset, out=data, casting='unsafe')

                # Cast to the output dtype and write.
                result = data.astype(dst_dtype, casting='unsafe', copy=False)
                dst.write(result)

                if MaskFlags.per_dataset in src.mask_flag_enums[0]:
                    dst.write_mask(src.read_masks()[0])

                # TODO: copy other properties (GCPs etc). Several other
                # programs need the same utility.

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/convert.py
NikolaBlagojevic/pyrecodes
"""Fetch and edit raster dataset metadata from the command line."""


from collections import OrderedDict
import json
import warnings

import click

import rasterio
import rasterio.crs
from rasterio.crs import CRS
from rasterio.enums import ColorInterp
from rasterio.errors import CRSError
from rasterio.rio import options
from rasterio.transform import guard_transform


# Handlers for info module options.


def all_handler(ctx, param, value):
    """Get tags from a template file or command line."""
    if ctx.obj and ctx.obj.get('like') and value is not None:
        ctx.obj['all_like'] = value
        value = ctx.obj.get('like')
    return value


def crs_handler(ctx, param, value):
    """Get crs value from a template file or command line."""
    retval = options.from_like_context(ctx, param, value)
    if retval is None and value:
        try:
            retval = json.loads(value)
        except ValueError:
            retval = value
        try:
            if isinstance(retval, dict):
                retval = CRS(retval)
            else:
                retval = CRS.from_string(retval)
        except CRSError:
            raise click.BadParameter(
                "'%s' is not a recognized CRS." % retval,
                param=param, param_hint='crs')
    return retval


def tags_handler(ctx, param, value):
    """Get tags from a template file or command line."""
    retval = options.from_like_context(ctx, param, value)
    if retval is None and value:
        try:
            retval = dict(p.split('=') for p in value)
        except Exception:
            raise click.BadParameter(
                "'%s' contains a malformed tag." % value,
                param=param, param_hint='transform')
    return retval


def transform_handler(ctx, param, value):
    """Get transform value from a template file or command line."""
    retval = options.from_like_context(ctx, param, value)
    if retval is None and value:
        try:
            value = json.loads(value)
        except ValueError:
            pass
        try:
            retval = guard_transform(value)
        except Exception:
            raise click.BadParameter(
                "'%s' is not recognized as an Affine array." % value,
                param=param, param_hint='transform')
    return retval


def colorinterp_handler(ctx, param, value):

    """Validate a string like ``red,green,blue,alpha`` and convert to
    a tuple.  Also handle ``RGB`` and ``RGBA``.
    """

    if value is None:
        return value
    # Using '--like'
    elif value.lower() == 'like':
        return options.from_like_context(ctx, param, value)
    elif value.lower() == 'rgb':
        return ColorInterp.red, ColorInterp.green, ColorInterp.blue
    elif value.lower() == 'rgba':
        return ColorInterp.red, ColorInterp.green, ColorInterp.blue, ColorInterp.alpha
    else:
        colorinterp = tuple(value.split(','))
        for ci in colorinterp:
            if ci not in ColorInterp.__members__:
                raise click.BadParameter(
                    "color interpretation '{ci}' is invalid.  Must be one of: "
                    "{valid}".format(
                        ci=ci, valid=', '.join(ColorInterp.__members__)))
        return tuple(ColorInterp[ci] for ci in colorinterp)


@click.command('edit-info', short_help="Edit dataset metadata.")
@options.file_in_arg
@options.bidx_opt
@options.edit_nodata_opt
@click.option('--unset-nodata', default=False, is_flag=True,
              help="Unset the dataset's nodata value.")
@click.option('--crs', callback=crs_handler, default=None,
              help="New coordinate reference system")
@click.option('--unset-crs', default=False, is_flag=True,
              help="Unset the dataset's CRS value.")
@click.option('--transform', callback=transform_handler,
              help="New affine transform matrix")
@click.option('--units', help="Edit units of a band (requires --bidx)")
@click.option('--description',
              help="Edit description of a band (requires --bidx)")
@click.option('--tag', 'tags', callback=tags_handler, multiple=True,
              metavar='KEY=VAL', help="New tag.")
@click.option('--all', 'allmd', callback=all_handler, flag_value='like',
              is_eager=True, default=False,
              help="Copy all metadata items from the template file.")
@click.option(
    '--colorinterp', callback=colorinterp_handler,
    metavar="name[,name,...]|RGB|RGBA|like",
    help="Set color interpretation for all bands like 'red,green,blue,alpha'. "
         "Can also use 'RGBA' as shorthand for 'red,green,blue,alpha' and "
         "'RGB' for the same sans alpha band.  Use 'like' to inherit color "
         "interpretation from '--like'.")
@options.like_opt
@click.pass_context
def edit(ctx, input, bidx, nodata, unset_nodata, crs, unset_crs, transform,
         units, description, tags, allmd, like, colorinterp):
    """Edit a dataset's metadata: coordinate reference system, affine
    transformation matrix, nodata value, and tags.

    The coordinate reference system may be either a PROJ.4 or EPSG:nnnn
    string,

      --crs 'EPSG:4326'

    or a JSON text-encoded PROJ.4 object.

      --crs '{"proj": "utm", "zone": 18, ...}'

    Transforms are JSON-encoded Affine objects like:

      --transform '[300.038, 0.0, 101985.0, 0.0, -300.042, 2826915.0]'

    Prior to Rasterio 1.0 GDAL geotransforms were supported for --transform,
    but are no longer supported.

    Metadata items may also be read from an existing dataset using a
    combination of the --like option with at least one of --all,
    `--crs like`, `--nodata like`, and `--transform like`.

      rio edit-info example.tif --like template.tif --all

    To get just the transform from the template:

      rio edit-info example.tif --like template.tif --transform like

    """
    import numpy as np

    def in_dtype_range(value, dtype):
        kind = np.dtype(dtype).kind
        if kind == 'f' and (np.isnan(value) or np.isinf(value)):
            return True
        infos = {'c': np.finfo, 'f': np.finfo, 'i': np.iinfo,
                 'u': np.iinfo}
        rng = infos[kind](dtype)
        return rng.min <= value <= rng.max

    # If '--all' is given before '--like' on the commandline then 'allmd'
    # is the string 'like'.  This is caused by '--like' not having an
    # opportunity to populate metadata before '--all' is evaluated.
    if allmd == 'like':
        allmd = ctx.obj['like']

    with ctx.obj['env'], rasterio.open(input, 'r+') as dst:

        if allmd:
            nodata = allmd['nodata']
            crs = allmd['crs']
            transform = allmd['transform']
            tags = allmd['tags']
            colorinterp = allmd['colorinterp']

        if unset_nodata and nodata is not None:
            raise click.BadParameter(
                "--unset-nodata and --nodata cannot be used together."
            )

        if unset_crs and crs:
            raise click.BadParameter("--unset-crs and --crs cannot be used together.")

        if unset_nodata:
            # Setting nodata to None will raise NotImplementedError
            # if GDALDeleteRasterNoDataValue() isn't present in the
            # GDAL library.
            try:
                dst.nodata = None
            except NotImplementedError as exc:  # pragma: no cover
                raise click.ClickException(str(exc))

        elif nodata is not None:
            dtype = dst.dtypes[0]
            if nodata is not None and not in_dtype_range(nodata, dtype):
                raise click.BadParameter(
                    "outside the range of the file's data type (%s)." % dtype,
                    param=nodata,
                    param_hint="nodata",
                )
            dst.nodata = nodata

        if unset_crs:
            dst.crs = None
        elif crs:
            dst.crs = crs

        if transform:
            dst.transform = transform

        if tags:
            dst.update_tags(**tags)

        if units:
            dst.set_band_unit(bidx, units)

        if description:
            dst.set_band_description(bidx, description)

        if colorinterp:
            if like and len(colorinterp) != dst.count:
                raise click.ClickException(
                    "When using '--like' for color interpretation the "
                    "template and target images must have the same number "
                    "of bands.  Found {template} color interpretations for "
                    "template image and {target} bands in target "
                    "image.".format(
                        template=len(colorinterp),
                        target=dst.count))
            try:
                dst.colorinterp = colorinterp
            except ValueError as e:
                raise click.ClickException(str(e))

    # Post check - ensure that crs was unset properly
    if unset_crs:
        with ctx.obj['env'], rasterio.open(input, 'r') as src:
            if src.crs:
                warnings.warn(
                    'CRS was not unset. Availability of his functionality '
                    'differs depending on GDAL version and driver')

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/edit_info.py
NikolaBlagojevic/pyrecodes
"""Fetch and edit raster dataset metadata from the command line."""

import json
import os

import click

from rasterio._env import GDALDataFinder, PROJDataFinder


@click.command(short_help="Print information about the Rasterio environment.")
@click.option(
    "--formats",
    "key",
    flag_value="formats",
    default=True,
    help="Enumerate the available formats.",
)
@click.option(
    "--credentials",
    "key",
    flag_value="credentials",
    default=False,
    help="Print credentials.",
)
@click.option(
    "--gdal-data",
    "key",
    flag_value="gdal_data",
    default=False,
    help="Print GDAL data path.",
)
@click.option(
    "--proj-data",
    "key",
    flag_value="proj_data",
    default=False,
    help="Print PROJ data path.",
)
@click.pass_context
def env(ctx, key):
    """Print information about the Rasterio environment."""
    with ctx.obj["env"] as env:
        if key == "credentials":
            click.echo(json.dumps(env.session.credentials))
        elif key == "gdal_data":
            click.echo(os.environ.get("GDAL_DATA") or GDALDataFinder().search())
        elif key == "proj_data":
            click.echo(
                os.environ.get("PROJ_DATA", os.environ.get("PROJ_LIB"))
                or PROJDataFinder().search()
            )
        else:
            for k, v in sorted(env.drivers().items()):
                click.echo("{0}: {1}".format(k, v))

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/env.py
NikolaBlagojevic/pyrecodes
"""
Helper objects used by multiple CLI commands.
"""

import json
import os

import click

from rasterio.errors import FileOverwriteError


def coords(obj):
    """Yield all coordinate coordinate tuples from a geometry or feature.
    From python-geojson package."""
    if isinstance(obj, (tuple, list)):
        coordinates = obj
    elif 'geometry' in obj:
        coordinates = obj['geometry']['coordinates']
    else:
        coordinates = obj.get('coordinates', obj)
    for e in coordinates:
        if isinstance(e, (float, int)):
            yield tuple(coordinates)
            break
        else:
            for f in coords(e):
                yield f


def write_features(
        fobj, collection, sequence=False, geojson_type='feature', use_rs=False,
        **dump_kwds):
    """Read an iterator of (feat, bbox) pairs and write to file using
    the selected modes."""
    # Sequence of features expressed as bbox, feature, or collection.
    if sequence:
        for feat in collection():
            xs, ys = zip(*coords(feat))
            bbox = (min(xs), min(ys), max(xs), max(ys))
            if use_rs:
                fobj.write(u'\u001e')
            if geojson_type == 'bbox':
                fobj.write(json.dumps(bbox, **dump_kwds))
            else:
                fobj.write(json.dumps(feat, **dump_kwds))
            fobj.write('\n')

    # Aggregate all features into a single object expressed as
    # bbox or collection.
    else:
        features = list(collection())
        if geojson_type == 'bbox':
            fobj.write(json.dumps(collection.bbox, **dump_kwds))
        else:
            fobj.write(json.dumps({
                'bbox': collection.bbox,
                'type': 'FeatureCollection',
                'features': features},
                **dump_kwds))
        fobj.write('\n')


def resolve_inout(
    input=None, output=None, files=None, overwrite=False, num_inputs=None
):
    """Resolves inputs and outputs from standard args and options.

    Parameters
    ----------
    input : str
        A single input filename, optional.
    output : str
        A single output filename, optional.
    files : str
        A sequence of filenames in which the last is the output filename.
    overwrite : bool
        Whether to force overwriting the output file.
    num_inputs : int
        Raise exceptions if the number of resolved input files is higher
        or lower than this number.

    Returns
    -------
    tuple (str, list of str)
        The resolved output filename and input filenames as a tuple of
        length 2.

    If provided, the output file may be overwritten. An output
    file extracted from files will not be overwritten unless
    overwrite is True.

    Raises
    ------
    click.BadParameter

    """
    resolved_output = output or (files[-1] if files else None)

    if not overwrite and resolved_output and os.path.exists(resolved_output):
        raise FileOverwriteError(
            "file exists and won't be overwritten without use of the `--overwrite` option."
        )

    resolved_inputs = (
        [input] if input else [] +
        list(files[:-1 if not output else None]) if files else [])

    if num_inputs is not None:
        if len(resolved_inputs) < num_inputs:
            raise click.BadParameter("Insufficient inputs")
        elif len(resolved_inputs) > num_inputs:
            raise click.BadParameter("Too many inputs")

    return resolved_output, resolved_inputs


def to_lower(ctx, param, value):
    """Click callback, converts values to lowercase."""
    return value.lower()

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/helpers.py
NikolaBlagojevic/pyrecodes
"""Command access to dataset metadata, stats, and more."""


import json

from attr import asdict
import click

import rasterio
from rasterio.rio import options
from rasterio.transform import from_gcps


@click.command(short_help="Print information about a data file.")
@options.file_in_arg
@click.option('--meta', 'aspect', flag_value='meta', default=True,
              help="Show data file structure (default).")
@click.option('--tags', 'aspect', flag_value='tags',
              help="Show data file tags.")
@click.option('--namespace', help="Select a tag namespace.")
@click.option('--indent', default=None, type=int,
              help="Indentation level for pretty printed output")
# Options to pick out a single metadata item and print it as
# a string.
@click.option('--count', 'meta_member', flag_value='count',
              help="Print the count of bands.")
@click.option('-t', '--dtype', 'meta_member', flag_value='dtype',
              help="Print the dtype name.")
@click.option('--nodata', 'meta_member', flag_value='nodata',
              help="Print the nodata value.")
@click.option('-f', '--format', '--driver', 'meta_member', flag_value='driver',
              help="Print the format driver.")
@click.option('--shape', 'meta_member', flag_value='shape',
              help="Print the (height, width) shape.")
@click.option('--height', 'meta_member', flag_value='height',
              help="Print the height (number of rows).")
@click.option('--width', 'meta_member', flag_value='width',
              help="Print the width (number of columns).")
@click.option('--crs', 'meta_member', flag_value='crs',
              help="Print the CRS as a PROJ.4 string.")
@click.option('--bounds', 'meta_member', flag_value='bounds',
              help="Print the boundary coordinates "
                   "(left, bottom, right, top).")
@click.option('-r', '--res', 'meta_member', flag_value='res',
              help="Print pixel width and height.")
@click.option('--lnglat', 'meta_member', flag_value='lnglat',
              help="Print longitude and latitude at center.")
@click.option('--stats', 'meta_member', flag_value='stats',
              help="Print statistics (min, max, mean) of a single band "
                   "(use --bidx).")
@click.option('--checksum', 'meta_member', flag_value='checksum',
              help="Print integer checksum of a single band "
                   "(use --bidx).")
@click.option('--subdatasets', 'meta_member', flag_value='subdatasets',
              help="Print subdataset identifiers.")
@click.option('-v', '--tell-me-more', '--verbose', 'verbose', is_flag=True,
              help="Output extra information.")
@options.bidx_opt
@options.masked_opt
@click.pass_context
def info(ctx, input, aspect, indent, namespace, meta_member, verbose, bidx,
         masked):
    """Print metadata about the dataset as JSON.

    Optionally print a single metadata item as a string.
    """
    with ctx.obj['env'], rasterio.open(input) as src:

        info = dict(src.profile)
        info['shape'] = (info['height'], info['width'])
        info['bounds'] = src.bounds

        if src.crs:
            epsg = src.crs.to_epsg()
            if epsg:
                info['crs'] = 'EPSG:{}'.format(epsg)
            else:
                info['crs'] = src.crs.to_string()
        else:
            info['crs'] = None

        info['res'] = src.res
        info['colorinterp'] = [ci.name for ci in src.colorinterp]
        info['units'] = [units or None for units in src.units]
        info['descriptions'] = src.descriptions
        info['indexes'] = src.indexes
        info['mask_flags'] = [[
            flag.name for flag in flags] for flags in src.mask_flag_enums]

        if src.crs:
            info['lnglat'] = src.lnglat()

        gcps, gcps_crs = src.gcps

        if gcps:
            info['gcps'] = {'points': [p.asdict() for p in gcps]}
            if gcps_crs:
                epsg = gcps_crs.to_epsg()
                if epsg:
                    info['gcps']['crs'] = 'EPSG:{}'.format(epsg)
                else:
                    info['gcps']['crs'] = src.crs.to_string()
            else:
                info['gcps']['crs'] = None

            info['gcps']['transform'] = from_gcps(gcps)

        if verbose:
            stats = [asdict(src.statistics(bidx)) for bidx in src.indexes]
            info['stats'] = stats
            info['checksum'] = [src.checksum(i) for i in src.indexes]

        if aspect == 'meta':
            if meta_member == 'subdatasets':
                for name in src.subdatasets:
                    click.echo(name)
            elif meta_member == 'stats':
                st = src.statistics(bidx)
                click.echo("{st.min} {st.max} {st.mean} {st.std}".format(st=st))
            elif meta_member == 'checksum':
                click.echo(str(src.checksum(bidx)))
            elif meta_member:
                if isinstance(info[meta_member], (list, tuple)):
                    click.echo(" ".join(map(str, info[meta_member])))
                else:
                    click.echo(info[meta_member])
            else:
                click.echo(json.dumps(info, sort_keys=True, indent=indent))

        elif aspect == 'tags':
            click.echo(
                json.dumps(src.tags(ns=namespace), indent=indent))

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/info.py
NikolaBlagojevic/pyrecodes
"""Fetch and edit raster dataset metadata from the command line."""
from __future__ import absolute_import

import code
import logging
import sys
import collections
import warnings

import numpy as np
import click

from . import options
import rasterio
from rasterio.plot import show, show_hist

try:
    import matplotlib.pyplot as plt
except ImportError:  # pragma: no cover
    plt = None
except RuntimeError as e:  # pragma: no cover
    # Certain environment configurations can trigger a RuntimeError like:

    # Trying to import matplotlibRuntimeError: Python is not installed as a
    # framework. The Mac OS X backend will not be able to function correctly
    # if Python is not installed as a framework. See the Python ...
    warnings.warn(str(e), RuntimeWarning, stacklevel=2)
    plt = None


Stats = collections.namedtuple('Stats', ['min', 'max', 'mean'])

# Collect dictionary of functions for use in the interpreter in main()
funcs = locals()


def stats(dataset):
    """Return a tuple with raster min, max, and mean."""
    if isinstance(dataset, tuple):
        arr = dataset[0].read(dataset[1])
    else:
        arr = dataset
    return Stats(np.min(arr), np.max(arr), np.mean(arr))


def main(banner, dataset, alt_interpreter=None):
    """Main entry point for use with python interpreter."""
    local = dict(funcs, src=dataset, np=np, rio=rasterio, plt=plt)
    if not alt_interpreter:
        code.interact(banner, local=local)
    elif alt_interpreter == 'ipython':  # pragma: no cover
        import IPython
        IPython.InteractiveShell.banner1 = banner
        IPython.start_ipython(argv=[], user_ns=local)
    else:
        raise ValueError("Unsupported interpreter '%s'" % alt_interpreter)

    return 0


@click.command(short_help="Open a data file and start an interpreter.")
@options.file_in_arg
@click.option('--ipython', 'interpreter', flag_value='ipython',
              help="Use IPython as interpreter.")
@click.option(
    '-m',
    '--mode',
    type=click.Choice(['r', 'r+']),
    default='r',
    help="File mode (default 'r').")
@click.pass_context
def insp(ctx, input, mode, interpreter):
    """Open the input file in a Python interpreter."""
    logger = logging.getLogger()
    try:
        with ctx.obj['env']:
            with rasterio.open(input, mode) as src:
                main(
                    'Rasterio %s Interactive Inspector (Python %s)\n'
                    'Type "src.meta", "src.read(1)", or "help(src)" '
                    'for more information.' % (
                        rasterio.__version__,
                        '.'.join(map(str, sys.version_info[:3]))),
                    src, interpreter)
    except Exception:
        logger.exception("Exception caught during processing")
        raise click.Abort()

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/insp.py
NikolaBlagojevic/pyrecodes
"""
Main command group for Rasterio's CLI.

Subcommands developed as a part of the Rasterio package have their own
modules under ``rasterio.rio`` (like ``rasterio/rio/info.py``) and are
registered in the 'rasterio.rio_commands' entry point group in
Rasterio's ``setup.py``:

    entry_points='''
        [console_scripts]
        rio=rasterio.rio.main:main_group

        [rasterio.rio_commands]
        bounds=rasterio.rio.bounds:bounds
        calc=rasterio.rio.calc:calc
        ...

Users may create their own ``rio`` subcommands by writing modules that
register entry points in Rasterio's 'rasterio.rio_plugins' group. See
for example https://github.com/sgillies/rio-plugin-example, which has
been published to PyPI as ``rio-metasay``.

There's no advantage to making a ``rio`` subcommand which doesn't
import rasterio. But if you are using rasterio, you may profit from
Rasterio's CLI infrastructure and the network of existing commands.
Please add yours to the registry

  https://github.com/rasterio/rasterio/wiki/Rio-plugin-registry

so that other ``rio`` users may find it.
"""


import logging
from pkg_resources import iter_entry_points
import sys

from click_plugins import with_plugins
import click
import cligj

from . import options
import rasterio
from rasterio.session import AWSSession


def configure_logging(verbosity):
    log_level = max(10, 30 - 10 * verbosity)
    logging.basicConfig(stream=sys.stderr, level=log_level)


def gdal_version_cb(ctx, param, value):
    if not value or ctx.resilient_parsing:
        return

    click.echo("{0}".format(rasterio.__gdal_version__), color=ctx.color)
    ctx.exit()

def show_versions_cb(ctx, param, value):
    if not value or ctx.resilient_parsing:
        return

    rasterio.show_versions()
    ctx.exit()



@with_plugins(
    ep
    for ep in list(iter_entry_points("rasterio.rio_commands"))
    + list(iter_entry_points("rasterio.rio_plugins"))
)
@click.group()
@cligj.verbose_opt
@cligj.quiet_opt
@click.option(
    "--aws-profile", help="Select a profile from the AWS credentials file"
)
@click.option("--aws-no-sign-requests", is_flag=True, help="Make requests anonymously")
@click.option(
    "--aws-requester-pays", is_flag=True, help="Requester pays data transfer costs"
)
@click.version_option(version=rasterio.__version__, message="%(version)s")
@click.option("--gdal-version", is_eager=True, is_flag=True, callback=gdal_version_cb)
@click.option("--show-versions", help="Show dependency versions", is_eager=True, is_flag=True, callback=show_versions_cb)
@click.pass_context
def main_group(
    ctx,
    verbose,
    quiet,
    aws_profile,
    aws_no_sign_requests,
    aws_requester_pays,
    gdal_version,
    show_versions,
):
    """Rasterio command line interface.
    """
    verbosity = verbose - quiet
    configure_logging(verbosity)
    ctx.obj = {}
    ctx.obj["verbosity"] = verbosity
    ctx.obj["aws_profile"] = aws_profile
    envopts = {"CPL_DEBUG": (verbosity > 2)}
    if aws_profile or aws_no_sign_requests or aws_requester_pays:
        ctx.obj["env"] = rasterio.Env(
            session=AWSSession(
                profile_name=aws_profile,
                aws_unsigned=aws_no_sign_requests,
                requester_pays=aws_requester_pays,
            ), **envopts)
    else:
        ctx.obj["env"] = rasterio.Env(**envopts)

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/main.py
NikolaBlagojevic/pyrecodes
import json
import logging

import click

from .helpers import resolve_inout
from . import options
import rasterio
import rasterio.shutil
from rasterio.mask import mask as mask_tool

logger = logging.getLogger(__name__)


# Mask command
@click.command(short_help='Mask in raster using features.')
@options.files_inout_arg
@options.output_opt
@click.option('-j', '--geojson-mask', 'geojson_mask',
              type=click.Path(), default=None,
              help='GeoJSON file to use for masking raster.  Use "-" to read '
                   'from stdin.  If not provided, original raster will be '
                   'returned')
@options.format_opt
@options.all_touched_opt
@click.option('--crop', is_flag=True, default=False,
              help='Crop output raster to the extent of the geometries. '
                   'GeoJSON must overlap input raster to use --crop')
@click.option('-i', '--invert', is_flag=True, default=False,
              help='Inverts the mask, so that areas covered by features are'
                   'masked out and areas not covered are retained.  Ignored '
                   'if using --crop')
@options.overwrite_opt
@options.creation_options
@click.pass_context
def mask(
        ctx,
        files,
        output,
        geojson_mask,
        driver,
        all_touched,
        crop,
        invert,
        overwrite,
        creation_options):
    """Masks in raster using GeoJSON features (masks out all areas not covered
    by features), and optionally crops the output raster to the extent of the
    features.  Features are assumed to be in the same coordinate reference
    system as the input raster.

    GeoJSON must be the first input file or provided from stdin:

    > rio mask input.tif output.tif --geojson-mask features.json

    > rio mask input.tif output.tif --geojson-mask - < features.json

    If the output raster exists, it will be completely overwritten with the
    results of this operation.

    The result is always equal to or within the bounds of the input raster.

    --crop and --invert options are mutually exclusive.

    --crop option is not valid if features are completely outside extent of
    input raster.
    """

    output, files = resolve_inout(
        files=files, output=output, overwrite=overwrite)
    input = files[0]

    if geojson_mask is None:
        click.echo('No GeoJSON provided, INPUT will be copied to OUTPUT',
                   err=True)
        rasterio.shutil.copyfiles(input, output)
        return

    if crop and invert:
        click.echo('Invert option ignored when using --crop', err=True)
        invert = False

    with ctx.obj['env']:
        try:
            with click.open_file(geojson_mask) as fh:
                geojson = json.loads(fh.read())
        except ValueError:
            raise click.BadParameter('GeoJSON could not be read from '
                                     '--geojson-mask or stdin',
                                     param_hint='--geojson-mask')

        if 'features' in geojson:
            geometries = [f['geometry'] for f in geojson['features']]
        elif 'geometry' in geojson:
            geometries = (geojson['geometry'], )
        elif 'coordinates' in geojson:
            geometries = (geojson, )
        else:
            raise click.BadParameter('Invalid GeoJSON', param=input,
                                     param_hint='input')

        with rasterio.open(input) as src:
            try:
                out_image, out_transform = mask_tool(src, geometries,
                                                     crop=crop, invert=invert,
                                                     all_touched=all_touched)
            except ValueError as e:
                if e.args[0] == 'Input shapes do not overlap raster.':
                    if crop:
                        raise click.BadParameter('not allowed for GeoJSON '
                                                 'outside the extent of the '
                                                 'input raster',
                                                 param=crop,
                                                 param_hint='--crop')
                else:  # pragma: no cover
                    raise e

            profile = src.profile
            profile.update(**creation_options)
            profile.pop("driver", None)
            if driver:
                profile["driver"] = driver
            profile.update({
                'driver': driver,
                'height': out_image.shape[1],
                'width': out_image.shape[2],
                'transform': out_transform
            })

            with rasterio.open(output, 'w', **profile) as out:
                out.write(out_image)

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/mask.py
NikolaBlagojevic/pyrecodes
"""$ rio merge"""

import warnings

import click

from rasterio.enums import Resampling
from rasterio.errors import RasterioDeprecationWarning
from rasterio.rio import options
from rasterio.rio.helpers import resolve_inout


def deprecated_precision(*args):
    warnings.warn(
        "The --precision option is unused, deprecated, and will be removed in 2.0.0.",
        RasterioDeprecationWarning,
    )
    return None


@click.command(short_help="Merge a stack of raster datasets.")
@options.files_inout_arg
@options.output_opt
@options.format_opt
@options.bounds_opt
@options.resolution_opt
@click.option('--resampling',
              type=click.Choice([r.name for r in Resampling if r.value <= 7]),
              default='nearest', help="Resampling method.",
              show_default=True)
@options.nodata_opt
@options.dtype_opt
@options.bidx_mult_opt
@options.overwrite_opt
@click.option(
    "--precision",
    type=int,
    default=None,
    callback=deprecated_precision,
    help="Unused, deprecated, and will be removed in 2.0.0.",
)
@options.creation_options
@click.pass_context
def merge(
    ctx,
    files,
    output,
    driver,
    bounds,
    res,
    resampling,
    nodata,
    dtype,
    bidx,
    overwrite,
    precision,
    creation_options,
):
    """Copy valid pixels from input files to an output file.

    All files must have the same number of bands, data type, and
    coordinate reference system.

    Input files are merged in their listed order using the reverse
    painter's algorithm. If the output file exists, its values will be
    overwritten by input values.

    Geospatial bounds and resolution of a new output file in the
    units of the input file coordinate reference system may be provided
    and are otherwise taken from the first input file.

    Note: --res changed from 2 parameters in 0.25.

    \b
      --res 0.1 0.1  => --res 0.1 (square)
      --res 0.1 0.2  => --res 0.1 --res 0.2  (rectangular)

    """
    from rasterio.merge import merge as merge_tool

    output, files = resolve_inout(
        files=files, output=output, overwrite=overwrite)

    resampling = Resampling[resampling]
    if driver:
        creation_options.update(driver=driver)

    with ctx.obj["env"]:
        merge_tool(
            files,
            bounds=bounds,
            res=res,
            nodata=nodata,
            dtype=dtype,
            indexes=(bidx or None),
            resampling=resampling,
            dst_path=output,
            dst_kwds=creation_options,
        )

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/merge.py
NikolaBlagojevic/pyrecodes
"""
Registry of common rio CLI options.  See cligj for more options.

.. code-block:: none

    -a, --all: Use all pixels touched by features.  In rio-mask, rio-rasterize
    --as-mask/--not-as-mask: interpret band as mask or not.  In rio-shapes
    --band/--mask: use band or mask.  In rio-shapes
    --bbox:
    -b, --bidx: band index(es) (singular or multiple value versions).
        In rio-info, rio-sample, rio-shapes, rio-stack (different usages)
    --bounds: bounds in world coordinates.
        In rio-info, rio-rasterize (different usages)
    --count: count of bands.  In rio-info
    --crop: Crop raster to extent of features.  In rio-mask
    --crs: CRS of input raster.  In rio-info
    --default-value: default for rasterized pixels.  In rio-rasterize
    --dimensions: Output width, height.  In rio-rasterize
    --dst-crs: destination CRS.  In rio-transform
    --fill: fill value for pixels not covered by features.  In rio-rasterize
    --formats: list available formats.  In rio-info
    --height: height of raster.  In rio-info
    -i, --invert: Invert mask created from features: In rio-mask
    -j, --geojson-mask: GeoJSON for masking raster.  In rio-mask
    --lnglat: geograhpic coordinates of center of raster.  In rio-info
    --masked/--not-masked: read masked data from source file.
        In rio-calc, rio-info
    -m, --mode: output file mode (r, r+).  In rio-insp
    --name: input file name alias.  In rio-calc
    --nodata: nodata value.  In rio-info, rio-merge (different usages)
    --photometric: photometric interpretation.  In rio-stack
    --property: GeoJSON property to use as values for rasterize.  In rio-rasterize
    -r, --res: output resolution.
        In rio-info, rio-rasterize (different usages.  TODO: try to combine
        usages, prefer rio-rasterize version)
    --sampling: Inverse of sampling fraction.  In rio-shapes
    --shape: shape (width, height) of band.  In rio-info
    --src-crs: source CRS.
        In rio-insp, rio-rasterize (different usages.  TODO: consolidate usages)
    --stats: print raster stats.  In rio-inf
    -t, --dtype: data type.  In rio-calc, rio-info (different usages)
    --width: width of raster.  In rio-info
    --with-nodata/--without-nodata: include nodata regions or not.  In rio-shapes.
    -v, --tell-me-more, --verbose
    --vfs: virtual file system.
"""

import logging
import os
import re

import click

import rasterio
import rasterio.shutil
from rasterio._path import _parse_path, _UnparsedPath


logger = logging.getLogger(__name__)


def _cb_key_val(ctx, param, value):

    """
    click callback to validate `--opt KEY1=VAL1 --opt KEY2=VAL2` and collect
    in a dictionary like the one below, which is what the CLI function receives.
    If no value or `None` is received then an empty dictionary is returned.

        {
            'KEY1': 'VAL1',
            'KEY2': 'VAL2'
        }

    Note: `==VAL` breaks this as `str.split('=', 1)` is used.
    """

    if not value:
        return {}
    else:
        out = {}
        for pair in value:
            if '=' not in pair:
                raise click.BadParameter(
                    "Invalid syntax for KEY=VAL arg: {}".format(pair))
            else:
                k, v = pair.split('=', 1)
                k = k.lower()
                v = v.lower()
                out[k] = None if v.lower() in ['none', 'null', 'nil', 'nada'] else v
        return out


def abspath_forward_slashes(path):
    """Return forward-slashed version of os.path.abspath"""
    return '/'.join(os.path.abspath(path).split(os.path.sep))


def file_in_handler(ctx, param, value):
    """Normalize ordinary filesystem and VFS paths"""
    try:
        path = _parse_path(value)

        if isinstance(path, _UnparsedPath):

            if os.path.exists(path.path) and rasterio.shutil.exists(value):
                return abspath_forward_slashes(path.path)
            else:
                return path.name

        elif path.scheme and path.is_remote:
            return path.name

        elif path.archive:
            if os.path.exists(path.archive) and rasterio.shutil.exists(value):
                archive = abspath_forward_slashes(path.archive)
                return "{}://{}!{}".format(path.scheme, archive, path.path)
            else:
                raise OSError(
                    "Input archive {} does not exist".format(path.archive))

        else:
            if os.path.exists(path.path) and rasterio.shutil.exists(value):
                return abspath_forward_slashes(path.path)
            else:
                raise OSError(
                    "Input file {} does not exist".format(path.path))

    except Exception:
        raise click.BadParameter("{} is not a valid input file".format(value))


def files_in_handler(ctx, param, value):
    """Process and validate input file names"""
    return tuple(file_in_handler(ctx, param, item) for item in value)


def files_inout_handler(ctx, param, value):
    """Process and validate input file names"""
    return tuple(file_in_handler(ctx, param, item) for item in value[:-1]) + tuple(value[-1:])


def from_like_context(ctx, param, value):
    """Return the value for an option from the context if the option
    or `--all` is given, else return None."""
    if ctx.obj and ctx.obj.get('like') and (
            value == 'like' or ctx.obj.get('all_like')):
        return ctx.obj['like'][param.name]
    else:
        return None


def like_handler(ctx, param, value):
    """Copy a dataset's meta property to the command context for access
    from other callbacks."""
    if ctx.obj is None:
        ctx.obj = {}
    if value:
        with rasterio.open(value) as src:
            metadata = src.meta
            ctx.obj['like'] = metadata
            ctx.obj['like']['transform'] = metadata['transform']
            ctx.obj['like']['tags'] = src.tags()
            ctx.obj['like']['colorinterp'] = src.colorinterp
        return True
    else:  # pragma: no cover
        return False


def nodata_handler(ctx, param, value):
    """Return a float or None"""
    if value is None or value.lower() in ["null", "nil", "none", "nada"]:
        return None
    else:
        try:
            return float(value)
        except (TypeError, ValueError):
            raise click.BadParameter(
                "{!r} is not a number".format(value), param=param, param_hint="nodata"
            )


def edit_nodata_handler(ctx, param, value):
    """Get nodata value from a template file or command line.

    Expected values are 'like', 'null', a numeric value, 'nan', or None.

    Returns
    -------
    float or None

    Raises
    ------
    click.BadParameter

    """
    if value == "like" or value is None:
        retval = from_like_context(ctx, param, value)
        if retval is not None:
            return retval

    return nodata_handler(ctx, param, value)


def bounds_handler(ctx, param, value):
    """Handle different forms of bounds."""
    retval = from_like_context(ctx, param, value)
    if retval is None and value is not None:
        try:
            value = value.strip(', []')
            retval = tuple(float(x) for x in re.split(r'[,\s]+', value))
            assert len(retval) == 4
            return retval
        except Exception:
            raise click.BadParameter(
                "{0!r} is not a valid bounding box representation".format(
                    value))
    else:  # pragma: no cover
        return retval


# Singular input file
file_in_arg = click.argument('INPUT', callback=file_in_handler)

# Singular output file
file_out_arg = click.argument(
    'OUTPUT',
    type=click.Path(resolve_path=True))

# Multiple input files.
files_in_arg = click.argument(
    'files',
    nargs=-1,
    type=click.Path(),
    required=True,
    metavar="INPUTS...",
    callback=files_in_handler)

# Multiple files, last of which is an output file.
files_inout_arg = click.argument(
    'files',
    nargs=-1,
    type=click.Path(),
    required=True,
    metavar="INPUTS... OUTPUT",
    callback=files_inout_handler)

bidx_opt = click.option(
    '-b', '--bidx',
    type=int,
    default=1,
    help="Input file band index (default: 1).")

bidx_mult_opt = click.option(
    '-b', '--bidx',
    type=int,
    multiple=True,
    help="Indexes of input file bands.")

# The 'magic' --bidx option is especially for rio-stack, which supports
# selection of multiple bands from multiple files. It has 'str' type
# instead of 'int' to support ranges in a syntax like `--bidx 1..3`.
bidx_magic_opt = click.option(
    '-b', '--bidx',
    multiple=True,
    help="Indexes of input file bands.")

# TODO: may be better suited to cligj
bounds_opt = click.option(
    '--bounds', default=None, callback=bounds_handler,
    help='Bounds: "left bottom right top" or "[left, bottom, right, top]".')

dimensions_opt = click.option(
    '--dimensions',
    nargs=2, type=int, default=None,
    help='Output dataset width, height in number of pixels.')

dtype_opt = click.option(
    '-t', '--dtype',
    type=click.Choice([
        'ubyte', 'uint8', 'uint16', 'int16', 'uint32', 'int32',
        'float32', 'float64']),
    default=None,
    help="Output data type.")

like_file_opt = click.option(
    '--like',
    type=click.Path(exists=True),
    help='Raster dataset to use as a template for obtaining affine '
         'transform (bounds and resolution), crs, data type, and driver '
         'used to create the output.')

masked_opt = click.option(
    '--masked/--not-masked',
    default=True,
    help="Evaluate expressions using masked arrays (the default) or ordinary "
         "numpy arrays.")

output_opt = click.option(
    '-o', '--output',
    default=None,
    type=click.Path(resolve_path=True),
    help="Path to output file (optional alternative to a positional arg).")

resolution_opt = click.option(
    '-r', '--res',
    multiple=True, type=float, default=None,
    help='Output dataset resolution in units of coordinate '
         'reference system. Pixels assumed to be square if this option '
         'is used once, otherwise use: '
         '--res pixel_width --res pixel_height.')

creation_options = click.option(
    '--co', '--profile', 'creation_options',
    metavar='NAME=VALUE',
    multiple=True,
    callback=_cb_key_val,
    help="Driver specific creation options. "
         "See the documentation for the selected output driver for "
         "more information.")

rgb_opt = click.option(
    '--rgb', 'photometric',
    flag_value='rgb',
    default=False,
    help="Set RGB photometric interpretation.")

overwrite_opt = click.option(
    '--overwrite', 'overwrite',
    is_flag=True, type=bool, default=False,
    help="Always overwrite an existing output file.")

nodata_opt = click.option(
    "--nodata",
    callback=nodata_handler,
    default=None,
    metavar="NUMBER|nan",
    help="Set a Nodata value.",
)

edit_nodata_opt = click.option(
    "--nodata",
    callback=edit_nodata_handler,
    default=None,
    metavar="NUMBER|nan|null",
    help="Modify the Nodata value.",
)

like_opt = click.option(
    '--like',
    type=click.Path(exists=True),
    callback=like_handler,
    is_eager=True,
    help="Raster dataset to use as a template for obtaining affine "
         "transform (bounds and resolution), crs, and nodata values.")

all_touched_opt = click.option(
    '-a', '--all', '--all_touched', 'all_touched',
    is_flag=True,
    default=False,
    help='Use all pixels touched by features, otherwise (default) use only '
         'pixels whose center is within the polygon or that are selected by '
         'Bresenhams line algorithm')

# Feature collection or feature sequence switch.
sequence_opt = click.option(
    '--sequence/--collection',
    default=True,
    help="Write a LF-delimited sequence of texts containing individual "
         "objects (the default) or write a single JSON text containing a "
         "feature collection object.")

format_opt = click.option(
    '-f', '--format', '--driver', 'driver',
    help="Output format driver")

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/options.py
NikolaBlagojevic/pyrecodes
# coding: utf-8
"""Manage overviews of a dataset."""

from functools import reduce
import logging
import operator

import click

from . import options
import rasterio
from rasterio.enums import _OverviewResampling as OverviewResampling


def build_handler(ctx, param, value):
    if value:
        try:
            if '^' in value:
                base, exp_range = value.split('^')
                exp_min, exp_max = (int(v) for v in exp_range.split('..'))
                value = [pow(int(base), k) for k in range(exp_min, exp_max + 1)]
            elif ',' in value:
                value = [int(v) for v in value.split(',')]
            elif value == "auto":
                pass
            else:
                raise Exception
        except Exception:
            raise click.BadParameter(u"must match 'n,n,n,…', 'n^n..n', or 'auto'.")
    return value


def get_maximum_overview_level(width, height, minsize=256):
    """
    Calculate the maximum overview level of a dataset at which
    the smallest overview is smaller than `minsize`.

    Attributes
    ----------
    width : int
        Width of the dataset.
    height : int
        Height of the dataset.
    minsize : int (default: 256)
        Minimum overview size.

    Returns
    -------
    overview_level: int
        overview level.

    """
    overview_level = 0
    overview_factor = 1
    while min(width // overview_factor, height // overview_factor) > minsize:
        overview_factor *= 2
        overview_level += 1

    return overview_level


@click.command('overview', short_help="Construct overviews in an existing dataset.")
@options.file_in_arg
@click.option('--build', callback=build_handler, metavar=u"f1,f2,…|b^min..max|auto",
              help="A sequence of decimation factors specified as "
                   "comma-separated list of numbers or a base and range of "
                   "exponents, or 'auto' to automatically determine the maximum factor.")
@click.option('--ls', help="Print the overviews for each band.",
              is_flag=True, default=False)
@click.option('--rebuild', help="Reconstruct existing overviews.",
              is_flag=True, default=False)
@click.option('--resampling', help="Resampling algorithm.",
              type=click.Choice([it.name for it in OverviewResampling]),
              default='nearest', show_default=True)
@click.pass_context
def overview(ctx, input, build, ls, rebuild, resampling):
    """Construct overviews in an existing dataset.

    A pyramid of overviews computed once and stored in the dataset can
    improve performance in some applications.

    The decimation levels at which to build overviews can be specified as
    a comma separated list

      rio overview --build 2,4,8,16

    or a base and range of exponents

      rio overview --build 2^1..4

    or 'auto' to automatically determine the maximum decimation level at
    which the smallest overview is smaller than 256 pixels in size.

      rio overview --build auto

    Note that overviews can not currently be removed and are not
    automatically updated when the dataset's primary bands are
    modified.

    Information about existing overviews can be printed using the --ls
    option.

      rio overview --ls

    """
    with ctx.obj['env']:
        if ls:
            with rasterio.open(input, 'r') as dst:
                resampling_method = dst.tags(
                    ns='rio_overview').get('resampling') or 'unknown'

                click.echo("Overview factors:")
                for idx in dst.indexes:
                    click.echo("  Band %d: %s (method: '%s')" % (
                        idx, dst.overviews(idx) or 'None', resampling_method))
        elif rebuild:
            with rasterio.open(input, 'r+') as dst:
                # Build the same overviews for all bands.
                factors = reduce(
                    operator.or_,
                    [set(dst.overviews(i)) for i in dst.indexes])

                # Attempt to recover the resampling method from dataset tags.
                resampling_method = dst.tags(
                    ns='rio_overview').get('resampling') or resampling

                dst.build_overviews(
                    list(factors), OverviewResampling[resampling_method])

        elif build:
            with rasterio.open(input, 'r+') as dst:
                if build == "auto":
                    overview_level = get_maximum_overview_level(dst.width, dst.height)
                    build = [2 ** j for j in range(1, overview_level + 1)]
                dst.build_overviews(build, OverviewResampling[resampling])

                # Save the resampling method to a tag.
                dst.update_tags(ns='rio_overview', resampling=resampling)

        else:
            raise click.UsageError(
                "Please specify --ls, --rebuild, or --build ...")

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/overview.py
NikolaBlagojevic/pyrecodes
"""$ rio rasterize"""


import json
import logging
from math import ceil
import os

from affine import Affine
import click
import cligj

import rasterio
from rasterio.errors import CRSError
from rasterio.coords import disjoint_bounds
from rasterio.rio import options
from rasterio.rio.helpers import resolve_inout
import rasterio.shutil


logger = logging.getLogger(__name__)


def files_handler(ctx, param, value):
    """Process and validate input file names"""
    return value


# Unlike the version in cligj, this one doesn't require values.
files_inout_arg = click.argument(
    'files',
    nargs=-1,
    type=click.Path(),
    metavar="INPUTS... OUTPUT",
    callback=files_handler)


@click.command(short_help='Rasterize features.')
@files_inout_arg
@options.output_opt
@options.format_opt
@options.like_file_opt
@options.bounds_opt
@options.dimensions_opt
@options.resolution_opt
@click.option('--src-crs', '--src_crs', 'src_crs', default=None,
              help='Source coordinate reference system.  Limited to EPSG '
              'codes for now.  Used as output coordinate system if output '
              'does not exist or --like option is not used. '
              'Default: EPSG:4326')
@options.all_touched_opt
@click.option('--default-value', '--default_value', 'default_value',
              type=float, default=1, help='Default value for rasterized pixels')
@click.option('--fill', type=float, default=0,
              help='Fill value for all pixels not overlapping features.  Will '
              'be evaluated as NoData pixels for output.  Default: 0')
@click.option('--property', 'prop', type=str, default=None, help='Property in '
              'GeoJSON features to use for rasterized values.  Any features '
              'that lack this property will be given --default_value instead.')
@options.overwrite_opt
@options.nodata_opt
@options.creation_options
@click.pass_context
def rasterize(
        ctx,
        files,
        output,
        driver,
        like,
        bounds,
        dimensions,
        res,
        src_crs,
        all_touched,
        default_value,
        fill,
        prop,
        overwrite,
        nodata,
        creation_options):
    """Rasterize GeoJSON into a new or existing raster.

    If the output raster exists, rio-rasterize will rasterize feature
    values into all bands of that raster.  The GeoJSON is assumed to be
    in the same coordinate reference system as the output unless
    --src-crs is provided.

    --default_value or property values when using --property must be
    using a data type valid for the data type of that raster.

    If a template raster is provided using the --like option, the affine
    transform and data type from that raster will be used to create the
    output.  Only a single band will be output.

    The GeoJSON is assumed to be in the same coordinate reference system
    unless --src-crs is provided.

    --default_value or property values when using --property must be
    using a data type valid for the data type of that raster.

    --driver, --bounds, --dimensions, --res, --nodata are ignored when
    output exists or --like raster is provided

    If the output does not exist and --like raster is not provided, the
    input GeoJSON will be used to determine the bounds of the output
    unless provided using --bounds.

    --dimensions or --res are required in this case.

    If --res is provided, the bottom and right coordinates of bounds are
    ignored.

    Note
    ----

    The GeoJSON is not projected to match the coordinate reference
    system of the output or --like rasters at this time.  This
    functionality may be added in the future.

    """

    from rasterio.crs import CRS
    from rasterio.features import rasterize
    from rasterio.features import bounds as calculate_bounds

    output, files = resolve_inout(
        files=files, output=output, overwrite=overwrite)

    bad_param = click.BadParameter('invalid CRS.  Must be an EPSG code.',
                                   ctx, param=src_crs, param_hint='--src_crs')
    has_src_crs = src_crs is not None
    try:
        src_crs = CRS.from_string(src_crs) if has_src_crs else CRS.from_string('EPSG:4326')
    except CRSError:
        raise bad_param

    # If values are actually meant to be integers, we need to cast them
    # as such or rasterize creates floating point outputs
    if default_value == int(default_value):
        default_value = int(default_value)
    if fill == int(fill):
        fill = int(fill)

    with ctx.obj['env']:

        def feature_value(feature):
            if prop and 'properties' in feature:
                return feature['properties'].get(prop, default_value)
            return default_value

        with click.open_file(files.pop(0) if files else '-') as gj_f:
            geojson = json.loads(gj_f.read())
        if 'features' in geojson:
            geometries = []
            for f in geojson['features']:
                geometries.append((f['geometry'], feature_value(f)))
        elif 'geometry' in geojson:
            geometries = ((geojson['geometry'], feature_value(geojson)), )
        else:
            raise click.BadParameter('Invalid GeoJSON', param=input,
                                     param_hint='input')

        geojson_bounds = geojson.get('bbox', calculate_bounds(geojson))

        if rasterio.shutil.exists(output):
            with rasterio.open(output, 'r+') as out:
                if has_src_crs and src_crs != out.crs:
                    raise click.BadParameter('GeoJSON does not match crs of '
                                             'existing output raster',
                                             param='input', param_hint='input')

                if disjoint_bounds(geojson_bounds, out.bounds):
                    click.echo("GeoJSON outside bounds of existing output "
                               "raster. Are they in different coordinate "
                               "reference systems?",
                               err=True)

                meta = out.meta

                result = rasterize(
                    geometries,
                    out_shape=(meta['height'], meta['width']),
                    transform=meta.get('affine', meta['transform']),
                    all_touched=all_touched,
                    dtype=meta.get('dtype', None),
                    default_value=default_value,
                    fill=fill)

                for bidx in range(1, meta['count'] + 1):
                    data = out.read(bidx, masked=True)
                    # Burn in any non-fill pixels, and update mask accordingly
                    ne = result != fill
                    data[ne] = result[ne]
                    if data.mask.any():
                        data.mask[ne] = False
                    out.write(data, indexes=bidx)

        else:
            if like is not None:
                template_ds = rasterio.open(like)

                if has_src_crs and src_crs != template_ds.crs:
                    raise click.BadParameter('GeoJSON does not match crs of '
                                             '--like raster',
                                             param='input', param_hint='input')

                if disjoint_bounds(geojson_bounds, template_ds.bounds):
                    click.echo("GeoJSON outside bounds of --like raster. "
                               "Are they in different coordinate reference "
                               "systems?",
                               err=True)

                kwargs = template_ds.profile
                kwargs['count'] = 1
                kwargs['transform'] = template_ds.transform

                template_ds.close()

            else:
                bounds = bounds or geojson_bounds

                if src_crs.is_geographic:
                    if (bounds[0] < -180 or bounds[2] > 180 or
                            bounds[1] < -80 or bounds[3] > 80):
                        raise click.BadParameter(
                            "Bounds are beyond the valid extent for "
                            "EPSG:4326.",
                            ctx, param=bounds, param_hint='--bounds')

                if dimensions:
                    width, height = dimensions
                    res = (
                        (bounds[2] - bounds[0]) / float(width),
                        (bounds[3] - bounds[1]) / float(height)
                    )

                else:
                    if not res:
                        raise click.BadParameter(
                            'pixel dimensions are required',
                            ctx, param=res, param_hint='--res')

                    elif len(res) == 1:
                        res = (res[0], res[0])

                    width = max(int(ceil((bounds[2] - bounds[0]) /
                                float(res[0]))), 1)
                    height = max(int(ceil((bounds[3] - bounds[1]) /
                                 float(res[1]))), 1)

                kwargs = {
                    'count': 1,
                    'crs': src_crs,
                    'width': width,
                    'height': height,
                    'transform': Affine(res[0], 0, bounds[0], 0, -res[1],
                                        bounds[3]),
                    'driver': driver
                }
                if driver:
                    kwargs["driver"] = driver

            kwargs.update(**creation_options)

            if nodata is not None:
                kwargs['nodata'] = nodata

            result = rasterize(
                geometries,
                out_shape=(kwargs['height'], kwargs['width']),
                transform=kwargs['transform'],
                all_touched=all_touched,
                dtype=kwargs.get('dtype', None),
                default_value=default_value,
                fill=fill)

            if 'dtype' not in kwargs:
                kwargs['dtype'] = result.dtype

            with rasterio.open(output, 'w', **kwargs) as out:
                out.write(result, indexes=1)

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/rasterize.py
NikolaBlagojevic/pyrecodes
"""``$ rio rm``"""


import click

import rasterio.shutil
from rasterio.errors import DriverRegistrationError, RasterioIOError


@click.command()
@click.argument('path')
@click.confirmation_option(
    help="Confirm delete without prompting",
    prompt="Are you sure you want to delete the dataset?",
    expose_value=True)
@click.option(
    '-f', '--format', '--driver', 'driver',
    help="Explicitly delete with this driver rather than probing for the "
         "appropriate driver.")
def rm(path, yes, driver):

    """Delete a dataset.

    Invoking the shell's '$ rm <path>' on a dataset can be used to
    delete a dataset referenced by a file path, but it won't handle
    deleting side car files.  This command is aware of datasets and
    their sidecar files.
    """

    try:
        rasterio.shutil.delete(path, driver=driver)
    except (DriverRegistrationError, RasterioIOError) as e:
        raise click.ClickException(str(e))

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/rm.py
NikolaBlagojevic/pyrecodes
import json

import click

import rasterio


@click.command(short_help="Sample a dataset.")
@click.argument('files', nargs=-1, required=True, metavar='FILE "[x, y]"')
@click.option('-b', '--bidx', default=None, help="Indexes of input file bands.")
@click.pass_context
def sample(ctx, files, bidx):
    """Sample a dataset at one or more points

    Sampling points (x, y) encoded as JSON arrays, in the coordinate
    reference system of the dataset, are read from the second
    positional argument or stdin. Values of the dataset's bands
    are also encoded as JSON arrays and are written to stdout.

    Example:

    \b
        $ cat << EOF | rio sample tests/data/RGB.byte.tif
        > [220650, 2719200]
        > [219650, 2718200]
        > EOF
        [28, 29, 27]
        [25, 29, 19]

    By default, rio-sample will sample all bands. Optionally, bands
    may be specified using a simple syntax:

      --bidx N samples the Nth band (first band is 1).

      --bidx M,N,0 samples bands M, N, and O.

      --bidx M..O samples bands M-O, inclusive.

      --bidx ..N samples all bands up to and including N.

      --bidx N.. samples all bands from N to the end.

    Example:

    \b
        $ cat << EOF | rio sample tests/data/RGB.byte.tif --bidx ..2
        > [220650, 2719200]
        > [219650, 2718200]
        > EOF
        [28, 29]
        [25, 29]

    """
    files = list(files)
    source_path = files.pop(0)
    input = files.pop(0) if files else '-'

    # Handle the case of file, stream, or string input.
    try:
        points = click.open_file(input).readlines()
    except OSError:
        points = [input]

    with ctx.obj["env"]:
        with rasterio.open(source_path) as src:
            if bidx is None:
                indexes = src.indexes
            elif ".." in bidx:
                start, stop = map(lambda x: int(x) if x else None, bidx.split(".."))
                if start is None:
                    start = 1
                indexes = src.indexes[slice(start - 1, stop)]
            else:
                indexes = list(map(int, bidx.split(",")))

            for vals in src.sample(
                (json.loads(line) for line in points), indexes=indexes
            ):
                click.echo(json.dumps(vals.tolist()))

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/sample.py
NikolaBlagojevic/pyrecodes
"""$ rio shapes"""

from __future__ import division


import logging

import click
import cligj
import rasterio

from rasterio.rio import options
from rasterio.features import dataset_features
from rasterio.rio.helpers import write_features

logger = logging.getLogger(__name__)


@click.command(short_help="Write shapes extracted from bands or masks.")
@options.file_in_arg
@options.output_opt
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@cligj.projection_geographic_opt
@cligj.projection_projected_opt
@options.sequence_opt
@cligj.use_rs_opt
@cligj.geojson_type_feature_opt(True)
@cligj.geojson_type_bbox_opt(False)
@click.option('--band/--mask', default=True,
              help="Choose to extract from a band (the default) or a mask.")
@click.option('--bidx', 'bandidx', type=int, default=None,
              help="Index of the band or mask that is the source of shapes.")
@click.option('--sampling', type=int, default=1,
              help="Inverse of the sampling fraction; "
                   "a value of 10 decimates.")
@click.option('--with-nodata/--without-nodata', default=False,
              help="Include or do not include (the default) nodata regions.")
@click.option('--as-mask/--not-as-mask', default=False,
              help="Interpret a band as a mask and output only one class of "
                   "valid data shapes.")
@click.pass_context
def shapes(
        ctx, input, output, precision, indent, compact, projection, sequence,
        use_rs, geojson_type, band, bandidx, sampling, with_nodata, as_mask):
    """Extracts shapes from one band or mask of a dataset and writes
    them out as GeoJSON. Unless otherwise specified, the shapes will be
    transformed to WGS 84 coordinates.

    The default action of this command is to extract shapes from the
    first band of the input dataset. The shapes are polygons bounding
    contiguous regions (or features) of the same raster value. This
    command performs poorly for int16 or float type datasets.

    Bands other than the first can be specified using the `--bidx`
    option:

      $ rio shapes --bidx 3 tests/data/RGB.byte.tif

    The valid data footprint of a dataset's i-th band can be extracted
    by using the `--mask` and `--bidx` options:

      $ rio shapes --mask --bidx 1 tests/data/RGB.byte.tif

    Omitting the `--bidx` option results in a footprint extracted from
    the conjunction of all band masks. This is generally smaller than
    any individual band's footprint.

    A dataset band may be analyzed as though it were a binary mask with
    the `--as-mask` option:

      $ rio shapes --as-mask --bidx 1 tests/data/RGB.byte.tif
    """
    # These import numpy, which we don't want to do unless it's needed.
    dump_kwds = {'sort_keys': True}
    if indent:
        dump_kwds['indent'] = indent
    if compact:
        dump_kwds['separators'] = (',', ':')

    stdout = click.open_file(
        output, 'w') if output else click.get_text_stream('stdout')

    bidx = 1 if bandidx is None and band else bandidx

    if not sequence:
        geojson_type = 'collection'

    geographic = True if projection == 'geographic' else False

    with ctx.obj["env"] as env:
        with rasterio.open(input) as src:
            write_features(
                stdout,
                feature_gen(
                    src,
                    env,
                    bidx,
                    sampling=sampling,
                    band=band,
                    as_mask=as_mask,
                    with_nodata=with_nodata,
                    geographic=geographic,
                    precision=precision,
                ),
                sequence=sequence,
                geojson_type=geojson_type,
                use_rs=use_rs,
                **dump_kwds
            )


def feature_gen(src, env, *args, **kwargs):
    class Collection:

        def __init__(self, env):
            self.bboxes = []
            self.env = env

        @property
        def bbox(self):
            minxs, minys, maxxs, maxys = zip(*self.bboxes)
            return min(minxs), min(minys), max(maxxs), max(maxys)

        def __call__(self):
            for f in dataset_features(src, *args, **kwargs):
                self.bboxes.append(f['bbox'])
                yield f

    return Collection(env)

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/shapes.py
NikolaBlagojevic/pyrecodes
"""$ rio stack"""

from collections.abc import Iterable
from itertools import zip_longest
import logging

import click

import rasterio
from rasterio.rio import options
from rasterio.rio.helpers import resolve_inout


@click.command(short_help="Stack a number of bands into a multiband dataset.")
@options.files_inout_arg
@options.output_opt
@options.format_opt
@options.bidx_magic_opt
@options.rgb_opt
@options.overwrite_opt
@options.creation_options
@click.pass_context
def stack(ctx, files, output, driver, bidx, photometric, overwrite,
          creation_options):
    """Stack a number of bands from one or more input files into a
    multiband dataset.

    Input datasets must be of a kind: same data type, dimensions, etc. The
    output is cloned from the first input.

    By default, rio-stack will take all bands from each input and write them
    in same order to the output. Optionally, bands for each input may be
    specified using a simple syntax:

      --bidx N takes the Nth band from the input (first band is 1).

      --bidx M,N,0 takes bands M, N, and O.

      --bidx M..O takes bands M-O, inclusive.

      --bidx ..N takes all bands up to and including N.

      --bidx N.. takes all bands from N to the end.

    Examples, using the Rasterio testing dataset, which produce a copy.

      rio stack RGB.byte.tif -o stacked.tif

      rio stack RGB.byte.tif --bidx 1,2,3 -o stacked.tif

      rio stack RGB.byte.tif --bidx 1..3 -o stacked.tif

      rio stack RGB.byte.tif --bidx ..2 RGB.byte.tif --bidx 3.. -o stacked.tif

    """
    logger = logging.getLogger(__name__)
    try:
        with ctx.obj['env']:
            output, files = resolve_inout(files=files, output=output,
                                          overwrite=overwrite)
            output_count = 0
            indexes = []
            for path, item in zip_longest(files, bidx, fillvalue=None):
                with rasterio.open(path) as src:
                    src_indexes = src.indexes
                if item is None:
                    indexes.append(src_indexes)
                    output_count += len(src_indexes)
                elif '..' in item:
                    start, stop = map(
                        lambda x: int(x) if x else None, item.split('..'))
                    if start is None:
                        start = 1
                    indexes.append(src_indexes[slice(start - 1, stop)])
                    output_count += len(src_indexes[slice(start - 1, stop)])
                else:
                    parts = list(map(int, item.split(',')))
                    if len(parts) == 1:
                        indexes.append(parts[0])
                        output_count += 1
                    else:
                        parts = list(parts)
                        indexes.append(parts)
                        output_count += len(parts)

            with rasterio.open(files[0]) as first:
                kwargs = first.meta
                kwargs.update(**creation_options)

            if driver:
                kwargs["driver"] = driver

            kwargs.update(count=output_count)

            if photometric:
                kwargs['photometric'] = photometric

            with rasterio.open(output, 'w', **kwargs) as dst:
                dst_idx = 1
                for path, index in zip(files, indexes):
                    with rasterio.open(path) as src:
                        if isinstance(index, int):
                            data = src.read(index)
                            dst.write(data, dst_idx)
                            dst_idx += 1
                        elif isinstance(index, Iterable):
                            data = src.read(index)
                            dst.write(data, range(dst_idx, dst_idx + len(index)))
                            dst_idx += len(index)

    except Exception:
        logger.exception("Exception caught during processing")
        raise click.Abort()

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/stack.py
NikolaBlagojevic/pyrecodes
"""Fetch and edit raster dataset metadata from the command line."""

import json

import click
from cligj import precision_opt

import rasterio.shutil


@click.command(short_help="Transform coordinates.")
@click.argument('INPUT', default='-', required=False)
@click.option('--src-crs', '--src_crs', default='EPSG:4326',
              help="Source CRS.")
@click.option('--dst-crs', '--dst_crs', default='EPSG:4326',
              help="Destination CRS.")
@precision_opt
@click.pass_context
def transform(ctx, input, src_crs, dst_crs, precision):
    """Transform coordinates between coordinate reference systems.

    JSON arrays of coordinates, interleaved, are read from stdin.
    Aarrays of transformed coordinates are written to stdout.

    To transform a longitude, latitude point (EPSG:4326 is the default)
    to another coordinate system with 2 decimal places of output
    precision, do the following.

    \b
        $ cat << EOF | rio transform --dst-crs EPSG:32618 --precision 2
        > [-78.0 23.0]
        > EOF
        [192457.13, 2546667.68]

    """
    import rasterio.warp

    # Handle the case of file, stream, or string input.
    try:
        src = click.open_file(input).readlines()
    except OSError:
        src = [input]

    with ctx.obj["env"]:

        if src_crs.startswith("EPSG"):
            src_crs = {"init": src_crs}
        elif rasterio.shutil.exists(src_crs):
            with rasterio.open(src_crs) as f:
                src_crs = f.crs

        if dst_crs.startswith("EPSG"):
            dst_crs = {"init": dst_crs}
        elif rasterio.shutil.exists(dst_crs):
            with rasterio.open(dst_crs) as f:
                dst_crs = f.crs

        for line in src:
            coords = json.loads(line)
            xs = coords[::2]
            ys = coords[1::2]
            xs, ys = rasterio.warp.transform(src_crs, dst_crs, xs, ys)

            if precision >= 0:
                xs = [round(v, precision) for v in xs]
                ys = [round(v, precision) for v in ys]

            result = [0] * len(coords)
            result[::2] = xs
            result[1::2] = ys
            click.echo(json.dumps(result))

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/transform.py
NikolaBlagojevic/pyrecodes
"""$ rio warp"""

import logging
from math import ceil, floor
import sys

import click

import rasterio
from rasterio.crs import CRS
from rasterio.env import setenv
from rasterio.errors import CRSError
from rasterio.rio import options
from rasterio.rio.helpers import resolve_inout
from rasterio.rio.options import _cb_key_val
from rasterio.transform import Affine
from rasterio.warp import (
    reproject, Resampling, SUPPORTED_RESAMPLING, transform_bounds,
    aligned_target, calculate_default_transform as calcdt)

logger = logging.getLogger(__name__)

# Improper usage of rio-warp can lead to accidental creation of
# extremely large datasets. We'll put a hard limit on the size of
# datasets and raise a usage error if the limits are exceeded.
MAX_OUTPUT_WIDTH = 100000
MAX_OUTPUT_HEIGHT = 100000


@click.command(short_help='Warp a raster dataset.')
@options.files_inout_arg
@options.output_opt
@options.format_opt
@click.option(
    '--like',
    type=click.Path(exists=True),
    help='Raster dataset to use as a template for obtaining affine '
         'transform (bounds and resolution), and crs.')
@click.option('--dst-crs', default=None,
              help='Target coordinate reference system.')
@options.dimensions_opt
@click.option(
    '--src-bounds',
    nargs=4, type=float, default=None,
    help="Determine output extent from source bounds: left bottom right top "
         ". Cannot be used with destination --bounds")
@click.option(
    '--bounds', '--dst-bounds', 'dst_bounds', nargs=4, type=float, default=None,
    help="Determine output extent from destination bounds: left bottom right top")
@options.resolution_opt
@click.option(
    "--resampling",
    type=click.Choice([r.name for r in SUPPORTED_RESAMPLING]),
    default="nearest",
    help="Resampling method.",
    show_default=True,
)
@click.option(
    "--src-nodata",
    default=None,
    show_default=True,
    type=float,
    help="Manually override source nodata",
)
@click.option(
    "--dst-nodata",
    default=None,
    show_default=True,
    type=float,
    help="Manually override destination nodata",
)
@click.option("--threads", type=int, default=1, help="Number of processing threads.")
@click.option(
    "--check-invert-proj/--no-check-invert-proj",
    default=True,
    help="Constrain output to valid coordinate region in dst-crs",
)
@click.option(
    "--target-aligned-pixels/--no-target-aligned-pixels",
    default=False,
    help="align the output bounds based on the resolution",
)
@options.overwrite_opt
@options.creation_options
@click.option(
    "--to",
    "--wo",
    "--transformer-option",
    "--warper-option",
    "warper_options",
    metavar="NAME=VALUE",
    multiple=True,
    callback=_cb_key_val,
    help="GDAL warper and coordinate transformer options.",
)
@click.pass_context
def warp(
    ctx,
    files,
    output,
    driver,
    like,
    dst_crs,
    dimensions,
    src_bounds,
    dst_bounds,
    res,
    resampling,
    src_nodata,
    dst_nodata,
    threads,
    check_invert_proj,
    overwrite,
    creation_options,
    target_aligned_pixels,
    warper_options,
):
    """
    Warp a raster dataset.

    If a template raster is provided using the --like option, the
    coordinate reference system, affine transform, and dimensions of
    that raster will be used for the output.  In this case --dst-crs,
    --bounds, --res, and --dimensions options are not applicable and
    an exception will be raised.

    \b
        $ rio warp input.tif output.tif --like template.tif

    The output coordinate reference system may be either a PROJ.4 or
    EPSG:nnnn string,

    \b
        --dst-crs EPSG:4326
        --dst-crs '+proj=longlat +ellps=WGS84 +datum=WGS84'

    or a JSON text-encoded PROJ.4 object.

    \b
        --dst-crs '{"proj": "utm", "zone": 18, ...}'

    If --dimensions are provided, --res and --bounds are not applicable and an
    exception will be raised.
    Resolution is calculated based on the relationship between the
    raster bounds in the target coordinate system and the dimensions,
    and may produce rectangular rather than square pixels.

    \b
        $ rio warp input.tif output.tif --dimensions 100 200 \\
        > --dst-crs EPSG:4326

    If --bounds are provided, --res is required if --dst-crs is provided
    (defaults to source raster resolution otherwise).

    \b
        $ rio warp input.tif output.tif \\
        > --bounds -78 22 -76 24 --res 0.1 --dst-crs EPSG:4326

    """
    output, files = resolve_inout(
        files=files, output=output, overwrite=overwrite)

    resampling = Resampling[resampling]  # get integer code for method

    if not len(res):
        # Click sets this as an empty tuple if not provided
        res = None
    else:
        # Expand one value to two if needed
        res = (res[0], res[0]) if len(res) == 1 else res

    if target_aligned_pixels:
        if not res:
            raise click.BadParameter(
                '--target-aligned-pixels requires a specified resolution')

    # Check invalid parameter combinations
    if like:
        invalid_combos = (dimensions, dst_bounds, dst_crs, res)
        if any(p for p in invalid_combos if p is not None):
            raise click.BadParameter(
                "--like cannot be used with any of --dimensions, --bounds, "
                "--dst-crs, or --res")

    elif dimensions:
        invalid_combos = (dst_bounds, res)
        if any(p for p in invalid_combos if p is not None):
            raise click.BadParameter(
                "--dimensions cannot be used with --bounds or --res")

    with ctx.obj['env']:
        setenv(CHECK_WITH_INVERT_PROJ=check_invert_proj)

        with rasterio.open(files[0]) as src:
            left, bottom, right, top = src.bounds

            out_kwargs = src.profile
            out_kwargs.pop("driver", None)
            if driver:
                out_kwargs["driver"] = driver

            # Sort out the bounds options.
            if src_bounds and dst_bounds:
                raise click.BadParameter(
                    "--src-bounds and destination --bounds may not be "
                    "specified simultaneously.")

            if like:
                with rasterio.open(like) as template_ds:
                    dst_crs = template_ds.crs
                    dst_transform = template_ds.transform
                    dst_height = template_ds.height
                    dst_width = template_ds.width

            elif dst_crs is not None:
                try:
                    dst_crs = CRS.from_string(dst_crs)
                except ValueError as err:
                    raise click.BadParameter(
                        str(err), param='dst_crs', param_hint='dst_crs')

                if dimensions:
                    # Calculate resolution appropriate for dimensions
                    # in target.
                    dst_width, dst_height = dimensions
                    bounds = src_bounds or src.bounds
                    try:
                        xmin, ymin, xmax, ymax = transform_bounds(
                            src.crs, dst_crs, *bounds)
                    except CRSError as err:
                        raise click.BadParameter(
                            str(err), param='dst_crs', param_hint='dst_crs')
                    dst_transform = Affine(
                        (xmax - xmin) / float(dst_width),
                        0, xmin, 0,
                        (ymin - ymax) / float(dst_height),
                        ymax
                    )

                elif src_bounds or dst_bounds:
                    if not res:
                        raise click.BadParameter(
                            "Required when using --bounds.",
                            param='res', param_hint='res')

                    if src_bounds:
                        try:
                            xmin, ymin, xmax, ymax = transform_bounds(
                                src.crs, dst_crs, *src_bounds)
                        except CRSError as err:
                            raise click.BadParameter(
                                str(err), param='dst_crs',
                                param_hint='dst_crs')
                    else:
                        xmin, ymin, xmax, ymax = dst_bounds

                    dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
                    dst_width = max(int(ceil((xmax - xmin) / res[0])), 1)
                    dst_height = max(int(ceil((ymax - ymin) / res[1])), 1)

                else:
                    try:
                        if src.transform.is_identity and src.gcps:
                            src_crs = src.gcps[1]
                            kwargs = {'gcps': src.gcps[0]}
                        else:
                            src_crs = src.crs
                            kwargs = src.bounds._asdict()
                        dst_transform, dst_width, dst_height = calcdt(
                            src_crs,
                            dst_crs,
                            src.width,
                            src.height,
                            resolution=res,
                            **kwargs,
                            **warper_options
                        )
                    except CRSError as err:
                        raise click.BadParameter(
                            str(err), param='dst_crs', param_hint='dst_crs')

            elif dimensions:
                # Same projection, different dimensions, calculate resolution.
                dst_crs = src.crs
                dst_width, dst_height = dimensions
                l, b, r, t = src_bounds or (left, bottom, right, top)
                dst_transform = Affine(
                    (r - l) / float(dst_width),
                    0, l, 0,
                    (b - t) / float(dst_height),
                    t
                )

            elif src_bounds or dst_bounds:
                # Same projection, different dimensions and possibly
                # different resolution.
                if not res:
                    res = (src.transform.a, -src.transform.e)

                dst_crs = src.crs
                xmin, ymin, xmax, ymax = (src_bounds or dst_bounds)
                dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
                dst_width = max(int(round((xmax - xmin) / res[0])), 1)
                dst_height = max(int(round((ymax - ymin) / res[1])), 1)

            elif res:
                # Same projection, different resolution.
                dst_crs = src.crs
                dst_transform = Affine(res[0], 0, left, 0, -res[1], top)
                dst_width = max(int(round((right - left) / res[0])), 1)
                dst_height = max(int(round((top - bottom) / res[1])), 1)

            else:
                dst_crs = src.crs
                inv_transform = ~src.transform
                eps = sys.float_info.epsilon
                c1, r1 = inv_transform * (left + eps, top + eps)
                c2, r2 = inv_transform * (right + eps, top + eps)
                c3, r3 = inv_transform * (right + eps, bottom + eps)
                c4, r4 = inv_transform * (left + eps, bottom + eps)
                col1 = min(c1, c2, c3, c4)
                col2 = max(c1, c2, c3, c4)
                row1 = min(r1, r2, r3, r4)
                row2 = max(r1, r2, r3, r4)
                col1 = floor(col1)
                col2 = ceil(col2)
                row1 = floor(row1)
                row2 = ceil(row2)
                px = (right - left) / (col2 - col1)
                py = (top - bottom) / (row2 - row1)
                res = max(px, py)
                dst_width = max(int(round((right - left) / res)), 1)
                dst_height = max(int(round((top - bottom) / res)), 1)
                dst_transform = Affine.translation(left, top) * Affine.scale(res, -res)

            if target_aligned_pixels:
                dst_transform, dst_width, dst_height = aligned_target(
                    dst_transform, dst_width, dst_height, res
                )

            # If src_nodata is not None, update the dst metadata NODATA
            # value to src_nodata (will be overridden by dst_nodata if it is not None
            if src_nodata is not None:
                # Update the dst nodata value
                out_kwargs.update(nodata=src_nodata)

            # Validate a manually set destination NODATA value
            # against the input datatype.
            if dst_nodata is not None:
                if src_nodata is None and src.meta['nodata'] is None:
                    raise click.BadParameter(
                        "--src-nodata must be provided because dst-nodata is not None")
                else:
                    # Update the dst nodata value
                    out_kwargs.update(nodata=dst_nodata)

            # When the bounds option is misused, extreme values of
            # destination width and height may result.
            if (dst_width < 0 or dst_height < 0 or
                    dst_width > MAX_OUTPUT_WIDTH or
                    dst_height > MAX_OUTPUT_HEIGHT):
                raise click.BadParameter(
                    "Invalid output dimensions: {0}.".format(
                        (dst_width, dst_height)))

            out_kwargs.update(
                crs=dst_crs,
                transform=dst_transform,
                width=dst_width,
                height=dst_height
            )

            # Adjust block size if necessary.
            if "blockxsize" in out_kwargs and dst_width < int(out_kwargs["blockxsize"]):
                del out_kwargs["blockxsize"]
                logger.warning(
                    "Blockxsize removed from creation options to accomodate small output width"
                )
            if "blockysize" in out_kwargs and dst_height < int(
                out_kwargs["blockysize"]
            ):
                del out_kwargs["blockysize"]
                logger.warning(
                    "Blockxsize removed from creation options to accomodate small output height"
                )

            out_kwargs.update(**creation_options)

            with rasterio.open(output, 'w', **out_kwargs) as dst:
                reproject(
                    source=rasterio.band(src, list(range(1, src.count + 1))),
                    destination=rasterio.band(
                        dst, list(range(1, src.count + 1))),
                    src_transform=src.transform,
                    src_crs=src.crs,
                    src_nodata=src_nodata,
                    dst_transform=out_kwargs['transform'],
                    dst_crs=out_kwargs['crs'],
                    dst_nodata=dst_nodata,
                    resampling=resampling,
                    num_threads=threads,
                    **warper_options
                )

env_pyrecodes/lib/python3.9/site-packages/rasterio/rio/warp.py
NikolaBlagojevic/pyrecodes
import attr
from collections import OrderedDict
from collections.abc import Iterable

@attr.s(slots=True)
class RPC:
    """Rational Polynomial Coefficients used to map (x, y, z) <-> (row, col) coordinates.
    
    This class contains a mapping between various RPC attributes and values. 

    Attributes
    ----------
    err_bias, err_rand: float, optional
        The RMS bias and random error in meters per horizontal axis of all points in image.
    lat_off, long_off, height_off: float
        Geodetic latitude, longitude, and height offset.
    lat_scale, long_scale, height_scale: float
        Geodetic latitude, longitude, and height scaling.
    line_off, samp_off: float
        Line (row) and sample (column) offset.
    line_scale, samp_scale: float
        Line (row) and sample (column) offset.
    line_num_coeff, line_den_coeff, samp_num_coeff, samp_den_coeff: list
        The twenty coefficients describing a numerator or denominator polynomial corresponding to line (row) or sample (col).
    """

    height_off = attr.ib()
    height_scale = attr.ib()
    lat_off = attr.ib()
    lat_scale = attr.ib()
    line_den_coeff = attr.ib()
    line_num_coeff = attr.ib()
    line_off = attr.ib()
    line_scale = attr.ib()
    long_off = attr.ib()
    long_scale = attr.ib()
    samp_den_coeff = attr.ib()
    samp_num_coeff = attr.ib()
    samp_off = attr.ib()
    samp_scale = attr.ib()
    err_bias = attr.ib(default=None)
    err_rand = attr.ib(default=None)

    def to_dict(self):
        """Return a dictionary representation of RPC"""
        return attr.asdict(self)

    def to_gdal(self):
        """Serialize RPC attribute name and values in a form expected by GDAL.
        
        Returns
        -------
        dict

        Notes
        -----
        The `err_bias` and `err_rand` are optional, and are not written to datasets by GDAL.
        """

        out = {
            "HEIGHT_OFF": str(self.height_off),
            "HEIGHT_SCALE": str(self.height_scale),
            "LAT_OFF": str(self.lat_off),
            "LAT_SCALE": str(self.lat_scale),
            "LINE_DEN_COEFF": " ".join(map(str, self.line_den_coeff)),
            "LINE_NUM_COEFF": " ".join(map(str, self.line_num_coeff)),
            "LINE_OFF": str(self.line_off),
            "LINE_SCALE": str(self.line_scale),
            "LONG_OFF": str(self.long_off),
            "LONG_SCALE": str(self.long_scale),
            "SAMP_DEN_COEFF": " ".join(map(str, self.samp_den_coeff)),
            "SAMP_NUM_COEFF": " ".join(map(str, self.samp_num_coeff)),
            "SAMP_OFF": str(self.samp_off),
            "SAMP_SCALE": str(self.samp_scale)
        }
        
        if self.err_bias:
            out.update(ERR_BIAS=str(self.err_bias))
        if self.err_rand:
            out.update(ERR_RAND=str(self.err_rand))
        
        return out

    @classmethod
    def from_gdal(cls, rpcs):
        """Deserialize dict values to float or list.


        Returns
        -------
        RPC
        """
        out = {}

        for key, val in rpcs.items():
            # Four items have 20 floats in their values.
            if key in {"LINE_NUM_COEFF", "LINE_DEN_COEFF", "SAMP_NUM_COEFF", "SAMP_DEN_COEFF"}:
                out[key] = [float(v) for v in val.split(maxsplit=20)[:20]]
            # All other items have one float in their values but might also contain non-conforming extra text.
            else:
                out[key] = float(val.split(maxsplit=1)[0])

        return cls(
            err_bias=out.get("ERR_BIAS"),
            err_rand=out.get("ERR_RAND"),
            height_off=out["HEIGHT_OFF"],
            height_scale=out["HEIGHT_SCALE"],
            lat_off=out["LAT_OFF"],
            lat_scale=out["LAT_SCALE"],
            line_den_coeff=out["LINE_DEN_COEFF"],
            line_num_coeff=out["LINE_NUM_COEFF"],
            line_off=out["LINE_OFF"],
            line_scale=out["LINE_SCALE"],
            long_off=out["LONG_OFF"],
            long_scale=out["LONG_SCALE"],
            samp_den_coeff=out["SAMP_DEN_COEFF"],
            samp_num_coeff=out["SAMP_NUM_COEFF"],
            samp_off=out["SAMP_OFF"],
            samp_scale=out["SAMP_SCALE"],
        )
env_pyrecodes/lib/python3.9/site-packages/rasterio/rpc.py
NikolaBlagojevic/pyrecodes
# Workaround for issue #378. A pure Python generator.

from itertools import zip_longest

import numpy as np

from rasterio.enums import MaskFlags
from rasterio.windows import Window
from rasterio.transform import rowcol


def _grouper(iterable, n, fillvalue=None):
    "Collect data into non-overlapping fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    # from itertools recipes
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)


def sample_gen(dataset, xy, indexes=None, masked=False):
    """Sample pixels from a dataset

    Parameters
    ----------
    dataset : rasterio Dataset
        Opened in "r" mode.
    xy : iterable
        Pairs of x, y coordinates in the dataset's reference system.
    indexes : int or list of int
        Indexes of dataset bands to sample.
    masked : bool, default: False
        Whether to mask samples that fall outside the extent of the
        dataset.

    Yields
    ------
    array
        A array of length equal to the number of specified indexes
        containing the dataset values for the bands corresponding to
        those indexes.

    """
    dt = dataset.transform
    read = dataset.read
    height = dataset.height
    width = dataset.width

    if indexes is None:
        indexes = dataset.indexes
    elif isinstance(indexes, int):
        indexes = [indexes]

    nodata = np.full(len(indexes), (dataset.nodata or 0),  dtype=dataset.dtypes[0])
    if masked:
        # Masks for masked arrays are inverted (False means valid)
        mask = [MaskFlags.all_valid not in dataset.mask_flag_enums[i-1] for i in indexes]
        nodata = np.ma.array(nodata, mask=mask)

    for pts in _grouper(xy, 256):
        pts = zip(*filter(lambda x: x is not None, pts))

        for row_off, col_off in zip(*rowcol(dt, *pts)):
            if row_off < 0 or col_off < 0 or row_off >= height or col_off >= width:
                yield nodata
            else:
                window = Window(col_off, row_off, 1, 1)
                data = read(indexes, window=window, masked=masked)
                yield data[:, 0, 0]

env_pyrecodes/lib/python3.9/site-packages/rasterio/sample.py
NikolaBlagojevic/pyrecodes
"""Abstraction for sessions in various clouds."""

import logging
import os
from types import SimpleNamespace

from rasterio._path import _parse_path, _UnparsedPath


log = logging.getLogger(__name__)

try:
    import boto3
except ImportError:
    log.debug("Could not import boto3, continuing with reduced functionality.")
    boto3 = None


class Session:
    """Base for classes that configure access to secured resources.

    Attributes
    ----------
    credentials : dict
        Keys and values for session credentials.

    Notes
    -----
    This class is not intended to be instantiated.

    """

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials

        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.

        Returns
        -------
        bool

        """
        return NotImplemented

    def get_credential_options(self):
        """Get credentials as GDAL configuration options

        Returns
        -------
        dict

        """
        return NotImplemented

    @staticmethod
    def from_foreign_session(session, cls=None):
        """Create a session object matching the foreign `session`.

        Parameters
        ----------
        session : obj
            A foreign session object.
        cls : Session class, optional
            The class to return.

        Returns
        -------
        Session

        """
        if not cls:
            return DummySession()
        else:
            return cls(session)

    @staticmethod
    def cls_from_path(path):
        """Find the session class suited to the data at `path`.

        Parameters
        ----------
        path : str
            A dataset path or identifier.

        Returns
        -------
        class

        """
        if not path:
            return DummySession

        path = _parse_path(path)

        if isinstance(path, _UnparsedPath) or path.is_local:
            return DummySession

        elif (
            path.scheme == "s3" or "amazonaws.com" in path.path
        ) and not "X-Amz-Signature" in path.path:
            if boto3 is not None:
                return AWSSession
            else:
                log.info("boto3 not available, falling back to a DummySession.")
                return DummySession

        elif path.scheme == "oss" or "aliyuncs.com" in path.path:
            return OSSSession

        elif path.path.startswith("/vsiswift/"):
            return SwiftSession

        elif path.scheme == "az":
            return AzureSession

        # This factory can be extended to other cloud providers here.
        # elif path.scheme == "cumulonimbus":  # for example.
        #     return CumulonimbusSession(*args, **kwargs)

        else:
            return DummySession

    @staticmethod
    def from_path(path, *args, **kwargs):
        """Create a session object suited to the data at `path`.

        Parameters
        ----------
        path : str
            A dataset path or identifier.
        args : sequence
            Positional arguments for the foreign session constructor.
        kwargs : dict
            Keyword arguments for the foreign session constructor.

        Returns
        -------
        Session

        """
        return Session.cls_from_path(path)(*args, **kwargs)

    @staticmethod
    def aws_or_dummy(*args, **kwargs):
        """Create an AWSSession if boto3 is available, else DummySession

        Parameters
        ----------
        path : str
            A dataset path or identifier.
        args : sequence
            Positional arguments for the foreign session constructor.
        kwargs : dict
            Keyword arguments for the foreign session constructor.

        Returns
        -------
        Session

        """
        if boto3 is not None:
            return AWSSession(*args, **kwargs)
        else:
            return DummySession(*args, **kwargs)

    @staticmethod
    def from_environ(*args, **kwargs):
        """Create a session object suited to the environment.

        Parameters
        ----------
        path : str
            A dataset path or identifier.
        args : sequence
            Positional arguments for the foreign session constructor.
        kwargs : dict
            Keyword arguments for the foreign session constructor.

        Returns
        -------
        Session

        """
        try:
            session = Session.aws_or_dummy(*args, **kwargs)
            session.credentials
        except RuntimeError as exc:
            log.warning("Credentials in environment have expired. Creating a DummySession.")
            session = DummySession(*args, **kwargs)
        return session


class DummySession(Session):
    """A dummy session.

    Attributes
    ----------
    credentials : dict
        The session credentials.

    """

    def __init__(self, *args, **kwargs):
        self._session = None
        self.credentials = {}

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials

        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.

        Returns
        -------
        bool

        """
        return True

    def get_credential_options(self):
        """Get credentials as GDAL configuration options

        Returns
        -------
        dict

        """
        return {}


class AWSSession(Session):
    """Configures access to secured resources stored in AWS S3.
    """

    def __init__(
            self, session=None, aws_unsigned=None, aws_access_key_id=None,
            aws_secret_access_key=None, aws_session_token=None,
            region_name=None, profile_name=None, endpoint_url=None,
            requester_pays=False):
        """Create a new AWS session

        Parameters
        ----------
        session : optional
            A boto3 session object.
        aws_unsigned : bool, optional (default: False)
            If True, requests will be unsigned.
        aws_access_key_id : str, optional
            An access key id, as per boto3.
        aws_secret_access_key : str, optional
            A secret access key, as per boto3.
        aws_session_token : str, optional
            A session token, as per boto3.
        region_name : str, optional
            A region name, as per boto3.
        profile_name : str, optional
            A shared credentials profile name, as per boto3.
        endpoint_url: str, optional
            An endpoint_url, as per GDAL's AWS_S3_ENPOINT
        requester_pays : bool, optional
            True if the requester agrees to pay transfer costs (default:
            False)
        """
        if aws_unsigned is None:
            aws_unsigned = parse_bool(os.getenv("AWS_NO_SIGN_REQUEST", False))

        if session:
            self._session = session
        elif aws_unsigned:
            self._session = SimpleNamespace(region_name=region_name)
        else:
            self._session = boto3.Session(
                aws_access_key_id=aws_access_key_id,
                aws_secret_access_key=aws_secret_access_key,
                aws_session_token=aws_session_token,
                region_name=region_name,
                profile_name=profile_name)

        self.requester_pays = requester_pays
        self.unsigned = aws_unsigned
        self.endpoint_url = endpoint_url
        self._creds = (
            self._session.get_credentials()
            if not self.unsigned
            else None
        )

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials

        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.

        Returns
        -------
        bool

        """
        return ('AWS_ACCESS_KEY_ID' in config and 'AWS_SECRET_ACCESS_KEY' in config) or 'AWS_NO_SIGN_REQUEST' in config

    @property
    def credentials(self):
        """The session credentials as a dict"""
        res = {}
        if self._creds:  # pragma: no branch
            frozen_creds = self._creds.get_frozen_credentials()
            if frozen_creds.access_key:  # pragma: no branch
                res['aws_access_key_id'] = frozen_creds.access_key
            if frozen_creds.secret_key:  # pragma: no branch
                res['aws_secret_access_key'] = frozen_creds.secret_key
            if frozen_creds.token:
                res['aws_session_token'] = frozen_creds.token
        if self._session.region_name:
            res['aws_region'] = self._session.region_name
        if self.requester_pays:
            res['aws_request_payer'] = 'requester'
        if self.endpoint_url:
            res['aws_s3_endpoint'] = self.endpoint_url
        return res

    def get_credential_options(self):
        """Get credentials as GDAL configuration options

        Returns
        -------
        dict

        """
        if self.unsigned:
            opts = {'AWS_NO_SIGN_REQUEST': 'YES'}
            opts.update({k.upper(): v for k, v in self.credentials.items()
                        if k in ('aws_region', 'aws_s3_endpoint')})
            return opts
        else:
            return {k.upper(): v for k, v in self.credentials.items()}


class OSSSession(Session):
    """Configures access to secured resources stored in Alibaba Cloud OSS.
    """
    def __init__(self, oss_access_key_id=None, oss_secret_access_key=None, oss_endpoint=None):
        """Create new Alibaba Cloud OSS session

        Parameters
        ----------
        oss_access_key_id: string, optional (default: None)
            An access key id
        oss_secret_access_key: string, optional (default: None)
            An secret access key
        oss_endpoint: string, optional (default: None)
            the region attached to the bucket
        """

        self._creds = {
            "oss_access_key_id": oss_access_key_id,
            "oss_secret_access_key": oss_secret_access_key,
            "oss_endpoint": oss_endpoint
        }

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials

        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.

        Returns
        -------
        bool

        """
        return 'OSS_ACCESS_KEY_ID' in config and 'OSS_SECRET_ACCESS_KEY' in config

    @property
    def credentials(self):
        """The session credentials as a dict"""
        return self._creds

    def get_credential_options(self):
        """Get credentials as GDAL configuration options

        Returns
        -------
        dict

        """
        return {k.upper(): v for k, v in self.credentials.items()}


class GSSession(Session):
    """Configures access to secured resources stored in Google Cloud Storage
    """
    def __init__(self, google_application_credentials=None):
        """Create new Google Cloude Storage session

        Parameters
        ----------
        google_application_credentials: string
            Path to the google application credentials JSON file.
        """

        if google_application_credentials is not None:
            self._creds = {'google_application_credentials': google_application_credentials}
        else:
            self._creds = {}

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials

        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.

        Returns
        -------
        bool

        """
        return 'GOOGLE_APPLICATION_CREDENTIALS' in config

    @property
    def credentials(self):
        """The session credentials as a dict"""
        return self._creds

    def get_credential_options(self):
        """Get credentials as GDAL configuration options

        Returns
        -------
        dict

        """
        return {k.upper(): v for k, v in self.credentials.items()}


class SwiftSession(Session):
    """Configures access to secured resources stored in OpenStack Swift Object Storage.
    """
    def __init__(self, session=None,
                swift_storage_url=None, swift_auth_token=None,
                swift_auth_v1_url=None, swift_user=None, swift_key=None):
        """Create new OpenStack Swift Object Storage Session.

        Three methods are possible:
            1. Create session by the swiftclient library.
            2. The SWIFT_STORAGE_URL and SWIFT_AUTH_TOKEN (this method is recommended by GDAL docs).
            3. The SWIFT_AUTH_V1_URL, SWIFT_USER and SWIFT_KEY (This depends on the swiftclient library).

        Parameters
        ----------
        session: optional
            A swiftclient connection object
        swift_storage_url:
            the storage URL
        swift_auth_token:
            the value of the x-auth-token authorization token
        swift_storage_url: string, optional
            authentication URL
        swift_user: string, optional
            user name to authenticate as
        swift_key: string, optional
            key/password to authenticate with

        Examples
        --------
        >>> import rasterio
        >>> from rasterio.session import SwiftSession
        >>> fp = '/vsiswift/bucket/key.tif'
        >>> conn = Connection(authurl='http://127.0.0.1:7777/auth/v1.0', user='test:tester', key='testing')
        >>> session = SwiftSession(conn)
        >>> with rasterio.Env(session):
        >>>     with rasterio.open(fp) as src:
        >>>         print(src.profile)

        """
        if swift_storage_url and swift_auth_token:
            self._creds = {
                "swift_storage_url": swift_storage_url,
                "swift_auth_token": swift_auth_token
            }
        else:
            from swiftclient.client import Connection

            if session:
                self._session = session
            else:
                self._session = Connection(
                    authurl=swift_auth_v1_url,
                    user=swift_user,
                    key=swift_key
                )
            self._creds = {
                "swift_storage_url": self._session.get_auth()[0],
                "swift_auth_token": self._session.get_auth()[1]
            }

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials
        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.
        Returns
        -------
        bool
        """
        return 'SWIFT_STORAGE_URL' in config and 'SWIFT_AUTH_TOKEN' in config

    @property
    def credentials(self):
        """The session credentials as a dict"""
        return self._creds

    def get_credential_options(self):
        """Get credentials as GDAL configuration options
        Returns
        -------
        dict
        """
        return {k.upper(): v for k, v in self.credentials.items()}


class AzureSession(Session):
    """Configures access to secured resources stored in Microsoft Azure Blob Storage.
    """
    def __init__(self, azure_storage_connection_string=None,
                 azure_storage_account=None, azure_storage_access_key=None,
                 azure_unsigned=False):
        """Create new Microsoft Azure Blob Storage session

        Parameters
        ----------
        azure_storage_connection_string: string
            A connection string contains both an account name and a secret key.
        azure_storage_account: string
            An account name
        azure_storage_access_key: string
            A secret key
        azure_unsigned : bool, optional (default: False)
            If True, requests will be unsigned.
        """

        self.unsigned = parse_bool(os.getenv("AZURE_NO_SIGN_REQUEST", azure_unsigned))
        self.storage_account = azure_storage_account or os.getenv("AZURE_STORAGE_ACCOUNT")
        self.storage_access_key = azure_storage_access_key or os.getenv("AZURE_STORAGE_ACCESS_KEY")

        if azure_storage_connection_string:
            self._creds = {
                "azure_storage_connection_string": azure_storage_connection_string
            }
        elif not self.unsigned:
            self._creds = {
                "azure_storage_account": self.storage_account,
                "azure_storage_access_key": self.storage_access_key
            }
        else:
            self._creds = {
                "azure_storage_account": self.storage_account
            }

    @classmethod
    def hascreds(cls, config):
        """Determine if the given configuration has proper credentials

        Parameters
        ----------
        cls : class
            A Session class.
        config : dict
            GDAL configuration as a dict.

        Returns
        -------
        bool

        """
        return (
            'AZURE_STORAGE_CONNECTION_STRING' in config
            or ('AZURE_STORAGE_ACCOUNT' in config and 'AZURE_STORAGE_ACCESS_KEY' in config)
            or ('AZURE_STORAGE_ACCOUNT' in config and 'AZURE_NO_SIGN_REQUEST' in config)
        )

    @property
    def credentials(self):
        """The session credentials as a dict"""
        return self._creds

    def get_credential_options(self):
        """Get credentials as GDAL configuration options

        Returns
        -------
        dict

        """
        if self.unsigned:
            return {
                'AZURE_NO_SIGN_REQUEST': 'YES',
                'AZURE_STORAGE_ACCOUNT': self.storage_account
            }
        else:
            return {k.upper(): v for k, v in self.credentials.items()}

def parse_bool(v):
    """CPLTestBool equivalent"""
    if isinstance(v, bool):
        return v
    if isinstance(v, str):
        return not(v.lower() in ("no", "false", "off", "0"))
    return bool(v)
env_pyrecodes/lib/python3.9/site-packages/rasterio/session.py
NikolaBlagojevic/pyrecodes
"""Rasterio tools module

See this RFC about Rasterio tools:
https://github.com/rasterio/rasterio/issues/1300.
"""

import json

import rasterio
from rasterio.features import dataset_features


class JSONSequenceTool:
    """Extracts data from a dataset file and saves a JSON sequence
    """

    def __init__(self, func):
        """Initialize tool

        Parameters
        ----------
        func : callable
            A function or other callable object that takes a dataset and
            yields JSON serializable objects.
        """
        self.func = func

    def __call__(self, src_path, dst_path, src_kwargs=None, dst_kwargs=None, func_args=None, func_kwargs=None, config=None):
        """Execute the tool's primary function and perform file I/O

        Parameters
        ----------
        src_path : str or PathLike object
            A dataset path or URL. Will be opened in "r" mode using
            src_opts.
        dst_path : str or Path-like object
            A path or or PathLike object. Will be opened in "w" mode.
        src_kwargs : dict
            Options that will be passed to rasterio.open when opening
            src.
        dst_kwargs : dict
            Options that will be passed to json.dumps when serializing
            output.
        func_args : sequence
            Extra positional arguments for the tool's primary function.
        func_kwargs : dict
            Extra keyword arguments for the tool's primary function.
        config : dict
            Rasterio Env options.

        Returns
        -------
        None

        Side effects
        ------------
        Writes sequences of JSON texts to the named output file.
        """

        src_kwargs = src_kwargs or {}
        dst_kwargs = dst_kwargs or {}
        func_args = func_args or []
        func_kwargs = func_kwargs or {}
        config = config or {}

        with rasterio.Env(**config):
            with open(dst_path, 'w') as fdst, rasterio.open(src_path, **src_kwargs) as dsrc:
                for obj in self.func(dsrc, *func_args, **func_kwargs):
                    fdst.write(json.dumps(obj, **dst_kwargs))


dataset_features_tool = JSONSequenceTool(dataset_features)

env_pyrecodes/lib/python3.9/site-packages/rasterio/tools.py
NikolaBlagojevic/pyrecodes
"""Geospatial transforms"""

from contextlib import ExitStack
from functools import partial
import math
import numpy as np
import warnings

from affine import Affine

from rasterio.env import env_ctx_if_needed
from rasterio._transform import (
    _transform_from_gcps,
    RPCTransformerBase,
    GCPTransformerBase,
)
from rasterio.enums import TransformDirection, TransformMethod
from rasterio.control import GroundControlPoint
from rasterio.rpc import RPC
from rasterio.errors import TransformError, RasterioDeprecationWarning

IDENTITY = Affine.identity()
GDAL_IDENTITY = IDENTITY.to_gdal()


class TransformMethodsMixin:
    """Mixin providing methods for calculations related
    to transforming between rows and columns of the raster
    array and the coordinates.

    These methods are wrappers for the functionality in
    `rasterio.transform` module.

    A subclass with this mixin MUST provide a `transform`
    property.

    """

    def xy(
        self,
        row,
        col,
        z=None,
        offset="center",
        transform_method=TransformMethod.affine,
        **rpc_options
    ):
        """Get the coordinates x, y of a pixel at row, col.

        The pixel's center is returned by default, but a corner can be returned
        by setting `offset` to one of `ul, ur, ll, lr`.

        Parameters
        ----------
        row : int
            Pixel row.
        col : int
            Pixel column.
        z : float, optional
            Height associated with coordinates. Primarily used for RPC based
            coordinate transformations. Ignored for affine based
            transformations. Default: 0.
        offset : str, optional
            Determines if the returned coordinates are for the center of the
            pixel or for a corner.
        transform_method: TransformMethod, optional
            The coordinate transformation method. Default: `TransformMethod.affine`.
        rpc_options: dict, optional
            Additional arguments passed to GDALCreateRPCTransformer

        Returns
        -------
        tuple
            x, y

        """
        transform = getattr(self, transform_method.value)
        if transform_method is TransformMethod.gcps:
            transform = transform[0]
        if not transform:
            raise AttributeError("Dataset has no {}".format(transform_method))
        return xy(transform, row, col, zs=z, offset=offset, **rpc_options)

    def index(
        self,
        x,
        y,
        z=None,
        op=math.floor,
        precision=None,
        transform_method=TransformMethod.affine,
        **rpc_options
    ):
        """Get the (row, col) index of the pixel containing (x, y).

        Parameters
        ----------
        x : float
            x value in coordinate reference system
        y : float
            y value in coordinate reference system
        z : float, optional
            Height associated with coordinates. Primarily used for RPC based
            coordinate transformations. Ignored for affine based
            transformations. Default: 0.
        op : function, optional (default: math.floor)
            Function to convert fractional pixels to whole numbers (floor,
            ceiling, round)
        transform_method: TransformMethod, optional
            The coordinate transformation method. Default: `TransformMethod.affine`.
        rpc_options: dict, optional
            Additional arguments passed to GDALCreateRPCTransformer
        precision : int, optional
            This parameter is unused, deprecated in rasterio 1.3.0, and
            will be removed in version 2.0.0.

        Returns
        -------
        tuple
            (row index, col index)

        """
        if precision is not None:
            warnings.warn(
                "The precision parameter is unused, deprecated, and will be removed in 2.0.0.",
                RasterioDeprecationWarning,
            )

        transform = getattr(self, transform_method.value)
        if transform_method is TransformMethod.gcps:
            transform = transform[0]
        if not transform:
            raise AttributeError("Dataset has no {}".format(transform_method))
        return rowcol(transform, x, y, zs=z, op=op, **rpc_options)


def get_transformer(transform, **rpc_options):
    """Return the appropriate transformer class"""
    if transform is None:
        raise ValueError("Invalid transform")
    if isinstance(transform, Affine):
        transformer_cls = partial(AffineTransformer, transform)
    elif isinstance(transform, RPC):
        transformer_cls = partial(RPCTransformer, transform, **rpc_options)
    else:
        transformer_cls = partial(GCPTransformer, transform)
    return transformer_cls


def tastes_like_gdal(seq):
    """Return True if `seq` matches the GDAL geotransform pattern."""
    return tuple(seq) == GDAL_IDENTITY or (
        seq[2] == seq[4] == 0.0 and seq[1] > 0 and seq[5] < 0)


def guard_transform(transform):
    """Return an Affine transformation instance."""
    if not isinstance(transform, Affine):
        if tastes_like_gdal(transform):
            raise TypeError(
                "GDAL-style transforms have been deprecated.  This "
                "exception will be raised for a period of time to highlight "
                "potentially confusing errors, but will eventually be removed.")
        else:
            transform = Affine(*transform)
    return transform


def from_origin(west, north, xsize, ysize):
    """Return an Affine transformation given upper left and pixel sizes.

    Return an Affine transformation for a georeferenced raster given
    the coordinates of its upper left corner `west`, `north` and pixel
    sizes `xsize`, `ysize`.

    """
    return Affine.translation(west, north) * Affine.scale(xsize, -ysize)


def from_bounds(west, south, east, north, width, height):
    """Return an Affine transformation given bounds, width and height.

    Return an Affine transformation for a georeferenced raster given
    its bounds `west`, `south`, `east`, `north` and its `width` and
    `height` in number of pixels.

    """
    return Affine.translation(west, north) * Affine.scale(
        (east - west) / width, (south - north) / height)


def array_bounds(height, width, transform):
    """Return the bounds of an array given height, width, and a transform.

    Return the `west, south, east, north` bounds of an array given
    its height, width, and an affine transform.

    """
    w, n = transform.xoff, transform.yoff
    e, s = transform * (width, height)
    return w, s, e, n


def xy(transform, rows, cols, zs=None, offset='center', **rpc_options):
    """Get the x and y coordinates of pixels at `rows` and `cols`.

    The pixel's center is returned by default, but a corner can be returned
    by setting `offset` to one of `ul, ur, ll, lr`.

    Supports affine, Ground Control Point (GCP), or Rational Polynomial
    Coefficients (RPC) based coordinate transformations.

    Parameters
    ----------
    transform : Affine or sequence of GroundControlPoint or RPC
        Transform suitable for input to AffineTransformer, GCPTransformer, or RPCTransformer.
    rows : list or int
        Pixel rows.
    cols : int or sequence of ints
        Pixel columns.
    zs : list or float, optional
        Height associated with coordinates. Primarily used for RPC based
        coordinate transformations. Ignored for affine based
        transformations. Default: 0.
    offset : str, optional
        Determines if the returned coordinates are for the center of the
        pixel or for a corner.
    rpc_options : dict, optional
        Additional arguments passed to GDALCreateRPCTransformer.

    Returns
    -------
    xs : float or list of floats
        x coordinates in coordinate reference system
    ys : float or list of floats
        y coordinates in coordinate reference system

    """
    transformer_cls = get_transformer(transform, **rpc_options)
    with transformer_cls() as transformer:
        return transformer.xy(rows, cols, zs=zs, offset=offset)


def rowcol(transform, xs, ys, zs=None, op=math.floor, precision=None, **rpc_options):
    """Get rows and cols of the pixels containing (x, y).

    Parameters
    ----------
    transform : Affine or sequence of GroundControlPoint or RPC
        Transform suitable for input to AffineTransformer, GCPTransformer, or RPCTransformer.
    xs : list or float
        x values in coordinate reference system.
    ys : list or float
        y values in coordinate reference system.
    zs : list or float, optional
        Height associated with coordinates. Primarily used for RPC based
        coordinate transformations. Ignored for affine based
        transformations. Default: 0.
    op : function
        Function to convert fractional pixels to whole numbers (floor, ceiling,
        round).
    precision : int or float, optional
        This parameter is unused, deprecated in rasterio 1.3.0, and
        will be removed in version 2.0.0.
    rpc_options : dict, optional
        Additional arguments passed to GDALCreateRPCTransformer.

    Returns
    -------
    rows : list of ints
        list of row indices
    cols : list of ints
        list of column indices

    """
    if precision is not None:
        warnings.warn(
            "The precision parameter is unused, deprecated, and will be removed in 2.0.0.",
            RasterioDeprecationWarning,
        )

    transformer_cls = get_transformer(transform, **rpc_options)
    with transformer_cls() as transformer:
        return transformer.rowcol(xs, ys, zs=zs, op=op)


def from_gcps(gcps):
    """Make an Affine transform from ground control points.

    Parameters
    ----------
    gcps : sequence of GroundControlPoint
        Such as the first item of a dataset's `gcps` property.

    Returns
    -------
    Affine

    """
    return Affine.from_gdal(*_transform_from_gcps(gcps))


class TransformerBase:
    """Generic GDAL transformer base class

    Notes
    -----
    Subclasses must have a _transformer attribute and implement a `_transform` method.

    """
    def __init__(self):
        self._transformer = None

    @staticmethod
    def _ensure_arr_input(xs, ys, zs=None):
        """Ensure all input coordinates are mapped to array-like objects

        Raises
        ------
        TransformError
            If input coordinates are not all of the same length
        """
        try:
            xs, ys, zs = np.broadcast_arrays(xs, ys, 0 if zs is None else zs)
        except ValueError as error:
            raise TransformError(
                "Input coordinates must be broadcastable to a 1d array"
            ) from error
        return np.atleast_1d(xs), np.atleast_1d(ys), np.atleast_1d(zs)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        pass

    def rowcol(self, xs, ys, zs=None, op=math.floor, precision=None):
        """Get rows and cols coordinates given geographic coordinates.

        Parameters
        ----------
        xs, ys : float or list of float
            Geographic coordinates
        zs : float or list of float, optional
            Height associated with coordinates. Primarily used for RPC based
            coordinate transformations. Ignored for affine based
            transformations. Default: 0.
        op : function, optional (default: math.floor)
            Function to convert fractional pixels to whole numbers (floor,
            ceiling, round)
        precision : int, optional (default: None)
            This parameter is unused, deprecated in rasterio 1.3.0, and
            will be removed in version 2.0.0.

        Raises
        ------
        ValueError
            If input coordinates are not all equal length

        Returns
        -------
        tuple of float or list of float.

        """
        if precision is not None:
            warnings.warn(
                "The precision parameter is unused, deprecated, and will be removed in 2.0.0.",
                RasterioDeprecationWarning,
            )

        AS_ARR = True if hasattr(xs, "__iter__") else False
        xs, ys, zs = self._ensure_arr_input(xs, ys, zs=zs)

        try:
            new_cols, new_rows = self._transform(
                xs, ys, zs, transform_direction=TransformDirection.reverse
            )

            if len(new_rows) == 1 and not AS_ARR:
                return (op(new_rows[0]), op(new_cols[0]))
            else:
                return ([op(r) for r in new_rows], [op(c) for c in new_cols])
        except TypeError:
            raise TransformError("Invalid inputs")

    def xy(self, rows, cols, zs=None, offset='center'):
        """
        Returns geographic coordinates given dataset rows and cols coordinates

        Parameters
        ----------
        rows, cols : int or list of int
            Image pixel coordinates
        zs : float or list of float, optional
            Height associated with coordinates. Primarily used for RPC based
            coordinate transformations. Ignored for affine based
            transformations. Default: 0.
        offset : str, optional
            Determines if the returned coordinates are for the center of the
            pixel or for a corner.
        Raises
        ------
        ValueError
            If input coordinates are not all equal length

        Returns
        -------
        tuple of float or list of float

        """
        AS_ARR = True if hasattr(rows, "__iter__") else False
        rows, cols, zs = self._ensure_arr_input(rows, cols, zs=zs)

        if offset == 'center':
            coff, roff = (0.5, 0.5)
        elif offset == 'ul':
            coff, roff = (0, 0)
        elif offset == 'ur':
            coff, roff = (1, 0)
        elif offset == 'll':
            coff, roff = (0, 1)
        elif offset == 'lr':
            coff, roff = (1, 1)
        else:
            raise TransformError("Invalid offset")

        # shift input coordinates according to offset
        T = IDENTITY.translation(coff, roff)
        offset_rows = []
        offset_cols = []
        try:
            for colrow in zip(cols, rows):
                offset_col, offset_row = T * colrow
                offset_rows.append(offset_row)
                offset_cols.append(offset_col)

            new_xs, new_ys = self._transform(
                offset_cols, offset_rows, zs, transform_direction=TransformDirection.forward
            )

            if len(new_xs) == 1 and not AS_ARR:
                return (new_xs[0], new_ys[0])
            else:
                return (new_xs, new_ys)
        except TypeError:
            raise TransformError("Invalid inputs")

    def _transform(self, xs, ys, zs, transform_direction):
        raise NotImplementedError


class GDALTransformerBase(TransformerBase):
    def __init__(self):
        super().__init__()
        self._env = ExitStack()

    def close(self):
        pass

    def __enter__(self):
        self._env.enter_context(env_ctx_if_needed())
        return self

    def __exit__(self, *args):
        self.close()
        self._env.close()


class AffineTransformer(TransformerBase):
    """A pure Python class related to affine based coordinate transformations."""
    def __init__(self, affine_transform):
        super().__init__()
        if not isinstance(affine_transform, Affine):
            raise ValueError("Not an affine transform")
        self._transformer = affine_transform

    def _transform(self, xs, ys, zs, transform_direction):
        resxs = []
        resys = []

        if transform_direction is TransformDirection.forward:
            transform = self._transformer
        elif transform_direction is TransformDirection.reverse:
            transform = ~self._transformer

        for x, y in zip(xs, ys):
            resx, resy = transform * (x, y)
            resxs.append(resx)
            resys.append(resy)

        return (resxs, resys)

    def __repr__(self):
        return "<AffineTransformer>"


class RPCTransformer(RPCTransformerBase, GDALTransformerBase):
    """
    Class related to Rational Polynomial Coeffecients (RPCs) based
    coordinate transformations.

    Uses GDALCreateRPCTransformer and GDALRPCTransform for computations. Options
    for GDALCreateRPCTransformer may be passed using `rpc_options`.
    Ensure that GDAL transformer objects are destroyed by calling `close()`
    method or using context manager interface.

    """
    def __init__(self, rpcs, **rpc_options):
        if not isinstance(rpcs, (RPC, dict)):
            raise ValueError("RPCTransformer requires RPC")
        super().__init__(rpcs, **rpc_options)

    def __repr__(self):
        return "<{} RPCTransformer>".format(
            self.closed and 'closed' or 'open')


class GCPTransformer(GCPTransformerBase, GDALTransformerBase):
    """
    Class related to Ground Control Point (GCPs) based
    coordinate transformations.

    Uses GDALCreateGCPTransformer and GDALGCPTransform for computations.
    Ensure that GDAL transformer objects are destroyed by calling `close()`
    method or using context manager interface.

    """
    def __init__(self, gcps):
        if len(gcps) and not isinstance(gcps[0], GroundControlPoint):
            raise ValueError("GCPTransformer requires sequence of GroundControlPoint")
        super().__init__(gcps)

    def __repr__(self):
        return "<{} GCPTransformer>".format(
            self.closed and 'closed' or 'open')

env_pyrecodes/lib/python3.9/site-packages/rasterio/transform.py
NikolaBlagojevic/pyrecodes
"""rasterio.vrt: a module concerned with GDAL VRTs"""

import xml.etree.ElementTree as ET

import rasterio
from rasterio._warp import WarpedVRTReaderBase
from rasterio.dtypes import _gdal_typename
from rasterio.enums import MaskFlags
from rasterio._path import _parse_path
from rasterio.transform import TransformMethodsMixin
from rasterio.windows import WindowMethodsMixin


class WarpedVRT(WarpedVRTReaderBase, WindowMethodsMixin,
                TransformMethodsMixin):
    """A virtual warped dataset.

    Abstracts the details of raster warping and allows access to data
    that is reprojected when read.

    This class is backed by an in-memory GDAL VRTWarpedDataset VRT file.

    Parameters
    ----------
    src_dataset : dataset object
        The warp source.
    src_crs : CRS or str, optional
        Overrides the coordinate reference system of `src_dataset`.
    src_transfrom : Affine, optional
        Overrides the transform of `src_dataset`.
    src_nodata : float, optional
        Overrides the nodata value of `src_dataset`, which is the
        default.
    crs : CRS or str, optional
        The coordinate reference system at the end of the warp
        operation.  Default: the crs of `src_dataset`. dst_crs was
        a deprecated alias for this parameter.
    transform : Affine, optional
        The transform for the virtual dataset. Default: will be
        computed from the attributes of `src_dataset`. dst_transform
        was a deprecated alias for this parameter.
    height, width: int, optional
        The dimensions of the virtual dataset. Defaults: will be
        computed from the attributes of `src_dataset`. dst_height
        and dst_width were deprecated alias for these parameters.
    nodata : float, optional
        Nodata value for the virtual dataset. Default: the nodata
        value of `src_dataset` or 0.0. dst_nodata was a deprecated
        alias for this parameter.
    resampling : Resampling, optional
        Warp resampling algorithm. Default: `Resampling.nearest`.
    tolerance : float, optional
        The maximum error tolerance in input pixels when
        approximating the warp transformation. Default: 0.125,
        or one-eigth of a pixel.
    src_alpha : int, optional
        Index of a source band to use as an alpha band for warping.
    add_alpha : bool, optional
        Whether to add an alpha masking band to the virtual dataset.
        Default: False. This option will cause deletion of the VRT
        nodata value.
    init_dest_nodata : bool, optional
        Whether or not to initialize output to `nodata`. Default:
        True.
    warp_mem_limit : int, optional
        The warp operation's memory limit in MB. The default (0)
        means 64 MB with GDAL 2.2.
    dtype : str, optional
        The working data type for warp operation and output.
    warp_extras : dict
        GDAL extra warp options. See
        https://gdal.org/doxygen/structGDALWarpOptions.html.

    Attributes
    ----------
    src_dataset : dataset
        The dataset object to be virtually warped.
    resampling : int
        One of the values from rasterio.enums.Resampling. The default is
        `Resampling.nearest`.
    tolerance : float
        The maximum error tolerance in input pixels when approximating
        the warp transformation. The default is 0.125.
    src_nodata: int or float, optional
        The source nodata value.  Pixels with this value will not be
        used for interpolation. If not set, it will be default to the
        nodata value of the source image, if available.
    dst_nodata: int or float, optional
        The nodata value used to initialize the destination; it will
        remain in all areas not covered by the reprojected source.
        Defaults to the value of src_nodata, or 0 (gdal default).
    working_dtype : str, optional
        The working data type for warp operation and output.
    warp_extras : dict
        GDAL extra warp options. See
        https://gdal.org/doxygen/structGDALWarpOptions.html.

    Examples
    --------

    >>> with rasterio.open('tests/data/RGB.byte.tif') as src:
    ...     with WarpedVRT(src, crs='EPSG:3857') as vrt:
    ...         data = vrt.read()

    """

    def __repr__(self):
        return "<{} WarpedVRT name='{}' mode='{}'>".format(
            self.closed and 'closed' or 'open', self.name, self.mode)

    def __enter__(self):
        self.start()
        return self

    def __exit__(self, *args, **kwargs):
        if not self._closed:
            self.close()

    def __del__(self):
        if not self._closed:
            self.close()


def _boundless_vrt_doc(
        src_dataset, nodata=None, background=None, hidenodata=False,
        width=None, height=None, transform=None, masked=False):
    """Make a VRT XML document.

    Parameters
    ----------
    src_dataset : Dataset
        The dataset to wrap.
    background : int or float, optional
        The background fill value for the boundless VRT.
    masked : bool
        If True, the src_dataset is replaced by its valid data mask.

    Returns
    -------
    str
        An XML text string.
    """

    nodata = nodata or src_dataset.nodata
    width = width or src_dataset.width
    height = height or src_dataset.height
    transform = transform or src_dataset.transform

    vrtdataset = ET.Element('VRTDataset')
    vrtdataset.attrib['rasterYSize'] = str(height)
    vrtdataset.attrib['rasterXSize'] = str(width)
    srs = ET.SubElement(vrtdataset, 'SRS')
    srs.text = src_dataset.crs.wkt if src_dataset.crs else ""
    geotransform = ET.SubElement(vrtdataset, 'GeoTransform')
    geotransform.text = ','.join([str(v) for v in transform.to_gdal()])

    for bidx, ci, block_shape, dtype in zip(src_dataset.indexes, src_dataset.colorinterp, src_dataset.block_shapes, src_dataset.dtypes):
        vrtrasterband = ET.SubElement(vrtdataset, 'VRTRasterBand')
        vrtrasterband.attrib['dataType'] = _gdal_typename(dtype)
        vrtrasterband.attrib['band'] = str(bidx)

        if background is not None or nodata is not None:
            nodatavalue = ET.SubElement(vrtrasterband, 'NoDataValue')
            nodatavalue.text = str(background or nodata)

            if hidenodata:
                hidenodatavalue = ET.SubElement(vrtrasterband, 'HideNoDataValue')
                hidenodatavalue.text = "1"

        colorinterp = ET.SubElement(vrtrasterband, 'ColorInterp')
        colorinterp.text = ci.name.capitalize()

        complexsource = ET.SubElement(vrtrasterband, 'ComplexSource')
        sourcefilename = ET.SubElement(complexsource, 'SourceFilename')
        sourcefilename.attrib['relativeToVRT'] = "0"
        sourcefilename.attrib["shared"] = "0"
        sourcefilename.text = _parse_path(src_dataset.name).as_vsi()
        sourceband = ET.SubElement(complexsource, 'SourceBand')
        sourceband.text = str(bidx)
        sourceproperties = ET.SubElement(complexsource, 'SourceProperties')
        sourceproperties.attrib['RasterXSize'] = str(width)
        sourceproperties.attrib['RasterYSize'] = str(height)
        sourceproperties.attrib['dataType'] = _gdal_typename(dtype)
        sourceproperties.attrib['BlockYSize'] = str(block_shape[0])
        sourceproperties.attrib['BlockXSize'] = str(block_shape[1])
        srcrect = ET.SubElement(complexsource, 'SrcRect')
        srcrect.attrib['xOff'] = '0'
        srcrect.attrib['yOff'] = '0'
        srcrect.attrib['xSize'] = str(src_dataset.width)
        srcrect.attrib['ySize'] = str(src_dataset.height)
        dstrect = ET.SubElement(complexsource, 'DstRect')
        dstrect.attrib['xOff'] = str((src_dataset.transform.xoff - transform.xoff) / transform.a)
        dstrect.attrib['yOff'] = str((src_dataset.transform.yoff - transform.yoff) / transform.e)
        dstrect.attrib['xSize'] = str(src_dataset.width * src_dataset.transform.a / transform.a)
        dstrect.attrib['ySize'] = str(src_dataset.height * src_dataset.transform.e / transform.e)

        if src_dataset.nodata is not None:
            nodata_elem = ET.SubElement(complexsource, 'NODATA')
            nodata_elem.text = str(src_dataset.nodata)

        if src_dataset.options is not None:
            openoptions = ET.SubElement(complexsource, 'OpenOptions')
            for ookey, oovalue in src_dataset.options.items():
                ooi = ET.SubElement(openoptions, 'OOI')
                ooi.attrib['key'] = str(ookey)
                ooi.text = str(oovalue)

        # Effectively replaces all values of the source dataset with
        # 255.  Due to GDAL optimizations, the source dataset will not
        # be read, so we get a performance improvement.
        if masked:
            scaleratio = ET.SubElement(complexsource, 'ScaleRatio')
            scaleratio.text = '0'
            scaleoffset = ET.SubElement(complexsource, 'ScaleOffset')
            scaleoffset.text = '255'

    if all(MaskFlags.per_dataset in flags for flags in src_dataset.mask_flag_enums):
        maskband = ET.SubElement(vrtdataset, 'MaskBand')
        vrtrasterband = ET.SubElement(maskband, 'VRTRasterBand')
        vrtrasterband.attrib['dataType'] = 'Byte'

        simplesource = ET.SubElement(vrtrasterband, 'SimpleSource')
        sourcefilename = ET.SubElement(simplesource, 'SourceFilename')
        sourcefilename.attrib['relativeToVRT'] = "0"
        sourcefilename.attrib["shared"] = "0"
        sourcefilename.text = _parse_path(src_dataset.name).as_vsi()

        sourceband = ET.SubElement(simplesource, 'SourceBand')
        sourceband.text = 'mask,1'
        sourceproperties = ET.SubElement(simplesource, 'SourceProperties')
        sourceproperties.attrib['RasterXSize'] = str(width)
        sourceproperties.attrib['RasterYSize'] = str(height)
        sourceproperties.attrib['dataType'] = 'Byte'
        sourceproperties.attrib['BlockYSize'] = str(block_shape[0])
        sourceproperties.attrib['BlockXSize'] = str(block_shape[1])
        srcrect = ET.SubElement(simplesource, 'SrcRect')
        srcrect.attrib['xOff'] = '0'
        srcrect.attrib['yOff'] = '0'
        srcrect.attrib['xSize'] = str(src_dataset.width)
        srcrect.attrib['ySize'] = str(src_dataset.height)
        dstrect = ET.SubElement(simplesource, 'DstRect')
        dstrect.attrib['xOff'] = str((src_dataset.transform.xoff - transform.xoff) / transform.a)
        dstrect.attrib['yOff'] = str((src_dataset.transform.yoff - transform.yoff) / transform.e)
        dstrect.attrib['xSize'] = str(src_dataset.width)
        dstrect.attrib['ySize'] = str(src_dataset.height)

    return ET.tostring(vrtdataset).decode('ascii')

env_pyrecodes/lib/python3.9/site-packages/rasterio/vrt.py
NikolaBlagojevic/pyrecodes
"""Raster warping and reprojection."""

from math import ceil, floor

from affine import Affine
import numpy as np

import rasterio

from rasterio._base import _transform
from rasterio.crs import CRS
from rasterio.enums import Resampling
from rasterio.env import ensure_env, require_gdal_version
from rasterio.errors import TransformError, RPCError
from rasterio.transform import array_bounds
from rasterio._warp import (
    _calculate_default_transform,
    _reproject,
    _transform_bounds,
    _transform_geom,
    SUPPORTED_RESAMPLING,
)

@ensure_env
def transform(src_crs, dst_crs, xs, ys, zs=None):

    """Transform vectors from source to target coordinate reference system.

    Transform vectors of x, y and optionally z from source
    coordinate reference system into target.

    Parameters
    ------------
    src_crs: CRS or dict
        Source coordinate reference system, as a rasterio CRS object.
        Example: CRS({'init': 'EPSG:4326'})
    dst_crs: CRS or dict
        Target coordinate reference system.
    xs: array_like
        Contains x values.  Will be cast to double floating point values.
    ys:  array_like
        Contains y values.
    zs: array_like, optional
        Contains z values.  Assumed to be all 0 if absent.

    Returns
    ---------
    out: tuple of array_like, (xs, ys, [zs])
        Tuple of x, y, and optionally z vectors, transformed into the target
        coordinate reference system.

    """
    if len(xs) != len(ys):
        raise TransformError("xs and ys arrays must be the same length")
    elif zs is not None and len(xs) != len(zs):
        raise TransformError("zs, xs, and ys arrays must be the same length")
    if len(xs) == 0:
        return ([], [], []) if zs is not None else ([], [])
    else:
        return _transform(src_crs, dst_crs, xs, ys, zs)


@ensure_env
@require_gdal_version('2.1', param='antimeridian_cutting', values=[False],
                      is_max_version=True,
                      reason="Antimeridian cutting is always enabled on "
                             "GDAL >= 2.2")
def transform_geom(
        src_crs,
        dst_crs,
        geom,
        antimeridian_cutting=True,
        antimeridian_offset=10.0,
        precision=-1):
    """Transform geometry from source coordinate reference system into target.

    Parameters
    ------------
    src_crs: CRS or dict
        Source coordinate reference system, in rasterio dict format.
        Example: CRS({'init': 'EPSG:4326'})
    dst_crs: CRS or dict
        Target coordinate reference system.
    geom: GeoJSON like dict object or iterable of GeoJSON like objects.
    antimeridian_cutting: bool, optional
        If True, cut geometries at the antimeridian, otherwise geometries
        will not be cut (default).  If False and GDAL is 2.2.0 or newer
        an exception is raised.  Antimeridian cutting is always on as of
        GDAL 2.2.0 but this could produce an unexpected geometry.
    antimeridian_offset: float
        Offset from the antimeridian in degrees (default: 10) within which
        any geometries will be split.
    precision: float
        If >= 0, geometry coordinates will be rounded to this number of decimal
        places after the transform operation, otherwise original coordinate
        values will be preserved (default).

    Returns
    ---------
    out: GeoJSON like dict object or list of GeoJSON like objects.
        Transformed geometry(s) in GeoJSON dict format
    """

    return _transform_geom(
        src_crs,
        dst_crs,
        geom,
        antimeridian_cutting,
        antimeridian_offset,
        precision)


def transform_bounds(
        src_crs,
        dst_crs,
        left,
        bottom,
        right,
        top,
        densify_pts=21):
    """Transform bounds from src_crs to dst_crs.

    Optionally densifying the edges (to account for nonlinear transformations
    along these edges) and extracting the outermost bounds.

    Note: antimeridian support added in version 1.3.0

    Parameters
    ----------
    src_crs: CRS or dict
        Source coordinate reference system, in rasterio dict format.
        Example: CRS({'init': 'EPSG:4326'})
    dst_crs: CRS or dict
        Target coordinate reference system.
    left, bottom, right, top: float
        Bounding coordinates in src_crs, from the bounds property of a raster.
    densify_pts: uint, optional
        Number of points to add to each edge to account for nonlinear
        edges produced by the transform process.  Large numbers will produce
        worse performance.  Default: 21 (gdal default).

    Returns
    -------
    left, bottom, right, top: float
        Outermost coordinates in target coordinate reference system.
    """
    src_crs = CRS.from_user_input(src_crs)
    dst_crs = CRS.from_user_input(dst_crs)
    return _transform_bounds(
        src_crs,
        dst_crs,
        left,
        bottom,
        right,
        top,
        densify_pts,
    )


@ensure_env
def reproject(source, destination=None, src_transform=None, gcps=None, rpcs=None,
              src_crs=None, src_nodata=None, dst_transform=None, dst_crs=None,
              dst_nodata=None, dst_resolution=None, src_alpha=0, dst_alpha=0,
              resampling=Resampling.nearest, num_threads=1,
              init_dest_nodata=True, warp_mem_limit=0, **kwargs):
    """Reproject a source raster to a destination raster.

    If the source and destination are ndarrays, coordinate reference
    system definitions and affine transformation parameters or ground
    control points (gcps) are required for reprojection.

    If the source and destination are rasterio Bands, shorthand for
    bands of datasets on disk, the coordinate reference systems and
    transforms or GCPs will be read from the appropriate datasets.

    Parameters
    ------------
    source: ndarray or Band
        The source is a 2 or 3-D ndarray, or a single or a multiple
        Rasterio Band object. The dimensionality of source
        and destination must match, i.e., for multiband reprojection
        the lengths of the first axes of the source and destination
        must be the same.
    destination: ndarray or Band, optional
        The destination is a 2 or 3-D ndarray, or a single or a multiple
        Rasterio Band object. The dimensionality of source
        and destination must match, i.e., for multiband reprojection
        the lengths of the first axes of the source and destination
        must be the same.
    src_transform: affine.Affine(), optional
        Source affine transformation. Required if source and
        destination are ndarrays. Will be derived from source if it is
        a rasterio Band. An error will be raised if this parameter is
        defined together with gcps.
    gcps: sequence of GroundControlPoint, optional
        Ground control points for the source. An error will be raised
        if this parameter is defined together with src_transform or rpcs.
    rpcs: RPC or dict, optional
        Rational polynomial coefficients for the source. An error will
        be raised if this parameter is defined together with src_transform
        or gcps.
    src_crs: CRS or dict, optional
        Source coordinate reference system, in rasterio dict format.
        Required if source and destination are ndarrays.
        Will be derived from source if it is a rasterio Band.
        Example: CRS({'init': 'EPSG:4326'})
    src_nodata: int or float, optional
        The source nodata value. Pixels with this value will not be
        used for interpolation. If not set, it will default to the
        nodata value of the source image if a masked ndarray or
        rasterio band, if available.
    dst_transform: affine.Affine(), optional
        Target affine transformation. Required if source and
        destination are ndarrays. Will be derived from target if it is
        a rasterio Band.
    dst_crs: CRS or dict, optional
        Target coordinate reference system. Required if source and
        destination are ndarrays. Will be derived from target if it
        is a rasterio Band.
    dst_nodata: int or float, optional
        The nodata value used to initialize the destination; it will
        remain in all areas not covered by the reprojected source.
        Defaults to the nodata value of the destination image (if set),
        the value of src_nodata, or 0 (GDAL default).
    dst_resolution: tuple (x resolution, y resolution) or float, optional
        Target resolution, in units of target coordinate reference
        system.
    src_alpha : int, optional
        Index of a band to use as the alpha band when warping.
    dst_alpha : int, optional
        Index of a band to use as the alpha band when warping.
    resampling: int, rasterio.enums.Resampling
        Resampling method to use.
        Default is :attr:`rasterio.enums.Resampling.nearest`.
        An exception will be raised for a method not supported by the running
        version of GDAL.
    num_threads : int, optional
        The number of warp worker threads. Default: 1.
    init_dest_nodata: bool
        Flag to specify initialization of nodata in destination;
        prevents overwrite of previous warps. Defaults to True.
    warp_mem_limit : int, optional
        The warp operation memory limit in MB. Larger values allow the
        warp operation to be carried out in fewer chunks. The amount of
        memory required to warp a 3-band uint8 2000 row x 2000 col
        raster to a destination of the same size is approximately
        56 MB. The default (0) means 64 MB with GDAL 2.2.
    kwargs:  dict, optional
        Additional arguments passed to both the image to image
        transformer :cpp:func:`GDALCreateGenImgProjTransformer2` (for example,
        MAX_GCP_ORDER=2) and the :cpp:struct:`GDALWarpOptions` (for example,
        INIT_DEST=NO_DATA).

    Returns
    ---------
    destination: ndarray or Band
        The transformed narray or Band.
    dst_transform: Affine
        THe affine transformation matrix of the destination.
    """

    # Only one type of georeferencing is permitted.
    if (src_transform and gcps) or (src_transform and rpcs) or (gcps and rpcs):
        raise ValueError("src_transform, gcps, and rpcs are mutually "
                         "exclusive parameters and may not be used together.")

    # Guard against invalid or unsupported resampling algorithms.
    try:
        if resampling == 7:
            raise ValueError("Gauss resampling is not supported")

        Resampling(resampling)

    except ValueError:
        raise ValueError(
            "resampling must be one of: {0}".format(", ".join(
                ['Resampling.{0}'.format(r.name) for r in
                 SUPPORTED_RESAMPLING])))

    if destination is None and dst_transform is not None:
        raise ValueError("Must provide destination if dst_transform is provided.")

    # calculate the destination transform if not provided
    if dst_transform is None and (destination is None or isinstance(destination, np.ndarray)):
        src_bounds = tuple([None] * 4)
        if isinstance(source, np.ndarray):
            if source.ndim == 3:
                src_count, src_height, src_width = source.shape
            else:
                src_count = 1
                src_height, src_width = source.shape

            # try to compute src_bounds if we don't have gcps
            if not (gcps or rpcs):
                src_bounds = array_bounds(src_height, src_width, src_transform)
        else:
            src_rdr, src_bidx, _, src_shape = source

            # dataset.bounds will return raster extents in pixel coordinates
            # if dataset does not have geotransform, which is not useful here.
            if not (src_rdr.transform.is_identity and src_rdr.crs is None):
                src_bounds = src_rdr.bounds

            src_crs = src_crs or src_rdr.crs

            # raise exception when reprojecting with rpcs using a CRS that is not EPSG:4326
            if rpcs:
                if isinstance(src_crs, str):
                    src_crs_obj = rasterio.crs.CRS.from_string(src_crs)
                else:
                    src_crs_obj = src_crs
                if src_crs is not None and src_crs_obj.to_epsg() != 4326:
                    raise RPCError("Reprojecting with rational polynomial coefficients using source CRS other than EPSG:4326")

            if isinstance(src_bidx, int):
                src_bidx = [src_bidx]

            src_count = len(src_bidx)
            src_height, src_width = src_shape
            gcps = src_rdr.gcps[0] if src_rdr.gcps[0] and not rpcs else None

        dst_height = None
        dst_width = None
        dst_count = src_count
        if isinstance(destination, np.ndarray):
            if destination.ndim == 3:
                dst_count, dst_height, dst_width = destination.shape
            else:
                dst_count = 1
                dst_height, dst_width = destination.shape

        left, bottom, right, top = src_bounds
        dst_transform, dst_width, dst_height = calculate_default_transform(
            src_crs=src_crs, dst_crs=dst_crs, width=src_width, height=src_height,
            left=left, bottom=bottom, right=right, top=top,
            gcps=gcps, rpcs=rpcs, dst_width=dst_width, dst_height=dst_height,
            resolution=dst_resolution)

        if destination is None:
            destination = np.empty((int(dst_count), int(dst_height), int(dst_width)),
                                   dtype=source.dtype)

    # Call the function in our extension module.
    _reproject(
        source, destination, src_transform=src_transform, gcps=gcps, rpcs=rpcs,
        src_crs=src_crs, src_nodata=src_nodata, dst_transform=dst_transform,
        dst_crs=dst_crs, dst_nodata=dst_nodata, dst_alpha=dst_alpha,
        src_alpha=src_alpha, resampling=resampling,
        init_dest_nodata=init_dest_nodata, num_threads=num_threads,
        warp_mem_limit=warp_mem_limit, **kwargs)

    return destination, dst_transform


def aligned_target(transform, width, height, resolution):
    """Aligns target to specified resolution

    Parameters
    ----------
    transform : Affine
        Input affine transformation matrix
    width, height: int
        Input dimensions
    resolution: tuple (x resolution, y resolution) or float
        Target resolution, in units of target coordinate reference
        system.

    Returns
    -------
    transform: Affine
        Output affine transformation matrix
    width, height: int
        Output dimensions

    """
    if isinstance(resolution, (float, int)):
        res = (float(resolution), float(resolution))
    else:
        res = resolution

    xmin = transform.xoff
    ymin = transform.yoff + height * transform.e
    xmax = transform.xoff + width * transform.a
    ymax = transform.yoff

    xmin = floor(xmin / res[0]) * res[0]
    xmax = ceil(xmax / res[0]) * res[0]
    ymin = floor(ymin / res[1]) * res[1]
    ymax = ceil(ymax / res[1]) * res[1]
    dst_transform = Affine(res[0], 0, xmin, 0, -res[1], ymax)
    dst_width = max(int(ceil((xmax - xmin) / res[0])), 1)
    dst_height = max(int(ceil((ymax - ymin) / res[1])), 1)

    return dst_transform, dst_width, dst_height


@ensure_env
def calculate_default_transform(
        src_crs, dst_crs, width, height, left=None, bottom=None, right=None,
        top=None, gcps=None, rpcs=None, resolution=None, dst_width=None, dst_height=None, **kwargs):
    """Output dimensions and transform for a reprojection.

    Source and destination coordinate reference systems and output
    width and height are the first four, required, parameters. Source
    georeferencing can be specified using either ground control points
    (gcps) or spatial bounds (left, bottom, right, top). These two
    forms of georeferencing are mutually exclusive.

    The destination transform is anchored at the left, top coordinate.

    Destination width and height (and resolution if not provided), are
    calculated using GDAL's method for suggest warp output.

    Parameters
    ----------
    src_crs: CRS or dict
        Source coordinate reference system, in rasterio dict format.
        Example: CRS({'init': 'EPSG:4326'})
    dst_crs: CRS or dict
        Target coordinate reference system.
    width, height: int
        Source raster width and height.
    left, bottom, right, top: float, optional
        Bounding coordinates in src_crs, from the bounds property of a
        raster. Required unless using gcps.
    gcps: sequence of GroundControlPoint, optional
        Instead of a bounding box for the source, a sequence of ground
        control points may be provided.
    rpcs: RPC or dict, optional
        Instead of a bounding box for the source, rational polynomial
        coefficients may be provided.
    resolution: tuple (x resolution, y resolution) or float, optional
        Target resolution, in units of target coordinate reference
        system.
    dst_width, dst_height: int, optional
        Output file size in pixels and lines. Cannot be used together
        with resolution.
    kwargs:  dict, optional
        Additional arguments passed to transformation function.

    Returns
    -------
    transform: Affine
        Output affine transformation matrix
    width, height: int
        Output dimensions

    Notes
    -----
    Some behavior of this function is determined by the
    CHECK_WITH_INVERT_PROJ environment variable:

        YES
            constrain output raster to extents that can be inverted
            avoids visual artifacts and coordinate discontinuties.
        NO
            reproject coordinates beyond valid bound limits
    """
    if any(x is not None for x in (left, bottom, right, top)) and gcps:
        raise ValueError("Bounding values and ground control points may not"
                         " be used together.")
    if any(x is not None for x in (left, bottom, right, top)) and rpcs:
        raise ValueError("Bounding values and rational polynomial coefficients may not"
                         " be used together.")

    if any(x is None for x in (left, bottom, right, top)) and not (gcps or rpcs):
        raise ValueError("Either four bounding values, ground control points,"
                         " or rational polynomial coefficients must be specified")

    if gcps and rpcs:
        raise ValueError("ground control points and rational polynomial",
                         " coefficients may not be used together.")

    if (dst_width is None) != (dst_height is None):
        raise ValueError("Either dst_width and dst_height must be specified "
                         "or none of them.")

    if all(x is not None for x in (dst_width, dst_height)):
        dimensions = (dst_width, dst_height)
    else:
        dimensions = None

    if resolution and dimensions:
        raise ValueError("Resolution cannot be used with dst_width and dst_height.")

    dst_affine, dst_width, dst_height = _calculate_default_transform(
        src_crs, dst_crs, width, height, left, bottom, right, top, gcps, rpcs, **kwargs
    )

    # If resolution is specified, Keep upper-left anchored
    # adjust the transform resolutions
    # adjust the width/height by the ratio of estimated:specified res (ceil'd)
    if resolution:
        # resolutions argument into tuple
        try:
            res = (float(resolution), float(resolution))
        except TypeError:
            res = (resolution[0], resolution[0]) \
                if len(resolution) == 1 else resolution[0:2]

        # Assume yres is provided as positive,
        # needs to be negative for north-up affine
        xres = res[0]
        yres = -res[1]

        xratio = dst_affine.a / xres
        yratio = dst_affine.e / yres

        dst_affine = Affine(xres, dst_affine.b, dst_affine.c,
                            dst_affine.d, yres, dst_affine.f)

        dst_width = ceil(dst_width * xratio)
        dst_height = ceil(dst_height * yratio)

    if dimensions:
        xratio = dst_width / dimensions[0]
        yratio = dst_height / dimensions[1]

        dst_width = dimensions[0]
        dst_height = dimensions[1]

        dst_affine = Affine(dst_affine.a * xratio, dst_affine.b, dst_affine.c,
                            dst_affine.d, dst_affine.e * yratio, dst_affine.f)

    return dst_affine, dst_width, dst_height

env_pyrecodes/lib/python3.9/site-packages/rasterio/warp.py
NikolaBlagojevic/pyrecodes
"""Window utilities and related functions.

A window is an instance of Window

    Window(column_offset, row_offset, width, height)

or a 2D N-D array indexer in the form of a tuple.

    ((row_start, row_stop), (col_start, col_stop))

The latter can be evaluated within the context of a given height and
width and a boolean flag specifying whether the evaluation is boundless
or not. If boundless=True, negative index values do not mean index from
the end of the array dimension as they do in the boundless=False case.

The newer float precision read-write window capabilities of Rasterio
require instances of Window to be used.
"""

import collections
from collections.abc import Iterable
import functools
import math
import warnings

from affine import Affine
import attr
import numpy as np

from rasterio.errors import WindowError, RasterioDeprecationWarning
from rasterio.transform import rowcol, guard_transform


class WindowMethodsMixin:
    """Mixin providing methods for window-related calculations.
    These methods are wrappers for the functionality in
    `rasterio.windows` module.

    A subclass with this mixin MUST provide the following
    properties: `transform`, `height` and `width`.

    """

    def window(self, left, bottom, right, top, precision=None):
        """Get the window corresponding to the bounding coordinates.

        The resulting window is not cropped to the row and column
        limits of the dataset.

        Parameters
        ----------
        left: float
            Left (west) bounding coordinate
        bottom: float
            Bottom (south) bounding coordinate
        right: float
            Right (east) bounding coordinate
        top: float
            Top (north) bounding coordinate
        precision: int, optional
            This parameter is unused, deprecated in rasterio 1.3.0, and
            will be removed in version 2.0.0.

        Returns
        -------
        window: Window

        """
        if precision is not None:
            warnings.warn(
                "The precision parameter is unused, deprecated, and will be removed in 2.0.0.",
                RasterioDeprecationWarning,
            )

        return from_bounds(
            left,
            bottom,
            right,
            top,
            transform=guard_transform(self.transform),
        )

    def window_transform(self, window):
        """Get the affine transform for a dataset window.

        Parameters
        ----------
        window: rasterio.windows.Window
            Dataset window

        Returns
        -------
        transform: Affine
            The affine transform matrix for the given window

        """
        gtransform = guard_transform(self.transform)
        return transform(window, gtransform)

    def window_bounds(self, window):
        """Get the bounds of a window

        Parameters
        ----------
        window: rasterio.windows.Window
            Dataset window

        Returns
        -------
        bounds : tuple
            x_min, y_min, x_max, y_max for the given window

        """
        transform = guard_transform(self.transform)
        return bounds(window, transform)


def iter_args(function):
    """Decorator to allow function to take either ``*args`` or
    a single iterable which gets expanded to ``*args``.
    """
    @functools.wraps(function)
    def wrapper(*args, **kwargs):
        if len(args) == 1 and isinstance(args[0], Iterable):
            return function(*args[0])
        else:
            return function(*args)
    return wrapper


def toranges(window):
    """Normalize Windows to range tuples"""
    if isinstance(window, Window):
        return window.toranges()
    else:
        return window


def get_data_window(arr, nodata=None):
    """Window covering the input array's valid data pixels.

    Parameters
    ----------
    arr: numpy ndarray, <= 3 dimensions
    nodata: number
        If None, will either return a full window if arr is not a masked
        array, or will use the mask to determine non-nodata pixels.
        If provided, it must be a number within the valid range of the
        dtype of the input array.

    Returns
    -------
    Window
    """
    if not 0 < arr.ndim <=3 :
        raise WindowError(
            "get_data_window input array must have 1, 2, or 3 dimensions")

    # If nodata is defined, construct mask from that value
    # Otherwise retrieve mask from array (if it is masked)
    # Finally try returning a full window (nodata=None and nothing in arr is masked)
    if nodata is not None:
        if np.isnan(nodata):
            arr_mask = ~np.isnan(arr)
        else:
            arr_mask = arr != nodata
    elif np.ma.is_masked(arr):
        arr_mask = ~np.ma.getmask(arr)
    else:
        if arr.ndim == 1:
            full_window = ((0, arr.size), (0, 0))
        else:
            full_window = ((0, arr.shape[-2]), (0, arr.shape[-1]))
        return Window.from_slices(*full_window)

    if arr.ndim == 3:
        arr_mask = np.any(arr_mask, axis=0)

    # We only have 1 or 2 dimension cases to process
    v = []
    for nz in arr_mask.nonzero():
        if nz.size:
            v.append((nz.min(), nz.max() + 1))
        else:
            v.append((0, 0))

    if arr_mask.ndim == 1:
        v.append((0, 0))

    return Window.from_slices(*v)


def _compute_union(w1, w2):
    """Compute the union of two windows"""
    col_off = min(w1.col_off, w2.col_off)
    row_off = min(w1.row_off, w2.row_off)
    width = max(w1.col_off+w1.width, w2.col_off+w2.width) - col_off
    height = max(w1.row_off+w1.height, w2.row_off+w2.height) - row_off
    return col_off, row_off, width, height


def _union(w1, w2):
    coeffs = _compute_union(w1, w2)
    return Window(*coeffs)


@iter_args
def union(*windows):
    """
    Union windows and return the outermost extent they cover.

    Parameters
    ----------
    windows: sequence
        One or more Windows.

    Returns
    -------
    Window
    """
    return functools.reduce(_union, windows)


@iter_args
def intersection(*windows):
    """Innermost extent of window intersections.

    Will raise WindowError if windows do not intersect.

    Parameters
    ----------
    windows: sequence
        One or more Windows.

    Returns
    -------
    Window
    """
    return functools.reduce(_intersection, windows)


def _compute_intersection(w1, w2):
    """ Compute intersection of window 1 and window 2"""
    col_off = max(w1.col_off, w2.col_off)
    row_off = max(w1.row_off, w2.row_off)
    width = min(w1.col_off+w1.width, w2.col_off+w2.width) - col_off
    height = min(w1.row_off+w1.height, w2.row_off+w2.height) - row_off
    return col_off, row_off, width, height


def _intersection(w1, w2):
    """ Compute intersection of window 1 and window 2"""
    coeffs = _compute_intersection(w1, w2)
    if coeffs[2] > 0 and coeffs[3] > 0:
        return Window(*coeffs)
    else:
        raise WindowError(f"Intersection is empty {w1} {w2}")


@iter_args
def intersect(*windows):
    """Test if all given windows intersect.

    Parameters
    ----------
    windows: sequence
        One or more Windows.

    Returns
    -------
    bool
        True if all windows intersect.
    """
    try:
        intersection(*windows)
        return True
    except WindowError:
        return False


def from_bounds(
    left, bottom, right, top, transform=None, height=None, width=None, precision=None
):
    """Get the window corresponding to the bounding coordinates.

    Parameters
    ----------
    left: float, required
        Left (west) bounding coordinates
    bottom: float, required
        Bottom (south) bounding coordinates
    right: float, required
        Right (east) bounding coordinates
    top: float, required
        Top (north) bounding coordinates
    transform: Affine, required
        Affine transform matrix.
    precision, height, width: int, optional
        These parameters are unused, deprecated in rasterio 1.3.0, and
        will be removed in version 2.0.0.

    Returns
    -------
    Window
        A new Window.

    Raises
    ------
    WindowError
        If a window can't be calculated.

    """
    if height is not None or width is not None or precision is not None:
        warnings.warn(
            "The height, width, and precision parameters are unused, deprecated, and will be removed in 2.0.0.",
            RasterioDeprecationWarning,
        )

    if not isinstance(transform, Affine):  # TODO: RPCs?
        raise WindowError("A transform object is required to calculate the window")
    if (right - left) / transform.a < 0:
        raise WindowError("Bounds and transform are inconsistent")
    if (bottom - top) / transform.e < 0:
        raise WindowError("Bounds and transform are inconsistent")

    rows, cols = rowcol(
        transform,
        [left, right, right, left],
        [top, top, bottom, bottom],
        op=float,
    )
    row_start, row_stop = min(rows), max(rows)
    col_start, col_stop = min(cols), max(cols)

    return Window(
        col_off=col_start,
        row_off=row_start,
        width=max(col_stop - col_start, 0.0),
        height=max(row_stop - row_start, 0.0),
    )


def transform(window, transform):
    """Construct an affine transform matrix relative to a window.

    Parameters
    ----------
    window: Window
        The input window.
    transform: Affine
        an affine transform matrix.

    Returns
    -------
    Affine
        The affine transform matrix for the given window

    """
    window = evaluate(window, height=0, width=0)
    x, y = transform * (window.col_off or 0.0, window.row_off or 0.0)
    return Affine.translation(
        x - transform.c, y - transform.f) * transform


def bounds(window, transform, height=0, width=0):
    """Get the spatial bounds of a window.

    Parameters
    ----------
    window: Window
        The input window.
    transform: Affine
        an affine transform matrix.

    Returns
    -------
    left, bottom, right, top: float
        A tuple of spatial coordinate bounding values.
    """
    window = evaluate(window, height=height, width=width)

    row_min = window.row_off
    row_max = row_min + window.height
    col_min = window.col_off
    col_max = col_min + window.width

    left, bottom = transform * (col_min, row_max)
    right, top = transform * (col_max, row_min)
    return left, bottom, right, top


def crop(window, height, width):
    """Crops a window to given height and width.

    Parameters
    ----------
    window : Window.
        The input window.
    height, width : int
        The number of rows and cols in the cropped window.

    Returns
    -------
    Window
        A new Window object.
    """
    window = evaluate(window, height=height, width=width)

    row_start = min(max(window.row_off, 0), height)
    col_start = min(max(window.col_off, 0), width)
    row_stop = max(0, min(window.row_off + window.height, height))
    col_stop = max(0, min(window.col_off + window.width, width))

    return Window(col_start, row_start, col_stop - col_start,
                  row_stop - row_start)


def evaluate(window, height, width, boundless=False):
    """Evaluates a window tuple that may contain relative index values.

    The height and width of the array the window targets is the context
    for evaluation.

    Parameters
    ----------
    window: Window or tuple of (rows, cols).
        The input window.
    height, width: int
        The number of rows or columns in the array that the window
        targets.

    Returns
    -------
    Window
        A new Window object with absolute index values.
    """
    if isinstance(window, Window):
        return window
    else:
        rows, cols = window
        return Window.from_slices(rows=rows, cols=cols, height=height,
                                  width=width, boundless=boundless)


def shape(window, height=-1, width=-1):
    """The shape of a window.

    height and width arguments are optional if there are no negative
    values in the window.

    Parameters
    ----------
    window: Window
        The input window.
    height, width : int, optional
        The number of rows or columns in the array that the window
        targets.

    Returns
    -------
    num_rows, num_cols
        The number of rows and columns of the window.
    """
    evaluated = evaluate(window, height, width)
    return evaluated.height, evaluated.width


def window_index(window, height=0, width=0):
    """Construct a pair of slice objects for ndarray indexing

    Starting indexes are rounded down, Stopping indexes are rounded up.

    Parameters
    ----------
    window: Window
        The input window.

    Returns
    -------
    row_slice, col_slice: slice
        A pair of slices in row, column order

    """
    window = evaluate(window, height=height, width=width)
    return window.toslices()


def round_window_to_full_blocks(window, block_shapes, height=0, width=0):
    """Round window to include full expanse of intersecting tiles.

    Parameters
    ----------
    window: Window
        The input window.

    block_shapes : tuple of block shapes
        The input raster's block shape. All bands must have the same
        block/stripe structure

    Returns
    -------
    Window
    """
    if len(set(block_shapes)) != 1:  # pragma: no cover
        raise WindowError(
            "All bands must have the same block/stripe structure")

    window = evaluate(window, height=height, width=width)

    height_shape = block_shapes[0][0]
    width_shape = block_shapes[0][1]

    (row_start, row_stop), (col_start, col_stop) = window.toranges()

    row_min = int(row_start // height_shape) * height_shape
    row_max = int(row_stop // height_shape) * height_shape + \
        (height_shape if row_stop % height_shape != 0 else 0)

    col_min = int(col_start // width_shape) * width_shape
    col_max = int(col_stop // width_shape) * width_shape + \
        (width_shape if col_stop % width_shape != 0 else 0)

    return Window(col_min, row_min, col_max - col_min, row_max - row_min)


def validate_length_value(instance, attribute, value):
    if value and value < 0:
        raise ValueError("Number of columns or rows must be non-negative")


@attr.s(slots=True,
        frozen=True)
class Window:
    """Windows are rectangular subsets of rasters.

    This class abstracts the 2-tuples mentioned in the module docstring
    and adds methods and new constructors.

    Attributes
    ----------
    col_off, row_off: float
        The offset for the window.
    width, height: float
        Lengths of the window.

    Notes
    -----
    Previously the lengths were called 'num_cols' and 'num_rows' but
    this is a bit confusing in the new float precision world and the
    attributes have been changed. The originals are deprecated.
    """
    col_off = attr.ib()
    row_off = attr.ib()
    width = attr.ib(validator=validate_length_value)
    height = attr.ib(validator=validate_length_value)

    def __repr__(self):
        """Return a nicely formatted representation string"""
        return (
            "Window(col_off={self.col_off}, row_off={self.row_off}, "
            "width={self.width}, height={self.height})").format(
                self=self)

    def flatten(self):
        """A flattened form of the window.

        Returns
        -------
        col_off, row_off, width, height: float
            Window offsets and lengths.
        """
        return (self.col_off, self.row_off, self.width, self.height)

    def todict(self):
        """A mapping of attribute names and values.

        Returns
        -------
        dict
        """
        return collections.OrderedDict(
            col_off=self.col_off, row_off=self.row_off, width=self.width,
            height=self.height)

    def toranges(self):
        """Makes an equivalent pair of range tuples"""
        return (
            (self.row_off, self.row_off + self.height),
            (self.col_off, self.col_off + self.width))

    def toslices(self):
        """Slice objects for use as an ndarray indexer.

        Returns
        -------
        row_slice, col_slice: slice
            A pair of slices in row, column order

        """
        (r0, r1), (c0, c1) = self.toranges()

        if r0 < 0:
            r0 = 0
        if r1 < 0:
            r1 = 0
        if c0 < 0:
            c0 = 0
        if c1 < 0:
            c1 = 0

        return (
            slice(int(math.floor(r0)), int(math.ceil(r1))),
            slice(int(math.floor(c0)), int(math.ceil(c1))),
        )

    @classmethod
    def from_slices(cls, rows, cols, height=-1, width=-1, boundless=False):
        """Construct a Window from row and column slices or tuples / lists of
        start and stop indexes. Converts the rows and cols to offsets, height,
        and width.

        In general, indexes are defined relative to the upper left corner of
        the dataset: rows=(0, 10), cols=(0, 4) defines a window that is 4
        columns wide and 10 rows high starting from the upper left.

        Start indexes may be `None` and will default to 0.
        Stop indexes may be `None` and will default to width or height, which
        must be provided in this case.

        Negative start indexes are evaluated relative to the lower right of the
        dataset: rows=(-2, None), cols=(-2, None) defines a window that is 2
        rows high and 2 columns wide starting from the bottom right.

        Parameters
        ----------
        rows, cols: slice, tuple, or list
            Slices or 2 element tuples/lists containing start, stop indexes.
        height, width: float
            A shape to resolve relative values against. Only used when a start
            or stop index is negative or a stop index is None.
        boundless: bool, optional
            Whether the inputs are bounded (default) or not.

        Returns
        -------
        Window
        """

        # Normalize to slices
        if isinstance(rows, (tuple, list)):
            if len(rows) != 2:
                raise WindowError("rows must have a start and stop index")
            rows = slice(*rows)

        elif not isinstance(rows, slice):
            raise WindowError("rows must be a slice, tuple, or list")

        if isinstance(cols, (tuple, list)):
            if len(cols) != 2:
                raise WindowError("cols must have a start and stop index")
            cols = slice(*cols)

        elif not isinstance(cols, slice):
            raise WindowError("cols must be a slice, tuple, or list")

        # Height and width are required if stop indices are implicit
        if rows.stop is None and height < 0:
            raise WindowError("height is required if row stop index is None")

        if cols.stop is None and width < 0:
            raise WindowError("width is required if col stop index is None")

        # Convert implicit indices to offsets, height, and width
        row_off = 0.0 if rows.start is None else rows.start
        row_stop = height if rows.stop is None else rows.stop

        col_off = 0.0 if cols.start is None else cols.start
        col_stop = width if cols.stop is None else cols.stop

        if not boundless:
            if (row_off < 0 or row_stop < 0):
                if height < 0:
                    raise WindowError("height is required when providing "
                                      "negative indexes")

                if row_off < 0:
                    row_off += height

                if row_stop < 0:
                    row_stop += height

            if (col_off < 0 or col_stop < 0):
                if width < 0:
                    raise WindowError("width is required when providing "
                                      "negative indexes")

                if col_off < 0:
                    col_off += width

                if col_stop < 0:
                    col_stop += width

        num_cols = max(col_stop - col_off, 0.0)
        num_rows = max(row_stop - row_off, 0.0)

        return cls(col_off=col_off, row_off=row_off, width=num_cols,
                   height=num_rows)

    def round_lengths(self, **kwds):
        """Return a copy with width and height rounded.

        Lengths are rounded to the preceding (floor) or succeeding (ceil)
        whole number. The offsets are not changed.

        Parameters
        ----------
        kwds : dict
            Collects keyword arguments that are no longer used.

        Returns
        -------
        Window

        """
        operator = lambda x: int(math.floor(x + 0.5))
        width = operator(self.width)
        height = operator(self.height)
        return Window(self.col_off, self.row_off, width, height)

    def round_shape(self, **kwds):
        warnings.warn(
            "round_shape is deprecated and will be removed in Rasterio 2.0.0.",
            RasterioDeprecationWarning,
        )
        return self.round_lengths(**kwds)

    def round_offsets(self, **kwds):
        """Return a copy with column and row offsets rounded.

        Offsets are rounded to the preceding (floor) or succeeding (ceil)
        whole number. The lengths are not changed.

        Parameters
        ----------
        kwds : dict
            Collects keyword arguments that are no longer used.

        Returns
        -------
        Window

        """
        operator = lambda x: int(math.floor(x + 0.001))
        row_off = operator(self.row_off)
        col_off = operator(self.col_off)
        return Window(col_off, row_off, self.width, self.height)

    def crop(self, height, width):
        """Return a copy cropped to height and width"""
        return crop(self, height, width)

    def intersection(self, other):
        """Return the intersection of this window and another

        Parameters
        ----------

        other: Window
            Another window

        Returns
        -------
        Window
        """
        return intersection([self, other])

env_pyrecodes/lib/python3.9/site-packages/rasterio/windows.py
NikolaBlagojevic/pyrecodes
# Copyright 2014 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Dict, Optional, Set

import nh3


ALLOWED_TAGS = {
    # Bleach Defaults
    "a", "abbr", "acronym", "b", "blockquote", "code", "em", "i", "li", "ol",
    "strong", "ul",

    # Custom Additions
    "br", "caption", "cite", "col", "colgroup", "dd", "del", "details", "div",
    "dl", "dt", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "img", "p", "pre",
    "span", "sub", "summary", "sup", "table", "tbody", "td", "th", "thead",
    "tr", "tt", "kbd", "var", "input", "section", "aside", "nav", "s", "figure",
    "figcaption",
}

ALLOWED_ATTRIBUTES = {
    # Bleach Defaults
    "a": {"href", "title"},
    "abbr": {"title"},
    "acronym": {"title"},

    # Custom Additions
    "*": {"id"},
    "hr": {"class"},
    "img": {"src", "width", "height", "alt", "align", "class"},
    "span": {"class"},
    "th": {"align", "class"},
    "td": {"align", "colspan", "rowspan"},
    "div": {"align", "class"},
    "h1": {"align"},
    "h2": {"align"},
    "h3": {"align"},
    "h4": {"align"},
    "h5": {"align"},
    "h6": {"align"},
    "code": {"class"},
    "p": {"align", "class"},
    "pre": {"lang"},
    "ol": {"start"},
    "input": {"type", "checked", "disabled"},
    "aside": {"class"},
    "dd": {"class"},
    "dl": {"class"},
    "dt": {"class"},
    "ul": {"class"},
    "nav": {"class"},
    "figure": {"class"},
}


def clean(
    html: str,
    tags: Optional[Set[str]] = None,
    attributes: Optional[Dict[str, Set[str]]] = None
) -> Optional[str]:
    if tags is None:
        tags = ALLOWED_TAGS
    if attributes is None:
        attributes = ALLOWED_ATTRIBUTES

    try:
        cleaned = nh3.clean(
            html,
            tags=ALLOWED_TAGS,
            attributes=ALLOWED_ATTRIBUTES,
            link_rel="nofollow",
            url_schemes={"http", "https", "mailto"},
        )

        return cleaned
    except ValueError:
        return None

env_pyrecodes/lib/python3.9/site-packages/readme_renderer/clean.py
NikolaBlagojevic/pyrecodes
"""
Cross-specification, implementation-agnostic JSON referencing.
"""
from referencing._core import Anchor, Registry, Resource, Specification

__all__ = ["Anchor", "Registry", "Resource", "Specification"]

env_pyrecodes/lib/python3.9/site-packages/referencing/__init__.py
NikolaBlagojevic/pyrecodes
from __future__ import annotations

from typing import NoReturn, TypeVar

from attrs import define as _define, frozen as _frozen

_T = TypeVar("_T")


def define(cls: type[_T]) -> type[_T]:  # pragma: no cover
    cls.__init_subclass__ = _do_not_subclass
    return _define(cls)


def frozen(cls: type[_T]) -> type[_T]:
    cls.__init_subclass__ = _do_not_subclass
    return _frozen(cls)


class UnsupportedSubclassing(Exception):
    pass


@staticmethod
def _do_not_subclass() -> NoReturn:  # pragma: no cover
    raise UnsupportedSubclassing(
        "Subclassing is not part of referencing's public API. "
        "If no other suitable API exists for what you're trying to do, "
        "feel free to file an issue asking for one.",
    )

env_pyrecodes/lib/python3.9/site-packages/referencing/_attrs.py
NikolaBlagojevic/pyrecodes
from __future__ import annotations

from collections.abc import Iterable, Iterator, Sequence
from typing import Any, Callable, ClassVar, Generic, Protocol, TypeVar
from urllib.parse import unquote, urldefrag, urljoin

from attrs import evolve, field
from rpds import HashTrieMap, HashTrieSet, List

from referencing import exceptions
from referencing._attrs import frozen
from referencing.typing import URI, Anchor as AnchorType, D, Mapping, Retrieve

EMPTY_UNCRAWLED: HashTrieSet[URI] = HashTrieSet()
EMPTY_PREVIOUS_RESOLVERS: List[URI] = List()


class _MaybeInSubresource(Protocol[D]):
    def __call__(
        self,
        segments: Sequence[int | str],
        resolver: Resolver[D],
        subresource: Resource[D],
    ) -> Resolver[D]:
        ...


@frozen
class Specification(Generic[D]):
    """
    A specification which defines referencing behavior.

    The various methods of a `Specification` allow for varying referencing
    behavior across JSON Schema specification versions, etc.
    """

    #: A short human-readable name for the specification, used for debugging.
    name: str

    #: Find the ID of a given document.
    id_of: Callable[[D], URI | None]

    #: Retrieve the subresources of the given document (without traversing into
    #: the subresources themselves).
    subresources_of: Callable[[D], Iterable[D]]

    #: While resolving a JSON pointer, conditionally enter a subresource
    #: (if e.g. we have just entered a keyword whose value is a subresource)
    maybe_in_subresource: _MaybeInSubresource[D]

    #: Retrieve the anchors contained in the given document.
    _anchors_in: Callable[
        [Specification[D], D],
        Iterable[AnchorType[D]],
    ] = field(alias="anchors_in")

    #: An opaque specification where resources have no subresources
    #: nor internal identifiers.
    OPAQUE: ClassVar[Specification[Any]]

    def __repr__(self) -> str:
        return f"<Specification name={self.name!r}>"

    def anchors_in(self, contents: D):
        """
        Retrieve the anchors contained in the given document.
        """
        return self._anchors_in(self, contents)

    def create_resource(self, contents: D) -> Resource[D]:
        """
        Create a resource which is interpreted using this specification.
        """
        return Resource(contents=contents, specification=self)


Specification.OPAQUE = Specification(
    name="opaque",
    id_of=lambda contents: None,
    subresources_of=lambda contents: [],
    anchors_in=lambda specification, contents: [],
    maybe_in_subresource=lambda segments, resolver, subresource: resolver,
)


@frozen
class Resource(Generic[D]):
    r"""
    A document (deserialized JSON) with a concrete interpretation under a spec.

    In other words, a Python object, along with an instance of `Specification`
    which describes how the document interacts with referencing -- both
    internally (how it refers to other `Resource`\ s) and externally (how it
    should be identified such that it is referenceable by other documents).
    """

    contents: D
    _specification: Specification[D] = field(alias="specification")

    @classmethod
    def from_contents(
        cls,
        contents: D,
        default_specification: Specification[D] = None,  # type: ignore[reportGeneralTypeIssues]  # noqa: E501
    ) -> Resource[D]:
        """
        Attempt to discern which specification applies to the given contents.

        Raises:

            `CannotDetermineSpecification`

                if the given contents don't have any discernible
                information which could be used to guess which
                specification they identify as
        """
        specification = default_specification
        if isinstance(contents, Mapping):
            jsonschema_dialect_id = contents.get("$schema")  # type: ignore[reportUnknownMemberType]  # noqa: E501
            if jsonschema_dialect_id is not None:
                from referencing.jsonschema import specification_with

                specification = specification_with(
                    jsonschema_dialect_id,  # type: ignore[reportUnknownArgumentType]  # noqa: E501
                    default=default_specification,
                )

        if specification is None:  # type: ignore[reportUnnecessaryComparison]
            raise exceptions.CannotDetermineSpecification(contents)
        return cls(contents=contents, specification=specification)  # type: ignore[reportUnknownArgumentType]  # noqa: E501

    @classmethod
    def opaque(cls, contents: D) -> Resource[D]:
        """
        Create an opaque `Resource` -- i.e. one with opaque specification.

        See `Specification.OPAQUE` for details.
        """
        return Specification.OPAQUE.create_resource(contents=contents)

    def id(self) -> URI | None:
        """
        Retrieve this resource's (specification-specific) identifier.
        """
        id = self._specification.id_of(self.contents)
        if id is None:
            return
        return id.rstrip("#")

    def subresources(self) -> Iterable[Resource[D]]:
        """
        Retrieve this resource's subresources.
        """
        return (
            Resource.from_contents(
                each,
                default_specification=self._specification,
            )
            for each in self._specification.subresources_of(self.contents)
        )

    def anchors(self) -> Iterable[AnchorType[D]]:
        """
        Retrieve this resource's (specification-specific) identifier.
        """
        return self._specification.anchors_in(self.contents)

    def pointer(self, pointer: str, resolver: Resolver[D]) -> Resolved[D]:
        """
        Resolve the given JSON pointer.

        Raises:

            `exceptions.PointerToNowhere`

                if the pointer points to a location not present in the document
        """
        contents = self.contents
        segments: list[int | str] = []
        for segment in unquote(pointer[1:]).split("/"):
            if isinstance(contents, Sequence):
                segment = int(segment)
            else:
                segment = segment.replace("~1", "/").replace("~0", "~")
            try:
                contents = contents[segment]  # type: ignore[reportUnknownArgumentType]  # noqa: E501
            except LookupError:
                raise exceptions.PointerToNowhere(ref=pointer, resource=self)

            segments.append(segment)
            last = resolver
            resolver = self._specification.maybe_in_subresource(
                segments=segments,
                resolver=resolver,
                subresource=self._specification.create_resource(contents),  # type: ignore[reportUnknownArgumentType]  # noqa: E501
            )
            if resolver is not last:
                segments = []
        return Resolved(contents=contents, resolver=resolver)  # type: ignore[reportUnknownArgumentType]  # noqa: E501


def _fail_to_retrieve(uri: URI):
    raise exceptions.NoSuchResource(ref=uri)


@frozen
class Registry(Mapping[URI, Resource[D]]):
    r"""
    A registry of `Resource`\ s, each identified by their canonical URIs.

    Registries store a collection of in-memory resources, and optionally
    enable additional resources which may be stored elsewhere (e.g. in a
    database, a separate set of files, over the network, etc.).

    They also lazily walk their known resources, looking for subresources
    within them. In other words, subresources contained within any added
    resources will be retrievable via their own IDs (though this discovery of
    subresources will be delayed until necessary).

    Registries are immutable, and their methods return new instances of the
    registry with the additional resources added to them.

    The ``retrieve`` argument can be used to configure retrieval of resources
    dynamically, either over the network, from a database, or the like.
    Pass it a callable which will be called if any URI not present in the
    registry is accessed. It must either return a `Resource` or else raise a
    `NoSuchResource` exception indicating that the resource does not exist
    even according to the retrieval logic.
    """

    _resources: HashTrieMap[URI, Resource[D]] = field(
        default=HashTrieMap(),
        converter=HashTrieMap.convert,  # type: ignore[reportGeneralTypeIssues]  # noqa: E501
        alias="resources",
    )
    _anchors: HashTrieMap[tuple[URI, str], AnchorType[D]] = HashTrieMap()  # type: ignore[reportGeneralTypeIssues]  # noqa: E501
    _uncrawled: HashTrieSet[URI] = EMPTY_UNCRAWLED
    _retrieve: Retrieve[D] = field(default=_fail_to_retrieve, alias="retrieve")

    def __getitem__(self, uri: URI) -> Resource[D]:
        """
        Return the (already crawled) `Resource` identified by the given URI.
        """
        try:
            return self._resources[uri.rstrip("#")]
        except KeyError:
            raise exceptions.NoSuchResource(ref=uri)

    def __iter__(self) -> Iterator[URI]:
        """
        Iterate over all crawled URIs in the registry.
        """
        return iter(self._resources)

    def __len__(self) -> int:
        """
        Count the total number of fully crawled resources in this registry.
        """
        return len(self._resources)

    def __rmatmul__(
        self,
        new: Resource[D] | Iterable[Resource[D]],
    ) -> Registry[D]:
        """
        Create a new registry with resource(s) added using their internal IDs.

        Resources must have a internal IDs (e.g. the :kw:`$id` keyword in
        modern JSON Schema versions), otherwise an error will be raised.

        Both a single resource as well as an iterable of resources works, i.e.:

            * ``resource @ registry`` or

            * ``[iterable, of, multiple, resources] @ registry``

        which -- again, assuming the resources have internal IDs -- is
        equivalent to calling `Registry.with_resources` as such:

        .. code:: python

            registry.with_resources(
                (resource.id(), resource) for resource in new_resources
            )

        Raises:

            `NoInternalID`

                if the resource(s) in fact do not have IDs
        """
        if isinstance(new, Resource):
            new = (new,)

        resources = self._resources
        uncrawled = self._uncrawled
        for resource in new:
            id = resource.id()
            if id is None:
                raise exceptions.NoInternalID(resource=resource)
            uncrawled = uncrawled.insert(id)
            resources = resources.insert(id, resource)
        return evolve(self, resources=resources, uncrawled=uncrawled)

    def __repr__(self) -> str:
        size = len(self)
        pluralized = "resource" if size == 1 else "resources"
        if self._uncrawled:
            uncrawled = len(self._uncrawled)
            if uncrawled == size:
                summary = f"uncrawled {pluralized}"
            else:
                summary = f"{pluralized}, {uncrawled} uncrawled"
        else:
            summary = f"{pluralized}"
        return f"<Registry ({size} {summary})>"

    def get_or_retrieve(self, uri: URI) -> Retrieved[D, Resource[D]]:
        """
        Get a resource from the registry, crawling or retrieving if necessary.

        May involve crawling to find the given URI if it is not already known,
        so the returned object is a `Retrieved` object which contains both the
        resource value as well as the registry which ultimately contained it.
        """
        resource = self._resources.get(uri)
        if resource is not None:
            return Retrieved(registry=self, value=resource)

        registry = self.crawl()
        resource = registry._resources.get(uri)
        if resource is not None:
            return Retrieved(registry=registry, value=resource)

        try:
            resource = registry._retrieve(uri)
        except (
            exceptions.CannotDetermineSpecification,
            exceptions.NoSuchResource,
        ):
            raise
        except Exception:
            raise exceptions.Unretrievable(ref=uri)
        else:
            registry = registry.with_resource(uri, resource)
            return Retrieved(registry=registry, value=resource)

    def remove(self, uri: URI):
        """
        Return a registry with the resource identified by a given URI removed.
        """
        if uri not in self._resources:
            raise exceptions.NoSuchResource(ref=uri)

        return evolve(
            self,
            resources=self._resources.remove(uri),
            uncrawled=self._uncrawled.discard(uri),
            anchors=HashTrieMap(
                (k, v) for k, v in self._anchors.items() if k[0] != uri
            ),
        )

    def anchor(self, uri: URI, name: str):
        """
        Retrieve a given anchor from a resource which must already be crawled.
        """
        value = self._anchors.get((uri, name))
        if value is not None:
            return Retrieved(value=value, registry=self)

        registry = self.crawl()
        value = registry._anchors.get((uri, name))
        if value is not None:
            return Retrieved(value=value, registry=registry)

        resource = self[uri]
        canonical_uri = resource.id()
        if canonical_uri is not None:
            value = registry._anchors.get((canonical_uri, name))
            if value is not None:
                return Retrieved(value=value, registry=registry)

        if "/" in name:
            raise exceptions.InvalidAnchor(
                ref=uri,
                resource=resource,
                anchor=name,
            )
        raise exceptions.NoSuchAnchor(ref=uri, resource=resource, anchor=name)

    def contents(self, uri: URI) -> D:
        """
        Retrieve the (already crawled) contents identified by the given URI.
        """
        # Empty fragment URIs are equivalent to URIs without the fragment.
        # TODO: Is this true for non JSON Schema resources? Probably not.
        return self._resources[uri.rstrip("#")].contents

    def crawl(self) -> Registry[D]:
        """
        Crawl all added resources, discovering subresources.
        """
        resources = self._resources
        anchors = self._anchors
        uncrawled = [(uri, resources[uri]) for uri in self._uncrawled]
        while uncrawled:
            uri, resource = uncrawled.pop()

            id = resource.id()
            if id is not None:
                uri = urljoin(uri, id)
                resources = resources.insert(uri, resource)
            for each in resource.anchors():
                anchors = anchors.insert((uri, each.name), each)
            uncrawled.extend((uri, each) for each in resource.subresources())
        return evolve(
            self,
            resources=resources,
            anchors=anchors,
            uncrawled=EMPTY_UNCRAWLED,
        )

    def with_resource(self, uri: URI, resource: Resource[D]):
        """
        Add the given `Resource` to the registry, without crawling it.
        """
        return self.with_resources([(uri, resource)])

    def with_resources(
        self,
        pairs: Iterable[tuple[URI, Resource[D]]],
    ) -> Registry[D]:
        r"""
        Add the given `Resource`\ s to the registry, without crawling them.
        """
        resources = self._resources
        uncrawled = self._uncrawled
        for uri, resource in pairs:
            # Empty fragment URIs are equivalent to URIs without the fragment.
            # TODO: Is this true for non JSON Schema resources? Probably not.
            uri = uri.rstrip("#")
            uncrawled = uncrawled.insert(uri)
            resources = resources.insert(uri, resource)
        return evolve(self, resources=resources, uncrawled=uncrawled)

    def with_contents(
        self,
        pairs: Iterable[tuple[URI, D]],
        **kwargs: Any,
    ) -> Registry[D]:
        r"""
        Add the given contents to the registry, autodetecting when necessary.
        """
        return self.with_resources(
            (uri, Resource.from_contents(each, **kwargs))
            for uri, each in pairs
        )

    def combine(self, *registries: Registry[D]) -> Registry[D]:
        """
        Combine together one or more other registries, producing a unified one.
        """
        if registries == (self,):
            return self
        resources = self._resources
        anchors = self._anchors
        uncrawled = self._uncrawled
        retrieve = self._retrieve
        for registry in registries:
            resources = resources.update(registry._resources)  # type: ignore[reportUnknownMemberType]  # noqa: E501
            anchors = anchors.update(registry._anchors)  # type: ignore[reportUnknownMemberType]  # noqa: E501
            uncrawled = uncrawled.update(registry._uncrawled)

            if registry._retrieve is not _fail_to_retrieve:
                if registry._retrieve is not retrieve is not _fail_to_retrieve:
                    raise ValueError(
                        "Cannot combine registries with conflicting retrieval "
                        "functions.",
                    )
                retrieve = registry._retrieve
        return evolve(
            self,
            anchors=anchors,
            resources=resources,
            uncrawled=uncrawled,
            retrieve=retrieve,
        )

    def resolver(self, base_uri: URI = "") -> Resolver[D]:
        """
        Return a `Resolver` which resolves references against this registry.
        """
        return Resolver(base_uri=base_uri, registry=self)

    def resolver_with_root(self, resource: Resource[D]) -> Resolver[D]:
        """
        Return a `Resolver` with a specific root resource.
        """
        uri = resource.id() or ""
        return Resolver(
            base_uri=uri,
            registry=self.with_resource(uri, resource),
        )


#: An anchor or resource.
AnchorOrResource = TypeVar("AnchorOrResource", AnchorType[Any], Resource[Any])


@frozen
class Retrieved(Generic[D, AnchorOrResource]):
    """
    A value retrieved from a `Registry`.
    """

    value: AnchorOrResource
    registry: Registry[D]


@frozen
class Resolved(Generic[D]):
    """
    A reference resolved to its contents by a `Resolver`.
    """

    contents: D
    resolver: Resolver[D]


@frozen
class Resolver(Generic[D]):
    """
    A reference resolver.

    Resolvers help resolve references (including relative ones) by
    pairing a fixed base URI with a `Registry`.

    This object, under normal circumstances, is expected to be used by
    *implementers of libraries* built on top of `referencing` (e.g. JSON Schema
    implementations or other libraries resolving JSON references),
    not directly by end-users populating registries or while writing
    schemas or other resources.

    References are resolved against the base URI, and the combined URI
    is then looked up within the registry.

    The process of resolving a reference may itself involve calculating
    a *new* base URI for future reference resolution (e.g. if an
    intermediate resource sets a new base URI), or may involve encountering
    additional subresources and adding them to a new registry.
    """

    _base_uri: str = field(alias="base_uri")
    _registry: Registry[D] = field(alias="registry")
    _previous: List[URI] = field(default=List(), repr=False, alias="previous")

    def lookup(self, ref: URI) -> Resolved[D]:
        """
        Resolve the given reference to the resource it points to.

        Raises:

            `exceptions.Unresolvable`

                or a subclass thereof (see below) if the reference isn't
                resolvable

            `exceptions.NoSuchAnchor`

                if the reference is to a URI where a resource exists but
                contains a plain name fragment which does not exist within
                the resource

            `exceptions.PointerToNowhere`

                if the reference is to a URI where a resource exists but
                contains a JSON pointer to a location within the resource
                that does not exist
        """
        if ref.startswith("#"):
            uri, fragment = self._base_uri, ref[1:]
        else:
            uri, fragment = urldefrag(urljoin(self._base_uri, ref))
        try:
            retrieved = self._registry.get_or_retrieve(uri)
        except exceptions.NoSuchResource:
            raise exceptions.Unresolvable(ref=ref) from None
        except exceptions.Unretrievable:
            raise exceptions.Unresolvable(ref=ref)

        if fragment.startswith("/"):
            resolver = self._evolve(registry=retrieved.registry, base_uri=uri)
            return retrieved.value.pointer(pointer=fragment, resolver=resolver)

        if fragment:
            retrieved = retrieved.registry.anchor(uri, fragment)
            resolver = self._evolve(registry=retrieved.registry, base_uri=uri)
            return retrieved.value.resolve(resolver=resolver)

        resolver = self._evolve(registry=retrieved.registry, base_uri=uri)
        return Resolved(contents=retrieved.value.contents, resolver=resolver)

    def in_subresource(self, subresource: Resource[D]) -> Resolver[D]:
        """
        Create a resolver for a subresource (which may have a new base URI).
        """
        id = subresource.id()
        if id is None:
            return self
        return evolve(self, base_uri=urljoin(self._base_uri, id))

    def dynamic_scope(self) -> Iterable[tuple[URI, Registry[D]]]:
        """
        In specs with such a notion, return the URIs in the dynamic scope.
        """
        for uri in self._previous:
            yield uri, self._registry

    def _evolve(self, base_uri: str, **kwargs: Any):
        """
        Evolve, appending to the dynamic scope.
        """
        previous = self._previous
        if self._base_uri and (not previous or base_uri != self._base_uri):
            previous = previous.push_front(self._base_uri)
        return evolve(self, base_uri=base_uri, previous=previous, **kwargs)


@frozen
class Anchor(Generic[D]):
    """
    A simple anchor in a `Resource`.
    """

    name: str
    resource: Resource[D]

    def resolve(self, resolver: Resolver[D]):
        """
        Return the resource for this anchor.
        """
        return Resolved(contents=self.resource.contents, resolver=resolver)

env_pyrecodes/lib/python3.9/site-packages/referencing/_core.py
NikolaBlagojevic/pyrecodes
"""
Errors, oh no!
"""
from __future__ import annotations

from typing import TYPE_CHECKING, Any

import attrs

from referencing._attrs import frozen
from referencing.typing import URI

if TYPE_CHECKING:
    from referencing import Resource


@frozen
class NoSuchResource(KeyError):
    """
    The given URI is not present in a registry.

    Unlike most exceptions, this class *is* intended to be publicly
    instantiable and *is* part of the public API of the package.
    """

    ref: URI

    def __eq__(self, other: Any) -> bool:
        if self.__class__ is not other.__class__:
            return NotImplemented
        return attrs.astuple(self) == attrs.astuple(other)

    def __hash__(self) -> int:
        return hash(attrs.astuple(self))


@frozen
class NoInternalID(Exception):
    """
    A resource has no internal ID, but one is needed.

    E.g. in modern JSON Schema drafts, this is the :kw:`$id` keyword.

    One might be needed if a resource was to-be added to a registry but no
    other URI is available, and the resource doesn't declare its canonical URI.
    """

    resource: Resource[Any]

    def __eq__(self, other: Any) -> bool:
        if self.__class__ is not other.__class__:
            return NotImplemented
        return attrs.astuple(self) == attrs.astuple(other)

    def __hash__(self) -> int:
        return hash(attrs.astuple(self))


@frozen
class Unretrievable(KeyError):
    """
    The given URI is not present in a registry, and retrieving it failed.
    """

    ref: URI

    def __eq__(self, other: Any) -> bool:
        if self.__class__ is not other.__class__:
            return NotImplemented
        return attrs.astuple(self) == attrs.astuple(other)

    def __hash__(self) -> int:
        return hash(attrs.astuple(self))


@frozen
class CannotDetermineSpecification(Exception):
    """
    Attempting to detect the appropriate `Specification` failed.

    This happens if no discernible information is found in the contents of the
    new resource which would help identify it.
    """

    contents: Any

    def __eq__(self, other: Any) -> bool:
        if self.__class__ is not other.__class__:
            return NotImplemented
        return attrs.astuple(self) == attrs.astuple(other)

    def __hash__(self) -> int:
        return hash(attrs.astuple(self))


@attrs.frozen  # Because here we allow subclassing below.
class Unresolvable(Exception):
    """
    A reference was unresolvable.
    """

    ref: URI

    def __eq__(self, other: Any) -> bool:
        if self.__class__ is not other.__class__:
            return NotImplemented
        return attrs.astuple(self) == attrs.astuple(other)

    def __hash__(self) -> int:
        return hash(attrs.astuple(self))


@frozen
class PointerToNowhere(Unresolvable):
    """
    A JSON Pointer leads to a part of a document that does not exist.
    """

    resource: Resource[Any]

    def __str__(self) -> str:
        msg = f"{self.ref!r} does not exist within {self.resource.contents!r}"
        if self.ref == "/":
            msg += (
                ". The pointer '/' is a valid JSON Pointer but it points to "
                "an empty string property ''. If you intended to point "
                "to the entire resource, you should use '#'."
            )
        return msg


@frozen
class NoSuchAnchor(Unresolvable):
    """
    An anchor does not exist within a particular resource.
    """

    resource: Resource[Any]
    anchor: str

    def __str__(self) -> str:
        return (
            f"{self.anchor!r} does not exist within {self.resource.contents!r}"
        )


@frozen
class InvalidAnchor(Unresolvable):
    """
    An anchor which could never exist in a resource was dereferenced.

    It is somehow syntactically invalid.
    """

    resource: Resource[Any]
    anchor: str

    def __str__(self) -> str:
        return (
            f"'#{self.anchor}' is not a valid anchor, neither as a "
            "plain name anchor nor as a JSON Pointer. You may have intended "
            f"to use '#/{self.anchor}', as the slash is required *before each "
            "segment* of a JSON pointer."
        )

env_pyrecodes/lib/python3.9/site-packages/referencing/exceptions.py
NikolaBlagojevic/pyrecodes
"""
Referencing implementations for JSON Schema specs (historic & current).
"""

from __future__ import annotations

from collections.abc import Sequence, Set
from typing import Any, Iterable, Union

from referencing import Anchor, Registry, Resource, Specification, exceptions
from referencing._attrs import frozen
from referencing._core import Resolved as _Resolved, Resolver as _Resolver
from referencing.typing import URI, Anchor as AnchorType, Mapping

#: A JSON Schema which is a JSON object
ObjectSchema = Mapping[str, Any]

#: A JSON Schema of any kind
Schema = Union[bool, ObjectSchema]

#: A JSON Schema Registry
SchemaRegistry = Registry[Schema]


@frozen
class UnknownDialect(Exception):
    """
    A dialect identifier was found for a dialect unknown by this library.

    If it's a custom ("unofficial") dialect, be sure you've registered it.
    """

    uri: URI


def _dollar_id(contents: Schema) -> URI | None:
    if isinstance(contents, bool):
        return
    return contents.get("$id")


def _legacy_dollar_id(contents: Schema) -> URI | None:
    if isinstance(contents, bool) or "$ref" in contents:
        return
    id = contents.get("$id")
    if id is not None and not id.startswith("#"):
        return id


def _legacy_id(contents: ObjectSchema) -> URI | None:
    if "$ref" in contents:
        return
    id = contents.get("id")
    if id is not None and not id.startswith("#"):
        return id


def _anchor(
    specification: Specification[Schema],
    contents: Schema,
) -> Iterable[AnchorType[Schema]]:
    if isinstance(contents, bool):
        return
    anchor = contents.get("$anchor")
    if anchor is not None:
        yield Anchor(
            name=anchor,
            resource=specification.create_resource(contents),
        )

    dynamic_anchor = contents.get("$dynamicAnchor")
    if dynamic_anchor is not None:
        yield DynamicAnchor(
            name=dynamic_anchor,
            resource=specification.create_resource(contents),
        )


def _anchor_2019(
    specification: Specification[Schema],
    contents: Schema,
) -> Iterable[Anchor[Schema]]:
    if isinstance(contents, bool):
        return []
    anchor = contents.get("$anchor")
    if anchor is None:
        return []
    return [
        Anchor(
            name=anchor,
            resource=specification.create_resource(contents),
        ),
    ]


def _legacy_anchor_in_dollar_id(
    specification: Specification[Schema],
    contents: Schema,
) -> Iterable[Anchor[Schema]]:
    if isinstance(contents, bool):
        return []
    id = contents.get("$id", "")
    if not id.startswith("#"):
        return []
    return [
        Anchor(
            name=id[1:],
            resource=specification.create_resource(contents),
        ),
    ]


def _legacy_anchor_in_id(
    specification: Specification[ObjectSchema],
    contents: ObjectSchema,
) -> Iterable[Anchor[ObjectSchema]]:
    id = contents.get("id", "")
    if not id.startswith("#"):
        return []
    return [
        Anchor(
            name=id[1:],
            resource=specification.create_resource(contents),
        ),
    ]


def _subresources_of(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    """
    Create a callable returning JSON Schema specification-style subschemas.

    Relies on specifying the set of keywords containing subschemas in their
    values, in a subobject's values, or in a subarray.
    """

    def subresources_of(contents: Schema) -> Iterable[ObjectSchema]:
        if isinstance(contents, bool):
            return
        for each in in_value:
            if each in contents:
                yield contents[each]
        for each in in_subarray:
            if each in contents:
                yield from contents[each]
        for each in in_subvalues:
            if each in contents:
                yield from contents[each].values()

    return subresources_of


def _subresources_of_with_crazy_items(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    """
    Specifically handle older drafts where there are some funky keywords.
    """

    def subresources_of(contents: Schema) -> Iterable[ObjectSchema]:
        if isinstance(contents, bool):
            return
        for each in in_value:
            if each in contents:
                yield contents[each]
        for each in in_subarray:
            if each in contents:
                yield from contents[each]
        for each in in_subvalues:
            if each in contents:
                yield from contents[each].values()

        items = contents.get("items")
        if items is not None:
            if isinstance(items, Sequence):
                yield from items
            else:
                yield items

    return subresources_of


def _subresources_of_with_crazy_items_dependencies(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    """
    Specifically handle older drafts where there are some funky keywords.
    """

    def subresources_of(contents: Schema) -> Iterable[ObjectSchema]:
        if isinstance(contents, bool):
            return
        for each in in_value:
            if each in contents:
                yield contents[each]
        for each in in_subarray:
            if each in contents:
                yield from contents[each]
        for each in in_subvalues:
            if each in contents:
                yield from contents[each].values()

        items = contents.get("items")
        if items is not None:
            if isinstance(items, Sequence):
                yield from items
            else:
                yield items
        dependencies = contents.get("dependencies")
        if dependencies is not None:
            values = iter(dependencies.values())
            value = next(values, None)
            if isinstance(value, Mapping):
                yield value
                yield from values

    return subresources_of


def _subresources_of_with_crazy_aP_items_dependencies(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    """
    Specifically handle even older drafts where there are some funky keywords.
    """

    def subresources_of(contents: ObjectSchema) -> Iterable[ObjectSchema]:
        for each in in_value:
            if each in contents:
                yield contents[each]
        for each in in_subarray:
            if each in contents:
                yield from contents[each]
        for each in in_subvalues:
            if each in contents:
                yield from contents[each].values()

        items = contents.get("items")
        if items is not None:
            if isinstance(items, Sequence):
                yield from items
            else:
                yield items
        dependencies = contents.get("dependencies")
        if dependencies is not None:
            values = iter(dependencies.values())
            value = next(values, None)
            if isinstance(value, Mapping):
                yield value
                yield from values

        for each in "additionalItems", "additionalProperties":
            value = contents.get(each)
            if isinstance(value, Mapping):
                yield value

    return subresources_of


def _maybe_in_subresource(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    in_child = in_subvalues | in_subarray

    def maybe_in_subresource(
        segments: Sequence[int | str],
        resolver: _Resolver[Any],
        subresource: Resource[Any],
    ) -> _Resolver[Any]:
        _segments = iter(segments)
        for segment in _segments:
            if segment not in in_value and (
                segment not in in_child or next(_segments, None) is None
            ):
                return resolver
        return resolver.in_subresource(subresource)

    return maybe_in_subresource


def _maybe_in_subresource_crazy_items(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    in_child = in_subvalues | in_subarray

    def maybe_in_subresource(
        segments: Sequence[int | str],
        resolver: _Resolver[Any],
        subresource: Resource[Any],
    ) -> _Resolver[Any]:
        _segments = iter(segments)
        for segment in _segments:
            if segment == "items" and isinstance(
                subresource.contents,
                Mapping,
            ):
                return resolver.in_subresource(subresource)
            if segment not in in_value and (
                segment not in in_child or next(_segments, None) is None
            ):
                return resolver
        return resolver.in_subresource(subresource)

    return maybe_in_subresource


def _maybe_in_subresource_crazy_items_dependencies(
    in_value: Set[str] = frozenset(),
    in_subvalues: Set[str] = frozenset(),
    in_subarray: Set[str] = frozenset(),
):
    in_child = in_subvalues | in_subarray

    def maybe_in_subresource(
        segments: Sequence[int | str],
        resolver: _Resolver[Any],
        subresource: Resource[Any],
    ) -> _Resolver[Any]:
        _segments = iter(segments)
        for segment in _segments:
            if (
                segment == "items" or segment == "dependencies"
            ) and isinstance(subresource.contents, Mapping):
                return resolver.in_subresource(subresource)
            if segment not in in_value and (
                segment not in in_child or next(_segments, None) is None
            ):
                return resolver
        return resolver.in_subresource(subresource)

    return maybe_in_subresource


#: JSON Schema draft 2020-12
DRAFT202012 = Specification(
    name="draft2020-12",
    id_of=_dollar_id,
    subresources_of=_subresources_of(
        in_value={
            "additionalProperties",
            "contains",
            "contentSchema",
            "else",
            "if",
            "items",
            "not",
            "propertyNames",
            "then",
            "unevaluatedItems",
            "unevaluatedProperties",
        },
        in_subarray={"allOf", "anyOf", "oneOf", "prefixItems"},
        in_subvalues={
            "$defs",
            "dependentSchemas",
            "patternProperties",
            "properties",
        },
    ),
    anchors_in=_anchor,
    maybe_in_subresource=_maybe_in_subresource(
        in_value={
            "additionalProperties",
            "contains",
            "contentSchema",
            "else",
            "if",
            "items",
            "not",
            "propertyNames",
            "then",
            "unevaluatedItems",
            "unevaluatedProperties",
        },
        in_subarray={"allOf", "anyOf", "oneOf", "prefixItems"},
        in_subvalues={
            "$defs",
            "dependentSchemas",
            "patternProperties",
            "properties",
        },
    ),
)
#: JSON Schema draft 2019-09
DRAFT201909 = Specification(
    name="draft2019-09",
    id_of=_dollar_id,
    subresources_of=_subresources_of_with_crazy_items(
        in_value={
            "additionalItems",
            "additionalProperties",
            "contains",
            "contentSchema",
            "else",
            "if",
            "not",
            "propertyNames",
            "then",
            "unevaluatedItems",
            "unevaluatedProperties",
        },
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={
            "$defs",
            "dependentSchemas",
            "patternProperties",
            "properties",
        },
    ),
    anchors_in=_anchor_2019,
    maybe_in_subresource=_maybe_in_subresource_crazy_items(
        in_value={
            "additionalItems",
            "additionalProperties",
            "contains",
            "contentSchema",
            "else",
            "if",
            "not",
            "propertyNames",
            "then",
            "unevaluatedItems",
            "unevaluatedProperties",
        },
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={
            "$defs",
            "dependentSchemas",
            "patternProperties",
            "properties",
        },
    ),
)
#: JSON Schema draft 7
DRAFT7 = Specification(
    name="draft-07",
    id_of=_legacy_dollar_id,
    subresources_of=_subresources_of_with_crazy_items_dependencies(
        in_value={
            "additionalItems",
            "additionalProperties",
            "contains",
            "else",
            "if",
            "not",
            "propertyNames",
            "then",
        },
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
    anchors_in=_legacy_anchor_in_dollar_id,
    maybe_in_subresource=_maybe_in_subresource_crazy_items_dependencies(
        in_value={
            "additionalItems",
            "additionalProperties",
            "contains",
            "else",
            "if",
            "not",
            "propertyNames",
            "then",
        },
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
)
#: JSON Schema draft 6
DRAFT6 = Specification(
    name="draft-06",
    id_of=_legacy_dollar_id,
    subresources_of=_subresources_of_with_crazy_items_dependencies(
        in_value={
            "additionalItems",
            "additionalProperties",
            "contains",
            "not",
            "propertyNames",
        },
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
    anchors_in=_legacy_anchor_in_dollar_id,
    maybe_in_subresource=_maybe_in_subresource_crazy_items_dependencies(
        in_value={
            "additionalItems",
            "additionalProperties",
            "contains",
            "not",
            "propertyNames",
        },
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
)
#: JSON Schema draft 4
DRAFT4 = Specification(
    name="draft-04",
    id_of=_legacy_id,
    subresources_of=_subresources_of_with_crazy_aP_items_dependencies(
        in_value={"not"},
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
    anchors_in=_legacy_anchor_in_id,
    maybe_in_subresource=_maybe_in_subresource_crazy_items_dependencies(
        in_value={"additionalItems", "additionalProperties", "not"},
        in_subarray={"allOf", "anyOf", "oneOf"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
)
#: JSON Schema draft 3
DRAFT3 = Specification(
    name="draft-03",
    id_of=_legacy_id,
    subresources_of=_subresources_of_with_crazy_aP_items_dependencies(
        in_subarray={"extends"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
    anchors_in=_legacy_anchor_in_id,
    maybe_in_subresource=_maybe_in_subresource_crazy_items_dependencies(
        in_value={"additionalItems", "additionalProperties"},
        in_subarray={"extends"},
        in_subvalues={"definitions", "patternProperties", "properties"},
    ),
)


_SPECIFICATIONS: Registry[Specification[Schema]] = Registry(
    {  # type: ignore[reportGeneralTypeIssues]  # :/ internal vs external types
        dialect_id: Resource.opaque(specification)
        for dialect_id, specification in [
            ("https://json-schema.org/draft/2020-12/schema", DRAFT202012),
            ("https://json-schema.org/draft/2019-09/schema", DRAFT201909),
            ("http://json-schema.org/draft-07/schema", DRAFT7),
            ("http://json-schema.org/draft-06/schema", DRAFT6),
            ("http://json-schema.org/draft-04/schema", DRAFT4),
            ("http://json-schema.org/draft-03/schema", DRAFT3),
        ]
    },
)


def specification_with(
    dialect_id: URI,
    default: Specification[Any] = None,  # type: ignore[reportGeneralTypeIssues]  # noqa: E501
) -> Specification[Any]:
    """
    Retrieve the `Specification` with the given dialect identifier.

    Raises:

        `UnknownDialect`

            if the given ``dialect_id`` isn't known
    """
    resource = _SPECIFICATIONS.get(dialect_id.rstrip("#"))
    if resource is not None:
        return resource.contents
    if default is None:  # type: ignore[reportUnnecessaryComparison]
        raise UnknownDialect(dialect_id)
    return default


@frozen
class DynamicAnchor:
    """
    Dynamic anchors, introduced in draft 2020.
    """

    name: str
    resource: Resource[Schema]

    def resolve(self, resolver: _Resolver[Schema]) -> _Resolved[Schema]:
        """
        Resolve this anchor dynamically.
        """
        last = self.resource
        for uri, registry in resolver.dynamic_scope():
            try:
                anchor = registry.anchor(uri, self.name).value
            except exceptions.NoSuchAnchor:
                continue
            if isinstance(anchor, DynamicAnchor):
                last = anchor.resource
        return _Resolved(
            contents=last.contents,
            resolver=resolver.in_subresource(last),
        )


def lookup_recursive_ref(resolver: _Resolver[Schema]) -> _Resolved[Schema]:
    """
    Recursive references (via recursive anchors), present only in draft 2019.

    As per the 2019 specification (§ 8.2.4.2.1), only the ``#`` recursive
    reference is supported (and is therefore assumed to be the relevant
    reference).
    """
    resolved = resolver.lookup("#")
    if isinstance(resolved.contents, Mapping) and resolved.contents.get(
        "$recursiveAnchor",
    ):
        for uri, _ in resolver.dynamic_scope():
            next_resolved = resolver.lookup(uri)
            if not isinstance(
                next_resolved.contents,
                Mapping,
            ) or not next_resolved.contents.get("$recursiveAnchor"):
                break
            resolved = next_resolved
    return resolved

env_pyrecodes/lib/python3.9/site-packages/referencing/jsonschema.py
NikolaBlagojevic/pyrecodes
"""
Helpers related to (dynamic) resource retrieval.
"""
from __future__ import annotations

from functools import lru_cache
from typing import Callable, TypeVar
import json

from referencing import Resource
from referencing.typing import URI, D, Retrieve

#: A serialized document (e.g. a JSON string)
_T = TypeVar("_T")


def to_cached_resource(
    cache: Callable[[Retrieve[D]], Retrieve[D]] | None = None,
    loads: Callable[[_T], D] = json.loads,
    from_contents: Callable[[D], Resource[D]] = Resource.from_contents,
) -> Callable[[Callable[[URI], _T]], Retrieve[D]]:
    """
    Create a retriever which caches its return values from a simpler callable.

    Takes a function which returns things like serialized JSON (strings) and
    returns something suitable for passing to `Registry` as a retrieve
    function.

    This decorator both reduces a small bit of boilerplate for a common case
    (deserializing JSON from strings and creating `Resource` objects from the
    result) as well as makes the probable need for caching a bit easier.
    Retrievers which otherwise do expensive operations (like hitting the
    network) might otherwise be called repeatedly.

    Examples
    --------

    .. testcode::

        from referencing import Registry
        import referencing.retrieval


        @referencing.retrieval.to_cached_resource()
        def retrieve(uri: str):
            print(f"Retrieved {uri}")

            # Normally, go get some expensive JSON from the network, a file ...
            return '''
                {
                    "$schema": "https://json-schema.org/draft/2020-12/schema",
                    "foo": "bar"
                }
            '''

        one = Registry(retrieve=retrieve).get_or_retrieve("urn:example:foo")
        print(one.value.contents["foo"])

        # Retrieving the same URI again reuses the same value (and thus doesn't
        # print another retrieval message here)
        two = Registry(retrieve=retrieve).get_or_retrieve("urn:example:foo")
        print(two.value.contents["foo"])

    .. testoutput::

        Retrieved urn:example:foo
        bar
        bar

    """
    if cache is None:
        cache = lru_cache(maxsize=None)

    def decorator(retrieve: Callable[[URI], _T]):
        @cache
        def cached_retrieve(uri: URI):
            response = retrieve(uri)
            contents = loads(response)
            return from_contents(contents)

        return cached_retrieve

    return decorator

env_pyrecodes/lib/python3.9/site-packages/referencing/retrieval.py
NikolaBlagojevic/pyrecodes
from rpds import HashTrieMap
import pytest

from referencing import Anchor, Registry, Resource, Specification, exceptions
from referencing.jsonschema import DRAFT202012

ID_AND_CHILDREN = Specification(
    name="id-and-children",
    id_of=lambda contents: contents.get("ID"),
    subresources_of=lambda contents: contents.get("children", []),
    anchors_in=lambda specification, contents: [
        Anchor(
            name=name,
            resource=specification.create_resource(contents=each),
        )
        for name, each in contents.get("anchors", {}).items()
    ],
    maybe_in_subresource=lambda segments, resolver, subresource: (
        resolver.in_subresource(subresource)
        if not len(segments) % 2
        and all(each == "children" for each in segments[::2])
        else resolver
    ),
)


def blow_up(uri):  # pragma: no cover
    """
    A retriever suitable for use in tests which expect it never to be used.
    """
    raise RuntimeError("This retrieve function expects to never be called!")


class TestRegistry:
    def test_with_resource(self):
        """
        Adding a resource to the registry then allows re-retrieving it.
        """

        resource = Resource.opaque(contents={"foo": "bar"})
        uri = "urn:example"
        registry = Registry().with_resource(uri=uri, resource=resource)
        assert registry[uri] is resource

    def test_with_resources(self):
        """
        Adding multiple resources to the registry is like adding each one.
        """

        one = Resource.opaque(contents={})
        two = Resource(contents={"foo": "bar"}, specification=ID_AND_CHILDREN)
        registry = Registry().with_resources(
            [
                ("http://example.com/1", one),
                ("http://example.com/foo/bar", two),
            ],
        )
        assert registry == Registry().with_resource(
            uri="http://example.com/1",
            resource=one,
        ).with_resource(
            uri="http://example.com/foo/bar",
            resource=two,
        )

    def test_matmul_resource(self):
        uri = "urn:example:resource"
        resource = ID_AND_CHILDREN.create_resource({"ID": uri, "foo": 12})
        registry = resource @ Registry()
        assert registry == Registry().with_resource(uri, resource)

    def test_matmul_many_resources(self):
        one_uri = "urn:example:one"
        one = ID_AND_CHILDREN.create_resource({"ID": one_uri, "foo": 12})

        two_uri = "urn:example:two"
        two = ID_AND_CHILDREN.create_resource({"ID": two_uri, "foo": 12})

        registry = [one, two] @ Registry()
        assert registry == Registry().with_resources(
            [(one_uri, one), (two_uri, two)],
        )

    def test_matmul_resource_without_id(self):
        resource = Resource.opaque(contents={"foo": "bar"})
        with pytest.raises(exceptions.NoInternalID) as e:
            resource @ Registry()
        assert e.value == exceptions.NoInternalID(resource=resource)

    def test_with_contents_from_json_schema(self):
        uri = "urn:example"
        schema = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        registry = Registry().with_contents([(uri, schema)])

        expected = Resource(contents=schema, specification=DRAFT202012)
        assert registry[uri] == expected

    def test_with_contents_and_default_specification(self):
        uri = "urn:example"
        registry = Registry().with_contents(
            [(uri, {"foo": "bar"})],
            default_specification=Specification.OPAQUE,
        )
        assert registry[uri] == Resource.opaque({"foo": "bar"})

    def test_len(self):
        total = 5
        registry = Registry().with_contents(
            [(str(i), {"foo": "bar"}) for i in range(total)],
            default_specification=Specification.OPAQUE,
        )
        assert len(registry) == total

    def test_iter(self):
        registry = Registry().with_contents(
            [(str(i), {"foo": "bar"}) for i in range(8)],
            default_specification=Specification.OPAQUE,
        )
        assert set(registry) == {str(i) for i in range(8)}

    def test_crawl_still_has_top_level_resource(self):
        resource = Resource.opaque({"foo": "bar"})
        uri = "urn:example"
        registry = Registry({uri: resource}).crawl()
        assert registry[uri] is resource

    def test_crawl_finds_a_subresource(self):
        child_id = "urn:child"
        root = ID_AND_CHILDREN.create_resource(
            {"ID": "urn:root", "children": [{"ID": child_id, "foo": 12}]},
        )
        registry = root @ Registry()
        with pytest.raises(LookupError):
            registry[child_id]

        expected = ID_AND_CHILDREN.create_resource({"ID": child_id, "foo": 12})
        assert registry.crawl()[child_id] == expected

    def test_crawl_finds_anchors_with_id(self):
        resource = ID_AND_CHILDREN.create_resource(
            {"ID": "urn:bar", "anchors": {"foo": 12}},
        )
        registry = resource @ Registry()

        assert registry.crawl().anchor(resource.id(), "foo").value == Anchor(
            name="foo",
            resource=ID_AND_CHILDREN.create_resource(12),
        )

    def test_crawl_finds_anchors_no_id(self):
        resource = ID_AND_CHILDREN.create_resource({"anchors": {"foo": 12}})
        registry = Registry().with_resource("urn:root", resource)

        assert registry.crawl().anchor("urn:root", "foo").value == Anchor(
            name="foo",
            resource=ID_AND_CHILDREN.create_resource(12),
        )

    def test_contents(self):
        resource = Resource.opaque({"foo": "bar"})
        uri = "urn:example"
        registry = Registry().with_resource(uri, resource)
        assert registry.contents(uri) == {"foo": "bar"}

    def test_getitem_strips_empty_fragments(self):
        uri = "http://example.com/"
        resource = ID_AND_CHILDREN.create_resource({"ID": uri + "#"})
        registry = resource @ Registry()
        assert registry[uri] == registry[uri + "#"] == resource

    def test_contents_strips_empty_fragments(self):
        uri = "http://example.com/"
        resource = ID_AND_CHILDREN.create_resource({"ID": uri + "#"})
        registry = resource @ Registry()
        assert (
            registry.contents(uri)
            == registry.contents(uri + "#")
            == {"ID": uri + "#"}
        )

    def test_crawled_anchor(self):
        resource = ID_AND_CHILDREN.create_resource({"anchors": {"foo": "bar"}})
        registry = Registry().with_resource("urn:example", resource)
        retrieved = registry.anchor("urn:example", "foo")
        assert retrieved.value == Anchor(
            name="foo",
            resource=ID_AND_CHILDREN.create_resource("bar"),
        )
        assert retrieved.registry == registry.crawl()

    def test_anchor_in_nonexistent_resource(self):
        registry = Registry()
        with pytest.raises(exceptions.NoSuchResource) as e:
            registry.anchor("urn:example", "foo")
        assert e.value == exceptions.NoSuchResource(ref="urn:example")

    def test_init(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        registry = Registry(
            {
                "http://example.com/1": one,
                "http://example.com/foo/bar": two,
            },
        )
        assert (
            registry
            == Registry()
            .with_resources(
                [
                    ("http://example.com/1", one),
                    ("http://example.com/foo/bar", two),
                ],
            )
            .crawl()
        )

    def test_dict_conversion(self):
        """
        Passing a `dict` to `Registry` gets converted to a `HashTrieMap`.

        So continuing to use the registry works.
        """

        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        registry = Registry(
            {"http://example.com/1": one},
        ).with_resource("http://example.com/foo/bar", two)
        assert (
            registry.crawl()
            == Registry()
            .with_resources(
                [
                    ("http://example.com/1", one),
                    ("http://example.com/foo/bar", two),
                ],
            )
            .crawl()
        )

    def test_no_such_resource(self):
        registry = Registry()
        with pytest.raises(exceptions.NoSuchResource) as e:
            registry["urn:bigboom"]
        assert e.value == exceptions.NoSuchResource(ref="urn:bigboom")

    def test_combine(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        three = ID_AND_CHILDREN.create_resource({"baz": "quux"})
        four = ID_AND_CHILDREN.create_resource({"anchors": {"foo": 12}})

        first = Registry({"http://example.com/1": one})
        second = Registry().with_resource("http://example.com/foo/bar", two)
        third = Registry(
            {
                "http://example.com/1": one,
                "http://example.com/baz": three,
            },
        )
        fourth = (
            Registry()
            .with_resource(
                "http://example.com/foo/quux",
                four,
            )
            .crawl()
        )
        assert first.combine(second, third, fourth) == Registry(
            [
                ("http://example.com/1", one),
                ("http://example.com/baz", three),
                ("http://example.com/foo/quux", four),
            ],
            anchors=HashTrieMap(
                {
                    ("http://example.com/foo/quux", "foo"): Anchor(
                        name="foo",
                        resource=ID_AND_CHILDREN.create_resource(12),
                    ),
                },
            ),
        ).with_resource("http://example.com/foo/bar", two)

    def test_combine_self(self):
        """
        Combining a registry with itself short-circuits.

        This is a performance optimization -- otherwise we do lots more work
        (in jsonschema this seems to correspond to making the test suite take
         *3x* longer).
        """

        registry = Registry({"urn:foo": "bar"})
        assert registry.combine(registry) is registry

    def test_combine_with_uncrawled_resources(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        three = ID_AND_CHILDREN.create_resource({"baz": "quux"})

        first = Registry().with_resource("http://example.com/1", one)
        second = Registry().with_resource("http://example.com/foo/bar", two)
        third = Registry(
            {
                "http://example.com/1": one,
                "http://example.com/baz": three,
            },
        )
        expected = Registry(
            [
                ("http://example.com/1", one),
                ("http://example.com/foo/bar", two),
                ("http://example.com/baz", three),
            ],
        )
        combined = first.combine(second, third)
        assert combined != expected
        assert combined.crawl() == expected

    def test_combine_with_single_retrieve(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        three = ID_AND_CHILDREN.create_resource({"baz": "quux"})

        def retrieve(uri):  # pragma: no cover
            pass

        first = Registry().with_resource("http://example.com/1", one)
        second = Registry(
            retrieve=retrieve,
        ).with_resource("http://example.com/2", two)
        third = Registry().with_resource("http://example.com/3", three)

        assert first.combine(second, third) == Registry(
            retrieve=retrieve,
        ).with_resources(
            [
                ("http://example.com/1", one),
                ("http://example.com/2", two),
                ("http://example.com/3", three),
            ],
        )
        assert second.combine(first, third) == Registry(
            retrieve=retrieve,
        ).with_resources(
            [
                ("http://example.com/1", one),
                ("http://example.com/2", two),
                ("http://example.com/3", three),
            ],
        )

    def test_combine_with_common_retrieve(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        three = ID_AND_CHILDREN.create_resource({"baz": "quux"})

        def retrieve(uri):  # pragma: no cover
            pass

        first = Registry(retrieve=retrieve).with_resource(
            "http://example.com/1",
            one,
        )
        second = Registry(
            retrieve=retrieve,
        ).with_resource("http://example.com/2", two)
        third = Registry(retrieve=retrieve).with_resource(
            "http://example.com/3",
            three,
        )

        assert first.combine(second, third) == Registry(
            retrieve=retrieve,
        ).with_resources(
            [
                ("http://example.com/1", one),
                ("http://example.com/2", two),
                ("http://example.com/3", three),
            ],
        )
        assert second.combine(first, third) == Registry(
            retrieve=retrieve,
        ).with_resources(
            [
                ("http://example.com/1", one),
                ("http://example.com/2", two),
                ("http://example.com/3", three),
            ],
        )

    def test_combine_conflicting_retrieve(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        three = ID_AND_CHILDREN.create_resource({"baz": "quux"})

        def foo_retrieve(uri):  # pragma: no cover
            pass

        def bar_retrieve(uri):  # pragma: no cover
            pass

        first = Registry(retrieve=foo_retrieve).with_resource(
            "http://example.com/1",
            one,
        )
        second = Registry().with_resource("http://example.com/2", two)
        third = Registry(retrieve=bar_retrieve).with_resource(
            "http://example.com/3",
            three,
        )

        with pytest.raises(Exception, match="conflict.*retriev"):
            first.combine(second, third)

    def test_remove(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        registry = Registry({"urn:foo": one, "urn:bar": two})
        assert registry.remove("urn:foo") == Registry({"urn:bar": two})

    def test_remove_uncrawled(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        registry = Registry().with_resources(
            [("urn:foo", one), ("urn:bar", two)],
        )
        assert registry.remove("urn:foo") == Registry().with_resource(
            "urn:bar",
            two,
        )

    def test_remove_with_anchors(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"anchors": {"foo": "bar"}})
        registry = (
            Registry()
            .with_resources(
                [("urn:foo", one), ("urn:bar", two)],
            )
            .crawl()
        )
        assert (
            registry.remove("urn:bar")
            == Registry()
            .with_resource(
                "urn:foo",
                one,
            )
            .crawl()
        )

    def test_remove_nonexistent_uri(self):
        with pytest.raises(exceptions.NoSuchResource) as e:
            Registry().remove("urn:doesNotExist")
        assert e.value == exceptions.NoSuchResource(ref="urn:doesNotExist")

    def test_retrieve(self):
        foo = Resource.opaque({"foo": "bar"})
        registry = Registry(retrieve=lambda uri: foo)
        assert registry.get_or_retrieve("urn:example").value == foo

    def test_retrieve_arbitrary_exception(self):
        foo = Resource.opaque({"foo": "bar"})

        def retrieve(uri):
            if uri == "urn:succeed":
                return foo
            raise Exception("Oh no!")

        registry = Registry(retrieve=retrieve)
        assert registry.get_or_retrieve("urn:succeed").value == foo
        with pytest.raises(exceptions.Unretrievable):
            registry.get_or_retrieve("urn:uhoh")

    def test_retrieve_no_such_resource(self):
        foo = Resource.opaque({"foo": "bar"})

        def retrieve(uri):
            if uri == "urn:succeed":
                return foo
            raise exceptions.NoSuchResource(ref=uri)

        registry = Registry(retrieve=retrieve)
        assert registry.get_or_retrieve("urn:succeed").value == foo
        with pytest.raises(exceptions.NoSuchResource):
            registry.get_or_retrieve("urn:uhoh")

    def test_retrieve_cannot_determine_specification(self):
        def retrieve(uri):
            return Resource.from_contents({})

        registry = Registry(retrieve=retrieve)
        with pytest.raises(exceptions.CannotDetermineSpecification):
            registry.get_or_retrieve("urn:uhoh")

    def test_retrieve_already_available_resource(self):
        foo = Resource.opaque({"foo": "bar"})
        registry = Registry({"urn:example": foo}, retrieve=blow_up)
        assert registry["urn:example"] == foo
        assert registry.get_or_retrieve("urn:example").value == foo

    def test_retrieve_first_checks_crawlable_resource(self):
        child = ID_AND_CHILDREN.create_resource({"ID": "urn:child", "foo": 12})
        root = ID_AND_CHILDREN.create_resource({"children": [child.contents]})
        registry = Registry(retrieve=blow_up).with_resource("urn:root", root)
        assert registry.crawl()["urn:child"] == child

    def test_resolver(self):
        one = Resource.opaque(contents={})
        registry = Registry({"http://example.com": one})
        resolver = registry.resolver(base_uri="http://example.com")
        assert resolver.lookup("#").contents == {}

    def test_resolver_with_root_identified(self):
        root = ID_AND_CHILDREN.create_resource({"ID": "http://example.com"})
        resolver = Registry().resolver_with_root(root)
        assert resolver.lookup("http://example.com").contents == root.contents
        assert resolver.lookup("#").contents == root.contents

    def test_resolver_with_root_unidentified(self):
        root = Resource.opaque(contents={})
        resolver = Registry().resolver_with_root(root)
        assert resolver.lookup("#").contents == root.contents

    def test_repr(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        registry = Registry().with_resources(
            [
                ("http://example.com/1", one),
                ("http://example.com/foo/bar", two),
            ],
        )
        assert repr(registry) == "<Registry (2 uncrawled resources)>"
        assert repr(registry.crawl()) == "<Registry (2 resources)>"

    def test_repr_mixed_crawled(self):
        one = Resource.opaque(contents={})
        two = ID_AND_CHILDREN.create_resource({"foo": "bar"})
        registry = (
            Registry(
                {"http://example.com/1": one},
            )
            .crawl()
            .with_resource(uri="http://example.com/foo/bar", resource=two)
        )
        assert repr(registry) == "<Registry (2 resources, 1 uncrawled)>"

    def test_repr_one_resource(self):
        registry = Registry().with_resource(
            uri="http://example.com/1",
            resource=Resource.opaque(contents={}),
        )
        assert repr(registry) == "<Registry (1 uncrawled resource)>"

    def test_repr_empty(self):
        assert repr(Registry()) == "<Registry (0 resources)>"


class TestResource:
    def test_from_contents_from_json_schema(self):
        schema = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        resource = Resource.from_contents(schema)
        assert resource == Resource(contents=schema, specification=DRAFT202012)

    def test_from_contents_with_no_discernible_information(self):
        """
        Creating a resource with no discernible way to see what
        specification it belongs to (e.g. no ``$schema`` keyword for JSON
        Schema) raises an error.
        """

        with pytest.raises(exceptions.CannotDetermineSpecification):
            Resource.from_contents({"foo": "bar"})

    def test_from_contents_with_no_discernible_information_and_default(self):
        resource = Resource.from_contents(
            {"foo": "bar"},
            default_specification=Specification.OPAQUE,
        )
        assert resource == Resource.opaque(contents={"foo": "bar"})

    def test_from_contents_unneeded_default(self):
        schema = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        resource = Resource.from_contents(
            schema,
            default_specification=Specification.OPAQUE,
        )
        assert resource == Resource(
            contents=schema,
            specification=DRAFT202012,
        )

    def test_non_mapping_from_contents(self):
        resource = Resource.from_contents(
            True,
            default_specification=ID_AND_CHILDREN,
        )
        assert resource == Resource(
            contents=True,
            specification=ID_AND_CHILDREN,
        )

    def test_from_contents_with_fallback(self):
        resource = Resource.from_contents(
            {"foo": "bar"},
            default_specification=Specification.OPAQUE,
        )
        assert resource == Resource.opaque(contents={"foo": "bar"})

    def test_id_delegates_to_specification(self):
        specification = Specification(
            name="",
            id_of=lambda contents: "urn:fixedID",
            subresources_of=lambda contents: [],
            anchors_in=lambda specification, contents: [],
            maybe_in_subresource=(
                lambda segments, resolver, subresource: resolver
            ),
        )
        resource = Resource(
            contents={"foo": "baz"},
            specification=specification,
        )
        assert resource.id() == "urn:fixedID"

    def test_id_strips_empty_fragment(self):
        uri = "http://example.com/"
        root = ID_AND_CHILDREN.create_resource({"ID": uri + "#"})
        assert root.id() == uri

    def test_subresources_delegates_to_specification(self):
        resource = ID_AND_CHILDREN.create_resource({"children": [{}, 12]})
        assert list(resource.subresources()) == [
            ID_AND_CHILDREN.create_resource(each) for each in [{}, 12]
        ]

    def test_subresource_with_different_specification(self):
        schema = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        resource = ID_AND_CHILDREN.create_resource({"children": [schema]})
        assert list(resource.subresources()) == [
            DRAFT202012.create_resource(schema),
        ]

    def test_anchors_delegates_to_specification(self):
        resource = ID_AND_CHILDREN.create_resource(
            {"anchors": {"foo": {}, "bar": 1, "baz": ""}},
        )
        assert list(resource.anchors()) == [
            Anchor(name="foo", resource=ID_AND_CHILDREN.create_resource({})),
            Anchor(name="bar", resource=ID_AND_CHILDREN.create_resource(1)),
            Anchor(name="baz", resource=ID_AND_CHILDREN.create_resource("")),
        ]

    def test_pointer_to_mapping(self):
        resource = Resource.opaque(contents={"foo": "baz"})
        resolver = Registry().resolver()
        assert resource.pointer("/foo", resolver=resolver).contents == "baz"

    def test_pointer_to_array(self):
        resource = Resource.opaque(contents={"foo": {"bar": [3]}})
        resolver = Registry().resolver()
        assert resource.pointer("/foo/bar/0", resolver=resolver).contents == 3

    def test_opaque(self):
        contents = {"foo": "bar"}
        assert Resource.opaque(contents) == Resource(
            contents=contents,
            specification=Specification.OPAQUE,
        )


class TestResolver:
    def test_lookup_exact_uri(self):
        resource = Resource.opaque(contents={"foo": "baz"})
        resolver = Registry({"http://example.com/1": resource}).resolver()
        resolved = resolver.lookup("http://example.com/1")
        assert resolved.contents == resource.contents

    def test_lookup_subresource(self):
        root = ID_AND_CHILDREN.create_resource(
            {
                "ID": "http://example.com/",
                "children": [
                    {"ID": "http://example.com/a", "foo": 12},
                ],
            },
        )
        registry = root @ Registry()
        resolved = registry.resolver().lookup("http://example.com/a")
        assert resolved.contents == {"ID": "http://example.com/a", "foo": 12}

    def test_lookup_anchor_with_id(self):
        root = ID_AND_CHILDREN.create_resource(
            {
                "ID": "http://example.com/",
                "anchors": {"foo": 12},
            },
        )
        registry = root @ Registry()
        resolved = registry.resolver().lookup("http://example.com/#foo")
        assert resolved.contents == 12

    def test_lookup_anchor_without_id(self):
        root = ID_AND_CHILDREN.create_resource({"anchors": {"foo": 12}})
        resolver = Registry().with_resource("urn:example", root).resolver()
        resolved = resolver.lookup("urn:example#foo")
        assert resolved.contents == 12

    def test_lookup_unknown_reference(self):
        resolver = Registry().resolver()
        ref = "http://example.com/does/not/exist"
        with pytest.raises(exceptions.Unresolvable) as e:
            resolver.lookup(ref)
        assert e.value == exceptions.Unresolvable(ref=ref)

    def test_lookup_non_existent_pointer(self):
        resource = Resource.opaque({"foo": {}})
        resolver = Registry({"http://example.com/1": resource}).resolver()
        ref = "http://example.com/1#/foo/bar"
        with pytest.raises(exceptions.Unresolvable) as e:
            resolver.lookup(ref)
        assert e.value == exceptions.PointerToNowhere(
            ref="/foo/bar",
            resource=resource,
        )
        assert str(e.value) == "'/foo/bar' does not exist within {'foo': {}}"

    def test_lookup_non_existent_pointer_to_array_index(self):
        resource = Resource.opaque([1, 2, 4, 8])
        resolver = Registry({"http://example.com/1": resource}).resolver()
        ref = "http://example.com/1#/10"
        with pytest.raises(exceptions.Unresolvable) as e:
            resolver.lookup(ref)
        assert e.value == exceptions.PointerToNowhere(
            ref="/10",
            resource=resource,
        )

    def test_lookup_pointer_to_empty_string(self):
        resolver = Registry().resolver_with_root(Resource.opaque({"": {}}))
        assert resolver.lookup("#/").contents == {}

    def test_lookup_non_existent_pointer_to_empty_string(self):
        resource = Resource.opaque({"foo": {}})
        resolver = Registry().resolver_with_root(resource)
        with pytest.raises(
            exceptions.Unresolvable,
            match="^'/' does not exist within {'foo': {}}.*'#'",
        ) as e:
            resolver.lookup("#/")
        assert e.value == exceptions.PointerToNowhere(
            ref="/",
            resource=resource,
        )

    def test_lookup_non_existent_anchor(self):
        root = ID_AND_CHILDREN.create_resource({"anchors": {}})
        resolver = Registry().with_resource("urn:example", root).resolver()
        resolved = resolver.lookup("urn:example")
        assert resolved.contents == root.contents

        ref = "urn:example#noSuchAnchor"
        with pytest.raises(exceptions.Unresolvable) as e:
            resolver.lookup(ref)
        assert "'noSuchAnchor' does not exist" in str(e.value)
        assert e.value == exceptions.NoSuchAnchor(
            ref="urn:example",
            resource=root,
            anchor="noSuchAnchor",
        )

    def test_lookup_invalid_JSON_pointerish_anchor(self):
        resolver = Registry().resolver_with_root(
            ID_AND_CHILDREN.create_resource(
                {
                    "ID": "http://example.com/",
                    "foo": {"bar": 12},
                },
            ),
        )

        valid = resolver.lookup("#/foo/bar")
        assert valid.contents == 12

        with pytest.raises(exceptions.InvalidAnchor) as e:
            resolver.lookup("#foo/bar")
        assert " '#/foo/bar'" in str(e.value)

    def test_lookup_retrieved_resource(self):
        resource = Resource.opaque(contents={"foo": "baz"})
        resolver = Registry(retrieve=lambda uri: resource).resolver()
        resolved = resolver.lookup("http://example.com/")
        assert resolved.contents == resource.contents

    def test_lookup_failed_retrieved_resource(self):
        """
        Unretrievable exceptions are also wrapped in Unresolvable.
        """

        uri = "http://example.com/"

        registry = Registry(retrieve=blow_up)
        with pytest.raises(exceptions.Unretrievable):
            registry.get_or_retrieve(uri)

        resolver = registry.resolver()
        with pytest.raises(exceptions.Unresolvable):
            resolver.lookup(uri)

    def test_repeated_lookup_from_retrieved_resource(self):
        """
        A (custom-)retrieved resource is added to the registry returned by
        looking it up.
        """
        resource = Resource.opaque(contents={"foo": "baz"})
        once = [resource]

        def retrieve(uri: str):
            return once.pop()

        resolver = Registry(retrieve=retrieve).resolver()
        resolved = resolver.lookup("http://example.com/")
        assert resolved.contents == resource.contents

        resolved = resolved.resolver.lookup("http://example.com/")
        assert resolved.contents == resource.contents

    def test_repeated_anchor_lookup_from_retrieved_resource(self):
        resource = Resource.opaque(contents={"foo": "baz"})
        once = [resource]

        def retrieve(uri: str):
            return once.pop()

        resolver = Registry(retrieve=retrieve).resolver()
        resolved = resolver.lookup("http://example.com/")
        assert resolved.contents == resource.contents

        resolved = resolved.resolver.lookup("#")
        assert resolved.contents == resource.contents

    # FIXME: The tests below aren't really representable in the current
    #        suite, though we should probably think of ways to do so.

    def test_in_subresource(self):
        root = ID_AND_CHILDREN.create_resource(
            {
                "ID": "http://example.com/",
                "children": [
                    {
                        "ID": "child/",
                        "children": [{"ID": "grandchild"}],
                    },
                ],
            },
        )
        registry = root @ Registry()

        resolver = registry.resolver()
        first = resolver.lookup("http://example.com/")
        assert first.contents == root.contents

        with pytest.raises(exceptions.Unresolvable):
            first.resolver.lookup("grandchild")

        sub = first.resolver.in_subresource(
            ID_AND_CHILDREN.create_resource(first.contents["children"][0]),
        )
        second = sub.lookup("grandchild")
        assert second.contents == {"ID": "grandchild"}

    def test_in_pointer_subresource(self):
        root = ID_AND_CHILDREN.create_resource(
            {
                "ID": "http://example.com/",
                "children": [
                    {
                        "ID": "child/",
                        "children": [{"ID": "grandchild"}],
                    },
                ],
            },
        )
        registry = root @ Registry()

        resolver = registry.resolver()
        first = resolver.lookup("http://example.com/")
        assert first.contents == root.contents

        with pytest.raises(exceptions.Unresolvable):
            first.resolver.lookup("grandchild")

        second = first.resolver.lookup("#/children/0")
        third = second.resolver.lookup("grandchild")
        assert third.contents == {"ID": "grandchild"}

    def test_dynamic_scope(self):
        one = ID_AND_CHILDREN.create_resource(
            {
                "ID": "http://example.com/",
                "children": [
                    {
                        "ID": "child/",
                        "children": [{"ID": "grandchild"}],
                    },
                ],
            },
        )
        two = ID_AND_CHILDREN.create_resource(
            {
                "ID": "http://example.com/two",
                "children": [{"ID": "two-child/"}],
            },
        )
        registry = [one, two] @ Registry()

        resolver = registry.resolver()
        first = resolver.lookup("http://example.com/")
        second = first.resolver.lookup("#/children/0")
        third = second.resolver.lookup("grandchild")
        fourth = third.resolver.lookup("http://example.com/two")
        assert list(fourth.resolver.dynamic_scope()) == [
            ("http://example.com/child/grandchild", fourth.resolver._registry),
            ("http://example.com/child/", fourth.resolver._registry),
            ("http://example.com/", fourth.resolver._registry),
        ]
        assert list(third.resolver.dynamic_scope()) == [
            ("http://example.com/child/", third.resolver._registry),
            ("http://example.com/", third.resolver._registry),
        ]
        assert list(second.resolver.dynamic_scope()) == [
            ("http://example.com/", second.resolver._registry),
        ]
        assert list(first.resolver.dynamic_scope()) == []


class TestSpecification:
    def test_create_resource(self):
        specification = Specification(
            name="",
            id_of=lambda contents: "urn:fixedID",
            subresources_of=lambda contents: [],
            anchors_in=lambda specification, contents: [],
            maybe_in_subresource=(
                lambda segments, resolver, subresource: resolver
            ),
        )
        resource = specification.create_resource(contents={"foo": "baz"})
        assert resource == Resource(
            contents={"foo": "baz"},
            specification=specification,
        )
        assert resource.id() == "urn:fixedID"

    def test_repr(self):
        assert (
            repr(ID_AND_CHILDREN) == "<Specification name='id-and-children'>"
        )


class TestOpaqueSpecification:
    THINGS = [{"foo": "bar"}, True, 37, "foo", object()]

    @pytest.mark.parametrize("thing", THINGS)
    def test_no_id(self, thing):
        """
        An arbitrary thing has no ID.
        """

        assert Specification.OPAQUE.id_of(thing) is None

    @pytest.mark.parametrize("thing", THINGS)
    def test_no_subresources(self, thing):
        """
        An arbitrary thing has no subresources.
        """

        assert list(Specification.OPAQUE.subresources_of(thing)) == []

    @pytest.mark.parametrize("thing", THINGS)
    def test_no_anchors(self, thing):
        """
        An arbitrary thing has no anchors.
        """

        assert list(Specification.OPAQUE.anchors_in(thing)) == []


@pytest.mark.parametrize(
    "cls",
    [Anchor, Registry, Resource, Specification, exceptions.PointerToNowhere],
)
def test_nonsubclassable(cls):
    with pytest.raises(Exception, match="(?i)subclassing"):

        class Boom(cls):  # pragma: no cover
            pass

env_pyrecodes/lib/python3.9/site-packages/referencing/tests/test_core.py
NikolaBlagojevic/pyrecodes
import itertools

import pytest

from referencing import Resource, exceptions


def pairs(choices):
    return itertools.combinations(choices, 2)


TRUE = Resource.opaque(True)


thunks = (
    lambda: exceptions.CannotDetermineSpecification(TRUE),
    lambda: exceptions.NoSuchResource("urn:example:foo"),
    lambda: exceptions.NoInternalID(TRUE),
    lambda: exceptions.InvalidAnchor(resource=TRUE, anchor="foo", ref="a#b"),
    lambda: exceptions.NoSuchAnchor(resource=TRUE, anchor="foo", ref="a#b"),
    lambda: exceptions.PointerToNowhere(resource=TRUE, ref="urn:example:foo"),
    lambda: exceptions.Unresolvable("urn:example:foo"),
    lambda: exceptions.Unretrievable("urn:example:foo"),
)


@pytest.mark.parametrize("one, two", pairs(each() for each in thunks))
def test_eq_incompatible_types(one, two):
    assert one != two


@pytest.mark.parametrize("thunk", thunks)
def test_hash(thunk):
    assert thunk() in {thunk()}

env_pyrecodes/lib/python3.9/site-packages/referencing/tests/test_exceptions.py
NikolaBlagojevic/pyrecodes
import pytest

from referencing import Registry, Resource, Specification
import referencing.jsonschema


@pytest.mark.parametrize(
    "uri, expected",
    [
        (
            "https://json-schema.org/draft/2020-12/schema",
            referencing.jsonschema.DRAFT202012,
        ),
        (
            "https://json-schema.org/draft/2019-09/schema",
            referencing.jsonschema.DRAFT201909,
        ),
        (
            "http://json-schema.org/draft-07/schema#",
            referencing.jsonschema.DRAFT7,
        ),
        (
            "http://json-schema.org/draft-06/schema#",
            referencing.jsonschema.DRAFT6,
        ),
        (
            "http://json-schema.org/draft-04/schema#",
            referencing.jsonschema.DRAFT4,
        ),
        (
            "http://json-schema.org/draft-03/schema#",
            referencing.jsonschema.DRAFT3,
        ),
    ],
)
def test_schemas_with_explicit_schema_keywords_are_detected(uri, expected):
    """
    The $schema keyword in JSON Schema is a dialect identifier.
    """
    contents = {"$schema": uri}
    resource = Resource.from_contents(contents)
    assert resource == Resource(contents=contents, specification=expected)


def test_unknown_dialect():
    dialect_id = "http://example.com/unknown-json-schema-dialect-id"
    with pytest.raises(referencing.jsonschema.UnknownDialect) as excinfo:
        Resource.from_contents({"$schema": dialect_id})
    assert excinfo.value.uri == dialect_id


@pytest.mark.parametrize(
    "id, specification",
    [
        ("$id", referencing.jsonschema.DRAFT202012),
        ("$id", referencing.jsonschema.DRAFT201909),
        ("$id", referencing.jsonschema.DRAFT7),
        ("$id", referencing.jsonschema.DRAFT6),
        ("id", referencing.jsonschema.DRAFT4),
        ("id", referencing.jsonschema.DRAFT3),
    ],
)
def test_id_of_mapping(id, specification):
    uri = "http://example.com/some-schema"
    assert specification.id_of({id: uri}) == uri


@pytest.mark.parametrize(
    "specification",
    [
        referencing.jsonschema.DRAFT202012,
        referencing.jsonschema.DRAFT201909,
        referencing.jsonschema.DRAFT7,
        referencing.jsonschema.DRAFT6,
    ],
)
@pytest.mark.parametrize("value", [True, False])
def test_id_of_bool(specification, value):
    assert specification.id_of(value) is None


@pytest.mark.parametrize(
    "specification",
    [
        referencing.jsonschema.DRAFT202012,
        referencing.jsonschema.DRAFT201909,
        referencing.jsonschema.DRAFT7,
        referencing.jsonschema.DRAFT6,
    ],
)
@pytest.mark.parametrize("value", [True, False])
def test_anchors_in_bool(specification, value):
    assert list(specification.anchors_in(value)) == []


@pytest.mark.parametrize(
    "specification",
    [
        referencing.jsonschema.DRAFT202012,
        referencing.jsonschema.DRAFT201909,
        referencing.jsonschema.DRAFT7,
        referencing.jsonschema.DRAFT6,
    ],
)
@pytest.mark.parametrize("value", [True, False])
def test_subresources_of_bool(specification, value):
    assert list(specification.subresources_of(value)) == []


@pytest.mark.parametrize(
    "uri, expected",
    [
        (
            "https://json-schema.org/draft/2020-12/schema",
            referencing.jsonschema.DRAFT202012,
        ),
        (
            "https://json-schema.org/draft/2019-09/schema",
            referencing.jsonschema.DRAFT201909,
        ),
        (
            "http://json-schema.org/draft-07/schema#",
            referencing.jsonschema.DRAFT7,
        ),
        (
            "http://json-schema.org/draft-06/schema#",
            referencing.jsonschema.DRAFT6,
        ),
        (
            "http://json-schema.org/draft-04/schema#",
            referencing.jsonschema.DRAFT4,
        ),
        (
            "http://json-schema.org/draft-03/schema#",
            referencing.jsonschema.DRAFT3,
        ),
    ],
)
def test_specification_with(uri, expected):
    assert referencing.jsonschema.specification_with(uri) == expected


@pytest.mark.parametrize(
    "uri, expected",
    [
        (
            "http://json-schema.org/draft-07/schema",
            referencing.jsonschema.DRAFT7,
        ),
        (
            "http://json-schema.org/draft-06/schema",
            referencing.jsonschema.DRAFT6,
        ),
        (
            "http://json-schema.org/draft-04/schema",
            referencing.jsonschema.DRAFT4,
        ),
        (
            "http://json-schema.org/draft-03/schema",
            referencing.jsonschema.DRAFT3,
        ),
    ],
)
def test_specification_with_no_empty_fragment(uri, expected):
    assert referencing.jsonschema.specification_with(uri) == expected


def test_specification_with_unknown_dialect():
    dialect_id = "http://example.com/unknown-json-schema-dialect-id"
    with pytest.raises(referencing.jsonschema.UnknownDialect) as excinfo:
        referencing.jsonschema.specification_with(dialect_id)
    assert excinfo.value.uri == dialect_id


def test_specification_with_default():
    dialect_id = "http://example.com/unknown-json-schema-dialect-id"
    specification = referencing.jsonschema.specification_with(
        dialect_id,
        default=Specification.OPAQUE,
    )
    assert specification is Specification.OPAQUE


# FIXME: The tests below should move to the referencing suite but I haven't yet
#        figured out how to represent dynamic (& recursive) ref lookups in it.
def test_lookup_trivial_dynamic_ref():
    one = referencing.jsonschema.DRAFT202012.create_resource(
        {"$dynamicAnchor": "foo"},
    )
    resolver = Registry().with_resource("http://example.com", one).resolver()
    resolved = resolver.lookup("http://example.com#foo")
    assert resolved.contents == one.contents


def test_multiple_lookup_trivial_dynamic_ref():
    TRUE = referencing.jsonschema.DRAFT202012.create_resource(True)
    root = referencing.jsonschema.DRAFT202012.create_resource(
        {
            "$id": "http://example.com",
            "$dynamicAnchor": "fooAnchor",
            "$defs": {
                "foo": {
                    "$id": "foo",
                    "$dynamicAnchor": "fooAnchor",
                    "$defs": {
                        "bar": True,
                        "baz": {
                            "$dynamicAnchor": "fooAnchor",
                        },
                    },
                },
            },
        },
    )
    resolver = (
        Registry()
        .with_resources(
            [
                ("http://example.com", root),
                ("http://example.com/foo/", TRUE),
                ("http://example.com/foo/bar", root),
            ],
        )
        .resolver()
    )

    first = resolver.lookup("http://example.com")
    second = first.resolver.lookup("foo/")
    resolver = second.resolver.lookup("bar").resolver
    fourth = resolver.lookup("#fooAnchor")
    assert fourth.contents == root.contents


def test_multiple_lookup_dynamic_ref_to_nondynamic_ref():
    one = referencing.jsonschema.DRAFT202012.create_resource(
        {"$anchor": "fooAnchor"},
    )
    two = referencing.jsonschema.DRAFT202012.create_resource(
        {
            "$id": "http://example.com",
            "$dynamicAnchor": "fooAnchor",
            "$defs": {
                "foo": {
                    "$id": "foo",
                    "$dynamicAnchor": "fooAnchor",
                    "$defs": {
                        "bar": True,
                        "baz": {
                            "$dynamicAnchor": "fooAnchor",
                        },
                    },
                },
            },
        },
    )
    resolver = (
        Registry()
        .with_resources(
            [
                ("http://example.com", two),
                ("http://example.com/foo/", one),
                ("http://example.com/foo/bar", two),
            ],
        )
        .resolver()
    )

    first = resolver.lookup("http://example.com")
    second = first.resolver.lookup("foo/")
    resolver = second.resolver.lookup("bar").resolver
    fourth = resolver.lookup("#fooAnchor")
    assert fourth.contents == two.contents


def test_lookup_trivial_recursive_ref():
    one = referencing.jsonschema.DRAFT201909.create_resource(
        {"$recursiveAnchor": True},
    )
    resolver = Registry().with_resource("http://example.com", one).resolver()
    first = resolver.lookup("http://example.com")
    resolved = referencing.jsonschema.lookup_recursive_ref(
        resolver=first.resolver,
    )
    assert resolved.contents == one.contents


def test_lookup_recursive_ref_to_bool():
    TRUE = referencing.jsonschema.DRAFT201909.create_resource(True)
    registry = Registry({"http://example.com": TRUE})
    resolved = referencing.jsonschema.lookup_recursive_ref(
        resolver=registry.resolver(base_uri="http://example.com"),
    )
    assert resolved.contents == TRUE.contents


def test_multiple_lookup_recursive_ref_to_bool():
    TRUE = referencing.jsonschema.DRAFT201909.create_resource(True)
    root = referencing.jsonschema.DRAFT201909.create_resource(
        {
            "$id": "http://example.com",
            "$recursiveAnchor": True,
            "$defs": {
                "foo": {
                    "$id": "foo",
                    "$recursiveAnchor": True,
                    "$defs": {
                        "bar": True,
                        "baz": {
                            "$recursiveAnchor": True,
                            "$anchor": "fooAnchor",
                        },
                    },
                },
            },
        },
    )
    resolver = (
        Registry()
        .with_resources(
            [
                ("http://example.com", root),
                ("http://example.com/foo/", TRUE),
                ("http://example.com/foo/bar", root),
            ],
        )
        .resolver()
    )

    first = resolver.lookup("http://example.com")
    second = first.resolver.lookup("foo/")
    resolver = second.resolver.lookup("bar").resolver
    fourth = referencing.jsonschema.lookup_recursive_ref(resolver=resolver)
    assert fourth.contents == root.contents


def test_multiple_lookup_recursive_ref_with_nonrecursive_ref():
    one = referencing.jsonschema.DRAFT201909.create_resource(
        {"$recursiveAnchor": True},
    )
    two = referencing.jsonschema.DRAFT201909.create_resource(
        {
            "$id": "http://example.com",
            "$recursiveAnchor": True,
            "$defs": {
                "foo": {
                    "$id": "foo",
                    "$recursiveAnchor": True,
                    "$defs": {
                        "bar": True,
                        "baz": {
                            "$recursiveAnchor": True,
                            "$anchor": "fooAnchor",
                        },
                    },
                },
            },
        },
    )
    three = referencing.jsonschema.DRAFT201909.create_resource(
        {"$recursiveAnchor": False},
    )
    resolver = (
        Registry()
        .with_resources(
            [
                ("http://example.com", three),
                ("http://example.com/foo/", two),
                ("http://example.com/foo/bar", one),
            ],
        )
        .resolver()
    )

    first = resolver.lookup("http://example.com")
    second = first.resolver.lookup("foo/")
    resolver = second.resolver.lookup("bar").resolver
    fourth = referencing.jsonschema.lookup_recursive_ref(resolver=resolver)
    assert fourth.contents == two.contents

env_pyrecodes/lib/python3.9/site-packages/referencing/tests/test_jsonschema.py
NikolaBlagojevic/pyrecodes
from pathlib import Path
import json
import os

import pytest

from referencing import Registry
from referencing.exceptions import Unresolvable
import referencing.jsonschema


class SuiteNotFound(Exception):
    def __str__(self):  # pragma: no cover
        return (
            "Cannot find the referencing suite. "
            "Set the REFERENCING_SUITE environment variable to the path to "
            "the suite, or run the test suite from alongside a full checkout "
            "of the git repository."
        )


if "REFERENCING_SUITE" in os.environ:  # pragma: no cover
    SUITE = Path(os.environ["REFERENCING_SUITE"]) / "tests"
else:
    SUITE = Path(__file__).parent.parent.parent / "suite/tests"
if not SUITE.is_dir():  # pragma: no cover
    raise SuiteNotFound()
DIALECT_IDS = json.loads(SUITE.joinpath("specifications.json").read_text())


@pytest.mark.parametrize(
    "test_path",
    [
        pytest.param(each, id=f"{each.parent.name}-{each.stem}")
        for each in SUITE.glob("*/**/*.json")
    ],
)
def test_referencing_suite(test_path, subtests):
    dialect_id = DIALECT_IDS[test_path.relative_to(SUITE).parts[0]]
    specification = referencing.jsonschema.specification_with(dialect_id)
    loaded = json.loads(test_path.read_text())
    registry = loaded["registry"]
    registry = Registry().with_resources(
        (uri, specification.create_resource(contents))
        for uri, contents in loaded["registry"].items()
    )
    for test in loaded["tests"]:
        with subtests.test(test=test):
            resolver = registry.resolver(base_uri=test.get("base_uri", ""))

            if test.get("error"):
                with pytest.raises(Unresolvable):
                    resolver.lookup(test["ref"])
            else:
                resolved = resolver.lookup(test["ref"])
                assert resolved.contents == test["target"]

                then = test.get("then")
                while then:  # pragma: no cover
                    with subtests.test(test=test, then=then):
                        resolved = resolved.resolver.lookup(then["ref"])
                        assert resolved.contents == then["target"]
                    then = then.get("then")

env_pyrecodes/lib/python3.9/site-packages/referencing/tests/test_referencing_suite.py
NikolaBlagojevic/pyrecodes
from functools import lru_cache
import json

import pytest

from referencing import Registry, Resource, exceptions
from referencing.jsonschema import DRAFT202012
from referencing.retrieval import to_cached_resource


class TestToCachedResource:
    def test_it_caches_retrieved_resources(self):
        contents = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        stack = [json.dumps(contents)]

        @to_cached_resource()
        def retrieve(uri):
            return stack.pop()

        registry = Registry(retrieve=retrieve)

        expected = Resource.from_contents(contents)

        got = registry.get_or_retrieve("urn:example:schema")
        assert got.value == expected

        # And a second time we get the same value.
        again = registry.get_or_retrieve("urn:example:schema")
        assert again.value is got.value

    def test_custom_loader(self):
        contents = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        stack = [json.dumps(contents)[::-1]]

        @to_cached_resource(loads=lambda s: json.loads(s[::-1]))
        def retrieve(uri):
            return stack.pop()

        registry = Registry(retrieve=retrieve)

        expected = Resource.from_contents(contents)

        got = registry.get_or_retrieve("urn:example:schema")
        assert got.value == expected

        # And a second time we get the same value.
        again = registry.get_or_retrieve("urn:example:schema")
        assert again.value is got.value

    def test_custom_from_contents(self):
        contents = {}
        stack = [json.dumps(contents)]

        @to_cached_resource(from_contents=DRAFT202012.create_resource)
        def retrieve(uri):
            return stack.pop()

        registry = Registry(retrieve=retrieve)

        expected = DRAFT202012.create_resource(contents)

        got = registry.get_or_retrieve("urn:example:schema")
        assert got.value == expected

        # And a second time we get the same value.
        again = registry.get_or_retrieve("urn:example:schema")
        assert again.value is got.value

    def test_custom_cache(self):
        schema = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
        mapping = {
            "urn:example:1": dict(schema, foo=1),
            "urn:example:2": dict(schema, foo=2),
            "urn:example:3": dict(schema, foo=3),
        }

        resources = {
            uri: Resource.from_contents(contents)
            for uri, contents in mapping.items()
        }

        @to_cached_resource(cache=lru_cache(maxsize=2))
        def retrieve(uri):
            return json.dumps(mapping.pop(uri))

        registry = Registry(retrieve=retrieve)

        got = registry.get_or_retrieve("urn:example:1")
        assert got.value == resources["urn:example:1"]
        assert registry.get_or_retrieve("urn:example:1").value is got.value
        assert registry.get_or_retrieve("urn:example:1").value is got.value

        got = registry.get_or_retrieve("urn:example:2")
        assert got.value == resources["urn:example:2"]
        assert registry.get_or_retrieve("urn:example:2").value is got.value
        assert registry.get_or_retrieve("urn:example:2").value is got.value

        # This still succeeds, but evicts the first URI
        got = registry.get_or_retrieve("urn:example:3")
        assert got.value == resources["urn:example:3"]
        assert registry.get_or_retrieve("urn:example:3").value is got.value
        assert registry.get_or_retrieve("urn:example:3").value is got.value

        # And now this fails (as we popped the value out of `mapping`)
        with pytest.raises(exceptions.Unretrievable):
            registry.get_or_retrieve("urn:example:1")

env_pyrecodes/lib/python3.9/site-packages/referencing/tests/test_retrieval.py
NikolaBlagojevic/pyrecodes
"""
Type-annotation related support for the referencing library.
"""
from __future__ import annotations

from typing import TYPE_CHECKING, Protocol, TypeVar

try:
    from collections.abc import Mapping as Mapping

    Mapping[str, str]
except TypeError:  # pragma: no cover
    from typing import Mapping as Mapping


if TYPE_CHECKING:
    from referencing._core import Resolved, Resolver, Resource

#: A URI which identifies a `Resource`.
URI = str

#: The type of documents within a registry.
D = TypeVar("D")


class Retrieve(Protocol[D]):
    """
    A retrieval callable, usable within a `Registry` for resource retrieval.

    Does not make assumptions about where the resource might be coming from.
    """

    def __call__(self, uri: URI) -> Resource[D]:
        """
        Retrieve the resource with the given URI.

        Raise `referencing.exceptions.NoSuchResource` if you wish to indicate
        the retriever cannot lookup the given URI.
        """
        ...


class Anchor(Protocol[D]):
    """
    An anchor within a `Resource`.

    Beyond "simple" anchors, some specifications like JSON Schema's 2020
    version have dynamic anchors.
    """

    @property
    def name(self) -> str:
        """
        Return the name of this anchor.
        """
        ...

    def resolve(self, resolver: Resolver[D]) -> Resolved[D]:
        """
        Return the resource for this anchor.
        """
        ...

env_pyrecodes/lib/python3.9/site-packages/referencing/typing.py
NikolaBlagojevic/pyrecodes
"""Private module full of compatibility hacks.

Primarily this is for downstream redistributions of requests that unvendor
urllib3 without providing a shim.

.. warning::

    This module is private. If you use it, and something breaks, you were
    warned
"""
import sys

import requests

try:
    from requests.packages.urllib3 import fields
    from requests.packages.urllib3 import filepost
    from requests.packages.urllib3 import poolmanager
except ImportError:
    from urllib3 import fields
    from urllib3 import filepost
    from urllib3 import poolmanager

try:
    from requests.packages.urllib3.connection import HTTPConnection
    from requests.packages.urllib3 import connection
except ImportError:
    try:
        from urllib3.connection import HTTPConnection
        from urllib3 import connection
    except ImportError:
        HTTPConnection = None
        connection = None


if requests.__build__ < 0x020300:
    timeout = None
else:
    try:
        from requests.packages.urllib3.util import timeout
    except ImportError:
        from urllib3.util import timeout

PY3 = sys.version_info > (3, 0)

if PY3:
    from collections.abc import Mapping, MutableMapping
    import queue
    from urllib.parse import urlencode, urljoin
else:
    from collections import Mapping, MutableMapping
    import Queue as queue
    from urllib import urlencode
    from urlparse import urljoin

try:
    basestring = basestring
except NameError:
    basestring = (str, bytes)


class HTTPHeaderDict(MutableMapping):
    """
    :param headers:
        An iterable of field-value pairs. Must not contain multiple field names
        when compared case-insensitively.

    :param kwargs:
        Additional field-value pairs to pass in to ``dict.update``.

    A ``dict`` like container for storing HTTP Headers.

    Field names are stored and compared case-insensitively in compliance with
    RFC 7230. Iteration provides the first case-sensitive key seen for each
    case-insensitive pair.

    Using ``__setitem__`` syntax overwrites fields that compare equal
    case-insensitively in order to maintain ``dict``'s api. For fields that
    compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
    in a loop.

    If multiple fields that are equal case-insensitively are passed to the
    constructor or ``.update``, the behavior is undefined and some will be
    lost.

    >>> headers = HTTPHeaderDict()
    >>> headers.add('Set-Cookie', 'foo=bar')
    >>> headers.add('set-cookie', 'baz=quxx')
    >>> headers['content-length'] = '7'
    >>> headers['SET-cookie']
    'foo=bar, baz=quxx'
    >>> headers['Content-Length']
    '7'
    """

    def __init__(self, headers=None, **kwargs):
        super(HTTPHeaderDict, self).__init__()
        self._container = {}
        if headers is not None:
            if isinstance(headers, HTTPHeaderDict):
                self._copy_from(headers)
            else:
                self.extend(headers)
        if kwargs:
            self.extend(kwargs)

    def __setitem__(self, key, val):
        self._container[key.lower()] = (key, val)
        return self._container[key.lower()]

    def __getitem__(self, key):
        val = self._container[key.lower()]
        return ', '.join(val[1:])

    def __delitem__(self, key):
        del self._container[key.lower()]

    def __contains__(self, key):
        return key.lower() in self._container

    def __eq__(self, other):
        if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
            return False
        if not isinstance(other, type(self)):
            other = type(self)(other)
        return ({k.lower(): v for k, v in self.itermerged()} ==
                {k.lower(): v for k, v in other.itermerged()})

    def __ne__(self, other):
        return not self.__eq__(other)

    if not PY3:  # Python 2
        iterkeys = MutableMapping.iterkeys
        itervalues = MutableMapping.itervalues

    __marker = object()

    def __len__(self):
        return len(self._container)

    def __iter__(self):
        # Only provide the originally cased names
        for vals in self._container.values():
            yield vals[0]

    def pop(self, key, default=__marker):
        """D.pop(k[,d]) -> v, remove specified key and return its value.

        If key is not found, d is returned if given, otherwise KeyError is
        raised.
        """
        # Using the MutableMapping function directly fails due to the private
        # marker.
        # Using ordinary dict.pop would expose the internal structures.
        # So let's reinvent the wheel.
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def discard(self, key):
        try:
            del self[key]
        except KeyError:
            pass

    def add(self, key, val):
        """Adds a (name, value) pair, doesn't overwrite the value if it already
        exists.

        >>> headers = HTTPHeaderDict(foo='bar')
        >>> headers.add('Foo', 'baz')
        >>> headers['foo']
        'bar, baz'
        """
        key_lower = key.lower()
        new_vals = key, val
        # Keep the common case aka no item present as fast as possible
        vals = self._container.setdefault(key_lower, new_vals)
        if new_vals is not vals:
            # new_vals was not inserted, as there was a previous one
            if isinstance(vals, list):
                # If already several items got inserted, we have a list
                vals.append(val)
            else:
                # vals should be a tuple then, i.e. only one item so far
                # Need to convert the tuple to list for further extension
                self._container[key_lower] = [vals[0], vals[1], val]

    def extend(self, *args, **kwargs):
        """Generic import function for any type of header-like object.
        Adapted version of MutableMapping.update in order to insert items
        with self.add instead of self.__setitem__
        """
        if len(args) > 1:
            raise TypeError("extend() takes at most 1 positional "
                            "arguments ({} given)".format(len(args)))
        other = args[0] if len(args) >= 1 else ()

        if isinstance(other, HTTPHeaderDict):
            for key, val in other.iteritems():
                self.add(key, val)
        elif isinstance(other, Mapping):
            for key in other:
                self.add(key, other[key])
        elif hasattr(other, "keys"):
            for key in other.keys():
                self.add(key, other[key])
        else:
            for key, value in other:
                self.add(key, value)

        for key, value in kwargs.items():
            self.add(key, value)

    def getlist(self, key):
        """Returns a list of all the values for the named field. Returns an
        empty list if the key doesn't exist."""
        try:
            vals = self._container[key.lower()]
        except KeyError:
            return []
        else:
            if isinstance(vals, tuple):
                return [vals[1]]
            else:
                return vals[1:]

    # Backwards compatibility for httplib
    getheaders = getlist
    getallmatchingheaders = getlist
    iget = getlist

    def __repr__(self):
        return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))

    def _copy_from(self, other):
        for key in other:
            val = other.getlist(key)
            if isinstance(val, list):
                # Don't need to convert tuples
                val = list(val)
            self._container[key.lower()] = [key] + val

    def copy(self):
        clone = type(self)()
        clone._copy_from(self)
        return clone

    def iteritems(self):
        """Iterate over all header lines, including duplicate ones."""
        for key in self:
            vals = self._container[key.lower()]
            for val in vals[1:]:
                yield vals[0], val

    def itermerged(self):
        """Iterate over all headers, merging duplicate ones together."""
        for key in self:
            val = self._container[key.lower()]
            yield val[0], ', '.join(val[1:])

    def items(self):
        return list(self.iteritems())

    @classmethod
    def from_httplib(cls, message):  # Python 2
        """Read headers from a Python 2 httplib message object."""
        # python2.7 does not expose a proper API for exporting multiheaders
        # efficiently. This function re-reads raw lines from the message
        # object and extracts the multiheaders properly.
        headers = []

        for line in message.headers:
            if line.startswith((' ', '\t')):
                key, value = headers[-1]
                headers[-1] = (key, value + '\r\n' + line.rstrip())
                continue

            key, value = line.split(':', 1)
            headers.append((key, value.strip()))

        return cls(headers)


__all__ = (
    'basestring',
    'connection',
    'fields',
    'filepost',
    'poolmanager',
    'timeout',
    'HTTPHeaderDict',
    'queue',
    'urlencode',
    'urljoin',
)

env_pyrecodes/lib/python3.9/site-packages/requests_toolbelt/_compat.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
"""The implementation of the SocketOptionsAdapter."""
import socket
import warnings
import sys

import requests
from requests import adapters

from .._compat import connection
from .._compat import poolmanager
from .. import exceptions as exc


class SocketOptionsAdapter(adapters.HTTPAdapter):
    """An adapter for requests that allows users to specify socket options.

    Since version 2.4.0 of requests, it is possible to specify a custom list
    of socket options that need to be set before establishing the connection.

    Example usage::

        >>> import socket
        >>> import requests
        >>> from requests_toolbelt.adapters import socket_options
        >>> s = requests.Session()
        >>> opts = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0)]
        >>> adapter = socket_options.SocketOptionsAdapter(socket_options=opts)
        >>> s.mount('http://', adapter)

    You can also take advantage of the list of default options on this class
    to keep using the original options in addition to your custom options. In
    that case, ``opts`` might look like::

        >>> opts = socket_options.SocketOptionsAdapter.default_options + opts

    """

    if connection is not None:
        default_options = getattr(
            connection.HTTPConnection,
            'default_socket_options',
            [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
        )
    else:
        default_options = []
        warnings.warn(exc.RequestsVersionTooOld,
                      "This version of Requests is only compatible with a "
                      "version of urllib3 which is too old to support "
                      "setting options on a socket. This adapter is "
                      "functionally useless.")

    def __init__(self, **kwargs):
        self.socket_options = kwargs.pop('socket_options',
                                         self.default_options)

        super(SocketOptionsAdapter, self).__init__(**kwargs)

    def init_poolmanager(self, connections, maxsize, block=False):
        if requests.__build__ >= 0x020400:
            # NOTE(Ian): Perhaps we should raise a warning
            self.poolmanager = poolmanager.PoolManager(
                num_pools=connections,
                maxsize=maxsize,
                block=block,
                socket_options=self.socket_options
            )
        else:
            super(SocketOptionsAdapter, self).init_poolmanager(
                connections, maxsize, block
            )


class TCPKeepAliveAdapter(SocketOptionsAdapter):
    """An adapter for requests that turns on TCP Keep-Alive by default.

    The adapter sets 4 socket options:

    - ``SOL_SOCKET`` ``SO_KEEPALIVE`` - This turns on TCP Keep-Alive
    - ``IPPROTO_TCP`` ``TCP_KEEPINTVL`` 20 - Sets the keep alive interval
    - ``IPPROTO_TCP`` ``TCP_KEEPCNT`` 5 - Sets the number of keep alive probes
    - ``IPPROTO_TCP`` ``TCP_KEEPIDLE`` 60 - Sets the keep alive time if the
      socket library has the ``TCP_KEEPIDLE`` constant

    The latter three can be overridden by keyword arguments (respectively):

    - ``interval``
    - ``count``
    - ``idle``

    You can use this adapter like so::

       >>> from requests_toolbelt.adapters import socket_options
       >>> tcp = socket_options.TCPKeepAliveAdapter(idle=120, interval=10)
       >>> s = requests.Session()
       >>> s.mount('http://', tcp)

    """

    def __init__(self, **kwargs):
        socket_options = kwargs.pop('socket_options',
                                    SocketOptionsAdapter.default_options)
        idle = kwargs.pop('idle', 60)
        interval = kwargs.pop('interval', 20)
        count = kwargs.pop('count', 5)
        socket_options = socket_options + [
            (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
        ]

        # NOTE(Ian): OSX does not have these constants defined, so we
        # set them conditionally.
        if getattr(socket, 'TCP_KEEPINTVL', None) is not None:
            socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
                                interval)]
        elif sys.platform == 'darwin':
            # On OSX, TCP_KEEPALIVE from netinet/tcp.h is not exported
            # by python's socket module
            TCP_KEEPALIVE = getattr(socket, 'TCP_KEEPALIVE', 0x10)
            socket_options += [(socket.IPPROTO_TCP, TCP_KEEPALIVE, interval)]

        if getattr(socket, 'TCP_KEEPCNT', None) is not None:
            socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, count)]

        if getattr(socket, 'TCP_KEEPIDLE', None) is not None:
            socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle)]

        super(TCPKeepAliveAdapter, self).__init__(
            socket_options=socket_options, **kwargs
        )

env_pyrecodes/lib/python3.9/site-packages/requests_toolbelt/adapters/socket_options.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
"""A X509Adapter for use with the requests library.

This file contains an implementation of the X509Adapter that will
allow users to authenticate a request using an arbitrary
X.509 certificate without needing to convert it to a .pem file

"""

from OpenSSL.crypto import PKey, X509
from cryptography import x509
from cryptography.hazmat.primitives.serialization import (load_pem_private_key,
                                                          load_der_private_key)
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.backends import default_backend

from datetime import datetime
from requests.adapters import HTTPAdapter
import requests

from .. import exceptions as exc

"""
importing the protocol constants from _ssl instead of ssl because only the
constants are needed and to handle issues caused by importing from ssl on
the 2.7.x line.
"""
try:
    from _ssl import PROTOCOL_TLS as PROTOCOL
except ImportError:
    from _ssl import PROTOCOL_SSLv23 as PROTOCOL


PyOpenSSLContext = None


class X509Adapter(HTTPAdapter):
    r"""Adapter for use with X.509 certificates.

    Provides an interface for Requests sessions to contact HTTPS urls and
    authenticate  with an X.509 cert by implementing the Transport Adapter
    interface. This class will need to be manually instantiated and mounted
    to the session

    :param pool_connections: The number of urllib3 connection pools to
           cache.
    :param pool_maxsize: The maximum number of connections to save in the
            pool.
    :param max_retries: The maximum number of retries each connection
        should attempt. Note, this applies only to failed DNS lookups,
        socket connections and connection timeouts, never to requests where
        data has made it to the server. By default, Requests does not retry
        failed connections. If you need granular control over the
        conditions under which we retry a request, import urllib3's
        ``Retry`` class and pass that instead.
    :param pool_block: Whether the connection pool should block for
            connections.

    :param bytes cert_bytes:
        bytes object containing contents of a cryptography.x509Certificate
        object using the encoding specified by the ``encoding`` parameter.
    :param bytes pk_bytes:
        bytes object containing contents of a object that implements
        ``cryptography.hazmat.primitives.serialization.PrivateFormat``
        using the encoding specified by the ``encoding`` parameter.
    :param password:
        string or utf8 encoded bytes containing the passphrase used for the
        private key. None if unencrypted. Defaults to None.
    :param encoding:
        Enumeration detailing the encoding method used on the ``cert_bytes``
        parameter. Can be either PEM or DER. Defaults to PEM.
    :type encoding:
        :class: `cryptography.hazmat.primitives.serialization.Encoding`

    Usage::

      >>> import requests
      >>> from requests_toolbelt.adapters.x509 import X509Adapter
      >>> s = requests.Session()
      >>> a = X509Adapter(max_retries=3,
                cert_bytes=b'...', pk_bytes=b'...', encoding='...'
      >>> s.mount('https://', a)
    """

    def __init__(self, *args, **kwargs):
        self._import_pyopensslcontext()
        self._check_version()
        cert_bytes = kwargs.pop('cert_bytes', None)
        pk_bytes = kwargs.pop('pk_bytes', None)
        password = kwargs.pop('password', None)
        encoding = kwargs.pop('encoding', Encoding.PEM)

        password_bytes = None

        if cert_bytes is None or not isinstance(cert_bytes, bytes):
            raise ValueError('Invalid cert content provided. '
                             'You must provide an X.509 cert '
                             'formatted as a byte array.')
        if pk_bytes is None or not isinstance(pk_bytes, bytes):
            raise ValueError('Invalid private key content provided. '
                             'You must provide a private key '
                             'formatted as a byte array.')

        if isinstance(password, bytes):
            password_bytes = password
        elif password:
            password_bytes = password.encode('utf8')

        self.ssl_context = create_ssl_context(cert_bytes, pk_bytes,
                                              password_bytes, encoding)

        super(X509Adapter, self).__init__(*args, **kwargs)

    def init_poolmanager(self, *args, **kwargs):
        if self.ssl_context:
            kwargs['ssl_context'] = self.ssl_context
        return super(X509Adapter, self).init_poolmanager(*args, **kwargs)

    def proxy_manager_for(self, *args, **kwargs):
        if self.ssl_context:
            kwargs['ssl_context'] = self.ssl_context
        return super(X509Adapter, self).proxy_manager_for(*args, **kwargs)

    def _import_pyopensslcontext(self):
        global PyOpenSSLContext

        if requests.__build__ < 0x021200:
            PyOpenSSLContext = None
        else:
            try:
                from requests.packages.urllib3.contrib.pyopenssl \
                        import PyOpenSSLContext
            except ImportError:
                try:
                    from urllib3.contrib.pyopenssl import PyOpenSSLContext
                except ImportError:
                    PyOpenSSLContext = None

    def _check_version(self):
        if PyOpenSSLContext is None:
            raise exc.VersionMismatchError(
                "The X509Adapter requires at least Requests 2.12.0 to be "
                "installed. Version {} was found instead.".format(
                    requests.__version__
                )
            )


def check_cert_dates(cert):
    """Verify that the supplied client cert is not invalid."""

    now = datetime.utcnow()
    if cert.not_valid_after < now or cert.not_valid_before > now:
        raise ValueError('Client certificate expired: Not After: '
                         '{:%Y-%m-%d %H:%M:%SZ} '
                         'Not Before: {:%Y-%m-%d %H:%M:%SZ}'
                         .format(cert.not_valid_after, cert.not_valid_before))


def create_ssl_context(cert_byes, pk_bytes, password=None,
                       encoding=Encoding.PEM):
    """Create an SSL Context with the supplied cert/password.

    :param cert_bytes array of bytes containing the cert encoded
           using the method supplied in the ``encoding`` parameter
    :param pk_bytes array of bytes containing the private key encoded
           using the method supplied in the ``encoding`` parameter
    :param password array of bytes containing the passphrase to be used
           with the supplied private key. None if unencrypted.
           Defaults to None.
    :param encoding ``cryptography.hazmat.primitives.serialization.Encoding``
            details the encoding method used on the ``cert_bytes``  and
            ``pk_bytes`` parameters. Can be either PEM or DER.
            Defaults to PEM.
    """
    backend = default_backend()

    cert = None
    key = None
    if encoding == Encoding.PEM:
        cert = x509.load_pem_x509_certificate(cert_byes, backend)
        key = load_pem_private_key(pk_bytes, password, backend)
    elif encoding == Encoding.DER:
        cert = x509.load_der_x509_certificate(cert_byes, backend)
        key = load_der_private_key(pk_bytes, password, backend)
    else:
        raise ValueError('Invalid encoding provided: Must be PEM or DER')

    if not (cert and key):
        raise ValueError('Cert and key could not be parsed from '
                         'provided data')
    check_cert_dates(cert)
    ssl_context = PyOpenSSLContext(PROTOCOL)
    ssl_context._ctx.use_certificate(X509.from_cryptography(cert))
    ssl_context._ctx.use_privatekey(PKey.from_cryptography_key(key))
    return ssl_context

env_pyrecodes/lib/python3.9/site-packages/requests_toolbelt/adapters/x509.py
NikolaBlagojevic/pyrecodes
# -*- coding: utf-8 -*-
"""Collection of exceptions raised by requests-toolbelt."""


class StreamingError(Exception):
    """Used in :mod:`requests_toolbelt.downloadutils.stream`."""
    pass


class VersionMismatchError(Exception):
    """Used to indicate a version mismatch in the version of requests required.

    The feature in use requires a newer version of Requests to function
    appropriately but the version installed is not sufficient.
    """
    pass


class RequestsVersionTooOld(Warning):
    """Used to indicate that the Requests version is too old.

    If the version of Requests is too old to support a feature, we will issue
    this warning to the user.
    """
    pass

env_pyrecodes/lib/python3.9/site-packages/requests_toolbelt/exceptions.py
NikolaBlagojevic/pyrecodes
import requests

from ._compat import urljoin


class BaseUrlSession(requests.Session):
    """A Session with a URL that all requests will use as a base.

    Let's start by looking at a few examples:

    .. code-block:: python

        >>> from requests_toolbelt import sessions
        >>> s = sessions.BaseUrlSession(
        ...     base_url='https://example.com/resource/')
        >>> r = s.get('sub-resource/', params={'foo': 'bar'})
        >>> print(r.request.url)
        https://example.com/resource/sub-resource/?foo=bar

    Our call to the ``get`` method will make a request to the URL passed in
    when we created the Session and the partial resource name we provide.
    We implement this by overriding the ``request`` method of the Session.

    Likewise, we override the ``prepare_request`` method so you can construct
    a PreparedRequest in the same way:

    .. code-block:: python

        >>> from requests import Request
        >>> from requests_toolbelt import sessions
        >>> s = sessions.BaseUrlSession(
        ...     base_url='https://example.com/resource/')
        >>> request = Request(method='GET', url='sub-resource/')
        >>> prepared_request = s.prepare_request(request)
        >>> r = s.send(prepared_request)
        >>> print(r.request.url)
        https://example.com/resource/sub-resource

    .. note::

        The base URL that you provide and the path you provide are **very**
        important.

    Let's look at another *similar* example

    .. code-block:: python

        >>> from requests_toolbelt import sessions
        >>> s = sessions.BaseUrlSession(
        ...     base_url='https://example.com/resource/')
        >>> r = s.get('/sub-resource/', params={'foo': 'bar'})
        >>> print(r.request.url)
        https://example.com/sub-resource/?foo=bar

    The key difference here is that we called ``get`` with ``/sub-resource/``,
    i.e., there was a leading ``/``. This changes how we create the URL
    because we rely on :mod:`urllib.parse.urljoin`.

    To override how we generate the URL, sub-class this method and override the
    ``create_url`` method.

    Based on implementation from
    https://github.com/kennethreitz/requests/issues/2554#issuecomment-109341010
    """

    base_url = None

    def __init__(self, base_url=None):
        if base_url:
            self.base_url = base_url
        super(BaseUrlSession, self).__init__()

    def request(self, method, url, *args, **kwargs):
        """Send the request after generating the complete URL."""
        url = self.create_url(url)
        return super(BaseUrlSession, self).request(
            method, url, *args, **kwargs
        )

    def prepare_request(self, request, *args, **kwargs):
        """Prepare the request after generating the complete URL."""
        request.url = self.create_url(request.url)
        return super(BaseUrlSession, self).prepare_request(
            request, *args, **kwargs
        )

    def create_url(self, url):
        """Create the URL based off this partial path."""
        return urljoin(self.base_url, url)

env_pyrecodes/lib/python3.9/site-packages/requests_toolbelt/sessions.py
NikolaBlagojevic/pyrecodes
"""This module provides functions for dumping information about responses."""
import collections

from requests import compat


__all__ = ('dump_response', 'dump_all')

HTTP_VERSIONS = {
    9: b'0.9',
    10: b'1.0',
    11: b'1.1',
}

_PrefixSettings = collections.namedtuple('PrefixSettings',
                                         ['request', 'response'])


class PrefixSettings(_PrefixSettings):
    def __new__(cls, request, response):
        request = _coerce_to_bytes(request)
        response = _coerce_to_bytes(response)
        return super(PrefixSettings, cls).__new__(cls, request, response)


def _get_proxy_information(response):
    if getattr(response.connection, 'proxy_manager', False):
        proxy_info = {}
        request_url = response.request.url
        if request_url.startswith('https://'):
            proxy_info['method'] = 'CONNECT'

        proxy_info['request_path'] = request_url
        return proxy_info
    return None


def _format_header(name, value):
    return (_coerce_to_bytes(name) + b': ' + _coerce_to_bytes(value) +
            b'\r\n')


def _build_request_path(url, proxy_info):
    uri = compat.urlparse(url)
    proxy_url = proxy_info.get('request_path')
    if proxy_url is not None:
        request_path = _coerce_to_bytes(proxy_url)
        return request_path, uri

    request_path = _coerce_to_bytes(uri.path)
    if uri.query:
        request_path += b'?' + _coerce_to_bytes(uri.query)

    return request_path, uri


def _dump_request_data(request, prefixes, bytearr, proxy_info=None):
    if proxy_info is None:
        proxy_info = {}

    prefix = prefixes.request
    method = _coerce_to_bytes(proxy_info.pop('method', request.method))
    request_path, uri = _build_request_path(request.url, proxy_info)

    # <prefix><METHOD> <request-path> HTTP/1.1
    bytearr.extend(prefix + method + b' ' + request_path + b' HTTP/1.1\r\n')

    # <prefix>Host: <request-host> OR host header specified by user
    headers = request.headers.copy()
    host_header = _coerce_to_bytes(headers.pop('Host', uri.netloc))
    bytearr.extend(prefix + b'Host: ' + host_header + b'\r\n')

    for name, value in headers.items():
        bytearr.extend(prefix + _format_header(name, value))

    bytearr.extend(prefix + b'\r\n')
    if request.body:
        if isinstance(request.body, compat.basestring):
            bytearr.extend(prefix + _coerce_to_bytes(request.body))
        else:
            # In the event that the body is a file-like object, let's not try
            # to read everything into memory.
            bytearr.extend(b'<< Request body is not a string-like type >>')
        bytearr.extend(b'\r\n')
    bytearr.extend(b'\r\n')


def _dump_response_data(response, prefixes, bytearr):
    prefix = prefixes.response
    # Let's interact almost entirely with urllib3's response
    raw = response.raw

    # Let's convert the version int from httplib to bytes
    version_str = HTTP_VERSIONS.get(raw.version, b'?')

    # <prefix>HTTP/<version_str> <status_code> <reason>
    bytearr.extend(prefix + b'HTTP/' + version_str + b' ' +
                   str(raw.status).encode('ascii') + b' ' +
                   _coerce_to_bytes(response.reason) + b'\r\n')

    headers = raw.headers
    for name in headers.keys():
        for value in headers.getlist(name):
            bytearr.extend(prefix + _format_header(name, value))

    bytearr.extend(prefix + b'\r\n')

    bytearr.extend(response.content)


def _coerce_to_bytes(data):
    if not isinstance(data, bytes) and hasattr(data, 'encode'):
        data = data.encode('utf-8')
    # Don't bail out with an exception if data is None
    return data if data is not None else b''


def dump_response(response, request_prefix=b'< ', response_prefix=b'> ',
                  data_array=None):
    """Dump a single request-response cycle's information.

    This will take a response object and dump only the data that requests can
    see for that single request-response cycle.

    Example::

        import requests
        from requests_toolbelt.utils import dump

        resp = requests.get('https://api.github.com/users/sigmavirus24')
        data = dump.dump_response(resp)
        print(data.decode('utf-8'))

    :param response:
        The response to format
    :type response: :class:`requests.Response`
    :param request_prefix: (*optional*)
        Bytes to prefix each line of the request data
    :type request_prefix: :class:`bytes`
    :param response_prefix: (*optional*)
        Bytes to prefix each line of the response data
    :type response_prefix: :class:`bytes`
    :param data_array: (*optional*)
        Bytearray to which we append the request-response cycle data
    :type data_array: :class:`bytearray`
    :returns: Formatted bytes of request and response information.
    :rtype: :class:`bytearray`
    """
    data = data_array if data_array is not None else bytearray()
    prefixes = PrefixSettings(request_prefix, response_prefix)

    if not hasattr(response, 'request'):
        raise ValueError('Response has no associated request')

    proxy_info = _get_proxy_information(response)
    _dump_request_data(response.request, prefixes, data,
                       proxy_info=proxy_info)
    _dump_response_data(response, prefixes, data)
    return data


def dump_all(response, request_prefix=b'< ', response_prefix=b'> '):
    """Dump all requests and responses including redirects.

    This takes the response returned by requests and will dump all
    request-response pairs in the redirect history in order followed by the
    final request-response.

    Example::

        import requests
        from requests_toolbelt.utils import dump

        resp = requests.get('https://httpbin.org/redirect/5')
        data = dump.dump_all(resp)
        print(data.decode('utf-8'))

    :param response:
        The response to format
    :type response: :class:`requests.Response`
    :param request_prefix: (*optional*)
        Bytes to prefix each line of the request data
    :type request_prefix: :class:`bytes`
    :param response_prefix: (*optional*)
        Bytes to prefix each line of the response data
    :type response_prefix: :class:`bytes`
    :returns: Formatted bytes of request and response information.
    :rtype: :class:`bytearray`
    """
    data = bytearray()

    history = list(response.history[:])
    history.append(response)

    for response in history:
        dump_response(response, request_prefix, response_prefix, data)

    return data

env_pyrecodes/lib/python3.9/site-packages/requests_toolbelt/utils/dump.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2014 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
An implementation of semantics and validations described in RFC 3986.

See http://rfc3986.readthedocs.io/ for detailed documentation.

:copyright: (c) 2014 Rackspace
:license: Apache v2.0, see LICENSE for details
"""
from .api import iri_reference
from .api import IRIReference
from .api import is_valid_uri
from .api import normalize_uri
from .api import uri_reference
from .api import URIReference
from .api import urlparse
from .parseresult import ParseResult

__title__ = "rfc3986"
__author__ = "Ian Stapleton Cordasco"
__author_email__ = "graffatcolmingov@gmail.com"
__license__ = "Apache v2.0"
__copyright__ = "Copyright 2014 Rackspace; 2016 Ian Stapleton Cordasco"
__version__ = "2.0.0"

__all__ = (
    "ParseResult",
    "URIReference",
    "IRIReference",
    "is_valid_uri",
    "normalize_uri",
    "uri_reference",
    "iri_reference",
    "urlparse",
    "__title__",
    "__author__",
    "__author_email__",
    "__license__",
    "__copyright__",
    "__version__",
)

env_pyrecodes/lib/python3.9/site-packages/rfc3986/__init__.py
NikolaBlagojevic/pyrecodes
"""Module containing the implementation of the URIMixin class."""
import warnings

from . import exceptions as exc
from . import misc
from . import normalizers
from . import validators


class URIMixin:
    """Mixin with all shared methods for URIs and IRIs."""

    __hash__ = tuple.__hash__

    def authority_info(self):
        """Return a dictionary with the ``userinfo``, ``host``, and ``port``.

        If the authority is not valid, it will raise a
        :class:`~rfc3986.exceptions.InvalidAuthority` Exception.

        :returns:
            ``{'userinfo': 'username:password', 'host': 'www.example.com',
            'port': '80'}``
        :rtype: dict
        :raises rfc3986.exceptions.InvalidAuthority:
            If the authority is not ``None`` and can not be parsed.
        """
        if not self.authority:
            return {"userinfo": None, "host": None, "port": None}

        match = self._match_subauthority()

        if match is None:
            # In this case, we have an authority that was parsed from the URI
            # Reference, but it cannot be further parsed by our
            # misc.SUBAUTHORITY_MATCHER. In this case it must not be a valid
            # authority.
            raise exc.InvalidAuthority(self.authority.encode(self.encoding))

        # We had a match, now let's ensure that it is actually a valid host
        # address if it is IPv4
        matches = match.groupdict()
        host = matches.get("host")

        if (
            host
            and misc.IPv4_MATCHER.match(host)
            and not validators.valid_ipv4_host_address(host)
        ):
            # If we have a host, it appears to be IPv4 and it does not have
            # valid bytes, it is an InvalidAuthority.
            raise exc.InvalidAuthority(self.authority.encode(self.encoding))

        return matches

    def _match_subauthority(self):
        return misc.SUBAUTHORITY_MATCHER.match(self.authority)

    @property
    def host(self):
        """If present, a string representing the host."""
        try:
            authority = self.authority_info()
        except exc.InvalidAuthority:
            return None
        return authority["host"]

    @property
    def port(self):
        """If present, the port extracted from the authority."""
        try:
            authority = self.authority_info()
        except exc.InvalidAuthority:
            return None
        return authority["port"]

    @property
    def userinfo(self):
        """If present, the userinfo extracted from the authority."""
        try:
            authority = self.authority_info()
        except exc.InvalidAuthority:
            return None
        return authority["userinfo"]

    def is_absolute(self):
        """Determine if this URI Reference is an absolute URI.

        See http://tools.ietf.org/html/rfc3986#section-4.3 for explanation.

        :returns: ``True`` if it is an absolute URI, ``False`` otherwise.
        :rtype: bool
        """
        return bool(misc.ABSOLUTE_URI_MATCHER.match(self.unsplit()))

    def is_valid(self, **kwargs):
        """Determine if the URI is valid.

        .. deprecated:: 1.1.0

            Use the :class:`~rfc3986.validators.Validator` object instead.

        :param bool require_scheme: Set to ``True`` if you wish to require the
            presence of the scheme component.
        :param bool require_authority: Set to ``True`` if you wish to require
            the presence of the authority component.
        :param bool require_path: Set to ``True`` if you wish to require the
            presence of the path component.
        :param bool require_query: Set to ``True`` if you wish to require the
            presence of the query component.
        :param bool require_fragment: Set to ``True`` if you wish to require
            the presence of the fragment component.
        :returns: ``True`` if the URI is valid. ``False`` otherwise.
        :rtype: bool
        """
        warnings.warn(
            "Please use rfc3986.validators.Validator instead. "
            "This method will be eventually removed.",
            DeprecationWarning,
        )
        validators = [
            (self.scheme_is_valid, kwargs.get("require_scheme", False)),
            (self.authority_is_valid, kwargs.get("require_authority", False)),
            (self.path_is_valid, kwargs.get("require_path", False)),
            (self.query_is_valid, kwargs.get("require_query", False)),
            (self.fragment_is_valid, kwargs.get("require_fragment", False)),
        ]
        return all(v(r) for v, r in validators)

    def authority_is_valid(self, require=False):
        """Determine if the authority component is valid.

        .. deprecated:: 1.1.0

            Use the :class:`~rfc3986.validators.Validator` object instead.

        :param bool require:
            Set to ``True`` to require the presence of this component.
        :returns:
            ``True`` if the authority is valid. ``False`` otherwise.
        :rtype:
            bool
        """
        warnings.warn(
            "Please use rfc3986.validators.Validator instead. "
            "This method will be eventually removed.",
            DeprecationWarning,
        )
        try:
            self.authority_info()
        except exc.InvalidAuthority:
            return False

        return validators.authority_is_valid(
            self.authority,
            host=self.host,
            require=require,
        )

    def scheme_is_valid(self, require=False):
        """Determine if the scheme component is valid.

        .. deprecated:: 1.1.0

            Use the :class:`~rfc3986.validators.Validator` object instead.

        :param str require: Set to ``True`` to require the presence of this
            component.
        :returns: ``True`` if the scheme is valid. ``False`` otherwise.
        :rtype: bool
        """
        warnings.warn(
            "Please use rfc3986.validators.Validator instead. "
            "This method will be eventually removed.",
            DeprecationWarning,
        )
        return validators.scheme_is_valid(self.scheme, require)

    def path_is_valid(self, require=False):
        """Determine if the path component is valid.

        .. deprecated:: 1.1.0

            Use the :class:`~rfc3986.validators.Validator` object instead.

        :param str require: Set to ``True`` to require the presence of this
            component.
        :returns: ``True`` if the path is valid. ``False`` otherwise.
        :rtype: bool
        """
        warnings.warn(
            "Please use rfc3986.validators.Validator instead. "
            "This method will be eventually removed.",
            DeprecationWarning,
        )
        return validators.path_is_valid(self.path, require)

    def query_is_valid(self, require=False):
        """Determine if the query component is valid.

        .. deprecated:: 1.1.0

            Use the :class:`~rfc3986.validators.Validator` object instead.

        :param str require: Set to ``True`` to require the presence of this
            component.
        :returns: ``True`` if the query is valid. ``False`` otherwise.
        :rtype: bool
        """
        warnings.warn(
            "Please use rfc3986.validators.Validator instead. "
            "This method will be eventually removed.",
            DeprecationWarning,
        )
        return validators.query_is_valid(self.query, require)

    def fragment_is_valid(self, require=False):
        """Determine if the fragment component is valid.

        .. deprecated:: 1.1.0

            Use the Validator object instead.

        :param str require: Set to ``True`` to require the presence of this
            component.
        :returns: ``True`` if the fragment is valid. ``False`` otherwise.
        :rtype: bool
        """
        warnings.warn(
            "Please use rfc3986.validators.Validator instead. "
            "This method will be eventually removed.",
            DeprecationWarning,
        )
        return validators.fragment_is_valid(self.fragment, require)

    def normalized_equality(self, other_ref):
        """Compare this URIReference to another URIReference.

        :param URIReference other_ref: (required), The reference with which
            we're comparing.
        :returns: ``True`` if the references are equal, ``False`` otherwise.
        :rtype: bool
        """
        return tuple(self.normalize()) == tuple(other_ref.normalize())

    def resolve_with(self, base_uri, strict=False):
        """Use an absolute URI Reference to resolve this relative reference.

        Assuming this is a relative reference that you would like to resolve,
        use the provided base URI to resolve it.

        See http://tools.ietf.org/html/rfc3986#section-5 for more information.

        :param base_uri: Either a string or URIReference. It must be an
            absolute URI or it will raise an exception.
        :returns: A new URIReference which is the result of resolving this
            reference using ``base_uri``.
        :rtype: :class:`URIReference`
        :raises rfc3986.exceptions.ResolutionError:
            If the ``base_uri`` does not at least have a scheme.
        """
        if not isinstance(base_uri, URIMixin):
            base_uri = type(self).from_string(base_uri)

        if not base_uri.is_valid(require_scheme=True):
            raise exc.ResolutionError(base_uri)

        # This is optional per
        # http://tools.ietf.org/html/rfc3986#section-5.2.1
        base_uri = base_uri.normalize()

        # The reference we're resolving
        resolving = self

        if not strict and resolving.scheme == base_uri.scheme:
            resolving = resolving.copy_with(scheme=None)

        # http://tools.ietf.org/html/rfc3986#page-32
        if resolving.scheme is not None:
            target = resolving.copy_with(
                path=normalizers.normalize_path(resolving.path)
            )
        else:
            if resolving.authority is not None:
                target = resolving.copy_with(
                    scheme=base_uri.scheme,
                    path=normalizers.normalize_path(resolving.path),
                )
            else:
                if resolving.path is None:
                    if resolving.query is not None:
                        query = resolving.query
                    else:
                        query = base_uri.query
                    target = resolving.copy_with(
                        scheme=base_uri.scheme,
                        authority=base_uri.authority,
                        path=base_uri.path,
                        query=query,
                    )
                else:
                    if resolving.path.startswith("/"):
                        path = normalizers.normalize_path(resolving.path)
                    else:
                        path = normalizers.normalize_path(
                            misc.merge_paths(base_uri, resolving.path)
                        )
                    target = resolving.copy_with(
                        scheme=base_uri.scheme,
                        authority=base_uri.authority,
                        path=path,
                        query=resolving.query,
                    )
        return target

    def unsplit(self):
        """Create a URI string from the components.

        :returns: The URI Reference reconstituted as a string.
        :rtype: str
        """
        # See http://tools.ietf.org/html/rfc3986#section-5.3
        result_list = []
        if self.scheme:
            result_list.extend([self.scheme, ":"])
        if self.authority:
            result_list.extend(["//", self.authority])
        if self.path:
            result_list.append(self.path)
        if self.query is not None:
            result_list.extend(["?", self.query])
        if self.fragment is not None:
            result_list.extend(["#", self.fragment])
        return "".join(result_list)

    def copy_with(
        self,
        scheme=misc.UseExisting,
        authority=misc.UseExisting,
        path=misc.UseExisting,
        query=misc.UseExisting,
        fragment=misc.UseExisting,
    ):
        """Create a copy of this reference with the new components.

        :param str scheme:
            (optional) The scheme to use for the new reference.
        :param str authority:
            (optional) The authority to use for the new reference.
        :param str path:
            (optional) The path to use for the new reference.
        :param str query:
            (optional) The query to use for the new reference.
        :param str fragment:
            (optional) The fragment to use for the new reference.
        :returns:
            New URIReference with provided components.
        :rtype:
            URIReference
        """
        attributes = {
            "scheme": scheme,
            "authority": authority,
            "path": path,
            "query": query,
            "fragment": fragment,
        }
        for key, value in list(attributes.items()):
            if value is misc.UseExisting:
                del attributes[key]
        uri = self._replace(**attributes)
        uri.encoding = self.encoding
        return uri

env_pyrecodes/lib/python3.9/site-packages/rfc3986/_mixin.py
NikolaBlagojevic/pyrecodes
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for the regular expressions crafted from ABNF."""
import sys

# https://tools.ietf.org/html/rfc3986#page-13
GEN_DELIMS = GENERIC_DELIMITERS = ":/?#[]@"
GENERIC_DELIMITERS_SET = set(GENERIC_DELIMITERS)
# https://tools.ietf.org/html/rfc3986#page-13
SUB_DELIMS = SUB_DELIMITERS = "!$&'()*+,;="
SUB_DELIMITERS_SET = set(SUB_DELIMITERS)
# Escape the '*' for use in regular expressions
SUB_DELIMITERS_RE = r"!$&'()\*+,;="
RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET)
ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
DIGIT = "0123456789"
# https://tools.ietf.org/html/rfc3986#section-2.3
UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r"._!-~"
UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS)
NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET)
# We need to escape the '-' in this case:
UNRESERVED_RE = r"A-Za-z0-9._~\-"

# Percent encoded character values
PERCENT_ENCODED = PCT_ENCODED = "%[A-Fa-f0-9]{2}"
PCHAR = "([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":@]|%s)" % PCT_ENCODED

# NOTE(sigmavirus24): We're going to use more strict regular expressions
# than appear in Appendix B for scheme. This will prevent over-eager
# consuming of items that aren't schemes.
SCHEME_RE = "[a-zA-Z][a-zA-Z0-9+.-]*"
_AUTHORITY_RE = "[^\\\\/?#]*"
_PATH_RE = "[^?#]*"
_QUERY_RE = "[^#]*"
_FRAGMENT_RE = ".*"

# Extracted from http://tools.ietf.org/html/rfc3986#appendix-B
COMPONENT_PATTERN_DICT = {
    "scheme": SCHEME_RE,
    "authority": _AUTHORITY_RE,
    "path": _PATH_RE,
    "query": _QUERY_RE,
    "fragment": _FRAGMENT_RE,
}

# See http://tools.ietf.org/html/rfc3986#appendix-B
# In this case, we name each of the important matches so we can use
# SRE_Match#groupdict to parse the values out if we so choose. This is also
# modified to ignore other matches that are not important to the parsing of
# the reference so we can also simply use SRE_Match#groups.
URL_PARSING_RE = (
    r"(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?"
    r"(?P<path>{path})(?:\?(?P<query>{query}))?"
    r"(?:#(?P<fragment>{fragment}))?"
).format(**COMPONENT_PATTERN_DICT)


# #########################
# Authority Matcher Section
# #########################

# Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2
# The pattern for a regular name, e.g.,  www.google.com, api.github.com
REGULAR_NAME_RE = REG_NAME = "((?:{}|[{}])*)".format(
    "%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + UNRESERVED_RE
)
# The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1,
IPv4_RE = r"([0-9]{1,3}\.){3}[0-9]{1,3}"
# Hexadecimal characters used in each piece of an IPv6 address
HEXDIG_RE = "[0-9A-Fa-f]{1,4}"
# Least-significant 32 bits of an IPv6 address
LS32_RE = "({hex}:{hex}|{ipv4})".format(hex=HEXDIG_RE, ipv4=IPv4_RE)
# Substitutions into the following patterns for IPv6 patterns defined
# http://tools.ietf.org/html/rfc3986#page-20
_subs = {"hex": HEXDIG_RE, "ls32": LS32_RE}

# Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details
# about ABNF (Augmented Backus-Naur Form) use in the comments
variations = [
    #                            6( h16 ":" ) ls32
    "(%(hex)s:){6}%(ls32)s" % _subs,
    #                       "::" 5( h16 ":" ) ls32
    "::(%(hex)s:){5}%(ls32)s" % _subs,
    # [               h16 ] "::" 4( h16 ":" ) ls32
    "(%(hex)s)?::(%(hex)s:){4}%(ls32)s" % _subs,
    # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
    "((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s" % _subs,
    # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
    "((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s" % _subs,
    # [ *3( h16 ":" ) h16 ] "::"    h16 ":"   ls32
    "((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s" % _subs,
    # [ *4( h16 ":" ) h16 ] "::"              ls32
    "((%(hex)s:){0,4}%(hex)s)?::%(ls32)s" % _subs,
    # [ *5( h16 ":" ) h16 ] "::"              h16
    "((%(hex)s:){0,5}%(hex)s)?::%(hex)s" % _subs,
    # [ *6( h16 ":" ) h16 ] "::"
    "((%(hex)s:){0,6}%(hex)s)?::" % _subs,
]

IPv6_RE = "(({})|({})|({})|({})|({})|({})|({})|({})|({}))".format(*variations)

IPv_FUTURE_RE = r"v[0-9A-Fa-f]+\.[%s]+" % (
    UNRESERVED_RE + SUB_DELIMITERS_RE + ":"
)

# RFC 6874 Zone ID ABNF
ZONE_ID = "(?:[" + UNRESERVED_RE + "]|" + PCT_ENCODED + ")+"

IPv6_ADDRZ_RFC4007_RE = IPv6_RE + "(?:(?:%25|%)" + ZONE_ID + ")?"
IPv6_ADDRZ_RE = IPv6_RE + "(?:%25" + ZONE_ID + ")?"

IP_LITERAL_RE = r"\[({}|{})\]".format(
    IPv6_ADDRZ_RFC4007_RE,
    IPv_FUTURE_RE,
)

# Pattern for matching the host piece of the authority
HOST_RE = HOST_PATTERN = "({}|{}|{})".format(
    REG_NAME,
    IPv4_RE,
    IP_LITERAL_RE,
)
USERINFO_RE = (
    "^([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":]|%s)+" % (PCT_ENCODED)
)
PORT_RE = "[0-9]{1,5}"

# ####################
# Path Matcher Section
# ####################

# See http://tools.ietf.org/html/rfc3986#section-3.3 for more information
# about the path patterns defined below.
segments = {
    "segment": PCHAR + "*",
    # Non-zero length segment
    "segment-nz": PCHAR + "+",
    # Non-zero length segment without ":"
    "segment-nz-nc": PCHAR.replace(":", "") + "+",
}

# Path types taken from Section 3.3 (linked above)
PATH_EMPTY = "^$"
PATH_ROOTLESS = "%(segment-nz)s(/%(segment)s)*" % segments
PATH_NOSCHEME = "%(segment-nz-nc)s(/%(segment)s)*" % segments
PATH_ABSOLUTE = "/(%s)?" % PATH_ROOTLESS
PATH_ABEMPTY = "(/%(segment)s)*" % segments
PATH_RE = "^({}|{}|{}|{}|{})$".format(
    PATH_ABEMPTY,
    PATH_ABSOLUTE,
    PATH_NOSCHEME,
    PATH_ROOTLESS,
    PATH_EMPTY,
)

FRAGMENT_RE = QUERY_RE = (
    "^([/?:@" + UNRESERVED_RE + SUB_DELIMITERS_RE + "]|%s)*$" % PCT_ENCODED
)

# ##########################
# Relative reference matcher
# ##########################

# See http://tools.ietf.org/html/rfc3986#section-4.2 for details
RELATIVE_PART_RE = "(//{}{}|{}|{}|{})".format(
    COMPONENT_PATTERN_DICT["authority"],
    PATH_ABEMPTY,
    PATH_ABSOLUTE,
    PATH_NOSCHEME,
    PATH_EMPTY,
)

# See http://tools.ietf.org/html/rfc3986#section-3 for definition
HIER_PART_RE = "(//{}{}|{}|{}|{})".format(
    COMPONENT_PATTERN_DICT["authority"],
    PATH_ABEMPTY,
    PATH_ABSOLUTE,
    PATH_ROOTLESS,
    PATH_EMPTY,
)

# ###############
# IRIs / RFC 3987
# ###############

# Only wide-unicode gets the high-ranges of UCSCHAR
if sys.maxunicode > 0xFFFF:  # pragma: no cover
    IPRIVATE = "\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD"
    UCSCHAR_RE = (
        "\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
        "\U00010000-\U0001FFFD\U00020000-\U0002FFFD"
        "\U00030000-\U0003FFFD\U00040000-\U0004FFFD"
        "\U00050000-\U0005FFFD\U00060000-\U0006FFFD"
        "\U00070000-\U0007FFFD\U00080000-\U0008FFFD"
        "\U00090000-\U0009FFFD\U000A0000-\U000AFFFD"
        "\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD"
        "\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD"
    )
else:  # pragma: no cover
    IPRIVATE = "\uE000-\uF8FF"
    UCSCHAR_RE = "\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"

IUNRESERVED_RE = "A-Za-z0-9\\._~\\-" + UCSCHAR_RE
IPCHAR = "([" + IUNRESERVED_RE + SUB_DELIMITERS_RE + ":@]|%s)" % PCT_ENCODED

isegments = {
    "isegment": IPCHAR + "*",
    # Non-zero length segment
    "isegment-nz": IPCHAR + "+",
    # Non-zero length segment without ":"
    "isegment-nz-nc": IPCHAR.replace(":", "") + "+",
}

IPATH_ROOTLESS = "%(isegment-nz)s(/%(isegment)s)*" % isegments
IPATH_NOSCHEME = "%(isegment-nz-nc)s(/%(isegment)s)*" % isegments
IPATH_ABSOLUTE = "/(?:%s)?" % IPATH_ROOTLESS
IPATH_ABEMPTY = "(?:/%(isegment)s)*" % isegments
IPATH_RE = "^(?:{}|{}|{}|{}|{})$".format(
    IPATH_ABEMPTY,
    IPATH_ABSOLUTE,
    IPATH_NOSCHEME,
    IPATH_ROOTLESS,
    PATH_EMPTY,
)

IREGULAR_NAME_RE = IREG_NAME = "(?:{}|[{}])*".format(
    "%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + IUNRESERVED_RE
)

IHOST_RE = IHOST_PATTERN = "({}|{}|{})".format(
    IREG_NAME,
    IPv4_RE,
    IP_LITERAL_RE,
)

IUSERINFO_RE = (
    "^(?:[" + IUNRESERVED_RE + SUB_DELIMITERS_RE + ":]|%s)+" % (PCT_ENCODED)
)

IFRAGMENT_RE = (
    "^(?:[/?:@" + IUNRESERVED_RE + SUB_DELIMITERS_RE + "]|%s)*$" % PCT_ENCODED
)
IQUERY_RE = (
    "^(?:[/?:@"
    + IUNRESERVED_RE
    + SUB_DELIMITERS_RE
    + IPRIVATE
    + "]|%s)*$" % PCT_ENCODED
)

IRELATIVE_PART_RE = "(//{}{}|{}|{}|{})".format(
    COMPONENT_PATTERN_DICT["authority"],
    IPATH_ABEMPTY,
    IPATH_ABSOLUTE,
    IPATH_NOSCHEME,
    PATH_EMPTY,
)

IHIER_PART_RE = "(//{}{}|{}|{}|{})".format(
    COMPONENT_PATTERN_DICT["authority"],
    IPATH_ABEMPTY,
    IPATH_ABSOLUTE,
    IPATH_ROOTLESS,
    PATH_EMPTY,
)

env_pyrecodes/lib/python3.9/site-packages/rfc3986/abnf_regexp.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2014 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module containing the simple and functional API for rfc3986.

This module defines functions and provides access to the public attributes
and classes of rfc3986.
"""
from .iri import IRIReference
from .parseresult import ParseResult
from .uri import URIReference


def uri_reference(uri, encoding="utf-8"):
    """Parse a URI string into a URIReference.

    This is a convenience function. You could achieve the same end by using
    ``URIReference.from_string(uri)``.

    :param str uri: The URI which needs to be parsed into a reference.
    :param str encoding: The encoding of the string provided
    :returns: A parsed URI
    :rtype: :class:`URIReference`
    """
    return URIReference.from_string(uri, encoding)


def iri_reference(iri, encoding="utf-8"):
    """Parse a IRI string into an IRIReference.

    This is a convenience function. You could achieve the same end by using
    ``IRIReference.from_string(iri)``.

    :param str iri: The IRI which needs to be parsed into a reference.
    :param str encoding: The encoding of the string provided
    :returns: A parsed IRI
    :rtype: :class:`IRIReference`
    """
    return IRIReference.from_string(iri, encoding)


def is_valid_uri(uri, encoding="utf-8", **kwargs):
    """Determine if the URI given is valid.

    This is a convenience function. You could use either
    ``uri_reference(uri).is_valid()`` or
    ``URIReference.from_string(uri).is_valid()`` to achieve the same result.

    :param str uri: The URI to be validated.
    :param str encoding: The encoding of the string provided
    :param bool require_scheme: Set to ``True`` if you wish to require the
        presence of the scheme component.
    :param bool require_authority: Set to ``True`` if you wish to require the
        presence of the authority component.
    :param bool require_path: Set to ``True`` if you wish to require the
        presence of the path component.
    :param bool require_query: Set to ``True`` if you wish to require the
        presence of the query component.
    :param bool require_fragment: Set to ``True`` if you wish to require the
        presence of the fragment component.
    :returns: ``True`` if the URI is valid, ``False`` otherwise.
    :rtype: bool
    """
    return URIReference.from_string(uri, encoding).is_valid(**kwargs)


def normalize_uri(uri, encoding="utf-8"):
    """Normalize the given URI.

    This is a convenience function. You could use either
    ``uri_reference(uri).normalize().unsplit()`` or
    ``URIReference.from_string(uri).normalize().unsplit()`` instead.

    :param str uri: The URI to be normalized.
    :param str encoding: The encoding of the string provided
    :returns: The normalized URI.
    :rtype: str
    """
    normalized_reference = URIReference.from_string(uri, encoding).normalize()
    return normalized_reference.unsplit()


def urlparse(uri, encoding="utf-8"):
    """Parse a given URI and return a ParseResult.

    This is a partial replacement of the standard library's urlparse function.

    :param str uri: The URI to be parsed.
    :param str encoding: The encoding of the string provided.
    :returns: A parsed URI
    :rtype: :class:`~rfc3986.parseresult.ParseResult`
    """
    return ParseResult.from_string(uri, encoding, strict=False)

env_pyrecodes/lib/python3.9/site-packages/rfc3986/api.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2017 Ian Stapleton Cordasco
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing the logic for the URIBuilder object."""
from . import compat
from . import normalizers
from . import uri
from . import uri_reference


class URIBuilder:
    """Object to aid in building up a URI Reference from parts.

    .. note::

        This object should be instantiated by the user, but it's recommended
        that it is not provided with arguments. Instead, use the available
        method to populate the fields.

    """

    def __init__(
        self,
        scheme=None,
        userinfo=None,
        host=None,
        port=None,
        path=None,
        query=None,
        fragment=None,
    ):
        """Initialize our URI builder.

        :param str scheme:
            (optional)
        :param str userinfo:
            (optional)
        :param str host:
            (optional)
        :param int port:
            (optional)
        :param str path:
            (optional)
        :param str query:
            (optional)
        :param str fragment:
            (optional)
        """
        self.scheme = scheme
        self.userinfo = userinfo
        self.host = host
        self.port = port
        self.path = path
        self.query = query
        self.fragment = fragment

    def __repr__(self):
        """Provide a convenient view of our builder object."""
        formatstr = (
            "URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, "
            "host={b.host}, port={b.port}, path={b.path}, "
            "query={b.query}, fragment={b.fragment})"
        )
        return formatstr.format(b=self)

    @classmethod
    def from_uri(cls, reference):
        """Initialize the URI builder from another URI.

        Takes the given URI reference and creates a new URI builder instance
        populated with the values from the reference. If given a string it will
        try to convert it to a reference before constructing the builder.
        """
        if not isinstance(reference, uri.URIReference):
            reference = uri_reference(reference)
        return cls(
            scheme=reference.scheme,
            userinfo=reference.userinfo,
            host=reference.host,
            port=reference.port,
            path=reference.path,
            query=reference.query,
            fragment=reference.fragment,
        )

    def add_scheme(self, scheme):
        """Add a scheme to our builder object.

        After normalizing, this will generate a new URIBuilder instance with
        the specified scheme and all other attributes the same.

        .. code-block:: python

            >>> URIBuilder().add_scheme('HTTPS')
            URIBuilder(scheme='https', userinfo=None, host=None, port=None,
                    path=None, query=None, fragment=None)

        """
        scheme = normalizers.normalize_scheme(scheme)
        return URIBuilder(
            scheme=scheme,
            userinfo=self.userinfo,
            host=self.host,
            port=self.port,
            path=self.path,
            query=self.query,
            fragment=self.fragment,
        )

    def add_credentials(self, username, password):
        """Add credentials as the userinfo portion of the URI.

        .. code-block:: python

            >>> URIBuilder().add_credentials('root', 's3crete')
            URIBuilder(scheme=None, userinfo='root:s3crete', host=None,
                    port=None, path=None, query=None, fragment=None)

            >>> URIBuilder().add_credentials('root', None)
            URIBuilder(scheme=None, userinfo='root', host=None,
                    port=None, path=None, query=None, fragment=None)
        """
        if username is None:
            raise ValueError("Username cannot be None")
        userinfo = normalizers.normalize_username(username)

        if password is not None:
            userinfo = "{}:{}".format(
                userinfo,
                normalizers.normalize_password(password),
            )

        return URIBuilder(
            scheme=self.scheme,
            userinfo=userinfo,
            host=self.host,
            port=self.port,
            path=self.path,
            query=self.query,
            fragment=self.fragment,
        )

    def add_host(self, host):
        """Add hostname to the URI.

        .. code-block:: python

            >>> URIBuilder().add_host('google.com')
            URIBuilder(scheme=None, userinfo=None, host='google.com',
                    port=None, path=None, query=None, fragment=None)

        """
        return URIBuilder(
            scheme=self.scheme,
            userinfo=self.userinfo,
            host=normalizers.normalize_host(host),
            port=self.port,
            path=self.path,
            query=self.query,
            fragment=self.fragment,
        )

    def add_port(self, port):
        """Add port to the URI.

        .. code-block:: python

            >>> URIBuilder().add_port(80)
            URIBuilder(scheme=None, userinfo=None, host=None, port='80',
                    path=None, query=None, fragment=None)

            >>> URIBuilder().add_port(443)
            URIBuilder(scheme=None, userinfo=None, host=None, port='443',
                    path=None, query=None, fragment=None)

        """
        port_int = int(port)
        if port_int < 0:
            raise ValueError(
                "ports are not allowed to be negative. You provided {}".format(
                    port_int,
                )
            )
        if port_int > 65535:
            raise ValueError(
                "ports are not allowed to be larger than 65535. "
                "You provided {}".format(
                    port_int,
                )
            )

        return URIBuilder(
            scheme=self.scheme,
            userinfo=self.userinfo,
            host=self.host,
            port=f"{port_int}",
            path=self.path,
            query=self.query,
            fragment=self.fragment,
        )

    def add_path(self, path):
        """Add a path to the URI.

        .. code-block:: python

            >>> URIBuilder().add_path('sigmavirus24/rfc3985')
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path='/sigmavirus24/rfc3986', query=None, fragment=None)

            >>> URIBuilder().add_path('/checkout.php')
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path='/checkout.php', query=None, fragment=None)

        """
        if not path.startswith("/"):
            path = f"/{path}"

        return URIBuilder(
            scheme=self.scheme,
            userinfo=self.userinfo,
            host=self.host,
            port=self.port,
            path=normalizers.normalize_path(path),
            query=self.query,
            fragment=self.fragment,
        )

    def extend_path(self, path):
        """Extend the existing path value with the provided value.

        .. versionadded:: 1.5.0

        .. code-block:: python

            >>> URIBuilder(path="/users").extend_path("/sigmavirus24")
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path='/users/sigmavirus24', query=None, fragment=None)

            >>> URIBuilder(path="/users/").extend_path("/sigmavirus24")
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path='/users/sigmavirus24', query=None, fragment=None)

            >>> URIBuilder(path="/users/").extend_path("sigmavirus24")
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path='/users/sigmavirus24', query=None, fragment=None)

            >>> URIBuilder(path="/users").extend_path("sigmavirus24")
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path='/users/sigmavirus24', query=None, fragment=None)

        """
        existing_path = self.path or ""
        path = "{}/{}".format(existing_path.rstrip("/"), path.lstrip("/"))

        return self.add_path(path)

    def add_query_from(self, query_items):
        """Generate and add a query a dictionary or list of tuples.

        .. code-block:: python

            >>> URIBuilder().add_query_from({'a': 'b c'})
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path=None, query='a=b+c', fragment=None)

            >>> URIBuilder().add_query_from([('a', 'b c')])
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path=None, query='a=b+c', fragment=None)

        """
        query = normalizers.normalize_query(compat.urlencode(query_items))

        return URIBuilder(
            scheme=self.scheme,
            userinfo=self.userinfo,
            host=self.host,
            port=self.port,
            path=self.path,
            query=query,
            fragment=self.fragment,
        )

    def extend_query_with(self, query_items):
        """Extend the existing query string with the new query items.

        .. versionadded:: 1.5.0

        .. code-block:: python

            >>> URIBuilder(query='a=b+c').extend_query_with({'a': 'b c'})
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path=None, query='a=b+c&a=b+c', fragment=None)

            >>> URIBuilder(query='a=b+c').extend_query_with([('a', 'b c')])
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path=None, query='a=b+c&a=b+c', fragment=None)
        """
        original_query_items = compat.parse_qsl(self.query or "")
        if not isinstance(query_items, list):
            query_items = list(query_items.items())

        return self.add_query_from(original_query_items + query_items)

    def add_query(self, query):
        """Add a pre-formated query string to the URI.

        .. code-block:: python

            >>> URIBuilder().add_query('a=b&c=d')
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path=None, query='a=b&c=d', fragment=None)

        """
        return URIBuilder(
            scheme=self.scheme,
            userinfo=self.userinfo,
            host=self.host,
            port=self.port,
            path=self.path,
            query=normalizers.normalize_query(query),
            fragment=self.fragment,
        )

    def add_fragment(self, fragment):
        """Add a fragment to the URI.

        .. code-block:: python

            >>> URIBuilder().add_fragment('section-2.6.1')
            URIBuilder(scheme=None, userinfo=None, host=None, port=None,
                    path=None, query=None, fragment='section-2.6.1')

        """
        return URIBuilder(
            scheme=self.scheme,
            userinfo=self.userinfo,
            host=self.host,
            port=self.port,
            path=self.path,
            query=self.query,
            fragment=normalizers.normalize_fragment(fragment),
        )

    def finalize(self):
        """Create a URIReference from our builder.

        .. code-block:: python

            >>> URIBuilder().add_scheme('https').add_host('github.com'
            ...     ).add_path('sigmavirus24/rfc3986').finalize().unsplit()
            'https://github.com/sigmavirus24/rfc3986'

            >>> URIBuilder().add_scheme('https').add_host('github.com'
            ...     ).add_path('sigmavirus24/rfc3986').add_credentials(
            ...     'sigmavirus24', 'not-re@l').finalize().unsplit()
            'https://sigmavirus24:not-re%40l@github.com/sigmavirus24/rfc3986'

        """
        return uri.URIReference(
            self.scheme,
            normalizers.normalize_authority(
                (self.userinfo, self.host, self.port)
            ),
            self.path,
            self.query,
            self.fragment,
        )

    def geturl(self):
        """Generate the URL from this builder.

        .. versionadded:: 1.5.0

        This is an alternative to calling :meth:`finalize` and keeping the
        :class:`rfc3986.uri.URIReference` around.
        """
        return self.finalize().unsplit()

env_pyrecodes/lib/python3.9/site-packages/rfc3986/builder.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2014 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Compatibility module for Python 2 and 3 support."""
import sys

try:
    from urllib.parse import quote as urlquote
except ImportError:  # Python 2.x
    from urllib import quote as urlquote

try:
    from urllib.parse import parse_qsl
except ImportError:  # Python 2.x
    from urlparse import parse_qsl

try:
    from urllib.parse import urlencode
except ImportError:  # Python 2.x
    from urllib import urlencode

__all__ = (
    "to_bytes",
    "to_str",
    "urlquote",
    "urlencode",
    "parse_qsl",
)

PY3 = (3, 0) <= sys.version_info < (4, 0)
PY2 = (2, 6) <= sys.version_info < (2, 8)


if PY3:
    unicode = str  # Python 3.x


def to_str(b, encoding="utf-8"):
    """Ensure that b is text in the specified encoding."""
    if hasattr(b, "decode") and not isinstance(b, unicode):
        b = b.decode(encoding)
    return b


def to_bytes(s, encoding="utf-8"):
    """Ensure that s is converted to bytes from the encoding."""
    if hasattr(s, "encode") and not isinstance(s, bytes):
        s = s.encode(encoding)
    return s

env_pyrecodes/lib/python3.9/site-packages/rfc3986/compat.py
NikolaBlagojevic/pyrecodes
"""Exceptions module for rfc3986."""
from . import compat


class RFC3986Exception(Exception):
    """Base class for all rfc3986 exception classes."""

    pass


class InvalidAuthority(RFC3986Exception):
    """Exception when the authority string is invalid."""

    def __init__(self, authority):
        """Initialize the exception with the invalid authority."""
        super().__init__(
            f"The authority ({compat.to_str(authority)}) is not valid."
        )


class InvalidPort(RFC3986Exception):
    """Exception when the port is invalid."""

    def __init__(self, port):
        """Initialize the exception with the invalid port."""
        super().__init__(f'The port ("{port}") is not valid.')


class ResolutionError(RFC3986Exception):
    """Exception to indicate a failure to resolve a URI."""

    def __init__(self, uri):
        """Initialize the error with the failed URI."""
        super().__init__(
            "{} does not meet the requirements for resolution.".format(
                uri.unsplit()
            )
        )


class ValidationError(RFC3986Exception):
    """Exception raised during Validation of a URI."""

    pass


class MissingComponentError(ValidationError):
    """Exception raised when a required component is missing."""

    def __init__(self, uri, *component_names):
        """Initialize the error with the missing component name."""
        verb = "was"
        if len(component_names) > 1:
            verb = "were"

        self.uri = uri
        self.components = sorted(component_names)
        components = ", ".join(self.components)
        super().__init__(
            f"{components} {verb} required but missing",
            uri,
            self.components,
        )


class UnpermittedComponentError(ValidationError):
    """Exception raised when a component has an unpermitted value."""

    def __init__(self, component_name, component_value, allowed_values):
        """Initialize the error with the unpermitted component."""
        super().__init__(
            "{} was required to be one of {!r} but was {!r}".format(
                component_name,
                list(sorted(allowed_values)),
                component_value,
            ),
            component_name,
            component_value,
            allowed_values,
        )
        self.component_name = component_name
        self.component_value = component_value
        self.allowed_values = allowed_values


class PasswordForbidden(ValidationError):
    """Exception raised when a URL has a password in the userinfo section."""

    def __init__(self, uri):
        """Initialize the error with the URI that failed validation."""
        unsplit = getattr(uri, "unsplit", lambda: uri)
        super().__init__(
            '"{}" contained a password when validation forbade it'.format(
                unsplit()
            )
        )
        self.uri = uri


class InvalidComponentsError(ValidationError):
    """Exception raised when one or more components are invalid."""

    def __init__(self, uri, *component_names):
        """Initialize the error with the invalid component name(s)."""
        verb = "was"
        if len(component_names) > 1:
            verb = "were"

        self.uri = uri
        self.components = sorted(component_names)
        components = ", ".join(self.components)
        super().__init__(
            f"{components} {verb} found to be invalid",
            uri,
            self.components,
        )


class MissingDependencyError(RFC3986Exception):
    """Exception raised when an IRI is encoded without the 'idna' module."""

env_pyrecodes/lib/python3.9/site-packages/rfc3986/exceptions.py
NikolaBlagojevic/pyrecodes
"""Module containing the implementation of the IRIReference class."""
# Copyright (c) 2014 Rackspace
# Copyright (c) 2015 Ian Stapleton Cordasco
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple

from . import compat
from . import exceptions
from . import misc
from . import normalizers
from . import uri


try:
    import idna
except ImportError:  # pragma: no cover
    idna = None


class IRIReference(
    namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin
):
    """Immutable object representing a parsed IRI Reference.

    Can be encoded into an URIReference object via the procedure
    specified in RFC 3987 Section 3.1

     .. note::
        The IRI submodule is a new interface and may possibly change in
        the future. Check for changes to the interface when upgrading.
    """

    slots = ()

    def __new__(
        cls, scheme, authority, path, query, fragment, encoding="utf-8"
    ):
        """Create a new IRIReference."""
        ref = super().__new__(
            cls,
            scheme or None,
            authority or None,
            path or None,
            query,
            fragment,
        )
        ref.encoding = encoding
        return ref

    def __eq__(self, other):
        """Compare this reference to another."""
        other_ref = other
        if isinstance(other, tuple):
            other_ref = self.__class__(*other)
        elif not isinstance(other, IRIReference):
            try:
                other_ref = self.__class__.from_string(other)
            except TypeError:
                raise TypeError(
                    "Unable to compare {}() to {}()".format(
                        type(self).__name__, type(other).__name__
                    )
                )

        # See http://tools.ietf.org/html/rfc3986#section-6.2
        return tuple(self) == tuple(other_ref)

    def _match_subauthority(self):
        return misc.ISUBAUTHORITY_MATCHER.match(self.authority)

    @classmethod
    def from_string(cls, iri_string, encoding="utf-8"):
        """Parse a IRI reference from the given unicode IRI string.

        :param str iri_string: Unicode IRI to be parsed into a reference.
        :param str encoding: The encoding of the string provided
        :returns: :class:`IRIReference` or subclass thereof
        """
        iri_string = compat.to_str(iri_string, encoding)

        split_iri = misc.IRI_MATCHER.match(iri_string).groupdict()
        return cls(
            split_iri["scheme"],
            split_iri["authority"],
            normalizers.encode_component(split_iri["path"], encoding),
            normalizers.encode_component(split_iri["query"], encoding),
            normalizers.encode_component(split_iri["fragment"], encoding),
            encoding,
        )

    def encode(self, idna_encoder=None):  # noqa: C901
        """Encode an IRIReference into a URIReference instance.

        If the ``idna`` module is installed or the ``rfc3986[idna]``
        extra is used then unicode characters in the IRI host
        component will be encoded with IDNA2008.

        :param idna_encoder:
            Function that encodes each part of the host component
            If not given will raise an exception if the IRI
            contains a host component.
        :rtype: uri.URIReference
        :returns: A URI reference
        """
        authority = self.authority
        if authority:
            if idna_encoder is None:
                if idna is None:  # pragma: no cover
                    raise exceptions.MissingDependencyError(
                        "Could not import the 'idna' module "
                        "and the IRI hostname requires encoding"
                    )

                def idna_encoder(name):
                    if any(ord(c) > 128 for c in name):
                        try:
                            return idna.encode(
                                name.lower(), strict=True, std3_rules=True
                            )
                        except idna.IDNAError:
                            raise exceptions.InvalidAuthority(self.authority)
                    return name

            authority = ""
            if self.host:
                authority = ".".join(
                    [
                        compat.to_str(idna_encoder(part))
                        for part in self.host.split(".")
                    ]
                )

            if self.userinfo is not None:
                authority = (
                    normalizers.encode_component(self.userinfo, self.encoding)
                    + "@"
                    + authority
                )

            if self.port is not None:
                authority += ":" + str(self.port)

        return uri.URIReference(
            self.scheme,
            authority,
            path=self.path,
            query=self.query,
            fragment=self.fragment,
            encoding=self.encoding,
        )

env_pyrecodes/lib/python3.9/site-packages/rfc3986/iri.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2014 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module containing compiled regular expressions and constants.

This module contains important constants, patterns, and compiled regular
expressions for parsing and validating URIs and their components.
"""
import re

from . import abnf_regexp

# These are enumerated for the named tuple used as a superclass of
# URIReference
URI_COMPONENTS = ["scheme", "authority", "path", "query", "fragment"]

important_characters = {
    "generic_delimiters": abnf_regexp.GENERIC_DELIMITERS,
    "sub_delimiters": abnf_regexp.SUB_DELIMITERS,
    # We need to escape the '*' in this case
    "re_sub_delimiters": abnf_regexp.SUB_DELIMITERS_RE,
    "unreserved_chars": abnf_regexp.UNRESERVED_CHARS,
    # We need to escape the '-' in this case:
    "re_unreserved": abnf_regexp.UNRESERVED_RE,
}

# For details about delimiters and reserved characters, see:
# http://tools.ietf.org/html/rfc3986#section-2.2
GENERIC_DELIMITERS = abnf_regexp.GENERIC_DELIMITERS_SET
SUB_DELIMITERS = abnf_regexp.SUB_DELIMITERS_SET
RESERVED_CHARS = abnf_regexp.RESERVED_CHARS_SET
# For details about unreserved characters, see:
# http://tools.ietf.org/html/rfc3986#section-2.3
UNRESERVED_CHARS = abnf_regexp.UNRESERVED_CHARS_SET
NON_PCT_ENCODED = abnf_regexp.NON_PCT_ENCODED_SET

URI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE)

SUBAUTHORITY_MATCHER = re.compile(
    (
        "^(?:(?P<userinfo>{})@)?"  # userinfo
        "(?P<host>{})"  # host
        ":?(?P<port>{})?$"  # port
    ).format(
        abnf_regexp.USERINFO_RE, abnf_regexp.HOST_PATTERN, abnf_regexp.PORT_RE
    )
)


HOST_MATCHER = re.compile("^" + abnf_regexp.HOST_RE + "$")
IPv4_MATCHER = re.compile("^" + abnf_regexp.IPv4_RE + "$")
IPv6_MATCHER = re.compile(r"^\[" + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r"\]$")

# Used by host validator
IPv6_NO_RFC4007_MATCHER = re.compile(r"^\[%s\]$" % (abnf_regexp.IPv6_ADDRZ_RE))

# Matcher used to validate path components
PATH_MATCHER = re.compile(abnf_regexp.PATH_RE)


# ##################################
# Query and Fragment Matcher Section
# ##################################

QUERY_MATCHER = re.compile(abnf_regexp.QUERY_RE)

FRAGMENT_MATCHER = QUERY_MATCHER

# Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1
SCHEME_MATCHER = re.compile(f"^{abnf_regexp.SCHEME_RE}$")

RELATIVE_REF_MATCHER = re.compile(
    r"^%s(\?%s)?(#%s)?$"
    % (
        abnf_regexp.RELATIVE_PART_RE,
        abnf_regexp.QUERY_RE,
        abnf_regexp.FRAGMENT_RE,
    )
)

# See http://tools.ietf.org/html/rfc3986#section-4.3
ABSOLUTE_URI_MATCHER = re.compile(
    r"^%s:%s(\?%s)?$"
    % (
        abnf_regexp.COMPONENT_PATTERN_DICT["scheme"],
        abnf_regexp.HIER_PART_RE,
        abnf_regexp.QUERY_RE[1:-1],
    )
)

# ###############
# IRIs / RFC 3987
# ###############

IRI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE, re.UNICODE)

ISUBAUTHORITY_MATCHER = re.compile(
    (
        "^(?:(?P<userinfo>{})@)?"  # iuserinfo
        "(?P<host>{})"  # ihost
        ":?(?P<port>{})?$"  # port
    ).format(
        abnf_regexp.IUSERINFO_RE, abnf_regexp.IHOST_RE, abnf_regexp.PORT_RE
    ),
    re.UNICODE,
)


# Path merger as defined in http://tools.ietf.org/html/rfc3986#section-5.2.3
def merge_paths(base_uri, relative_path):
    """Merge a base URI's path with a relative URI's path."""
    if base_uri.path is None and base_uri.authority is not None:
        return "/" + relative_path
    else:
        path = base_uri.path or ""
        index = path.rfind("/")
        return path[:index] + "/" + relative_path


UseExisting = object()

env_pyrecodes/lib/python3.9/site-packages/rfc3986/misc.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2014 Rackspace
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module with functions to normalize components."""
import re

from . import compat
from . import misc


def normalize_scheme(scheme):
    """Normalize the scheme component."""
    return scheme.lower()


def normalize_authority(authority):
    """Normalize an authority tuple to a string."""
    userinfo, host, port = authority
    result = ""
    if userinfo:
        result += normalize_percent_characters(userinfo) + "@"
    if host:
        result += normalize_host(host)
    if port:
        result += ":" + port
    return result


def normalize_username(username):
    """Normalize a username to make it safe to include in userinfo."""
    return compat.urlquote(username)


def normalize_password(password):
    """Normalize a password to make safe for userinfo."""
    return compat.urlquote(password)


def normalize_host(host):
    """Normalize a host string."""
    if misc.IPv6_MATCHER.match(host):
        percent = host.find("%")
        if percent != -1:
            percent_25 = host.find("%25")

            # Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25'
            # from RFC 6874. If the host is '[<IPv6 addr>%25]' then we
            # assume RFC 4007 and normalize to '[<IPV6 addr>%2525]'
            if (
                percent_25 == -1
                or percent < percent_25
                or (percent == percent_25 and percent_25 == len(host) - 4)
            ):
                host = host.replace("%", "%25", 1)

            # Don't normalize the casing of the Zone ID
            return host[:percent].lower() + host[percent:]

    return host.lower()


def normalize_path(path):
    """Normalize the path string."""
    if not path:
        return path

    path = normalize_percent_characters(path)
    return remove_dot_segments(path)


def normalize_query(query):
    """Normalize the query string."""
    if not query:
        return query
    return normalize_percent_characters(query)


def normalize_fragment(fragment):
    """Normalize the fragment string."""
    if not fragment:
        return fragment
    return normalize_percent_characters(fragment)


PERCENT_MATCHER = re.compile("%[A-Fa-f0-9]{2}")


def normalize_percent_characters(s):
    """All percent characters should be upper-cased.

    For example, ``"%3afoo%DF%ab"`` should be turned into ``"%3Afoo%DF%AB"``.
    """
    matches = set(PERCENT_MATCHER.findall(s))
    for m in matches:
        if not m.isupper():
            s = s.replace(m, m.upper())
    return s


def remove_dot_segments(s):
    """Remove dot segments from the string.

    See also Section 5.2.4 of :rfc:`3986`.
    """
    # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
    segments = s.split("/")  # Turn the path into a list of segments
    output = []  # Initialize the variable to use to store output

    for segment in segments:
        # '.' is the current directory, so ignore it, it is superfluous
        if segment == ".":
            continue
        # Anything other than '..', should be appended to the output
        elif segment != "..":
            output.append(segment)
        # In this case segment == '..', if we can, we should pop the last
        # element
        elif output:
            output.pop()

    # If the path starts with '/' and the output is empty or the first string
    # is non-empty
    if s.startswith("/") and (not output or output[0]):
        output.insert(0, "")

    # If the path starts with '/.' or '/..' ensure we add one more empty
    # string to add a trailing '/'
    if s.endswith(("/.", "/..")):
        output.append("")

    return "/".join(output)


def encode_component(uri_component, encoding):
    """Encode the specific component in the provided encoding."""
    if uri_component is None:
        return uri_component

    # Try to see if the component we're encoding is already percent-encoded
    # so we can skip all '%' characters but still encode all others.
    percent_encodings = len(
        PERCENT_MATCHER.findall(compat.to_str(uri_component, encoding))
    )

    uri_bytes = compat.to_bytes(uri_component, encoding)
    is_percent_encoded = percent_encodings == uri_bytes.count(b"%")

    encoded_uri = bytearray()

    for i in range(0, len(uri_bytes)):
        # Will return a single character bytestring on both Python 2 & 3
        byte = uri_bytes[i : i + 1]
        byte_ord = ord(byte)
        if (is_percent_encoded and byte == b"%") or (
            byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED
        ):
            encoded_uri.extend(byte)
            continue
        encoded_uri.extend(f"%{byte_ord:02x}".encode().upper())

    return encoded_uri.decode(encoding)

env_pyrecodes/lib/python3.9/site-packages/rfc3986/normalizers.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2015 Ian Stapleton Cordasco
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing the urlparse compatibility logic."""
from collections import namedtuple

from . import compat
from . import exceptions
from . import misc
from . import normalizers
from . import uri

__all__ = ("ParseResult", "ParseResultBytes")

PARSED_COMPONENTS = (
    "scheme",
    "userinfo",
    "host",
    "port",
    "path",
    "query",
    "fragment",
)


class ParseResultMixin:
    def _generate_authority(self, attributes):
        # I swear I did not align the comparisons below. That's just how they
        # happened to align based on pep8 and attribute lengths.
        userinfo, host, port = (
            attributes[p] for p in ("userinfo", "host", "port")
        )
        if self.userinfo != userinfo or self.host != host or self.port != port:
            if port:
                port = f"{port}"
            return normalizers.normalize_authority(
                (
                    compat.to_str(userinfo, self.encoding),
                    compat.to_str(host, self.encoding),
                    port,
                )
            )
        if isinstance(self.authority, bytes):
            return self.authority.decode("utf-8")
        return self.authority

    def geturl(self):
        """Shim to match the standard library method."""
        return self.unsplit()

    @property
    def hostname(self):
        """Shim to match the standard library."""
        return self.host

    @property
    def netloc(self):
        """Shim to match the standard library."""
        return self.authority

    @property
    def params(self):
        """Shim to match the standard library."""
        return self.query


class ParseResult(
    namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin
):
    """Implementation of urlparse compatibility class.

    This uses the URIReference logic to handle compatibility with the
    urlparse.ParseResult class.
    """

    slots = ()

    def __new__(
        cls,
        scheme,
        userinfo,
        host,
        port,
        path,
        query,
        fragment,
        uri_ref,
        encoding="utf-8",
    ):
        """Create a new ParseResult."""
        parse_result = super().__new__(
            cls,
            scheme or None,
            userinfo or None,
            host,
            port or None,
            path or None,
            query,
            fragment,
        )
        parse_result.encoding = encoding
        parse_result.reference = uri_ref
        return parse_result

    @classmethod
    def from_parts(
        cls,
        scheme=None,
        userinfo=None,
        host=None,
        port=None,
        path=None,
        query=None,
        fragment=None,
        encoding="utf-8",
    ):
        """Create a ParseResult instance from its parts."""
        authority = ""
        if userinfo is not None:
            authority += userinfo + "@"
        if host is not None:
            authority += host
        if port is not None:
            authority += f":{port}"
        uri_ref = uri.URIReference(
            scheme=scheme,
            authority=authority,
            path=path,
            query=query,
            fragment=fragment,
            encoding=encoding,
        ).normalize()
        userinfo, host, port = authority_from(uri_ref, strict=True)
        return cls(
            scheme=uri_ref.scheme,
            userinfo=userinfo,
            host=host,
            port=port,
            path=uri_ref.path,
            query=uri_ref.query,
            fragment=uri_ref.fragment,
            uri_ref=uri_ref,
            encoding=encoding,
        )

    @classmethod
    def from_string(
        cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True
    ):
        """Parse a URI from the given unicode URI string.

        :param str uri_string: Unicode URI to be parsed into a reference.
        :param str encoding: The encoding of the string provided
        :param bool strict: Parse strictly according to :rfc:`3986` if True.
            If False, parse similarly to the standard library's urlparse
            function.
        :returns: :class:`ParseResult` or subclass thereof
        """
        reference = uri.URIReference.from_string(uri_string, encoding)
        if not lazy_normalize:
            reference = reference.normalize()
        userinfo, host, port = authority_from(reference, strict)

        return cls(
            scheme=reference.scheme,
            userinfo=userinfo,
            host=host,
            port=port,
            path=reference.path,
            query=reference.query,
            fragment=reference.fragment,
            uri_ref=reference,
            encoding=encoding,
        )

    @property
    def authority(self):
        """Return the normalized authority."""
        return self.reference.authority

    def copy_with(
        self,
        scheme=misc.UseExisting,
        userinfo=misc.UseExisting,
        host=misc.UseExisting,
        port=misc.UseExisting,
        path=misc.UseExisting,
        query=misc.UseExisting,
        fragment=misc.UseExisting,
    ):
        """Create a copy of this instance replacing with specified parts."""
        attributes = zip(
            PARSED_COMPONENTS,
            (scheme, userinfo, host, port, path, query, fragment),
        )
        attrs_dict = {}
        for name, value in attributes:
            if value is misc.UseExisting:
                value = getattr(self, name)
            attrs_dict[name] = value
        authority = self._generate_authority(attrs_dict)
        ref = self.reference.copy_with(
            scheme=attrs_dict["scheme"],
            authority=authority,
            path=attrs_dict["path"],
            query=attrs_dict["query"],
            fragment=attrs_dict["fragment"],
        )
        return ParseResult(uri_ref=ref, encoding=self.encoding, **attrs_dict)

    def encode(self, encoding=None):
        """Convert to an instance of ParseResultBytes."""
        encoding = encoding or self.encoding
        attrs = dict(
            zip(
                PARSED_COMPONENTS,
                (
                    attr.encode(encoding) if hasattr(attr, "encode") else attr
                    for attr in self
                ),
            )
        )
        return ParseResultBytes(
            uri_ref=self.reference, encoding=encoding, **attrs
        )

    def unsplit(self, use_idna=False):
        """Create a URI string from the components.

        :returns: The parsed URI reconstituted as a string.
        :rtype: str
        """
        parse_result = self
        if use_idna and self.host:
            hostbytes = self.host.encode("idna")
            host = hostbytes.decode(self.encoding)
            parse_result = self.copy_with(host=host)
        return parse_result.reference.unsplit()


class ParseResultBytes(
    namedtuple("ParseResultBytes", PARSED_COMPONENTS), ParseResultMixin
):
    """Compatibility shim for the urlparse.ParseResultBytes object."""

    def __new__(
        cls,
        scheme,
        userinfo,
        host,
        port,
        path,
        query,
        fragment,
        uri_ref,
        encoding="utf-8",
        lazy_normalize=True,
    ):
        """Create a new ParseResultBytes instance."""
        parse_result = super().__new__(
            cls,
            scheme or None,
            userinfo or None,
            host,
            port or None,
            path or None,
            query or None,
            fragment or None,
        )
        parse_result.encoding = encoding
        parse_result.reference = uri_ref
        parse_result.lazy_normalize = lazy_normalize
        return parse_result

    @classmethod
    def from_parts(
        cls,
        scheme=None,
        userinfo=None,
        host=None,
        port=None,
        path=None,
        query=None,
        fragment=None,
        encoding="utf-8",
        lazy_normalize=True,
    ):
        """Create a ParseResult instance from its parts."""
        authority = ""
        if userinfo is not None:
            authority += userinfo + "@"
        if host is not None:
            authority += host
        if port is not None:
            authority += f":{int(port)}"
        uri_ref = uri.URIReference(
            scheme=scheme,
            authority=authority,
            path=path,
            query=query,
            fragment=fragment,
            encoding=encoding,
        )
        if not lazy_normalize:
            uri_ref = uri_ref.normalize()
        to_bytes = compat.to_bytes
        userinfo, host, port = authority_from(uri_ref, strict=True)
        return cls(
            scheme=to_bytes(scheme, encoding),
            userinfo=to_bytes(userinfo, encoding),
            host=to_bytes(host, encoding),
            port=port,
            path=to_bytes(path, encoding),
            query=to_bytes(query, encoding),
            fragment=to_bytes(fragment, encoding),
            uri_ref=uri_ref,
            encoding=encoding,
            lazy_normalize=lazy_normalize,
        )

    @classmethod
    def from_string(
        cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True
    ):
        """Parse a URI from the given unicode URI string.

        :param str uri_string: Unicode URI to be parsed into a reference.
        :param str encoding: The encoding of the string provided
        :param bool strict: Parse strictly according to :rfc:`3986` if True.
            If False, parse similarly to the standard library's urlparse
            function.
        :returns: :class:`ParseResultBytes` or subclass thereof
        """
        reference = uri.URIReference.from_string(uri_string, encoding)
        if not lazy_normalize:
            reference = reference.normalize()
        userinfo, host, port = authority_from(reference, strict)

        to_bytes = compat.to_bytes
        return cls(
            scheme=to_bytes(reference.scheme, encoding),
            userinfo=to_bytes(userinfo, encoding),
            host=to_bytes(host, encoding),
            port=port,
            path=to_bytes(reference.path, encoding),
            query=to_bytes(reference.query, encoding),
            fragment=to_bytes(reference.fragment, encoding),
            uri_ref=reference,
            encoding=encoding,
            lazy_normalize=lazy_normalize,
        )

    @property
    def authority(self):
        """Return the normalized authority."""
        return self.reference.authority.encode(self.encoding)

    def copy_with(
        self,
        scheme=misc.UseExisting,
        userinfo=misc.UseExisting,
        host=misc.UseExisting,
        port=misc.UseExisting,
        path=misc.UseExisting,
        query=misc.UseExisting,
        fragment=misc.UseExisting,
        lazy_normalize=True,
    ):
        """Create a copy of this instance replacing with specified parts."""
        attributes = zip(
            PARSED_COMPONENTS,
            (scheme, userinfo, host, port, path, query, fragment),
        )
        attrs_dict = {}
        for name, value in attributes:
            if value is misc.UseExisting:
                value = getattr(self, name)
            if not isinstance(value, bytes) and hasattr(value, "encode"):
                value = value.encode(self.encoding)
            attrs_dict[name] = value
        authority = self._generate_authority(attrs_dict)
        to_str = compat.to_str
        ref = self.reference.copy_with(
            scheme=to_str(attrs_dict["scheme"], self.encoding),
            authority=to_str(authority, self.encoding),
            path=to_str(attrs_dict["path"], self.encoding),
            query=to_str(attrs_dict["query"], self.encoding),
            fragment=to_str(attrs_dict["fragment"], self.encoding),
        )
        if not lazy_normalize:
            ref = ref.normalize()
        return ParseResultBytes(
            uri_ref=ref,
            encoding=self.encoding,
            lazy_normalize=lazy_normalize,
            **attrs_dict,
        )

    def unsplit(self, use_idna=False):
        """Create a URI bytes object from the components.

        :returns: The parsed URI reconstituted as a string.
        :rtype: bytes
        """
        parse_result = self
        if use_idna and self.host:
            # self.host is bytes, to encode to idna, we need to decode it
            # first
            host = self.host.decode(self.encoding)
            hostbytes = host.encode("idna")
            parse_result = self.copy_with(host=hostbytes)
        if self.lazy_normalize:
            parse_result = parse_result.copy_with(lazy_normalize=False)
        uri = parse_result.reference.unsplit()
        return uri.encode(self.encoding)


def split_authority(authority):
    # Initialize our expected return values
    userinfo = host = port = None
    # Initialize an extra var we may need to use
    extra_host = None
    # Set-up rest in case there is no userinfo portion
    rest = authority

    if "@" in authority:
        userinfo, rest = authority.rsplit("@", 1)

    # Handle IPv6 host addresses
    if rest.startswith("["):
        host, rest = rest.split("]", 1)
        host += "]"

    if ":" in rest:
        extra_host, port = rest.split(":", 1)
    elif not host and rest:
        host = rest

    if extra_host and not host:
        host = extra_host

    return userinfo, host, port


def authority_from(reference, strict):
    try:
        subauthority = reference.authority_info()
    except exceptions.InvalidAuthority:
        if strict:
            raise
        userinfo, host, port = split_authority(reference.authority)
    else:
        # Thanks to Richard Barrell for this idea:
        # https://twitter.com/0x2ba22e11/status/617338811975139328
        userinfo, host, port = (
            subauthority.get(p) for p in ("userinfo", "host", "port")
        )

    if port:
        try:
            port = int(port)
        except ValueError:
            raise exceptions.InvalidPort(port)
    return userinfo, host, port

env_pyrecodes/lib/python3.9/site-packages/rfc3986/parseresult.py
NikolaBlagojevic/pyrecodes
"""Module containing the implementation of the URIReference class."""
# Copyright (c) 2014 Rackspace
# Copyright (c) 2015 Ian Stapleton Cordasco
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple

from . import compat
from . import misc
from . import normalizers
from ._mixin import URIMixin


class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
    """Immutable object representing a parsed URI Reference.

    .. note::

        This class is not intended to be directly instantiated by the user.

    This object exposes attributes for the following components of a
    URI:

    - scheme
    - authority
    - path
    - query
    - fragment

    .. attribute:: scheme

        The scheme that was parsed for the URI Reference. For example,
        ``http``, ``https``, ``smtp``, ``imap``, etc.

    .. attribute:: authority

        Component of the URI that contains the user information, host,
        and port sub-components. For example,
        ``google.com``, ``127.0.0.1:5000``, ``username@[::1]``,
        ``username:password@example.com:443``, etc.

    .. attribute:: path

        The path that was parsed for the given URI Reference. For example,
        ``/``, ``/index.php``, etc.

    .. attribute:: query

        The query component for a given URI Reference. For example, ``a=b``,
        ``a=b%20c``, ``a=b+c``, ``a=b,c=d,e=%20f``, etc.

    .. attribute:: fragment

        The fragment component of a URI. For example, ``section-3.1``.

    This class also provides extra attributes for easier access to information
    like the subcomponents of the authority component.

    .. attribute:: userinfo

        The user information parsed from the authority.

    .. attribute:: host

        The hostname, IPv4, or IPv6 address parsed from the authority.

    .. attribute:: port

        The port parsed from the authority.
    """

    slots = ()

    def __new__(
        cls, scheme, authority, path, query, fragment, encoding="utf-8"
    ):
        """Create a new URIReference."""
        ref = super().__new__(
            cls,
            scheme or None,
            authority or None,
            path or None,
            query,
            fragment,
        )
        ref.encoding = encoding
        return ref

    __hash__ = tuple.__hash__

    def __eq__(self, other):
        """Compare this reference to another."""
        other_ref = other
        if isinstance(other, tuple):
            other_ref = URIReference(*other)
        elif not isinstance(other, URIReference):
            try:
                other_ref = URIReference.from_string(other)
            except TypeError:
                raise TypeError(
                    "Unable to compare URIReference() to {}()".format(
                        type(other).__name__
                    )
                )

        # See http://tools.ietf.org/html/rfc3986#section-6.2
        naive_equality = tuple(self) == tuple(other_ref)
        return naive_equality or self.normalized_equality(other_ref)

    def normalize(self):
        """Normalize this reference as described in Section 6.2.2.

        This is not an in-place normalization. Instead this creates a new
        URIReference.

        :returns: A new reference object with normalized components.
        :rtype: URIReference
        """
        # See http://tools.ietf.org/html/rfc3986#section-6.2.2 for logic in
        # this method.
        return URIReference(
            normalizers.normalize_scheme(self.scheme or ""),
            normalizers.normalize_authority(
                (self.userinfo, self.host, self.port)
            ),
            normalizers.normalize_path(self.path or ""),
            normalizers.normalize_query(self.query),
            normalizers.normalize_fragment(self.fragment),
            self.encoding,
        )

    @classmethod
    def from_string(cls, uri_string, encoding="utf-8"):
        """Parse a URI reference from the given unicode URI string.

        :param str uri_string: Unicode URI to be parsed into a reference.
        :param str encoding: The encoding of the string provided
        :returns: :class:`URIReference` or subclass thereof
        """
        uri_string = compat.to_str(uri_string, encoding)

        split_uri = misc.URI_MATCHER.match(uri_string).groupdict()
        return cls(
            split_uri["scheme"],
            split_uri["authority"],
            normalizers.encode_component(split_uri["path"], encoding),
            normalizers.encode_component(split_uri["query"], encoding),
            normalizers.encode_component(split_uri["fragment"], encoding),
            encoding,
        )

env_pyrecodes/lib/python3.9/site-packages/rfc3986/uri.py
NikolaBlagojevic/pyrecodes
# Copyright (c) 2017 Ian Stapleton Cordasco
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module containing the validation logic for rfc3986."""
from . import exceptions
from . import misc
from . import normalizers


class Validator:
    """Object used to configure validation of all objects in rfc3986.

    .. versionadded:: 1.0

    Example usage::

         >>> from rfc3986 import api, validators
         >>> uri = api.uri_reference('https://github.com/')
         >>> validator = validators.Validator().require_presence_of(
         ...    'scheme', 'host', 'path',
         ... ).allow_schemes(
         ...    'http', 'https',
         ... ).allow_hosts(
         ...    '127.0.0.1', 'github.com',
         ... )
         >>> validator.validate(uri)
         >>> invalid_uri = rfc3986.uri_reference('imap://mail.google.com')
         >>> validator.validate(invalid_uri)
         Traceback (most recent call last):
         ...
         rfc3986.exceptions.MissingComponentError: ('path was required but
         missing', URIReference(scheme=u'imap', authority=u'mail.google.com',
         path=None, query=None, fragment=None), ['path'])

    """

    COMPONENT_NAMES = frozenset(
        ["scheme", "userinfo", "host", "port", "path", "query", "fragment"]
    )

    def __init__(self):
        """Initialize our default validations."""
        self.allowed_schemes = set()
        self.allowed_hosts = set()
        self.allowed_ports = set()
        self.allow_password = True
        self.required_components = {
            "scheme": False,
            "userinfo": False,
            "host": False,
            "port": False,
            "path": False,
            "query": False,
            "fragment": False,
        }
        self.validated_components = self.required_components.copy()

    def allow_schemes(self, *schemes):
        """Require the scheme to be one of the provided schemes.

        .. versionadded:: 1.0

        :param schemes:
            Schemes, without ``://`` that are allowed.
        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        for scheme in schemes:
            self.allowed_schemes.add(normalizers.normalize_scheme(scheme))
        return self

    def allow_hosts(self, *hosts):
        """Require the host to be one of the provided hosts.

        .. versionadded:: 1.0

        :param hosts:
            Hosts that are allowed.
        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        for host in hosts:
            self.allowed_hosts.add(normalizers.normalize_host(host))
        return self

    def allow_ports(self, *ports):
        """Require the port to be one of the provided ports.

        .. versionadded:: 1.0

        :param ports:
            Ports that are allowed.
        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        for port in ports:
            port_int = int(port, base=10)
            if 0 <= port_int <= 65535:
                self.allowed_ports.add(port)
        return self

    def allow_use_of_password(self):
        """Allow passwords to be present in the URI.

        .. versionadded:: 1.0

        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        self.allow_password = True
        return self

    def forbid_use_of_password(self):
        """Prevent passwords from being included in the URI.

        .. versionadded:: 1.0

        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        self.allow_password = False
        return self

    def check_validity_of(self, *components):
        """Check the validity of the components provided.

        This can be specified repeatedly.

        .. versionadded:: 1.1

        :param components:
            Names of components from :attr:`Validator.COMPONENT_NAMES`.
        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        components = [c.lower() for c in components]
        for component in components:
            if component not in self.COMPONENT_NAMES:
                raise ValueError(f'"{component}" is not a valid component')
        self.validated_components.update(
            {component: True for component in components}
        )
        return self

    def require_presence_of(self, *components):
        """Require the components provided.

        This can be specified repeatedly.

        .. versionadded:: 1.0

        :param components:
            Names of components from :attr:`Validator.COMPONENT_NAMES`.
        :returns:
            The validator instance.
        :rtype:
            Validator
        """
        components = [c.lower() for c in components]
        for component in components:
            if component not in self.COMPONENT_NAMES:
                raise ValueError(f'"{component}" is not a valid component')
        self.required_components.update(
            {component: True for component in components}
        )
        return self

    def validate(self, uri):
        """Check a URI for conditions specified on this validator.

        .. versionadded:: 1.0

        :param uri:
            Parsed URI to validate.
        :type uri:
            rfc3986.uri.URIReference
        :raises MissingComponentError:
            When a required component is missing.
        :raises UnpermittedComponentError:
            When a component is not one of those allowed.
        :raises PasswordForbidden:
            When a password is present in the userinfo component but is
            not permitted by configuration.
        :raises InvalidComponentsError:
            When a component was found to be invalid.
        """
        if not self.allow_password:
            check_password(uri)

        required_components = [
            component
            for component, required in self.required_components.items()
            if required
        ]
        validated_components = [
            component
            for component, required in self.validated_components.items()
            if required
        ]
        if required_components:
            ensure_required_components_exist(uri, required_components)
        if validated_components:
            ensure_components_are_valid(uri, validated_components)

        ensure_one_of(self.allowed_schemes, uri, "scheme")
        ensure_one_of(self.allowed_hosts, uri, "host")
        ensure_one_of(self.allowed_ports, uri, "port")


def check_password(uri):
    """Assert that there is no password present in the uri."""
    userinfo = uri.userinfo
    if not userinfo:
        return
    credentials = userinfo.split(":", 1)
    if len(credentials) <= 1:
        return
    raise exceptions.PasswordForbidden(uri)


def ensure_one_of(allowed_values, uri, attribute):
    """Assert that the uri's attribute is one of the allowed values."""
    value = getattr(uri, attribute)
    if value is not None and allowed_values and value not in allowed_values:
        raise exceptions.UnpermittedComponentError(
            attribute,
            value,
            allowed_values,
        )


def ensure_required_components_exist(uri, required_components):
    """Assert that all required components are present in the URI."""
    missing_components = sorted(
        component
        for component in required_components
        if getattr(uri, component) is None
    )
    if missing_components:
        raise exceptions.MissingComponentError(uri, *missing_components)


def is_valid(value, matcher, require):
    """Determine if a value is valid based on the provided matcher.

    :param str value:
        Value to validate.
    :param matcher:
        Compiled regular expression to use to validate the value.
    :param require:
        Whether or not the value is required.
    """
    if require:
        return value is not None and matcher.match(value)

    # require is False and value is not None
    return value is None or matcher.match(value)


def authority_is_valid(authority, host=None, require=False):
    """Determine if the authority string is valid.

    :param str authority:
        The authority to validate.
    :param str host:
        (optional) The host portion of the authority to validate.
    :param bool require:
        (optional) Specify if authority must not be None.
    :returns:
        ``True`` if valid, ``False`` otherwise
    :rtype:
        bool
    """
    validated = is_valid(authority, misc.SUBAUTHORITY_MATCHER, require)
    if validated and host is not None:
        return host_is_valid(host, require)
    return validated


def host_is_valid(host, require=False):
    """Determine if the host string is valid.

    :param str host:
        The host to validate.
    :param bool require:
        (optional) Specify if host must not be None.
    :returns:
        ``True`` if valid, ``False`` otherwise
    :rtype:
        bool
    """
    validated = is_valid(host, misc.HOST_MATCHER, require)
    if validated and host is not None and misc.IPv4_MATCHER.match(host):
        return valid_ipv4_host_address(host)
    elif validated and host is not None and misc.IPv6_MATCHER.match(host):
        return misc.IPv6_NO_RFC4007_MATCHER.match(host) is not None
    return validated


def scheme_is_valid(scheme, require=False):
    """Determine if the scheme is valid.

    :param str scheme:
        The scheme string to validate.
    :param bool require:
        (optional) Set to ``True`` to require the presence of a scheme.
    :returns:
        ``True`` if the scheme is valid. ``False`` otherwise.
    :rtype:
        bool
    """
    return is_valid(scheme, misc.SCHEME_MATCHER, require)


def path_is_valid(path, require=False):
    """Determine if the path component is valid.

    :param str path:
        The path string to validate.
    :param bool require:
        (optional) Set to ``True`` to require the presence of a path.
    :returns:
        ``True`` if the path is valid. ``False`` otherwise.
    :rtype:
        bool
    """
    return is_valid(path, misc.PATH_MATCHER, require)


def query_is_valid(query, require=False):
    """Determine if the query component is valid.

    :param str query:
        The query string to validate.
    :param bool require:
        (optional) Set to ``True`` to require the presence of a query.
    :returns:
        ``True`` if the query is valid. ``False`` otherwise.
    :rtype:
        bool
    """
    return is_valid(query, misc.QUERY_MATCHER, require)


def fragment_is_valid(fragment, require=False):
    """Determine if the fragment component is valid.

    :param str fragment:
        The fragment string to validate.
    :param bool require:
        (optional) Set to ``True`` to require the presence of a fragment.
    :returns:
        ``True`` if the fragment is valid. ``False`` otherwise.
    :rtype:
        bool
    """
    return is_valid(fragment, misc.FRAGMENT_MATCHER, require)


def valid_ipv4_host_address(host):
    """Determine if the given host is a valid IPv4 address."""
    # If the host exists, and it might be IPv4, check each byte in the
    # address.
    return all([0 <= int(byte, base=10) <= 255 for byte in host.split(".")])


_COMPONENT_VALIDATORS = {
    "scheme": scheme_is_valid,
    "path": path_is_valid,
    "query": query_is_valid,
    "fragment": fragment_is_valid,
}

_SUBAUTHORITY_VALIDATORS = {"userinfo", "host", "port"}


def subauthority_component_is_valid(uri, component):
    """Determine if the userinfo, host, and port are valid."""
    try:
        subauthority_dict = uri.authority_info()
    except exceptions.InvalidAuthority:
        return False

    # If we can parse the authority into sub-components and we're not
    # validating the port, we can assume it's valid.
    if component == "host":
        return host_is_valid(subauthority_dict["host"])
    elif component != "port":
        return True

    try:
        port = int(subauthority_dict["port"])
    except TypeError:
        # If the port wasn't provided it'll be None and int(None) raises a
        # TypeError
        return True

    return 0 <= port <= 65535


def ensure_components_are_valid(uri, validated_components):
    """Assert that all components are valid in the URI."""
    invalid_components = set()
    for component in validated_components:
        if component in _SUBAUTHORITY_VALIDATORS:
            if not subauthority_component_is_valid(uri, component):
                invalid_components.add(component)
            # Python's peephole optimizer means that while this continue *is*
            # actually executed, coverage.py cannot detect that. See also,
            # https://bitbucket.org/ned/coveragepy/issues/198/continue-marked-as-not-covered
            continue  # nocov: Python 2.7, 3.3, 3.4

        validator = _COMPONENT_VALIDATORS[component]
        if not validator(getattr(uri, component)):
            invalid_components.add(component)

    if invalid_components:
        raise exceptions.InvalidComponentsError(uri, *invalid_components)

env_pyrecodes/lib/python3.9/site-packages/rfc3986/validators.py
NikolaBlagojevic/pyrecodes
import inspect
import os
import platform
import sys
import threading
import zlib
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import datetime
from functools import wraps
from getpass import getpass
from html import escape
from inspect import isclass
from itertools import islice
from math import ceil
from time import monotonic
from types import FrameType, ModuleType, TracebackType
from typing import (
    IO,
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Mapping,
    NamedTuple,
    Optional,
    TextIO,
    Tuple,
    Type,
    Union,
    cast,
)

from rich._null_file import NULL_FILE

if sys.version_info >= (3, 8):
    from typing import Literal, Protocol, runtime_checkable
else:
    from typing_extensions import (
        Literal,
        Protocol,
        runtime_checkable,
    )  # pragma: no cover

from . import errors, themes
from ._emoji_replace import _emoji_replace
from ._export_format import CONSOLE_HTML_FORMAT, CONSOLE_SVG_FORMAT
from ._fileno import get_fileno
from ._log_render import FormatTimeCallable, LogRender
from .align import Align, AlignMethod
from .color import ColorSystem, blend_rgb
from .control import Control
from .emoji import EmojiVariant
from .highlighter import NullHighlighter, ReprHighlighter
from .markup import render as render_markup
from .measure import Measurement, measure_renderables
from .pager import Pager, SystemPager
from .pretty import Pretty, is_expandable
from .protocol import rich_cast
from .region import Region
from .scope import render_scope
from .screen import Screen
from .segment import Segment
from .style import Style, StyleType
from .styled import Styled
from .terminal_theme import DEFAULT_TERMINAL_THEME, SVG_EXPORT_THEME, TerminalTheme
from .text import Text, TextType
from .theme import Theme, ThemeStack

if TYPE_CHECKING:
    from ._windows import WindowsConsoleFeatures
    from .live import Live
    from .status import Status

JUPYTER_DEFAULT_COLUMNS = 115
JUPYTER_DEFAULT_LINES = 100
WINDOWS = platform.system() == "Windows"

HighlighterType = Callable[[Union[str, "Text"]], "Text"]
JustifyMethod = Literal["default", "left", "center", "right", "full"]
OverflowMethod = Literal["fold", "crop", "ellipsis", "ignore"]


class NoChange:
    pass


NO_CHANGE = NoChange()

try:
    _STDIN_FILENO = sys.__stdin__.fileno()
except Exception:
    _STDIN_FILENO = 0
try:
    _STDOUT_FILENO = sys.__stdout__.fileno()
except Exception:
    _STDOUT_FILENO = 1
try:
    _STDERR_FILENO = sys.__stderr__.fileno()
except Exception:
    _STDERR_FILENO = 2

_STD_STREAMS = (_STDIN_FILENO, _STDOUT_FILENO, _STDERR_FILENO)
_STD_STREAMS_OUTPUT = (_STDOUT_FILENO, _STDERR_FILENO)


_TERM_COLORS = {
    "kitty": ColorSystem.EIGHT_BIT,
    "256color": ColorSystem.EIGHT_BIT,
    "16color": ColorSystem.STANDARD,
}


class ConsoleDimensions(NamedTuple):
    """Size of the terminal."""

    width: int
    """The width of the console in 'cells'."""
    height: int
    """The height of the console in lines."""


@dataclass
class ConsoleOptions:
    """Options for __rich_console__ method."""

    size: ConsoleDimensions
    """Size of console."""
    legacy_windows: bool
    """legacy_windows: flag for legacy windows."""
    min_width: int
    """Minimum width of renderable."""
    max_width: int
    """Maximum width of renderable."""
    is_terminal: bool
    """True if the target is a terminal, otherwise False."""
    encoding: str
    """Encoding of terminal."""
    max_height: int
    """Height of container (starts as terminal)"""
    justify: Optional[JustifyMethod] = None
    """Justify value override for renderable."""
    overflow: Optional[OverflowMethod] = None
    """Overflow value override for renderable."""
    no_wrap: Optional[bool] = False
    """Disable wrapping for text."""
    highlight: Optional[bool] = None
    """Highlight override for render_str."""
    markup: Optional[bool] = None
    """Enable markup when rendering strings."""
    height: Optional[int] = None

    @property
    def ascii_only(self) -> bool:
        """Check if renderables should use ascii only."""
        return not self.encoding.startswith("utf")

    def copy(self) -> "ConsoleOptions":
        """Return a copy of the options.

        Returns:
            ConsoleOptions: a copy of self.
        """
        options: ConsoleOptions = ConsoleOptions.__new__(ConsoleOptions)
        options.__dict__ = self.__dict__.copy()
        return options

    def update(
        self,
        *,
        width: Union[int, NoChange] = NO_CHANGE,
        min_width: Union[int, NoChange] = NO_CHANGE,
        max_width: Union[int, NoChange] = NO_CHANGE,
        justify: Union[Optional[JustifyMethod], NoChange] = NO_CHANGE,
        overflow: Union[Optional[OverflowMethod], NoChange] = NO_CHANGE,
        no_wrap: Union[Optional[bool], NoChange] = NO_CHANGE,
        highlight: Union[Optional[bool], NoChange] = NO_CHANGE,
        markup: Union[Optional[bool], NoChange] = NO_CHANGE,
        height: Union[Optional[int], NoChange] = NO_CHANGE,
    ) -> "ConsoleOptions":
        """Update values, return a copy."""
        options = self.copy()
        if not isinstance(width, NoChange):
            options.min_width = options.max_width = max(0, width)
        if not isinstance(min_width, NoChange):
            options.min_width = min_width
        if not isinstance(max_width, NoChange):
            options.max_width = max_width
        if not isinstance(justify, NoChange):
            options.justify = justify
        if not isinstance(overflow, NoChange):
            options.overflow = overflow
        if not isinstance(no_wrap, NoChange):
            options.no_wrap = no_wrap
        if not isinstance(highlight, NoChange):
            options.highlight = highlight
        if not isinstance(markup, NoChange):
            options.markup = markup
        if not isinstance(height, NoChange):
            if height is not None:
                options.max_height = height
            options.height = None if height is None else max(0, height)
        return options

    def update_width(self, width: int) -> "ConsoleOptions":
        """Update just the width, return a copy.

        Args:
            width (int): New width (sets both min_width and max_width)

        Returns:
            ~ConsoleOptions: New console options instance.
        """
        options = self.copy()
        options.min_width = options.max_width = max(0, width)
        return options

    def update_height(self, height: int) -> "ConsoleOptions":
        """Update the height, and return a copy.

        Args:
            height (int): New height

        Returns:
            ~ConsoleOptions: New Console options instance.
        """
        options = self.copy()
        options.max_height = options.height = height
        return options

    def reset_height(self) -> "ConsoleOptions":
        """Return a copy of the options with height set to ``None``.

        Returns:
            ~ConsoleOptions: New console options instance.
        """
        options = self.copy()
        options.height = None
        return options

    def update_dimensions(self, width: int, height: int) -> "ConsoleOptions":
        """Update the width and height, and return a copy.

        Args:
            width (int): New width (sets both min_width and max_width).
            height (int): New height.

        Returns:
            ~ConsoleOptions: New console options instance.
        """
        options = self.copy()
        options.min_width = options.max_width = max(0, width)
        options.height = options.max_height = height
        return options


@runtime_checkable
class RichCast(Protocol):
    """An object that may be 'cast' to a console renderable."""

    def __rich__(
        self,
    ) -> Union["ConsoleRenderable", "RichCast", str]:  # pragma: no cover
        ...


@runtime_checkable
class ConsoleRenderable(Protocol):
    """An object that supports the console protocol."""

    def __rich_console__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> "RenderResult":  # pragma: no cover
        ...


# A type that may be rendered by Console.
RenderableType = Union[ConsoleRenderable, RichCast, str]
"""A string or any object that may be rendered by Rich."""

# The result of calling a __rich_console__ method.
RenderResult = Iterable[Union[RenderableType, Segment]]

_null_highlighter = NullHighlighter()


class CaptureError(Exception):
    """An error in the Capture context manager."""


class NewLine:
    """A renderable to generate new line(s)"""

    def __init__(self, count: int = 1) -> None:
        self.count = count

    def __rich_console__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> Iterable[Segment]:
        yield Segment("\n" * self.count)


class ScreenUpdate:
    """Render a list of lines at a given offset."""

    def __init__(self, lines: List[List[Segment]], x: int, y: int) -> None:
        self._lines = lines
        self.x = x
        self.y = y

    def __rich_console__(
        self, console: "Console", options: ConsoleOptions
    ) -> RenderResult:
        x = self.x
        move_to = Control.move_to
        for offset, line in enumerate(self._lines, self.y):
            yield move_to(x, offset)
            yield from line


class Capture:
    """Context manager to capture the result of printing to the console.
    See :meth:`~rich.console.Console.capture` for how to use.

    Args:
        console (Console): A console instance to capture output.
    """

    def __init__(self, console: "Console") -> None:
        self._console = console
        self._result: Optional[str] = None

    def __enter__(self) -> "Capture":
        self._console.begin_capture()
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        self._result = self._console.end_capture()

    def get(self) -> str:
        """Get the result of the capture."""
        if self._result is None:
            raise CaptureError(
                "Capture result is not available until context manager exits."
            )
        return self._result


class ThemeContext:
    """A context manager to use a temporary theme. See :meth:`~rich.console.Console.use_theme` for usage."""

    def __init__(self, console: "Console", theme: Theme, inherit: bool = True) -> None:
        self.console = console
        self.theme = theme
        self.inherit = inherit

    def __enter__(self) -> "ThemeContext":
        self.console.push_theme(self.theme)
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        self.console.pop_theme()


class PagerContext:
    """A context manager that 'pages' content. See :meth:`~rich.console.Console.pager` for usage."""

    def __init__(
        self,
        console: "Console",
        pager: Optional[Pager] = None,
        styles: bool = False,
        links: bool = False,
    ) -> None:
        self._console = console
        self.pager = SystemPager() if pager is None else pager
        self.styles = styles
        self.links = links

    def __enter__(self) -> "PagerContext":
        self._console._enter_buffer()
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        if exc_type is None:
            with self._console._lock:
                buffer: List[Segment] = self._console._buffer[:]
                del self._console._buffer[:]
                segments: Iterable[Segment] = buffer
                if not self.styles:
                    segments = Segment.strip_styles(segments)
                elif not self.links:
                    segments = Segment.strip_links(segments)
                content = self._console._render_buffer(segments)
            self.pager.show(content)
        self._console._exit_buffer()


class ScreenContext:
    """A context manager that enables an alternative screen. See :meth:`~rich.console.Console.screen` for usage."""

    def __init__(
        self, console: "Console", hide_cursor: bool, style: StyleType = ""
    ) -> None:
        self.console = console
        self.hide_cursor = hide_cursor
        self.screen = Screen(style=style)
        self._changed = False

    def update(
        self, *renderables: RenderableType, style: Optional[StyleType] = None
    ) -> None:
        """Update the screen.

        Args:
            renderable (RenderableType, optional): Optional renderable to replace current renderable,
                or None for no change. Defaults to None.
            style: (Style, optional): Replacement style, or None for no change. Defaults to None.
        """
        if renderables:
            self.screen.renderable = (
                Group(*renderables) if len(renderables) > 1 else renderables[0]
            )
        if style is not None:
            self.screen.style = style
        self.console.print(self.screen, end="")

    def __enter__(self) -> "ScreenContext":
        self._changed = self.console.set_alt_screen(True)
        if self._changed and self.hide_cursor:
            self.console.show_cursor(False)
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> None:
        if self._changed:
            self.console.set_alt_screen(False)
            if self.hide_cursor:
                self.console.show_cursor(True)


class Group:
    """Takes a group of renderables and returns a renderable object that renders the group.

    Args:
        renderables (Iterable[RenderableType]): An iterable of renderable objects.
        fit (bool, optional): Fit dimension of group to contents, or fill available space. Defaults to True.
    """

    def __init__(self, *renderables: "RenderableType", fit: bool = True) -> None:
        self._renderables = renderables
        self.fit = fit
        self._render: Optional[List[RenderableType]] = None

    @property
    def renderables(self) -> List["RenderableType"]:
        if self._render is None:
            self._render = list(self._renderables)
        return self._render

    def __rich_measure__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> "Measurement":
        if self.fit:
            return measure_renderables(console, options, self.renderables)
        else:
            return Measurement(options.max_width, options.max_width)

    def __rich_console__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> RenderResult:
        yield from self.renderables


def group(fit: bool = True) -> Callable[..., Callable[..., Group]]:
    """A decorator that turns an iterable of renderables in to a group.

    Args:
        fit (bool, optional): Fit dimension of group to contents, or fill available space. Defaults to True.
    """

    def decorator(
        method: Callable[..., Iterable[RenderableType]]
    ) -> Callable[..., Group]:
        """Convert a method that returns an iterable of renderables in to a Group."""

        @wraps(method)
        def _replace(*args: Any, **kwargs: Any) -> Group:
            renderables = method(*args, **kwargs)
            return Group(*renderables, fit=fit)

        return _replace

    return decorator


def _is_jupyter() -> bool:  # pragma: no cover
    """Check if we're running in a Jupyter notebook."""
    try:
        get_ipython  # type: ignore[name-defined]
    except NameError:
        return False
    ipython = get_ipython()  # type: ignore[name-defined]
    shell = ipython.__class__.__name__
    if (
        "google.colab" in str(ipython.__class__)
        or os.getenv("DATABRICKS_RUNTIME_VERSION")
        or shell == "ZMQInteractiveShell"
    ):
        return True  # Jupyter notebook or qtconsole
    elif shell == "TerminalInteractiveShell":
        return False  # Terminal running IPython
    else:
        return False  # Other type (?)


COLOR_SYSTEMS = {
    "standard": ColorSystem.STANDARD,
    "256": ColorSystem.EIGHT_BIT,
    "truecolor": ColorSystem.TRUECOLOR,
    "windows": ColorSystem.WINDOWS,
}

_COLOR_SYSTEMS_NAMES = {system: name for name, system in COLOR_SYSTEMS.items()}


@dataclass
class ConsoleThreadLocals(threading.local):
    """Thread local values for Console context."""

    theme_stack: ThemeStack
    buffer: List[Segment] = field(default_factory=list)
    buffer_index: int = 0


class RenderHook(ABC):
    """Provides hooks in to the render process."""

    @abstractmethod
    def process_renderables(
        self, renderables: List[ConsoleRenderable]
    ) -> List[ConsoleRenderable]:
        """Called with a list of objects to render.

        This method can return a new list of renderables, or modify and return the same list.

        Args:
            renderables (List[ConsoleRenderable]): A number of renderable objects.

        Returns:
            List[ConsoleRenderable]: A replacement list of renderables.
        """


_windows_console_features: Optional["WindowsConsoleFeatures"] = None


def get_windows_console_features() -> "WindowsConsoleFeatures":  # pragma: no cover
    global _windows_console_features
    if _windows_console_features is not None:
        return _windows_console_features
    from ._windows import get_windows_console_features

    _windows_console_features = get_windows_console_features()
    return _windows_console_features


def detect_legacy_windows() -> bool:
    """Detect legacy Windows."""
    return WINDOWS and not get_windows_console_features().vt


class Console:
    """A high level console interface.

    Args:
        color_system (str, optional): The color system supported by your terminal,
            either ``"standard"``, ``"256"`` or ``"truecolor"``. Leave as ``"auto"`` to autodetect.
        force_terminal (Optional[bool], optional): Enable/disable terminal control codes, or None to auto-detect terminal. Defaults to None.
        force_jupyter (Optional[bool], optional): Enable/disable Jupyter rendering, or None to auto-detect Jupyter. Defaults to None.
        force_interactive (Optional[bool], optional): Enable/disable interactive mode, or None to auto detect. Defaults to None.
        soft_wrap (Optional[bool], optional): Set soft wrap default on print method. Defaults to False.
        theme (Theme, optional): An optional style theme object, or ``None`` for default theme.
        stderr (bool, optional): Use stderr rather than stdout if ``file`` is not specified. Defaults to False.
        file (IO, optional): A file object where the console should write to. Defaults to stdout.
        quiet (bool, Optional): Boolean to suppress all output. Defaults to False.
        width (int, optional): The width of the terminal. Leave as default to auto-detect width.
        height (int, optional): The height of the terminal. Leave as default to auto-detect height.
        style (StyleType, optional): Style to apply to all output, or None for no style. Defaults to None.
        no_color (Optional[bool], optional): Enabled no color mode, or None to auto detect. Defaults to None.
        tab_size (int, optional): Number of spaces used to replace a tab character. Defaults to 8.
        record (bool, optional): Boolean to enable recording of terminal output,
            required to call :meth:`export_html`, :meth:`export_svg`, and :meth:`export_text`. Defaults to False.
        markup (bool, optional): Boolean to enable :ref:`console_markup`. Defaults to True.
        emoji (bool, optional): Enable emoji code. Defaults to True.
        emoji_variant (str, optional): Optional emoji variant, either "text" or "emoji". Defaults to None.
        highlight (bool, optional): Enable automatic highlighting. Defaults to True.
        log_time (bool, optional): Boolean to enable logging of time by :meth:`log` methods. Defaults to True.
        log_path (bool, optional): Boolean to enable the logging of the caller by :meth:`log`. Defaults to True.
        log_time_format (Union[str, TimeFormatterCallable], optional): If ``log_time`` is enabled, either string for strftime or callable that formats the time. Defaults to "[%X] ".
        highlighter (HighlighterType, optional): Default highlighter.
        legacy_windows (bool, optional): Enable legacy Windows mode, or ``None`` to auto detect. Defaults to ``None``.
        safe_box (bool, optional): Restrict box options that don't render on legacy Windows.
        get_datetime (Callable[[], datetime], optional): Callable that gets the current time as a datetime.datetime object (used by Console.log),
            or None for datetime.now.
        get_time (Callable[[], time], optional): Callable that gets the current time in seconds, default uses time.monotonic.
    """

    _environ: Mapping[str, str] = os.environ

    def __init__(
        self,
        *,
        color_system: Optional[
            Literal["auto", "standard", "256", "truecolor", "windows"]
        ] = "auto",
        force_terminal: Optional[bool] = None,
        force_jupyter: Optional[bool] = None,
        force_interactive: Optional[bool] = None,
        soft_wrap: bool = False,
        theme: Optional[Theme] = None,
        stderr: bool = False,
        file: Optional[IO[str]] = None,
        quiet: bool = False,
        width: Optional[int] = None,
        height: Optional[int] = None,
        style: Optional[StyleType] = None,
        no_color: Optional[bool] = None,
        tab_size: int = 8,
        record: bool = False,
        markup: bool = True,
        emoji: bool = True,
        emoji_variant: Optional[EmojiVariant] = None,
        highlight: bool = True,
        log_time: bool = True,
        log_path: bool = True,
        log_time_format: Union[str, FormatTimeCallable] = "[%X]",
        highlighter: Optional["HighlighterType"] = ReprHighlighter(),
        legacy_windows: Optional[bool] = None,
        safe_box: bool = True,
        get_datetime: Optional[Callable[[], datetime]] = None,
        get_time: Optional[Callable[[], float]] = None,
        _environ: Optional[Mapping[str, str]] = None,
    ):
        # Copy of os.environ allows us to replace it for testing
        if _environ is not None:
            self._environ = _environ

        self.is_jupyter = _is_jupyter() if force_jupyter is None else force_jupyter
        if self.is_jupyter:
            if width is None:
                jupyter_columns = self._environ.get("JUPYTER_COLUMNS")
                if jupyter_columns is not None and jupyter_columns.isdigit():
                    width = int(jupyter_columns)
                else:
                    width = JUPYTER_DEFAULT_COLUMNS
            if height is None:
                jupyter_lines = self._environ.get("JUPYTER_LINES")
                if jupyter_lines is not None and jupyter_lines.isdigit():
                    height = int(jupyter_lines)
                else:
                    height = JUPYTER_DEFAULT_LINES

        self.tab_size = tab_size
        self.record = record
        self._markup = markup
        self._emoji = emoji
        self._emoji_variant: Optional[EmojiVariant] = emoji_variant
        self._highlight = highlight
        self.legacy_windows: bool = (
            (detect_legacy_windows() and not self.is_jupyter)
            if legacy_windows is None
            else legacy_windows
        )

        if width is None:
            columns = self._environ.get("COLUMNS")
            if columns is not None and columns.isdigit():
                width = int(columns) - self.legacy_windows
        if height is None:
            lines = self._environ.get("LINES")
            if lines is not None and lines.isdigit():
                height = int(lines)

        self.soft_wrap = soft_wrap
        self._width = width
        self._height = height

        self._color_system: Optional[ColorSystem]

        self._force_terminal = None
        if force_terminal is not None:
            self._force_terminal = force_terminal

        self._file = file
        self.quiet = quiet
        self.stderr = stderr

        if color_system is None:
            self._color_system = None
        elif color_system == "auto":
            self._color_system = self._detect_color_system()
        else:
            self._color_system = COLOR_SYSTEMS[color_system]

        self._lock = threading.RLock()
        self._log_render = LogRender(
            show_time=log_time,
            show_path=log_path,
            time_format=log_time_format,
        )
        self.highlighter: HighlighterType = highlighter or _null_highlighter
        self.safe_box = safe_box
        self.get_datetime = get_datetime or datetime.now
        self.get_time = get_time or monotonic
        self.style = style
        self.no_color = (
            no_color if no_color is not None else "NO_COLOR" in self._environ
        )
        self.is_interactive = (
            (self.is_terminal and not self.is_dumb_terminal)
            if force_interactive is None
            else force_interactive
        )

        self._record_buffer_lock = threading.RLock()
        self._thread_locals = ConsoleThreadLocals(
            theme_stack=ThemeStack(themes.DEFAULT if theme is None else theme)
        )
        self._record_buffer: List[Segment] = []
        self._render_hooks: List[RenderHook] = []
        self._live: Optional["Live"] = None
        self._is_alt_screen = False

    def __repr__(self) -> str:
        return f"<console width={self.width} {self._color_system!s}>"

    @property
    def file(self) -> IO[str]:
        """Get the file object to write to."""
        file = self._file or (sys.stderr if self.stderr else sys.stdout)
        file = getattr(file, "rich_proxied_file", file)
        if file is None:
            file = NULL_FILE
        return file

    @file.setter
    def file(self, new_file: IO[str]) -> None:
        """Set a new file object."""
        self._file = new_file

    @property
    def _buffer(self) -> List[Segment]:
        """Get a thread local buffer."""
        return self._thread_locals.buffer

    @property
    def _buffer_index(self) -> int:
        """Get a thread local buffer."""
        return self._thread_locals.buffer_index

    @_buffer_index.setter
    def _buffer_index(self, value: int) -> None:
        self._thread_locals.buffer_index = value

    @property
    def _theme_stack(self) -> ThemeStack:
        """Get the thread local theme stack."""
        return self._thread_locals.theme_stack

    def _detect_color_system(self) -> Optional[ColorSystem]:
        """Detect color system from env vars."""
        if self.is_jupyter:
            return ColorSystem.TRUECOLOR
        if not self.is_terminal or self.is_dumb_terminal:
            return None
        if WINDOWS:  # pragma: no cover
            if self.legacy_windows:  # pragma: no cover
                return ColorSystem.WINDOWS
            windows_console_features = get_windows_console_features()
            return (
                ColorSystem.TRUECOLOR
                if windows_console_features.truecolor
                else ColorSystem.EIGHT_BIT
            )
        else:
            color_term = self._environ.get("COLORTERM", "").strip().lower()
            if color_term in ("truecolor", "24bit"):
                return ColorSystem.TRUECOLOR
            term = self._environ.get("TERM", "").strip().lower()
            _term_name, _hyphen, colors = term.rpartition("-")
            color_system = _TERM_COLORS.get(colors, ColorSystem.STANDARD)
            return color_system

    def _enter_buffer(self) -> None:
        """Enter in to a buffer context, and buffer all output."""
        self._buffer_index += 1

    def _exit_buffer(self) -> None:
        """Leave buffer context, and render content if required."""
        self._buffer_index -= 1
        self._check_buffer()

    def set_live(self, live: "Live") -> None:
        """Set Live instance. Used by Live context manager.

        Args:
            live (Live): Live instance using this Console.

        Raises:
            errors.LiveError: If this Console has a Live context currently active.
        """
        with self._lock:
            if self._live is not None:
                raise errors.LiveError("Only one live display may be active at once")
            self._live = live

    def clear_live(self) -> None:
        """Clear the Live instance."""
        with self._lock:
            self._live = None

    def push_render_hook(self, hook: RenderHook) -> None:
        """Add a new render hook to the stack.

        Args:
            hook (RenderHook): Render hook instance.
        """
        with self._lock:
            self._render_hooks.append(hook)

    def pop_render_hook(self) -> None:
        """Pop the last renderhook from the stack."""
        with self._lock:
            self._render_hooks.pop()

    def __enter__(self) -> "Console":
        """Own context manager to enter buffer context."""
        self._enter_buffer()
        return self

    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
        """Exit buffer context."""
        self._exit_buffer()

    def begin_capture(self) -> None:
        """Begin capturing console output. Call :meth:`end_capture` to exit capture mode and return output."""
        self._enter_buffer()

    def end_capture(self) -> str:
        """End capture mode and return captured string.

        Returns:
            str: Console output.
        """
        render_result = self._render_buffer(self._buffer)
        del self._buffer[:]
        self._exit_buffer()
        return render_result

    def push_theme(self, theme: Theme, *, inherit: bool = True) -> None:
        """Push a new theme on to the top of the stack, replacing the styles from the previous theme.
        Generally speaking, you should call :meth:`~rich.console.Console.use_theme` to get a context manager, rather
        than calling this method directly.

        Args:
            theme (Theme): A theme instance.
            inherit (bool, optional): Inherit existing styles. Defaults to True.
        """
        self._theme_stack.push_theme(theme, inherit=inherit)

    def pop_theme(self) -> None:
        """Remove theme from top of stack, restoring previous theme."""
        self._theme_stack.pop_theme()

    def use_theme(self, theme: Theme, *, inherit: bool = True) -> ThemeContext:
        """Use a different theme for the duration of the context manager.

        Args:
            theme (Theme): Theme instance to user.
            inherit (bool, optional): Inherit existing console styles. Defaults to True.

        Returns:
            ThemeContext: [description]
        """
        return ThemeContext(self, theme, inherit)

    @property
    def color_system(self) -> Optional[str]:
        """Get color system string.

        Returns:
            Optional[str]: "standard", "256" or "truecolor".
        """

        if self._color_system is not None:
            return _COLOR_SYSTEMS_NAMES[self._color_system]
        else:
            return None

    @property
    def encoding(self) -> str:
        """Get the encoding of the console file, e.g. ``"utf-8"``.

        Returns:
            str: A standard encoding string.
        """
        return (getattr(self.file, "encoding", "utf-8") or "utf-8").lower()

    @property
    def is_terminal(self) -> bool:
        """Check if the console is writing to a terminal.

        Returns:
            bool: True if the console writing to a device capable of
            understanding terminal codes, otherwise False.
        """
        if self._force_terminal is not None:
            return self._force_terminal

        if hasattr(sys.stdin, "__module__") and sys.stdin.__module__.startswith(
            "idlelib"
        ):
            # Return False for Idle which claims to be a tty but can't handle ansi codes
            return False

        if self.is_jupyter:
            # return False for Jupyter, which may have FORCE_COLOR set
            return False

        # If FORCE_COLOR env var has any value at all, we assume a terminal.
        force_color = self._environ.get("FORCE_COLOR")
        if force_color is not None:
            self._force_terminal = True
            return True

        isatty: Optional[Callable[[], bool]] = getattr(self.file, "isatty", None)
        try:
            return False if isatty is None else isatty()
        except ValueError:
            # in some situation (at the end of a pytest run for example) isatty() can raise
            # ValueError: I/O operation on closed file
            # return False because we aren't in a terminal anymore
            return False

    @property
    def is_dumb_terminal(self) -> bool:
        """Detect dumb terminal.

        Returns:
            bool: True if writing to a dumb terminal, otherwise False.

        """
        _term = self._environ.get("TERM", "")
        is_dumb = _term.lower() in ("dumb", "unknown")
        return self.is_terminal and is_dumb

    @property
    def options(self) -> ConsoleOptions:
        """Get default console options."""
        return ConsoleOptions(
            max_height=self.size.height,
            size=self.size,
            legacy_windows=self.legacy_windows,
            min_width=1,
            max_width=self.width,
            encoding=self.encoding,
            is_terminal=self.is_terminal,
        )

    @property
    def size(self) -> ConsoleDimensions:
        """Get the size of the console.

        Returns:
            ConsoleDimensions: A named tuple containing the dimensions.
        """

        if self._width is not None and self._height is not None:
            return ConsoleDimensions(self._width - self.legacy_windows, self._height)

        if self.is_dumb_terminal:
            return ConsoleDimensions(80, 25)

        width: Optional[int] = None
        height: Optional[int] = None

        if WINDOWS:  # pragma: no cover
            try:
                width, height = os.get_terminal_size()
            except (AttributeError, ValueError, OSError):  # Probably not a terminal
                pass
        else:
            for file_descriptor in _STD_STREAMS:
                try:
                    width, height = os.get_terminal_size(file_descriptor)
                except (AttributeError, ValueError, OSError):
                    pass
                else:
                    break

        columns = self._environ.get("COLUMNS")
        if columns is not None and columns.isdigit():
            width = int(columns)
        lines = self._environ.get("LINES")
        if lines is not None and lines.isdigit():
            height = int(lines)

        # get_terminal_size can report 0, 0 if run from pseudo-terminal
        width = width or 80
        height = height or 25
        return ConsoleDimensions(
            width - self.legacy_windows if self._width is None else self._width,
            height if self._height is None else self._height,
        )

    @size.setter
    def size(self, new_size: Tuple[int, int]) -> None:
        """Set a new size for the terminal.

        Args:
            new_size (Tuple[int, int]): New width and height.
        """
        width, height = new_size
        self._width = width
        self._height = height

    @property
    def width(self) -> int:
        """Get the width of the console.

        Returns:
            int: The width (in characters) of the console.
        """
        return self.size.width

    @width.setter
    def width(self, width: int) -> None:
        """Set width.

        Args:
            width (int): New width.
        """
        self._width = width

    @property
    def height(self) -> int:
        """Get the height of the console.

        Returns:
            int: The height (in lines) of the console.
        """
        return self.size.height

    @height.setter
    def height(self, height: int) -> None:
        """Set height.

        Args:
            height (int): new height.
        """
        self._height = height

    def bell(self) -> None:
        """Play a 'bell' sound (if supported by the terminal)."""
        self.control(Control.bell())

    def capture(self) -> Capture:
        """A context manager to *capture* the result of print() or log() in a string,
        rather than writing it to the console.

        Example:
            >>> from rich.console import Console
            >>> console = Console()
            >>> with console.capture() as capture:
            ...     console.print("[bold magenta]Hello World[/]")
            >>> print(capture.get())

        Returns:
            Capture: Context manager with disables writing to the terminal.
        """
        capture = Capture(self)
        return capture

    def pager(
        self, pager: Optional[Pager] = None, styles: bool = False, links: bool = False
    ) -> PagerContext:
        """A context manager to display anything printed within a "pager". The pager application
        is defined by the system and will typically support at least pressing a key to scroll.

        Args:
            pager (Pager, optional): A pager object, or None to use :class:`~rich.pager.SystemPager`. Defaults to None.
            styles (bool, optional): Show styles in pager. Defaults to False.
            links (bool, optional): Show links in pager. Defaults to False.

        Example:
            >>> from rich.console import Console
            >>> from rich.__main__ import make_test_card
            >>> console = Console()
            >>> with console.pager():
                    console.print(make_test_card())

        Returns:
            PagerContext: A context manager.
        """
        return PagerContext(self, pager=pager, styles=styles, links=links)

    def line(self, count: int = 1) -> None:
        """Write new line(s).

        Args:
            count (int, optional): Number of new lines. Defaults to 1.
        """

        assert count >= 0, "count must be >= 0"
        self.print(NewLine(count))

    def clear(self, home: bool = True) -> None:
        """Clear the screen.

        Args:
            home (bool, optional): Also move the cursor to 'home' position. Defaults to True.
        """
        if home:
            self.control(Control.clear(), Control.home())
        else:
            self.control(Control.clear())

    def status(
        self,
        status: RenderableType,
        *,
        spinner: str = "dots",
        spinner_style: StyleType = "status.spinner",
        speed: float = 1.0,
        refresh_per_second: float = 12.5,
    ) -> "Status":
        """Display a status and spinner.

        Args:
            status (RenderableType): A status renderable (str or Text typically).
            spinner (str, optional): Name of spinner animation (see python -m rich.spinner). Defaults to "dots".
            spinner_style (StyleType, optional): Style of spinner. Defaults to "status.spinner".
            speed (float, optional): Speed factor for spinner animation. Defaults to 1.0.
            refresh_per_second (float, optional): Number of refreshes per second. Defaults to 12.5.

        Returns:
            Status: A Status object that may be used as a context manager.
        """
        from .status import Status

        status_renderable = Status(
            status,
            console=self,
            spinner=spinner,
            spinner_style=spinner_style,
            speed=speed,
            refresh_per_second=refresh_per_second,
        )
        return status_renderable

    def show_cursor(self, show: bool = True) -> bool:
        """Show or hide the cursor.

        Args:
            show (bool, optional): Set visibility of the cursor.
        """
        if self.is_terminal:
            self.control(Control.show_cursor(show))
            return True
        return False

    def set_alt_screen(self, enable: bool = True) -> bool:
        """Enables alternative screen mode.

        Note, if you enable this mode, you should ensure that is disabled before
        the application exits. See :meth:`~rich.Console.screen` for a context manager
        that handles this for you.

        Args:
            enable (bool, optional): Enable (True) or disable (False) alternate screen. Defaults to True.

        Returns:
            bool: True if the control codes were written.

        """
        changed = False
        if self.is_terminal and not self.legacy_windows:
            self.control(Control.alt_screen(enable))
            changed = True
            self._is_alt_screen = enable
        return changed

    @property
    def is_alt_screen(self) -> bool:
        """Check if the alt screen was enabled.

        Returns:
            bool: True if the alt screen was enabled, otherwise False.
        """
        return self._is_alt_screen

    def set_window_title(self, title: str) -> bool:
        """Set the title of the console terminal window.

        Warning: There is no means within Rich of "resetting" the window title to its
        previous value, meaning the title you set will persist even after your application
        exits.

        ``fish`` shell resets the window title before and after each command by default,
        negating this issue. Windows Terminal and command prompt will also reset the title for you.
        Most other shells and terminals, however, do not do this.

        Some terminals may require configuration changes before you can set the title.
        Some terminals may not support setting the title at all.

        Other software (including the terminal itself, the shell, custom prompts, plugins, etc.)
        may also set the terminal window title. This could result in whatever value you write
        using this method being overwritten.

        Args:
            title (str): The new title of the terminal window.

        Returns:
            bool: True if the control code to change the terminal title was
                written, otherwise False. Note that a return value of True
                does not guarantee that the window title has actually changed,
                since the feature may be unsupported/disabled in some terminals.
        """
        if self.is_terminal:
            self.control(Control.title(title))
            return True
        return False

    def screen(
        self, hide_cursor: bool = True, style: Optional[StyleType] = None
    ) -> "ScreenContext":
        """Context manager to enable and disable 'alternative screen' mode.

        Args:
            hide_cursor (bool, optional): Also hide the cursor. Defaults to False.
            style (Style, optional): Optional style for screen. Defaults to None.

        Returns:
            ~ScreenContext: Context which enables alternate screen on enter, and disables it on exit.
        """
        return ScreenContext(self, hide_cursor=hide_cursor, style=style or "")

    def measure(
        self, renderable: RenderableType, *, options: Optional[ConsoleOptions] = None
    ) -> Measurement:
        """Measure a renderable. Returns a :class:`~rich.measure.Measurement` object which contains
        information regarding the number of characters required to print the renderable.

        Args:
            renderable (RenderableType): Any renderable or string.
            options (Optional[ConsoleOptions], optional): Options to use when measuring, or None
                to use default options. Defaults to None.

        Returns:
            Measurement: A measurement of the renderable.
        """
        measurement = Measurement.get(self, options or self.options, renderable)
        return measurement

    def render(
        self, renderable: RenderableType, options: Optional[ConsoleOptions] = None
    ) -> Iterable[Segment]:
        """Render an object in to an iterable of `Segment` instances.

        This method contains the logic for rendering objects with the console protocol.
        You are unlikely to need to use it directly, unless you are extending the library.

        Args:
            renderable (RenderableType): An object supporting the console protocol, or
                an object that may be converted to a string.
            options (ConsoleOptions, optional): An options object, or None to use self.options. Defaults to None.

        Returns:
            Iterable[Segment]: An iterable of segments that may be rendered.
        """

        _options = options or self.options
        if _options.max_width < 1:
            # No space to render anything. This prevents potential recursion errors.
            return
        render_iterable: RenderResult

        renderable = rich_cast(renderable)
        if hasattr(renderable, "__rich_console__") and not isclass(renderable):
            render_iterable = renderable.__rich_console__(self, _options)  # type: ignore[union-attr]
        elif isinstance(renderable, str):
            text_renderable = self.render_str(
                renderable, highlight=_options.highlight, markup=_options.markup
            )
            render_iterable = text_renderable.__rich_console__(self, _options)
        else:
            raise errors.NotRenderableError(
                f"Unable to render {renderable!r}; "
                "A str, Segment or object with __rich_console__ method is required"
            )

        try:
            iter_render = iter(render_iterable)
        except TypeError:
            raise errors.NotRenderableError(
                f"object {render_iterable!r} is not renderable"
            )
        _Segment = Segment
        _options = _options.reset_height()
        for render_output in iter_render:
            if isinstance(render_output, _Segment):
                yield render_output
            else:
                yield from self.render(render_output, _options)

    def render_lines(
        self,
        renderable: RenderableType,
        options: Optional[ConsoleOptions] = None,
        *,
        style: Optional[Style] = None,
        pad: bool = True,
        new_lines: bool = False,
    ) -> List[List[Segment]]:
        """Render objects in to a list of lines.

        The output of render_lines is useful when further formatting of rendered console text
        is required, such as the Panel class which draws a border around any renderable object.

        Args:
            renderable (RenderableType): Any object renderable in the console.
            options (Optional[ConsoleOptions], optional): Console options, or None to use self.options. Default to ``None``.
            style (Style, optional): Optional style to apply to renderables. Defaults to ``None``.
            pad (bool, optional): Pad lines shorter than render width. Defaults to ``True``.
            new_lines (bool, optional): Include "\n" characters at end of lines.

        Returns:
            List[List[Segment]]: A list of lines, where a line is a list of Segment objects.
        """
        with self._lock:
            render_options = options or self.options
            _rendered = self.render(renderable, render_options)
            if style:
                _rendered = Segment.apply_style(_rendered, style)

            render_height = render_options.height
            if render_height is not None:
                render_height = max(0, render_height)

            lines = list(
                islice(
                    Segment.split_and_crop_lines(
                        _rendered,
                        render_options.max_width,
                        include_new_lines=new_lines,
                        pad=pad,
                        style=style,
                    ),
                    None,
                    render_height,
                )
            )
            if render_options.height is not None:
                extra_lines = render_options.height - len(lines)
                if extra_lines > 0:
                    pad_line = [
                        [Segment(" " * render_options.max_width, style), Segment("\n")]
                        if new_lines
                        else [Segment(" " * render_options.max_width, style)]
                    ]
                    lines.extend(pad_line * extra_lines)

            return lines

    def render_str(
        self,
        text: str,
        *,
        style: Union[str, Style] = "",
        justify: Optional[JustifyMethod] = None,
        overflow: Optional[OverflowMethod] = None,
        emoji: Optional[bool] = None,
        markup: Optional[bool] = None,
        highlight: Optional[bool] = None,
        highlighter: Optional[HighlighterType] = None,
    ) -> "Text":
        """Convert a string to a Text instance. This is called automatically if
        you print or log a string.

        Args:
            text (str): Text to render.
            style (Union[str, Style], optional): Style to apply to rendered text.
            justify (str, optional): Justify method: "default", "left", "center", "full", or "right". Defaults to ``None``.
            overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to ``None``.
            emoji (Optional[bool], optional): Enable emoji, or ``None`` to use Console default.
            markup (Optional[bool], optional): Enable markup, or ``None`` to use Console default.
            highlight (Optional[bool], optional): Enable highlighting, or ``None`` to use Console default.
            highlighter (HighlighterType, optional): Optional highlighter to apply.
        Returns:
            ConsoleRenderable: Renderable object.

        """
        emoji_enabled = emoji or (emoji is None and self._emoji)
        markup_enabled = markup or (markup is None and self._markup)
        highlight_enabled = highlight or (highlight is None and self._highlight)

        if markup_enabled:
            rich_text = render_markup(
                text,
                style=style,
                emoji=emoji_enabled,
                emoji_variant=self._emoji_variant,
            )
            rich_text.justify = justify
            rich_text.overflow = overflow
        else:
            rich_text = Text(
                _emoji_replace(text, default_variant=self._emoji_variant)
                if emoji_enabled
                else text,
                justify=justify,
                overflow=overflow,
                style=style,
            )

        _highlighter = (highlighter or self.highlighter) if highlight_enabled else None
        if _highlighter is not None:
            highlight_text = _highlighter(str(rich_text))
            highlight_text.copy_styles(rich_text)
            return highlight_text

        return rich_text

    def get_style(
        self, name: Union[str, Style], *, default: Optional[Union[Style, str]] = None
    ) -> Style:
        """Get a Style instance by its theme name or parse a definition.

        Args:
            name (str): The name of a style or a style definition.

        Returns:
            Style: A Style object.

        Raises:
            MissingStyle: If no style could be parsed from name.

        """
        if isinstance(name, Style):
            return name

        try:
            style = self._theme_stack.get(name)
            if style is None:
                style = Style.parse(name)
            return style.copy() if style.link else style
        except errors.StyleSyntaxError as error:
            if default is not None:
                return self.get_style(default)
            raise errors.MissingStyle(
                f"Failed to get style {name!r}; {error}"
            ) from None

    def _collect_renderables(
        self,
        objects: Iterable[Any],
        sep: str,
        end: str,
        *,
        justify: Optional[JustifyMethod] = None,
        emoji: Optional[bool] = None,
        markup: Optional[bool] = None,
        highlight: Optional[bool] = None,
    ) -> List[ConsoleRenderable]:
        """Combine a number of renderables and text into one renderable.

        Args:
            objects (Iterable[Any]): Anything that Rich can render.
            sep (str): String to write between print data.
            end (str): String to write at end of print data.
            justify (str, optional): One of "left", "right", "center", or "full". Defaults to ``None``.
            emoji (Optional[bool], optional): Enable emoji code, or ``None`` to use console default.
            markup (Optional[bool], optional): Enable markup, or ``None`` to use console default.
            highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use console default.

        Returns:
            List[ConsoleRenderable]: A list of things to render.
        """
        renderables: List[ConsoleRenderable] = []
        _append = renderables.append
        text: List[Text] = []
        append_text = text.append

        append = _append
        if justify in ("left", "center", "right"):

            def align_append(renderable: RenderableType) -> None:
                _append(Align(renderable, cast(AlignMethod, justify)))

            append = align_append

        _highlighter: HighlighterType = _null_highlighter
        if highlight or (highlight is None and self._highlight):
            _highlighter = self.highlighter

        def check_text() -> None:
            if text:
                sep_text = Text(sep, justify=justify, end=end)
                append(sep_text.join(text))
                text.clear()

        for renderable in objects:
            renderable = rich_cast(renderable)
            if isinstance(renderable, str):
                append_text(
                    self.render_str(
                        renderable, emoji=emoji, markup=markup, highlighter=_highlighter
                    )
                )
            elif isinstance(renderable, Text):
                append_text(renderable)
            elif isinstance(renderable, ConsoleRenderable):
                check_text()
                append(renderable)
            elif is_expandable(renderable):
                check_text()
                append(Pretty(renderable, highlighter=_highlighter))
            else:
                append_text(_highlighter(str(renderable)))

        check_text()

        if self.style is not None:
            style = self.get_style(self.style)
            renderables = [Styled(renderable, style) for renderable in renderables]

        return renderables

    def rule(
        self,
        title: TextType = "",
        *,
        characters: str = "─",
        style: Union[str, Style] = "rule.line",
        align: AlignMethod = "center",
    ) -> None:
        """Draw a line with optional centered title.

        Args:
            title (str, optional): Text to render over the rule. Defaults to "".
            characters (str, optional): Character(s) to form the line. Defaults to "─".
            style (str, optional): Style of line. Defaults to "rule.line".
            align (str, optional): How to align the title, one of "left", "center", or "right". Defaults to "center".
        """
        from .rule import Rule

        rule = Rule(title=title, characters=characters, style=style, align=align)
        self.print(rule)

    def control(self, *control: Control) -> None:
        """Insert non-printing control codes.

        Args:
            control_codes (str): Control codes, such as those that may move the cursor.
        """
        if not self.is_dumb_terminal:
            with self:
                self._buffer.extend(_control.segment for _control in control)

    def out(
        self,
        *objects: Any,
        sep: str = " ",
        end: str = "\n",
        style: Optional[Union[str, Style]] = None,
        highlight: Optional[bool] = None,
    ) -> None:
        """Output to the terminal. This is a low-level way of writing to the terminal which unlike
        :meth:`~rich.console.Console.print` won't pretty print, wrap text, or apply markup, but will
        optionally apply highlighting and a basic style.

        Args:
            sep (str, optional): String to write between print data. Defaults to " ".
            end (str, optional): String to write at end of print data. Defaults to "\\\\n".
            style (Union[str, Style], optional): A style to apply to output. Defaults to None.
            highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use
                console default. Defaults to ``None``.
        """
        raw_output: str = sep.join(str(_object) for _object in objects)
        self.print(
            raw_output,
            style=style,
            highlight=highlight,
            emoji=False,
            markup=False,
            no_wrap=True,
            overflow="ignore",
            crop=False,
            end=end,
        )

    def print(
        self,
        *objects: Any,
        sep: str = " ",
        end: str = "\n",
        style: Optional[Union[str, Style]] = None,
        justify: Optional[JustifyMethod] = None,
        overflow: Optional[OverflowMethod] = None,
        no_wrap: Optional[bool] = None,
        emoji: Optional[bool] = None,
        markup: Optional[bool] = None,
        highlight: Optional[bool] = None,
        width: Optional[int] = None,
        height: Optional[int] = None,
        crop: bool = True,
        soft_wrap: Optional[bool] = None,
        new_line_start: bool = False,
    ) -> None:
        """Print to the console.

        Args:
            objects (positional args): Objects to log to the terminal.
            sep (str, optional): String to write between print data. Defaults to " ".
            end (str, optional): String to write at end of print data. Defaults to "\\\\n".
            style (Union[str, Style], optional): A style to apply to output. Defaults to None.
            justify (str, optional): Justify method: "default", "left", "right", "center", or "full". Defaults to ``None``.
            overflow (str, optional): Overflow method: "ignore", "crop", "fold", or "ellipsis". Defaults to None.
            no_wrap (Optional[bool], optional): Disable word wrapping. Defaults to None.
            emoji (Optional[bool], optional): Enable emoji code, or ``None`` to use console default. Defaults to ``None``.
            markup (Optional[bool], optional): Enable markup, or ``None`` to use console default. Defaults to ``None``.
            highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use console default. Defaults to ``None``.
            width (Optional[int], optional): Width of output, or ``None`` to auto-detect. Defaults to ``None``.
            crop (Optional[bool], optional): Crop output to width of terminal. Defaults to True.
            soft_wrap (bool, optional): Enable soft wrap mode which disables word wrapping and cropping of text or ``None`` for
                Console default. Defaults to ``None``.
            new_line_start (bool, False): Insert a new line at the start if the output contains more than one line. Defaults to ``False``.
        """
        if not objects:
            objects = (NewLine(),)

        if soft_wrap is None:
            soft_wrap = self.soft_wrap
        if soft_wrap:
            if no_wrap is None:
                no_wrap = True
            if overflow is None:
                overflow = "ignore"
            crop = False
        render_hooks = self._render_hooks[:]
        with self:
            renderables = self._collect_renderables(
                objects,
                sep,
                end,
                justify=justify,
                emoji=emoji,
                markup=markup,
                highlight=highlight,
            )
            for hook in render_hooks:
                renderables = hook.process_renderables(renderables)
            render_options = self.options.update(
                justify=justify,
                overflow=overflow,
                width=min(width, self.width) if width is not None else NO_CHANGE,
                height=height,
                no_wrap=no_wrap,
                markup=markup,
                highlight=highlight,
            )

            new_segments: List[Segment] = []
            extend = new_segments.extend
            render = self.render
            if style is None:
                for renderable in renderables:
                    extend(render(renderable, render_options))
            else:
                for renderable in renderables:
                    extend(
                        Segment.apply_style(
                            render(renderable, render_options), self.get_style(style)
                        )
                    )
            if new_line_start:
                if (
                    len("".join(segment.text for segment in new_segments).splitlines())
                    > 1
                ):
                    new_segments.insert(0, Segment.line())
            if crop:
                buffer_extend = self._buffer.extend
                for line in Segment.split_and_crop_lines(
                    new_segments, self.width, pad=False
                ):
                    buffer_extend(line)
            else:
                self._buffer.extend(new_segments)

    def print_json(
        self,
        json: Optional[str] = None,
        *,
        data: Any = None,
        indent: Union[None, int, str] = 2,
        highlight: bool = True,
        skip_keys: bool = False,
        ensure_ascii: bool = False,
        check_circular: bool = True,
        allow_nan: bool = True,
        default: Optional[Callable[[Any], Any]] = None,
        sort_keys: bool = False,
    ) -> None:
        """Pretty prints JSON. Output will be valid JSON.

        Args:
            json (Optional[str]): A string containing JSON.
            data (Any): If json is not supplied, then encode this data.
            indent (Union[None, int, str], optional): Number of spaces to indent. Defaults to 2.
            highlight (bool, optional): Enable highlighting of output: Defaults to True.
            skip_keys (bool, optional): Skip keys not of a basic type. Defaults to False.
            ensure_ascii (bool, optional): Escape all non-ascii characters. Defaults to False.
            check_circular (bool, optional): Check for circular references. Defaults to True.
            allow_nan (bool, optional): Allow NaN and Infinity values. Defaults to True.
            default (Callable, optional): A callable that converts values that can not be encoded
                in to something that can be JSON encoded. Defaults to None.
            sort_keys (bool, optional): Sort dictionary keys. Defaults to False.
        """
        from rich.json import JSON

        if json is None:
            json_renderable = JSON.from_data(
                data,
                indent=indent,
                highlight=highlight,
                skip_keys=skip_keys,
                ensure_ascii=ensure_ascii,
                check_circular=check_circular,
                allow_nan=allow_nan,
                default=default,
                sort_keys=sort_keys,
            )
        else:
            if not isinstance(json, str):
                raise TypeError(
                    f"json must be str. Did you mean print_json(data={json!r}) ?"
                )
            json_renderable = JSON(
                json,
                indent=indent,
                highlight=highlight,
                skip_keys=skip_keys,
                ensure_ascii=ensure_ascii,
                check_circular=check_circular,
                allow_nan=allow_nan,
                default=default,
                sort_keys=sort_keys,
            )
        self.print(json_renderable, soft_wrap=True)

    def update_screen(
        self,
        renderable: RenderableType,
        *,
        region: Optional[Region] = None,
        options: Optional[ConsoleOptions] = None,
    ) -> None:
        """Update the screen at a given offset.

        Args:
            renderable (RenderableType): A Rich renderable.
            region (Region, optional): Region of screen to update, or None for entire screen. Defaults to None.
            x (int, optional): x offset. Defaults to 0.
            y (int, optional): y offset. Defaults to 0.

        Raises:
            errors.NoAltScreen: If the Console isn't in alt screen mode.

        """
        if not self.is_alt_screen:
            raise errors.NoAltScreen("Alt screen must be enabled to call update_screen")
        render_options = options or self.options
        if region is None:
            x = y = 0
            render_options = render_options.update_dimensions(
                render_options.max_width, render_options.height or self.height
            )
        else:
            x, y, width, height = region
            render_options = render_options.update_dimensions(width, height)

        lines = self.render_lines(renderable, options=render_options)
        self.update_screen_lines(lines, x, y)

    def update_screen_lines(
        self, lines: List[List[Segment]], x: int = 0, y: int = 0
    ) -> None:
        """Update lines of the screen at a given offset.

        Args:
            lines (List[List[Segment]]): Rendered lines (as produced by :meth:`~rich.Console.render_lines`).
            x (int, optional): x offset (column no). Defaults to 0.
            y (int, optional): y offset (column no). Defaults to 0.

        Raises:
            errors.NoAltScreen: If the Console isn't in alt screen mode.
        """
        if not self.is_alt_screen:
            raise errors.NoAltScreen("Alt screen must be enabled to call update_screen")
        screen_update = ScreenUpdate(lines, x, y)
        segments = self.render(screen_update)
        self._buffer.extend(segments)
        self._check_buffer()

    def print_exception(
        self,
        *,
        width: Optional[int] = 100,
        extra_lines: int = 3,
        theme: Optional[str] = None,
        word_wrap: bool = False,
        show_locals: bool = False,
        suppress: Iterable[Union[str, ModuleType]] = (),
        max_frames: int = 100,
    ) -> None:
        """Prints a rich render of the last exception and traceback.

        Args:
            width (Optional[int], optional): Number of characters used to render code. Defaults to 100.
            extra_lines (int, optional): Additional lines of code to render. Defaults to 3.
            theme (str, optional): Override pygments theme used in traceback
            word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False.
            show_locals (bool, optional): Enable display of local variables. Defaults to False.
            suppress (Iterable[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback.
            max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100.
        """
        from .traceback import Traceback

        traceback = Traceback(
            width=width,
            extra_lines=extra_lines,
            theme=theme,
            word_wrap=word_wrap,
            show_locals=show_locals,
            suppress=suppress,
            max_frames=max_frames,
        )
        self.print(traceback)

    @staticmethod
    def _caller_frame_info(
        offset: int,
        currentframe: Callable[[], Optional[FrameType]] = inspect.currentframe,
    ) -> Tuple[str, int, Dict[str, Any]]:
        """Get caller frame information.

        Args:
            offset (int): the caller offset within the current frame stack.
            currentframe (Callable[[], Optional[FrameType]], optional): the callable to use to
                retrieve the current frame. Defaults to ``inspect.currentframe``.

        Returns:
            Tuple[str, int, Dict[str, Any]]: A tuple containing the filename, the line number and
                the dictionary of local variables associated with the caller frame.

        Raises:
            RuntimeError: If the stack offset is invalid.
        """
        # Ignore the frame of this local helper
        offset += 1

        frame = currentframe()
        if frame is not None:
            # Use the faster currentframe where implemented
            while offset and frame is not None:
                frame = frame.f_back
                offset -= 1
            assert frame is not None
            return frame.f_code.co_filename, frame.f_lineno, frame.f_locals
        else:
            # Fallback to the slower stack
            frame_info = inspect.stack()[offset]
            return frame_info.filename, frame_info.lineno, frame_info.frame.f_locals

    def log(
        self,
        *objects: Any,
        sep: str = " ",
        end: str = "\n",
        style: Optional[Union[str, Style]] = None,
        justify: Optional[JustifyMethod] = None,
        emoji: Optional[bool] = None,
        markup: Optional[bool] = None,
        highlight: Optional[bool] = None,
        log_locals: bool = False,
        _stack_offset: int = 1,
    ) -> None:
        """Log rich content to the terminal.

        Args:
            objects (positional args): Objects to log to the terminal.
            sep (str, optional): String to write between print data. Defaults to " ".
            end (str, optional): String to write at end of print data. Defaults to "\\\\n".
            style (Union[str, Style], optional): A style to apply to output. Defaults to None.
            justify (str, optional): One of "left", "right", "center", or "full". Defaults to ``None``.
            emoji (Optional[bool], optional): Enable emoji code, or ``None`` to use console default. Defaults to None.
            markup (Optional[bool], optional): Enable markup, or ``None`` to use console default. Defaults to None.
            highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use console default. Defaults to None.
            log_locals (bool, optional): Boolean to enable logging of locals where ``log()``
                was called. Defaults to False.
            _stack_offset (int, optional): Offset of caller from end of call stack. Defaults to 1.
        """
        if not objects:
            objects = (NewLine(),)

        render_hooks = self._render_hooks[:]

        with self:
            renderables = self._collect_renderables(
                objects,
                sep,
                end,
                justify=justify,
                emoji=emoji,
                markup=markup,
                highlight=highlight,
            )
            if style is not None:
                renderables = [Styled(renderable, style) for renderable in renderables]

            filename, line_no, locals = self._caller_frame_info(_stack_offset)
            link_path = None if filename.startswith("<") else os.path.abspath(filename)
            path = filename.rpartition(os.sep)[-1]
            if log_locals:
                locals_map = {
                    key: value
                    for key, value in locals.items()
                    if not key.startswith("__")
                }
                renderables.append(render_scope(locals_map, title="[i]locals"))

            renderables = [
                self._log_render(
                    self,
                    renderables,
                    log_time=self.get_datetime(),
                    path=path,
                    line_no=line_no,
                    link_path=link_path,
                )
            ]
            for hook in render_hooks:
                renderables = hook.process_renderables(renderables)
            new_segments: List[Segment] = []
            extend = new_segments.extend
            render = self.render
            render_options = self.options
            for renderable in renderables:
                extend(render(renderable, render_options))
            buffer_extend = self._buffer.extend
            for line in Segment.split_and_crop_lines(
                new_segments, self.width, pad=False
            ):
                buffer_extend(line)

    def _check_buffer(self) -> None:
        """Check if the buffer may be rendered. Render it if it can (e.g. Console.quiet is False)
        Rendering is supported on Windows, Unix and Jupyter environments. For
        legacy Windows consoles, the win32 API is called directly.
        This method will also record what it renders if recording is enabled via Console.record.
        """
        if self.quiet:
            del self._buffer[:]
            return
        with self._lock:
            if self.record:
                with self._record_buffer_lock:
                    self._record_buffer.extend(self._buffer[:])

            if self._buffer_index == 0:
                if self.is_jupyter:  # pragma: no cover
                    from .jupyter import display

                    display(self._buffer, self._render_buffer(self._buffer[:]))
                    del self._buffer[:]
                else:
                    if WINDOWS:
                        use_legacy_windows_render = False
                        if self.legacy_windows:
                            fileno = get_fileno(self.file)
                            if fileno is not None:
                                use_legacy_windows_render = (
                                    fileno in _STD_STREAMS_OUTPUT
                                )

                        if use_legacy_windows_render:
                            from rich._win32_console import LegacyWindowsTerm
                            from rich._windows_renderer import legacy_windows_render

                            buffer = self._buffer[:]
                            if self.no_color and self._color_system:
                                buffer = list(Segment.remove_color(buffer))

                            legacy_windows_render(buffer, LegacyWindowsTerm(self.file))
                        else:
                            # Either a non-std stream on legacy Windows, or modern Windows.
                            text = self._render_buffer(self._buffer[:])
                            # https://bugs.python.org/issue37871
                            # https://github.com/python/cpython/issues/82052
                            # We need to avoid writing more than 32Kb in a single write, due to the above bug
                            write = self.file.write
                            # Worse case scenario, every character is 4 bytes of utf-8
                            MAX_WRITE = 32 * 1024 // 4
                            try:
                                if len(text) <= MAX_WRITE:
                                    write(text)
                                else:
                                    batch: List[str] = []
                                    batch_append = batch.append
                                    size = 0
                                    for line in text.splitlines(True):
                                        if size + len(line) > MAX_WRITE and batch:
                                            write("".join(batch))
                                            batch.clear()
                                            size = 0
                                        batch_append(line)
                                        size += len(line)
                                    if batch:
                                        write("".join(batch))
                                        batch.clear()
                            except UnicodeEncodeError as error:
                                error.reason = f"{error.reason}\n*** You may need to add PYTHONIOENCODING=utf-8 to your environment ***"
                                raise
                    else:
                        text = self._render_buffer(self._buffer[:])
                        try:
                            self.file.write(text)
                        except UnicodeEncodeError as error:
                            error.reason = f"{error.reason}\n*** You may need to add PYTHONIOENCODING=utf-8 to your environment ***"
                            raise

                    self.file.flush()
                    del self._buffer[:]

    def _render_buffer(self, buffer: Iterable[Segment]) -> str:
        """Render buffered output, and clear buffer."""
        output: List[str] = []
        append = output.append
        color_system = self._color_system
        legacy_windows = self.legacy_windows
        not_terminal = not self.is_terminal
        if self.no_color and color_system:
            buffer = Segment.remove_color(buffer)
        for text, style, control in buffer:
            if style:
                append(
                    style.render(
                        text,
                        color_system=color_system,
                        legacy_windows=legacy_windows,
                    )
                )
            elif not (not_terminal and control):
                append(text)

        rendered = "".join(output)
        return rendered

    def input(
        self,
        prompt: TextType = "",
        *,
        markup: bool = True,
        emoji: bool = True,
        password: bool = False,
        stream: Optional[TextIO] = None,
    ) -> str:
        """Displays a prompt and waits for input from the user. The prompt may contain color / style.

        It works in the same way as Python's builtin :func:`input` function and provides elaborate line editing and history features if Python's builtin :mod:`readline` module is previously loaded.

        Args:
            prompt (Union[str, Text]): Text to render in the prompt.
            markup (bool, optional): Enable console markup (requires a str prompt). Defaults to True.
            emoji (bool, optional): Enable emoji (requires a str prompt). Defaults to True.
            password: (bool, optional): Hide typed text. Defaults to False.
            stream: (TextIO, optional): Optional file to read input from (rather than stdin). Defaults to None.

        Returns:
            str: Text read from stdin.
        """
        if prompt:
            self.print(prompt, markup=markup, emoji=emoji, end="")
        if password:
            result = getpass("", stream=stream)
        else:
            if stream:
                result = stream.readline()
            else:
                result = input()
        return result

    def export_text(self, *, clear: bool = True, styles: bool = False) -> str:
        """Generate text from console contents (requires record=True argument in constructor).

        Args:
            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``.
            styles (bool, optional): If ``True``, ansi escape codes will be included. ``False`` for plain text.
                Defaults to ``False``.

        Returns:
            str: String containing console contents.

        """
        assert (
            self.record
        ), "To export console contents set record=True in the constructor or instance"

        with self._record_buffer_lock:
            if styles:
                text = "".join(
                    (style.render(text) if style else text)
                    for text, style, _ in self._record_buffer
                )
            else:
                text = "".join(
                    segment.text
                    for segment in self._record_buffer
                    if not segment.control
                )
            if clear:
                del self._record_buffer[:]
        return text

    def save_text(self, path: str, *, clear: bool = True, styles: bool = False) -> None:
        """Generate text from console and save to a given location (requires record=True argument in constructor).

        Args:
            path (str): Path to write text files.
            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``.
            styles (bool, optional): If ``True``, ansi style codes will be included. ``False`` for plain text.
                Defaults to ``False``.

        """
        text = self.export_text(clear=clear, styles=styles)
        with open(path, "wt", encoding="utf-8") as write_file:
            write_file.write(text)

    def export_html(
        self,
        *,
        theme: Optional[TerminalTheme] = None,
        clear: bool = True,
        code_format: Optional[str] = None,
        inline_styles: bool = False,
    ) -> str:
        """Generate HTML from console contents (requires record=True argument in constructor).

        Args:
            theme (TerminalTheme, optional): TerminalTheme object containing console colors.
            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``.
            code_format (str, optional): Format string to render HTML. In addition to '{foreground}',
                '{background}', and '{code}', should contain '{stylesheet}' if inline_styles is ``False``.
            inline_styles (bool, optional): If ``True`` styles will be inlined in to spans, which makes files
                larger but easier to cut and paste markup. If ``False``, styles will be embedded in a style tag.
                Defaults to False.

        Returns:
            str: String containing console contents as HTML.
        """
        assert (
            self.record
        ), "To export console contents set record=True in the constructor or instance"
        fragments: List[str] = []
        append = fragments.append
        _theme = theme or DEFAULT_TERMINAL_THEME
        stylesheet = ""

        render_code_format = CONSOLE_HTML_FORMAT if code_format is None else code_format

        with self._record_buffer_lock:
            if inline_styles:
                for text, style, _ in Segment.filter_control(
                    Segment.simplify(self._record_buffer)
                ):
                    text = escape(text)
                    if style:
                        rule = style.get_html_style(_theme)
                        if style.link:
                            text = f'<a href="{style.link}">{text}</a>'
                        text = f'<span style="{rule}">{text}</span>' if rule else text
                    append(text)
            else:
                styles: Dict[str, int] = {}
                for text, style, _ in Segment.filter_control(
                    Segment.simplify(self._record_buffer)
                ):
                    text = escape(text)
                    if style:
                        rule = style.get_html_style(_theme)
                        style_number = styles.setdefault(rule, len(styles) + 1)
                        if style.link:
                            text = f'<a class="r{style_number}" href="{style.link}">{text}</a>'
                        else:
                            text = f'<span class="r{style_number}">{text}</span>'
                    append(text)
                stylesheet_rules: List[str] = []
                stylesheet_append = stylesheet_rules.append
                for style_rule, style_number in styles.items():
                    if style_rule:
                        stylesheet_append(f".r{style_number} {{{style_rule}}}")
                stylesheet = "\n".join(stylesheet_rules)

            rendered_code = render_code_format.format(
                code="".join(fragments),
                stylesheet=stylesheet,
                foreground=_theme.foreground_color.hex,
                background=_theme.background_color.hex,
            )
            if clear:
                del self._record_buffer[:]
        return rendered_code

    def save_html(
        self,
        path: str,
        *,
        theme: Optional[TerminalTheme] = None,
        clear: bool = True,
        code_format: str = CONSOLE_HTML_FORMAT,
        inline_styles: bool = False,
    ) -> None:
        """Generate HTML from console contents and write to a file (requires record=True argument in constructor).

        Args:
            path (str): Path to write html file.
            theme (TerminalTheme, optional): TerminalTheme object containing console colors.
            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``.
            code_format (str, optional): Format string to render HTML. In addition to '{foreground}',
                '{background}', and '{code}', should contain '{stylesheet}' if inline_styles is ``False``.
            inline_styles (bool, optional): If ``True`` styles will be inlined in to spans, which makes files
                larger but easier to cut and paste markup. If ``False``, styles will be embedded in a style tag.
                Defaults to False.

        """
        html = self.export_html(
            theme=theme,
            clear=clear,
            code_format=code_format,
            inline_styles=inline_styles,
        )
        with open(path, "wt", encoding="utf-8") as write_file:
            write_file.write(html)

    def export_svg(
        self,
        *,
        title: str = "Rich",
        theme: Optional[TerminalTheme] = None,
        clear: bool = True,
        code_format: str = CONSOLE_SVG_FORMAT,
        font_aspect_ratio: float = 0.61,
        unique_id: Optional[str] = None,
    ) -> str:
        """
        Generate an SVG from the console contents (requires record=True in Console constructor).

        Args:
            title (str, optional): The title of the tab in the output image
            theme (TerminalTheme, optional): The ``TerminalTheme`` object to use to style the terminal
            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``
            code_format (str, optional): Format string used to generate the SVG. Rich will inject a number of variables
                into the string in order to form the final SVG output. The default template used and the variables
                injected by Rich can be found by inspecting the ``console.CONSOLE_SVG_FORMAT`` variable.
            font_aspect_ratio (float, optional): The width to height ratio of the font used in the ``code_format``
                string. Defaults to 0.61, which is the width to height ratio of Fira Code (the default font).
                If you aren't specifying a different font inside ``code_format``, you probably don't need this.
            unique_id (str, optional): unique id that is used as the prefix for various elements (CSS styles, node
                ids). If not set, this defaults to a computed value based on the recorded content.
        """

        from rich.cells import cell_len

        style_cache: Dict[Style, str] = {}

        def get_svg_style(style: Style) -> str:
            """Convert a Style to CSS rules for SVG."""
            if style in style_cache:
                return style_cache[style]
            css_rules = []
            color = (
                _theme.foreground_color
                if (style.color is None or style.color.is_default)
                else style.color.get_truecolor(_theme)
            )
            bgcolor = (
                _theme.background_color
                if (style.bgcolor is None or style.bgcolor.is_default)
                else style.bgcolor.get_truecolor(_theme)
            )
            if style.reverse:
                color, bgcolor = bgcolor, color
            if style.dim:
                color = blend_rgb(color, bgcolor, 0.4)
            css_rules.append(f"fill: {color.hex}")
            if style.bold:
                css_rules.append("font-weight: bold")
            if style.italic:
                css_rules.append("font-style: italic;")
            if style.underline:
                css_rules.append("text-decoration: underline;")
            if style.strike:
                css_rules.append("text-decoration: line-through;")

            css = ";".join(css_rules)
            style_cache[style] = css
            return css

        _theme = theme or SVG_EXPORT_THEME

        width = self.width
        char_height = 20
        char_width = char_height * font_aspect_ratio
        line_height = char_height * 1.22

        margin_top = 1
        margin_right = 1
        margin_bottom = 1
        margin_left = 1

        padding_top = 40
        padding_right = 8
        padding_bottom = 8
        padding_left = 8

        padding_width = padding_left + padding_right
        padding_height = padding_top + padding_bottom
        margin_width = margin_left + margin_right
        margin_height = margin_top + margin_bottom

        text_backgrounds: List[str] = []
        text_group: List[str] = []
        classes: Dict[str, int] = {}
        style_no = 1

        def escape_text(text: str) -> str:
            """HTML escape text and replace spaces with nbsp."""
            return escape(text).replace(" ", "&#160;")

        def make_tag(
            name: str, content: Optional[str] = None, **attribs: object
        ) -> str:
            """Make a tag from name, content, and attributes."""

            def stringify(value: object) -> str:
                if isinstance(value, (float)):
                    return format(value, "g")
                return str(value)

            tag_attribs = " ".join(
                f'{k.lstrip("_").replace("_", "-")}="{stringify(v)}"'
                for k, v in attribs.items()
            )
            return (
                f"<{name} {tag_attribs}>{content}</{name}>"
                if content
                else f"<{name} {tag_attribs}/>"
            )

        with self._record_buffer_lock:
            segments = list(Segment.filter_control(self._record_buffer))
            if clear:
                self._record_buffer.clear()

        if unique_id is None:
            unique_id = "terminal-" + str(
                zlib.adler32(
                    ("".join(repr(segment) for segment in segments)).encode(
                        "utf-8",
                        "ignore",
                    )
                    + title.encode("utf-8", "ignore")
                )
            )
        y = 0
        for y, line in enumerate(Segment.split_and_crop_lines(segments, length=width)):
            x = 0
            for text, style, _control in line:
                style = style or Style()
                rules = get_svg_style(style)
                if rules not in classes:
                    classes[rules] = style_no
                    style_no += 1
                class_name = f"r{classes[rules]}"

                if style.reverse:
                    has_background = True
                    background = (
                        _theme.foreground_color.hex
                        if style.color is None
                        else style.color.get_truecolor(_theme).hex
                    )
                else:
                    bgcolor = style.bgcolor
                    has_background = bgcolor is not None and not bgcolor.is_default
                    background = (
                        _theme.background_color.hex
                        if style.bgcolor is None
                        else style.bgcolor.get_truecolor(_theme).hex
                    )

                text_length = cell_len(text)
                if has_background:
                    text_backgrounds.append(
                        make_tag(
                            "rect",
                            fill=background,
                            x=x * char_width,
                            y=y * line_height + 1.5,
                            width=char_width * text_length,
                            height=line_height + 0.25,
                            shape_rendering="crispEdges",
                        )
                    )

                if text != " " * len(text):
                    text_group.append(
                        make_tag(
                            "text",
                            escape_text(text),
                            _class=f"{unique_id}-{class_name}",
                            x=x * char_width,
                            y=y * line_height + char_height,
                            textLength=char_width * len(text),
                            clip_path=f"url(#{unique_id}-line-{y})",
                        )
                    )
                x += cell_len(text)

        line_offsets = [line_no * line_height + 1.5 for line_no in range(y)]
        lines = "\n".join(
            f"""<clipPath id="{unique_id}-line-{line_no}">
    {make_tag("rect", x=0, y=offset, width=char_width * width, height=line_height + 0.25)}
            </clipPath>"""
            for line_no, offset in enumerate(line_offsets)
        )

        styles = "\n".join(
            f".{unique_id}-r{rule_no} {{ {css} }}" for css, rule_no in classes.items()
        )
        backgrounds = "".join(text_backgrounds)
        matrix = "".join(text_group)

        terminal_width = ceil(width * char_width + padding_width)
        terminal_height = (y + 1) * line_height + padding_height
        chrome = make_tag(
            "rect",
            fill=_theme.background_color.hex,
            stroke="rgba(255,255,255,0.35)",
            stroke_width="1",
            x=margin_left,
            y=margin_top,
            width=terminal_width,
            height=terminal_height,
            rx=8,
        )

        title_color = _theme.foreground_color.hex
        if title:
            chrome += make_tag(
                "text",
                escape_text(title),
                _class=f"{unique_id}-title",
                fill=title_color,
                text_anchor="middle",
                x=terminal_width // 2,
                y=margin_top + char_height + 6,
            )
        chrome += f"""
            <g transform="translate(26,22)">
            <circle cx="0" cy="0" r="7" fill="#ff5f57"/>
            <circle cx="22" cy="0" r="7" fill="#febc2e"/>
            <circle cx="44" cy="0" r="7" fill="#28c840"/>
            </g>
        """

        svg = code_format.format(
            unique_id=unique_id,
            char_width=char_width,
            char_height=char_height,
            line_height=line_height,
            terminal_width=char_width * width - 1,
            terminal_height=(y + 1) * line_height - 1,
            width=terminal_width + margin_width,
            height=terminal_height + margin_height,
            terminal_x=margin_left + padding_left,
            terminal_y=margin_top + padding_top,
            styles=styles,
            chrome=chrome,
            backgrounds=backgrounds,
            matrix=matrix,
            lines=lines,
        )
        return svg

    def save_svg(
        self,
        path: str,
        *,
        title: str = "Rich",
        theme: Optional[TerminalTheme] = None,
        clear: bool = True,
        code_format: str = CONSOLE_SVG_FORMAT,
        font_aspect_ratio: float = 0.61,
        unique_id: Optional[str] = None,
    ) -> None:
        """Generate an SVG file from the console contents (requires record=True in Console constructor).

        Args:
            path (str): The path to write the SVG to.
            title (str, optional): The title of the tab in the output image
            theme (TerminalTheme, optional): The ``TerminalTheme`` object to use to style the terminal
            clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``
            code_format (str, optional): Format string used to generate the SVG. Rich will inject a number of variables
                into the string in order to form the final SVG output. The default template used and the variables
                injected by Rich can be found by inspecting the ``console.CONSOLE_SVG_FORMAT`` variable.
            font_aspect_ratio (float, optional): The width to height ratio of the font used in the ``code_format``
                string. Defaults to 0.61, which is the width to height ratio of Fira Code (the default font).
                If you aren't specifying a different font inside ``code_format``, you probably don't need this.
            unique_id (str, optional): unique id that is used as the prefix for various elements (CSS styles, node
                ids). If not set, this defaults to a computed value based on the recorded content.
        """
        svg = self.export_svg(
            title=title,
            theme=theme,
            clear=clear,
            code_format=code_format,
            font_aspect_ratio=font_aspect_ratio,
            unique_id=unique_id,
        )
        with open(path, "wt", encoding="utf-8") as write_file:
            write_file.write(svg)


def _svg_hash(svg_main_code: str) -> str:
    """Returns a unique hash for the given SVG main code.

    Args:
        svg_main_code (str): The content we're going to inject in the SVG envelope.

    Returns:
        str: a hash of the given content
    """
    return str(zlib.adler32(svg_main_code.encode()))


if __name__ == "__main__":  # pragma: no cover
    console = Console(record=True)

    console.log(
        "JSONRPC [i]request[/i]",
        5,
        1.3,
        True,
        False,
        None,
        {
            "jsonrpc": "2.0",
            "method": "subtract",
            "params": {"minuend": 42, "subtrahend": 23},
            "id": 3,
        },
    )

    console.log("Hello, World!", "{'a': 1}", repr(console))

    console.print(
        {
            "name": None,
            "empty": [],
            "quiz": {
                "sport": {
                    "answered": True,
                    "q1": {
                        "question": "Which one is correct team name in NBA?",
                        "options": [
                            "New York Bulls",
                            "Los Angeles Kings",
                            "Golden State Warriors",
                            "Huston Rocket",
                        ],
                        "answer": "Huston Rocket",
                    },
                },
                "maths": {
                    "answered": False,
                    "q1": {
                        "question": "5 + 7 = ?",
                        "options": [10, 11, 12, 13],
                        "answer": 12,
                    },
                    "q2": {
                        "question": "12 - 8 = ?",
                        "options": [1, 2, 3, 4],
                        "answer": 4,
                    },
                },
            },
        }
    )

env_pyrecodes/lib/python3.9/site-packages/rich/console.py
NikolaBlagojevic/pyrecodes
from __future__ import annotations

import sys
from typing import ClassVar, Dict, Iterable, List, Optional, Type, Union

from markdown_it import MarkdownIt
from markdown_it.token import Token

if sys.version_info >= (3, 8):
    from typing import get_args
else:
    from typing_extensions import get_args  # pragma: no cover

from rich.table import Table

from . import box
from ._loop import loop_first
from ._stack import Stack
from .console import Console, ConsoleOptions, JustifyMethod, RenderResult
from .containers import Renderables
from .jupyter import JupyterMixin
from .panel import Panel
from .rule import Rule
from .segment import Segment
from .style import Style, StyleStack
from .syntax import Syntax
from .text import Text, TextType


class MarkdownElement:
    new_line: ClassVar[bool] = True

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "MarkdownElement":
        """Factory to create markdown element,

        Args:
            markdown (Markdown): The parent Markdown object.
            token (Token): A node from markdown-it.

        Returns:
            MarkdownElement: A new markdown element
        """
        return cls()

    def on_enter(self, context: "MarkdownContext") -> None:
        """Called when the node is entered.

        Args:
            context (MarkdownContext): The markdown context.
        """

    def on_text(self, context: "MarkdownContext", text: TextType) -> None:
        """Called when text is parsed.

        Args:
            context (MarkdownContext): The markdown context.
        """

    def on_leave(self, context: "MarkdownContext") -> None:
        """Called when the parser leaves the element.

        Args:
            context (MarkdownContext): [description]
        """

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        """Called when a child element is closed.

        This method allows a parent element to take over rendering of its children.

        Args:
            context (MarkdownContext): The markdown context.
            child (MarkdownElement): The child markdown element.

        Returns:
            bool: Return True to render the element, or False to not render the element.
        """
        return True

    def __rich_console__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> "RenderResult":
        return ()


class UnknownElement(MarkdownElement):
    """An unknown element.

    Hopefully there will be no unknown elements, and we will have a MarkdownElement for
    everything in the document.

    """


class TextElement(MarkdownElement):
    """Base class for elements that render text."""

    style_name = "none"

    def on_enter(self, context: "MarkdownContext") -> None:
        self.style = context.enter_style(self.style_name)
        self.text = Text(justify="left")

    def on_text(self, context: "MarkdownContext", text: TextType) -> None:
        self.text.append(text, context.current_style if isinstance(text, str) else None)

    def on_leave(self, context: "MarkdownContext") -> None:
        context.leave_style()


class Paragraph(TextElement):
    """A Paragraph."""

    style_name = "markdown.paragraph"
    justify: JustifyMethod

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "Paragraph":
        return cls(justify=markdown.justify or "left")

    def __init__(self, justify: JustifyMethod) -> None:
        self.justify = justify

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        self.text.justify = self.justify
        yield self.text


class Heading(TextElement):
    """A heading."""

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "Heading":
        return cls(token.tag)

    def on_enter(self, context: "MarkdownContext") -> None:
        self.text = Text()
        context.enter_style(self.style_name)

    def __init__(self, tag: str) -> None:
        self.tag = tag
        self.style_name = f"markdown.{tag}"
        super().__init__()

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        text = self.text
        text.justify = "center"
        if self.tag == "h1":
            # Draw a border around h1s
            yield Panel(
                text,
                box=box.HEAVY,
                style="markdown.h1.border",
            )
        else:
            # Styled text for h2 and beyond
            if self.tag == "h2":
                yield Text("")
            yield text


class CodeBlock(TextElement):
    """A code block with syntax highlighting."""

    style_name = "markdown.code_block"

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "CodeBlock":
        node_info = token.info or ""
        lexer_name = node_info.partition(" ")[0]
        return cls(lexer_name or "text", markdown.code_theme)

    def __init__(self, lexer_name: str, theme: str) -> None:
        self.lexer_name = lexer_name
        self.theme = theme

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        code = str(self.text).rstrip()
        syntax = Syntax(
            code, self.lexer_name, theme=self.theme, word_wrap=True, padding=1
        )
        yield syntax


class BlockQuote(TextElement):
    """A block quote."""

    style_name = "markdown.block_quote"

    def __init__(self) -> None:
        self.elements: Renderables = Renderables()

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        self.elements.append(child)
        return False

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        render_options = options.update(width=options.max_width - 4)
        lines = console.render_lines(self.elements, render_options, style=self.style)
        style = self.style
        new_line = Segment("\n")
        padding = Segment("▌ ", style)
        for line in lines:
            yield padding
            yield from line
            yield new_line


class HorizontalRule(MarkdownElement):
    """A horizontal rule to divide sections."""

    new_line = False

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        style = console.get_style("markdown.hr", default="none")
        yield Rule(style=style)


class TableElement(MarkdownElement):
    """MarkdownElement corresponding to `table_open`."""

    def __init__(self) -> None:
        self.header: TableHeaderElement | None = None
        self.body: TableBodyElement | None = None

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        if isinstance(child, TableHeaderElement):
            self.header = child
        elif isinstance(child, TableBodyElement):
            self.body = child
        else:
            raise RuntimeError("Couldn't process markdown table.")
        return False

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        table = Table(box=box.SIMPLE_HEAVY)

        if self.header is not None and self.header.row is not None:
            for column in self.header.row.cells:
                table.add_column(column.content)

        if self.body is not None:
            for row in self.body.rows:
                row_content = [element.content for element in row.cells]
                table.add_row(*row_content)

        yield table


class TableHeaderElement(MarkdownElement):
    """MarkdownElement corresponding to `thead_open` and `thead_close`."""

    def __init__(self) -> None:
        self.row: TableRowElement | None = None

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        assert isinstance(child, TableRowElement)
        self.row = child
        return False


class TableBodyElement(MarkdownElement):
    """MarkdownElement corresponding to `tbody_open` and `tbody_close`."""

    def __init__(self) -> None:
        self.rows: list[TableRowElement] = []

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        assert isinstance(child, TableRowElement)
        self.rows.append(child)
        return False


class TableRowElement(MarkdownElement):
    """MarkdownElement corresponding to `tr_open` and `tr_close`."""

    def __init__(self) -> None:
        self.cells: List[TableDataElement] = []

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        assert isinstance(child, TableDataElement)
        self.cells.append(child)
        return False


class TableDataElement(MarkdownElement):
    """MarkdownElement corresponding to `td_open` and `td_close`
    and `th_open` and `th_close`."""

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "MarkdownElement":
        style = str(token.attrs.get("style")) or ""

        justify: JustifyMethod
        if "text-align:right" in style:
            justify = "right"
        elif "text-align:center" in style:
            justify = "center"
        elif "text-align:left" in style:
            justify = "left"
        else:
            justify = "default"

        assert justify in get_args(JustifyMethod)
        return cls(justify=justify)

    def __init__(self, justify: JustifyMethod) -> None:
        self.content: Text = Text("", justify=justify)
        self.justify = justify

    def on_text(self, context: "MarkdownContext", text: TextType) -> None:
        text = Text(text) if isinstance(text, str) else text
        text.stylize(context.current_style)
        self.content.append_text(text)


class ListElement(MarkdownElement):
    """A list element."""

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "ListElement":
        return cls(token.type, int(token.attrs.get("start", 1)))

    def __init__(self, list_type: str, list_start: int | None) -> None:
        self.items: List[ListItem] = []
        self.list_type = list_type
        self.list_start = list_start

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        assert isinstance(child, ListItem)
        self.items.append(child)
        return False

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        if self.list_type == "bullet_list_open":
            for item in self.items:
                yield from item.render_bullet(console, options)
        else:
            number = 1 if self.list_start is None else self.list_start
            last_number = number + len(self.items)
            for index, item in enumerate(self.items):
                yield from item.render_number(
                    console, options, number + index, last_number
                )


class ListItem(TextElement):
    """An item in a list."""

    style_name = "markdown.item"

    def __init__(self) -> None:
        self.elements: Renderables = Renderables()

    def on_child_close(
        self, context: "MarkdownContext", child: "MarkdownElement"
    ) -> bool:
        self.elements.append(child)
        return False

    def render_bullet(self, console: Console, options: ConsoleOptions) -> RenderResult:
        render_options = options.update(width=options.max_width - 3)
        lines = console.render_lines(self.elements, render_options, style=self.style)
        bullet_style = console.get_style("markdown.item.bullet", default="none")

        bullet = Segment(" • ", bullet_style)
        padding = Segment(" " * 3, bullet_style)
        new_line = Segment("\n")
        for first, line in loop_first(lines):
            yield bullet if first else padding
            yield from line
            yield new_line

    def render_number(
        self, console: Console, options: ConsoleOptions, number: int, last_number: int
    ) -> RenderResult:
        number_width = len(str(last_number)) + 2
        render_options = options.update(width=options.max_width - number_width)
        lines = console.render_lines(self.elements, render_options, style=self.style)
        number_style = console.get_style("markdown.item.number", default="none")

        new_line = Segment("\n")
        padding = Segment(" " * number_width, number_style)
        numeral = Segment(f"{number}".rjust(number_width - 1) + " ", number_style)
        for first, line in loop_first(lines):
            yield numeral if first else padding
            yield from line
            yield new_line


class Link(TextElement):
    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "MarkdownElement":
        url = token.attrs.get("href", "#")
        return cls(token.content, str(url))

    def __init__(self, text: str, href: str):
        self.text = Text(text)
        self.href = href


class ImageItem(TextElement):
    """Renders a placeholder for an image."""

    new_line = False

    @classmethod
    def create(cls, markdown: "Markdown", token: Token) -> "MarkdownElement":
        """Factory to create markdown element,

        Args:
            markdown (Markdown): The parent Markdown object.
            token (Any): A token from markdown-it.

        Returns:
            MarkdownElement: A new markdown element
        """
        return cls(str(token.attrs.get("src", "")), markdown.hyperlinks)

    def __init__(self, destination: str, hyperlinks: bool) -> None:
        self.destination = destination
        self.hyperlinks = hyperlinks
        self.link: Optional[str] = None
        super().__init__()

    def on_enter(self, context: "MarkdownContext") -> None:
        self.link = context.current_style.link
        self.text = Text(justify="left")
        super().on_enter(context)

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        link_style = Style(link=self.link or self.destination or None)
        title = self.text or Text(self.destination.strip("/").rsplit("/", 1)[-1])
        if self.hyperlinks:
            title.stylize(link_style)
        text = Text.assemble("🌆 ", title, " ", end="")
        yield text


class MarkdownContext:
    """Manages the console render state."""

    def __init__(
        self,
        console: Console,
        options: ConsoleOptions,
        style: Style,
        inline_code_lexer: Optional[str] = None,
        inline_code_theme: str = "monokai",
    ) -> None:
        self.console = console
        self.options = options
        self.style_stack: StyleStack = StyleStack(style)
        self.stack: Stack[MarkdownElement] = Stack()

        self._syntax: Optional[Syntax] = None
        if inline_code_lexer is not None:
            self._syntax = Syntax("", inline_code_lexer, theme=inline_code_theme)

    @property
    def current_style(self) -> Style:
        """Current style which is the product of all styles on the stack."""
        return self.style_stack.current

    def on_text(self, text: str, node_type: str) -> None:
        """Called when the parser visits text."""
        if node_type in {"fence", "code_inline"} and self._syntax is not None:
            highlight_text = self._syntax.highlight(text)
            highlight_text.rstrip()
            self.stack.top.on_text(
                self, Text.assemble(highlight_text, style=self.style_stack.current)
            )
        else:
            self.stack.top.on_text(self, text)

    def enter_style(self, style_name: Union[str, Style]) -> Style:
        """Enter a style context."""
        style = self.console.get_style(style_name, default="none")
        self.style_stack.push(style)
        return self.current_style

    def leave_style(self) -> Style:
        """Leave a style context."""
        style = self.style_stack.pop()
        return style


class Markdown(JupyterMixin):
    """A Markdown renderable.

    Args:
        markup (str): A string containing markdown.
        code_theme (str, optional): Pygments theme for code blocks. Defaults to "monokai".
        justify (JustifyMethod, optional): Justify value for paragraphs. Defaults to None.
        style (Union[str, Style], optional): Optional style to apply to markdown.
        hyperlinks (bool, optional): Enable hyperlinks. Defaults to ``True``.
        inline_code_lexer: (str, optional): Lexer to use if inline code highlighting is
            enabled. Defaults to None.
        inline_code_theme: (Optional[str], optional): Pygments theme for inline code
            highlighting, or None for no highlighting. Defaults to None.
    """

    elements: ClassVar[Dict[str, Type[MarkdownElement]]] = {
        "paragraph_open": Paragraph,
        "heading_open": Heading,
        "fence": CodeBlock,
        "code_block": CodeBlock,
        "blockquote_open": BlockQuote,
        "hr": HorizontalRule,
        "bullet_list_open": ListElement,
        "ordered_list_open": ListElement,
        "list_item_open": ListItem,
        "image": ImageItem,
        "table_open": TableElement,
        "tbody_open": TableBodyElement,
        "thead_open": TableHeaderElement,
        "tr_open": TableRowElement,
        "td_open": TableDataElement,
        "th_open": TableDataElement,
    }

    inlines = {"em", "strong", "code", "s"}

    def __init__(
        self,
        markup: str,
        code_theme: str = "monokai",
        justify: Optional[JustifyMethod] = None,
        style: Union[str, Style] = "none",
        hyperlinks: bool = True,
        inline_code_lexer: Optional[str] = None,
        inline_code_theme: Optional[str] = None,
    ) -> None:
        parser = MarkdownIt().enable("strikethrough").enable("table")
        self.markup = markup
        self.parsed = parser.parse(markup)
        self.code_theme = code_theme
        self.justify: Optional[JustifyMethod] = justify
        self.style = style
        self.hyperlinks = hyperlinks
        self.inline_code_lexer = inline_code_lexer
        self.inline_code_theme = inline_code_theme or code_theme

    def _flatten_tokens(self, tokens: Iterable[Token]) -> Iterable[Token]:
        """Flattens the token stream."""
        for token in tokens:
            is_fence = token.type == "fence"
            is_image = token.tag == "img"
            if token.children and not (is_image or is_fence):
                yield from self._flatten_tokens(token.children)
            else:
                yield token

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        """Render markdown to the console."""
        style = console.get_style(self.style, default="none")
        options = options.update(height=None)
        context = MarkdownContext(
            console,
            options,
            style,
            inline_code_lexer=self.inline_code_lexer,
            inline_code_theme=self.inline_code_theme,
        )
        tokens = self.parsed
        inline_style_tags = self.inlines
        new_line = False
        _new_line_segment = Segment.line()

        for token in self._flatten_tokens(tokens):
            node_type = token.type
            tag = token.tag

            entering = token.nesting == 1
            exiting = token.nesting == -1
            self_closing = token.nesting == 0

            if node_type == "text":
                context.on_text(token.content, node_type)
            elif node_type == "hardbreak":
                context.on_text("\n", node_type)
            elif node_type == "softbreak":
                context.on_text(" ", node_type)
            elif node_type == "link_open":
                href = str(token.attrs.get("href", ""))
                if self.hyperlinks:
                    link_style = console.get_style("markdown.link_url", default="none")
                    link_style += Style(link=href)
                    context.enter_style(link_style)
                else:
                    context.stack.push(Link.create(self, token))
            elif node_type == "link_close":
                if self.hyperlinks:
                    context.leave_style()
                else:
                    element = context.stack.pop()
                    assert isinstance(element, Link)
                    link_style = console.get_style("markdown.link", default="none")
                    context.enter_style(link_style)
                    context.on_text(element.text.plain, node_type)
                    context.leave_style()
                    context.on_text(" (", node_type)
                    link_url_style = console.get_style(
                        "markdown.link_url", default="none"
                    )
                    context.enter_style(link_url_style)
                    context.on_text(element.href, node_type)
                    context.leave_style()
                    context.on_text(")", node_type)
            elif (
                tag in inline_style_tags
                and node_type != "fence"
                and node_type != "code_block"
            ):
                if entering:
                    # If it's an opening inline token e.g. strong, em, etc.
                    # Then we move into a style context i.e. push to stack.
                    context.enter_style(f"markdown.{tag}")
                elif exiting:
                    # If it's a closing inline style, then we pop the style
                    # off of the stack, to move out of the context of it...
                    context.leave_style()
                else:
                    # If it's a self-closing inline style e.g. `code_inline`
                    context.enter_style(f"markdown.{tag}")
                    if token.content:
                        context.on_text(token.content, node_type)
                    context.leave_style()
            else:
                # Map the markdown tag -> MarkdownElement renderable
                element_class = self.elements.get(token.type) or UnknownElement
                element = element_class.create(self, token)

                if entering or self_closing:
                    context.stack.push(element)
                    element.on_enter(context)

                if exiting:  # CLOSING tag
                    element = context.stack.pop()

                    should_render = not context.stack or (
                        context.stack
                        and context.stack.top.on_child_close(context, element)
                    )

                    if should_render:
                        if new_line:
                            yield _new_line_segment

                        yield from console.render(element, context.options)
                elif self_closing:  # SELF-CLOSING tags (e.g. text, code, image)
                    context.stack.pop()
                    text = token.content
                    if text is not None:
                        element.on_text(context, text)

                    should_render = (
                        not context.stack
                        or context.stack
                        and context.stack.top.on_child_close(context, element)
                    )
                    if should_render:
                        if new_line:
                            yield _new_line_segment
                        yield from console.render(element, context.options)

                if exiting or self_closing:
                    element.on_leave(context)
                    new_line = element.new_line


if __name__ == "__main__":  # pragma: no cover
    import argparse
    import sys

    parser = argparse.ArgumentParser(
        description="Render Markdown to the console with Rich"
    )
    parser.add_argument(
        "path",
        metavar="PATH",
        help="path to markdown file, or - for stdin",
    )
    parser.add_argument(
        "-c",
        "--force-color",
        dest="force_color",
        action="store_true",
        default=None,
        help="force color for non-terminals",
    )
    parser.add_argument(
        "-t",
        "--code-theme",
        dest="code_theme",
        default="monokai",
        help="pygments code theme",
    )
    parser.add_argument(
        "-i",
        "--inline-code-lexer",
        dest="inline_code_lexer",
        default=None,
        help="inline_code_lexer",
    )
    parser.add_argument(
        "-y",
        "--hyperlinks",
        dest="hyperlinks",
        action="store_true",
        help="enable hyperlinks",
    )
    parser.add_argument(
        "-w",
        "--width",
        type=int,
        dest="width",
        default=None,
        help="width of output (default will auto-detect)",
    )
    parser.add_argument(
        "-j",
        "--justify",
        dest="justify",
        action="store_true",
        help="enable full text justify",
    )
    parser.add_argument(
        "-p",
        "--page",
        dest="page",
        action="store_true",
        help="use pager to scroll output",
    )
    args = parser.parse_args()

    from rich.console import Console

    if args.path == "-":
        markdown_body = sys.stdin.read()
    else:
        with open(args.path, "rt", encoding="utf-8") as markdown_file:
            markdown_body = markdown_file.read()

    markdown = Markdown(
        markdown_body,
        justify="full" if args.justify else "left",
        code_theme=args.code_theme,
        hyperlinks=args.hyperlinks,
        inline_code_lexer=args.inline_code_lexer,
    )
    if args.page:
        import io
        import pydoc

        fileio = io.StringIO()
        console = Console(
            file=fileio, force_terminal=args.force_color, width=args.width
        )
        console.print(markdown)
        pydoc.pager(fileio.getvalue())

    else:
        console = Console(
            force_terminal=args.force_color, width=args.width, record=True
        )
        console.print(markdown)

env_pyrecodes/lib/python3.9/site-packages/rich/markdown.py
NikolaBlagojevic/pyrecodes
import os.path
import platform
import re
import sys
import textwrap
from abc import ABC, abstractmethod
from pathlib import Path
from typing import (
    Any,
    Dict,
    Iterable,
    List,
    NamedTuple,
    Optional,
    Sequence,
    Set,
    Tuple,
    Type,
    Union,
)

from pygments.lexer import Lexer
from pygments.lexers import get_lexer_by_name, guess_lexer_for_filename
from pygments.style import Style as PygmentsStyle
from pygments.styles import get_style_by_name
from pygments.token import (
    Comment,
    Error,
    Generic,
    Keyword,
    Name,
    Number,
    Operator,
    String,
    Token,
    Whitespace,
)
from pygments.util import ClassNotFound

from rich.containers import Lines
from rich.padding import Padding, PaddingDimensions

from ._loop import loop_first
from .cells import cell_len
from .color import Color, blend_rgb
from .console import Console, ConsoleOptions, JustifyMethod, RenderResult
from .jupyter import JupyterMixin
from .measure import Measurement
from .segment import Segment, Segments
from .style import Style, StyleType
from .text import Text

TokenType = Tuple[str, ...]

WINDOWS = platform.system() == "Windows"
DEFAULT_THEME = "monokai"

# The following styles are based on https://github.com/pygments/pygments/blob/master/pygments/formatters/terminal.py
# A few modifications were made

ANSI_LIGHT: Dict[TokenType, Style] = {
    Token: Style(),
    Whitespace: Style(color="white"),
    Comment: Style(dim=True),
    Comment.Preproc: Style(color="cyan"),
    Keyword: Style(color="blue"),
    Keyword.Type: Style(color="cyan"),
    Operator.Word: Style(color="magenta"),
    Name.Builtin: Style(color="cyan"),
    Name.Function: Style(color="green"),
    Name.Namespace: Style(color="cyan", underline=True),
    Name.Class: Style(color="green", underline=True),
    Name.Exception: Style(color="cyan"),
    Name.Decorator: Style(color="magenta", bold=True),
    Name.Variable: Style(color="red"),
    Name.Constant: Style(color="red"),
    Name.Attribute: Style(color="cyan"),
    Name.Tag: Style(color="bright_blue"),
    String: Style(color="yellow"),
    Number: Style(color="blue"),
    Generic.Deleted: Style(color="bright_red"),
    Generic.Inserted: Style(color="green"),
    Generic.Heading: Style(bold=True),
    Generic.Subheading: Style(color="magenta", bold=True),
    Generic.Prompt: Style(bold=True),
    Generic.Error: Style(color="bright_red"),
    Error: Style(color="red", underline=True),
}

ANSI_DARK: Dict[TokenType, Style] = {
    Token: Style(),
    Whitespace: Style(color="bright_black"),
    Comment: Style(dim=True),
    Comment.Preproc: Style(color="bright_cyan"),
    Keyword: Style(color="bright_blue"),
    Keyword.Type: Style(color="bright_cyan"),
    Operator.Word: Style(color="bright_magenta"),
    Name.Builtin: Style(color="bright_cyan"),
    Name.Function: Style(color="bright_green"),
    Name.Namespace: Style(color="bright_cyan", underline=True),
    Name.Class: Style(color="bright_green", underline=True),
    Name.Exception: Style(color="bright_cyan"),
    Name.Decorator: Style(color="bright_magenta", bold=True),
    Name.Variable: Style(color="bright_red"),
    Name.Constant: Style(color="bright_red"),
    Name.Attribute: Style(color="bright_cyan"),
    Name.Tag: Style(color="bright_blue"),
    String: Style(color="yellow"),
    Number: Style(color="bright_blue"),
    Generic.Deleted: Style(color="bright_red"),
    Generic.Inserted: Style(color="bright_green"),
    Generic.Heading: Style(bold=True),
    Generic.Subheading: Style(color="bright_magenta", bold=True),
    Generic.Prompt: Style(bold=True),
    Generic.Error: Style(color="bright_red"),
    Error: Style(color="red", underline=True),
}

RICH_SYNTAX_THEMES = {"ansi_light": ANSI_LIGHT, "ansi_dark": ANSI_DARK}
NUMBERS_COLUMN_DEFAULT_PADDING = 2


class SyntaxTheme(ABC):
    """Base class for a syntax theme."""

    @abstractmethod
    def get_style_for_token(self, token_type: TokenType) -> Style:
        """Get a style for a given Pygments token."""
        raise NotImplementedError  # pragma: no cover

    @abstractmethod
    def get_background_style(self) -> Style:
        """Get the background color."""
        raise NotImplementedError  # pragma: no cover


class PygmentsSyntaxTheme(SyntaxTheme):
    """Syntax theme that delegates to Pygments theme."""

    def __init__(self, theme: Union[str, Type[PygmentsStyle]]) -> None:
        self._style_cache: Dict[TokenType, Style] = {}
        if isinstance(theme, str):
            try:
                self._pygments_style_class = get_style_by_name(theme)
            except ClassNotFound:
                self._pygments_style_class = get_style_by_name("default")
        else:
            self._pygments_style_class = theme

        self._background_color = self._pygments_style_class.background_color
        self._background_style = Style(bgcolor=self._background_color)

    def get_style_for_token(self, token_type: TokenType) -> Style:
        """Get a style from a Pygments class."""
        try:
            return self._style_cache[token_type]
        except KeyError:
            try:
                pygments_style = self._pygments_style_class.style_for_token(token_type)
            except KeyError:
                style = Style.null()
            else:
                color = pygments_style["color"]
                bgcolor = pygments_style["bgcolor"]
                style = Style(
                    color="#" + color if color else "#000000",
                    bgcolor="#" + bgcolor if bgcolor else self._background_color,
                    bold=pygments_style["bold"],
                    italic=pygments_style["italic"],
                    underline=pygments_style["underline"],
                )
            self._style_cache[token_type] = style
        return style

    def get_background_style(self) -> Style:
        return self._background_style


class ANSISyntaxTheme(SyntaxTheme):
    """Syntax theme to use standard colors."""

    def __init__(self, style_map: Dict[TokenType, Style]) -> None:
        self.style_map = style_map
        self._missing_style = Style.null()
        self._background_style = Style.null()
        self._style_cache: Dict[TokenType, Style] = {}

    def get_style_for_token(self, token_type: TokenType) -> Style:
        """Look up style in the style map."""
        try:
            return self._style_cache[token_type]
        except KeyError:
            # Styles form a hierarchy
            # We need to go from most to least specific
            # e.g. ("foo", "bar", "baz") to ("foo", "bar")  to ("foo",)
            get_style = self.style_map.get
            token = tuple(token_type)
            style = self._missing_style
            while token:
                _style = get_style(token)
                if _style is not None:
                    style = _style
                    break
                token = token[:-1]
            self._style_cache[token_type] = style
            return style

    def get_background_style(self) -> Style:
        return self._background_style


SyntaxPosition = Tuple[int, int]


class _SyntaxHighlightRange(NamedTuple):
    """
    A range to highlight in a Syntax object.
    `start` and `end` are 2-integers tuples, where the first integer is the line number
    (starting from 1) and the second integer is the column index (starting from 0).
    """

    style: StyleType
    start: SyntaxPosition
    end: SyntaxPosition


class Syntax(JupyterMixin):
    """Construct a Syntax object to render syntax highlighted code.

    Args:
        code (str): Code to highlight.
        lexer (Lexer | str): Lexer to use (see https://pygments.org/docs/lexers/)
        theme (str, optional): Color theme, aka Pygments style (see https://pygments.org/docs/styles/#getting-a-list-of-available-styles). Defaults to "monokai".
        dedent (bool, optional): Enable stripping of initial whitespace. Defaults to False.
        line_numbers (bool, optional): Enable rendering of line numbers. Defaults to False.
        start_line (int, optional): Starting number for line numbers. Defaults to 1.
        line_range (Tuple[int | None, int | None], optional): If given should be a tuple of the start and end line to render.
            A value of None in the tuple indicates the range is open in that direction.
        highlight_lines (Set[int]): A set of line numbers to highlight.
        code_width: Width of code to render (not including line numbers), or ``None`` to use all available width.
        tab_size (int, optional): Size of tabs. Defaults to 4.
        word_wrap (bool, optional): Enable word wrapping.
        background_color (str, optional): Optional background color, or None to use theme color. Defaults to None.
        indent_guides (bool, optional): Show indent guides. Defaults to False.
        padding (PaddingDimensions): Padding to apply around the syntax. Defaults to 0 (no padding).
    """

    _pygments_style_class: Type[PygmentsStyle]
    _theme: SyntaxTheme

    @classmethod
    def get_theme(cls, name: Union[str, SyntaxTheme]) -> SyntaxTheme:
        """Get a syntax theme instance."""
        if isinstance(name, SyntaxTheme):
            return name
        theme: SyntaxTheme
        if name in RICH_SYNTAX_THEMES:
            theme = ANSISyntaxTheme(RICH_SYNTAX_THEMES[name])
        else:
            theme = PygmentsSyntaxTheme(name)
        return theme

    def __init__(
        self,
        code: str,
        lexer: Union[Lexer, str],
        *,
        theme: Union[str, SyntaxTheme] = DEFAULT_THEME,
        dedent: bool = False,
        line_numbers: bool = False,
        start_line: int = 1,
        line_range: Optional[Tuple[Optional[int], Optional[int]]] = None,
        highlight_lines: Optional[Set[int]] = None,
        code_width: Optional[int] = None,
        tab_size: int = 4,
        word_wrap: bool = False,
        background_color: Optional[str] = None,
        indent_guides: bool = False,
        padding: PaddingDimensions = 0,
    ) -> None:
        self.code = code
        self._lexer = lexer
        self.dedent = dedent
        self.line_numbers = line_numbers
        self.start_line = start_line
        self.line_range = line_range
        self.highlight_lines = highlight_lines or set()
        self.code_width = code_width
        self.tab_size = tab_size
        self.word_wrap = word_wrap
        self.background_color = background_color
        self.background_style = (
            Style(bgcolor=background_color) if background_color else Style()
        )
        self.indent_guides = indent_guides
        self.padding = padding

        self._theme = self.get_theme(theme)
        self._stylized_ranges: List[_SyntaxHighlightRange] = []

    @classmethod
    def from_path(
        cls,
        path: str,
        encoding: str = "utf-8",
        lexer: Optional[Union[Lexer, str]] = None,
        theme: Union[str, SyntaxTheme] = DEFAULT_THEME,
        dedent: bool = False,
        line_numbers: bool = False,
        line_range: Optional[Tuple[int, int]] = None,
        start_line: int = 1,
        highlight_lines: Optional[Set[int]] = None,
        code_width: Optional[int] = None,
        tab_size: int = 4,
        word_wrap: bool = False,
        background_color: Optional[str] = None,
        indent_guides: bool = False,
        padding: PaddingDimensions = 0,
    ) -> "Syntax":
        """Construct a Syntax object from a file.

        Args:
            path (str): Path to file to highlight.
            encoding (str): Encoding of file.
            lexer (str | Lexer, optional): Lexer to use. If None, lexer will be auto-detected from path/file content.
            theme (str, optional): Color theme, aka Pygments style (see https://pygments.org/docs/styles/#getting-a-list-of-available-styles). Defaults to "emacs".
            dedent (bool, optional): Enable stripping of initial whitespace. Defaults to True.
            line_numbers (bool, optional): Enable rendering of line numbers. Defaults to False.
            start_line (int, optional): Starting number for line numbers. Defaults to 1.
            line_range (Tuple[int, int], optional): If given should be a tuple of the start and end line to render.
            highlight_lines (Set[int]): A set of line numbers to highlight.
            code_width: Width of code to render (not including line numbers), or ``None`` to use all available width.
            tab_size (int, optional): Size of tabs. Defaults to 4.
            word_wrap (bool, optional): Enable word wrapping of code.
            background_color (str, optional): Optional background color, or None to use theme color. Defaults to None.
            indent_guides (bool, optional): Show indent guides. Defaults to False.
            padding (PaddingDimensions): Padding to apply around the syntax. Defaults to 0 (no padding).

        Returns:
            [Syntax]: A Syntax object that may be printed to the console
        """
        code = Path(path).read_text(encoding=encoding)

        if not lexer:
            lexer = cls.guess_lexer(path, code=code)

        return cls(
            code,
            lexer,
            theme=theme,
            dedent=dedent,
            line_numbers=line_numbers,
            line_range=line_range,
            start_line=start_line,
            highlight_lines=highlight_lines,
            code_width=code_width,
            tab_size=tab_size,
            word_wrap=word_wrap,
            background_color=background_color,
            indent_guides=indent_guides,
            padding=padding,
        )

    @classmethod
    def guess_lexer(cls, path: str, code: Optional[str] = None) -> str:
        """Guess the alias of the Pygments lexer to use based on a path and an optional string of code.
        If code is supplied, it will use a combination of the code and the filename to determine the
        best lexer to use. For example, if the file is ``index.html`` and the file contains Django
        templating syntax, then "html+django" will be returned. If the file is ``index.html``, and no
        templating language is used, the "html" lexer will be used. If no string of code
        is supplied, the lexer will be chosen based on the file extension..

        Args:
             path (AnyStr): The path to the file containing the code you wish to know the lexer for.
             code (str, optional): Optional string of code that will be used as a fallback if no lexer
                is found for the supplied path.

        Returns:
            str: The name of the Pygments lexer that best matches the supplied path/code.
        """
        lexer: Optional[Lexer] = None
        lexer_name = "default"
        if code:
            try:
                lexer = guess_lexer_for_filename(path, code)
            except ClassNotFound:
                pass

        if not lexer:
            try:
                _, ext = os.path.splitext(path)
                if ext:
                    extension = ext.lstrip(".").lower()
                    lexer = get_lexer_by_name(extension)
            except ClassNotFound:
                pass

        if lexer:
            if lexer.aliases:
                lexer_name = lexer.aliases[0]
            else:
                lexer_name = lexer.name

        return lexer_name

    def _get_base_style(self) -> Style:
        """Get the base style."""
        default_style = self._theme.get_background_style() + self.background_style
        return default_style

    def _get_token_color(self, token_type: TokenType) -> Optional[Color]:
        """Get a color (if any) for the given token.

        Args:
            token_type (TokenType): A token type tuple from Pygments.

        Returns:
            Optional[Color]: Color from theme, or None for no color.
        """
        style = self._theme.get_style_for_token(token_type)
        return style.color

    @property
    def lexer(self) -> Optional[Lexer]:
        """The lexer for this syntax, or None if no lexer was found.

        Tries to find the lexer by name if a string was passed to the constructor.
        """

        if isinstance(self._lexer, Lexer):
            return self._lexer
        try:
            return get_lexer_by_name(
                self._lexer,
                stripnl=False,
                ensurenl=True,
                tabsize=self.tab_size,
            )
        except ClassNotFound:
            return None

    @property
    def default_lexer(self) -> Lexer:
        """A Pygments Lexer to use if one is not specified or invalid."""
        return get_lexer_by_name(
            "text",
            stripnl=False,
            ensurenl=True,
            tabsize=self.tab_size,
        )

    def highlight(
        self,
        code: str,
        line_range: Optional[Tuple[Optional[int], Optional[int]]] = None,
    ) -> Text:
        """Highlight code and return a Text instance.

        Args:
            code (str): Code to highlight.
            line_range(Tuple[int, int], optional): Optional line range to highlight.

        Returns:
            Text: A text instance containing highlighted syntax.
        """

        base_style = self._get_base_style()
        justify: JustifyMethod = (
            "default" if base_style.transparent_background else "left"
        )

        text = Text(
            justify=justify,
            style=base_style,
            tab_size=self.tab_size,
            no_wrap=not self.word_wrap,
        )
        _get_theme_style = self._theme.get_style_for_token

        lexer = self.lexer or self.default_lexer

        if lexer is None:
            text.append(code)
        else:
            if line_range:
                # More complicated path to only stylize a portion of the code
                # This speeds up further operations as there are less spans to process
                line_start, line_end = line_range

                def line_tokenize() -> Iterable[Tuple[Any, str]]:
                    """Split tokens to one per line."""
                    assert lexer  # required to make MyPy happy - we know lexer is not None at this point

                    for token_type, token in lexer.get_tokens(code):
                        while token:
                            line_token, new_line, token = token.partition("\n")
                            yield token_type, line_token + new_line

                def tokens_to_spans() -> Iterable[Tuple[str, Optional[Style]]]:
                    """Convert tokens to spans."""
                    tokens = iter(line_tokenize())
                    line_no = 0
                    _line_start = line_start - 1 if line_start else 0

                    # Skip over tokens until line start
                    while line_no < _line_start:
                        try:
                            _token_type, token = next(tokens)
                        except StopIteration:
                            break
                        yield (token, None)
                        if token.endswith("\n"):
                            line_no += 1
                    # Generate spans until line end
                    for token_type, token in tokens:
                        yield (token, _get_theme_style(token_type))
                        if token.endswith("\n"):
                            line_no += 1
                            if line_end and line_no >= line_end:
                                break

                text.append_tokens(tokens_to_spans())

            else:
                text.append_tokens(
                    (token, _get_theme_style(token_type))
                    for token_type, token in lexer.get_tokens(code)
                )
            if self.background_color is not None:
                text.stylize(f"on {self.background_color}")

        if self._stylized_ranges:
            self._apply_stylized_ranges(text)

        return text

    def stylize_range(
        self, style: StyleType, start: SyntaxPosition, end: SyntaxPosition
    ) -> None:
        """
        Adds a custom style on a part of the code, that will be applied to the syntax display when it's rendered.
        Line numbers are 1-based, while column indexes are 0-based.

        Args:
            style (StyleType): The style to apply.
            start (Tuple[int, int]): The start of the range, in the form `[line number, column index]`.
            end (Tuple[int, int]): The end of the range, in the form `[line number, column index]`.
        """
        self._stylized_ranges.append(_SyntaxHighlightRange(style, start, end))

    def _get_line_numbers_color(self, blend: float = 0.3) -> Color:
        background_style = self._theme.get_background_style() + self.background_style
        background_color = background_style.bgcolor
        if background_color is None or background_color.is_system_defined:
            return Color.default()
        foreground_color = self._get_token_color(Token.Text)
        if foreground_color is None or foreground_color.is_system_defined:
            return foreground_color or Color.default()
        new_color = blend_rgb(
            background_color.get_truecolor(),
            foreground_color.get_truecolor(),
            cross_fade=blend,
        )
        return Color.from_triplet(new_color)

    @property
    def _numbers_column_width(self) -> int:
        """Get the number of characters used to render the numbers column."""
        column_width = 0
        if self.line_numbers:
            column_width = (
                len(str(self.start_line + self.code.count("\n")))
                + NUMBERS_COLUMN_DEFAULT_PADDING
            )
        return column_width

    def _get_number_styles(self, console: Console) -> Tuple[Style, Style, Style]:
        """Get background, number, and highlight styles for line numbers."""
        background_style = self._get_base_style()
        if background_style.transparent_background:
            return Style.null(), Style(dim=True), Style.null()
        if console.color_system in ("256", "truecolor"):
            number_style = Style.chain(
                background_style,
                self._theme.get_style_for_token(Token.Text),
                Style(color=self._get_line_numbers_color()),
                self.background_style,
            )
            highlight_number_style = Style.chain(
                background_style,
                self._theme.get_style_for_token(Token.Text),
                Style(bold=True, color=self._get_line_numbers_color(0.9)),
                self.background_style,
            )
        else:
            number_style = background_style + Style(dim=True)
            highlight_number_style = background_style + Style(dim=False)
        return background_style, number_style, highlight_number_style

    def __rich_measure__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> "Measurement":
        _, right, _, left = Padding.unpack(self.padding)
        padding = left + right
        if self.code_width is not None:
            width = self.code_width + self._numbers_column_width + padding + 1
            return Measurement(self._numbers_column_width, width)
        lines = self.code.splitlines()
        width = (
            self._numbers_column_width
            + padding
            + (max(cell_len(line) for line in lines) if lines else 0)
        )
        if self.line_numbers:
            width += 1
        return Measurement(self._numbers_column_width, width)

    def __rich_console__(
        self, console: Console, options: ConsoleOptions
    ) -> RenderResult:
        segments = Segments(self._get_syntax(console, options))
        if self.padding:
            yield Padding(
                segments, style=self._theme.get_background_style(), pad=self.padding
            )
        else:
            yield segments

    def _get_syntax(
        self,
        console: Console,
        options: ConsoleOptions,
    ) -> Iterable[Segment]:
        """
        Get the Segments for the Syntax object, excluding any vertical/horizontal padding
        """
        transparent_background = self._get_base_style().transparent_background
        code_width = (
            (
                (options.max_width - self._numbers_column_width - 1)
                if self.line_numbers
                else options.max_width
            )
            if self.code_width is None
            else self.code_width
        )

        ends_on_nl, processed_code = self._process_code(self.code)
        text = self.highlight(processed_code, self.line_range)

        if not self.line_numbers and not self.word_wrap and not self.line_range:
            if not ends_on_nl:
                text.remove_suffix("\n")
            # Simple case of just rendering text
            style = (
                self._get_base_style()
                + self._theme.get_style_for_token(Comment)
                + Style(dim=True)
                + self.background_style
            )
            if self.indent_guides and not options.ascii_only:
                text = text.with_indent_guides(self.tab_size, style=style)
                text.overflow = "crop"
            if style.transparent_background:
                yield from console.render(
                    text, options=options.update(width=code_width)
                )
            else:
                syntax_lines = console.render_lines(
                    text,
                    options.update(width=code_width, height=None, justify="left"),
                    style=self.background_style,
                    pad=True,
                    new_lines=True,
                )
                for syntax_line in syntax_lines:
                    yield from syntax_line
            return

        start_line, end_line = self.line_range or (None, None)
        line_offset = 0
        if start_line:
            line_offset = max(0, start_line - 1)
        lines: Union[List[Text], Lines] = text.split("\n", allow_blank=ends_on_nl)
        if self.line_range:
            if line_offset > len(lines):
                return
            lines = lines[line_offset:end_line]

        if self.indent_guides and not options.ascii_only:
            style = (
                self._get_base_style()
                + self._theme.get_style_for_token(Comment)
                + Style(dim=True)
                + self.background_style
            )
            lines = (
                Text("\n")
                .join(lines)
                .with_indent_guides(self.tab_size, style=style + Style(italic=False))
                .split("\n", allow_blank=True)
            )

        numbers_column_width = self._numbers_column_width
        render_options = options.update(width=code_width)

        highlight_line = self.highlight_lines.__contains__
        _Segment = Segment
        new_line = _Segment("\n")

        line_pointer = "> " if options.legacy_windows else "❱ "

        (
            background_style,
            number_style,
            highlight_number_style,
        ) = self._get_number_styles(console)

        for line_no, line in enumerate(lines, self.start_line + line_offset):
            if self.word_wrap:
                wrapped_lines = console.render_lines(
                    line,
                    render_options.update(height=None, justify="left"),
                    style=background_style,
                    pad=not transparent_background,
                )
            else:
                segments = list(line.render(console, end=""))
                if options.no_wrap:
                    wrapped_lines = [segments]
                else:
                    wrapped_lines = [
                        _Segment.adjust_line_length(
                            segments,
                            render_options.max_width,
                            style=background_style,
                            pad=not transparent_background,
                        )
                    ]

            if self.line_numbers:
                wrapped_line_left_pad = _Segment(
                    " " * numbers_column_width + " ", background_style
                )
                for first, wrapped_line in loop_first(wrapped_lines):
                    if first:
                        line_column = str(line_no).rjust(numbers_column_width - 2) + " "
                        if highlight_line(line_no):
                            yield _Segment(line_pointer, Style(color="red"))
                            yield _Segment(line_column, highlight_number_style)
                        else:
                            yield _Segment("  ", highlight_number_style)
                            yield _Segment(line_column, number_style)
                    else:
                        yield wrapped_line_left_pad
                    yield from wrapped_line
                    yield new_line
            else:
                for wrapped_line in wrapped_lines:
                    yield from wrapped_line
                    yield new_line

    def _apply_stylized_ranges(self, text: Text) -> None:
        """
        Apply stylized ranges to a text instance,
        using the given code to determine the right portion to apply the style to.

        Args:
            text (Text): Text instance to apply the style to.
        """
        code = text.plain
        newlines_offsets = [
            # Let's add outer boundaries at each side of the list:
            0,
            # N.B. using "\n" here is much faster than using metacharacters such as "^" or "\Z":
            *[
                match.start() + 1
                for match in re.finditer("\n", code, flags=re.MULTILINE)
            ],
            len(code) + 1,
        ]

        for stylized_range in self._stylized_ranges:
            start = _get_code_index_for_syntax_position(
                newlines_offsets, stylized_range.start
            )
            end = _get_code_index_for_syntax_position(
                newlines_offsets, stylized_range.end
            )
            if start is not None and end is not None:
                text.stylize(stylized_range.style, start, end)

    def _process_code(self, code: str) -> Tuple[bool, str]:
        """
        Applies various processing to a raw code string
        (normalises it so it always ends with a line return, dedents it if necessary, etc.)

        Args:
            code (str): The raw code string to process

        Returns:
            Tuple[bool, str]: the boolean indicates whether the raw code ends with a line return,
                while the string is the processed code.
        """
        ends_on_nl = code.endswith("\n")
        processed_code = code if ends_on_nl else code + "\n"
        processed_code = (
            textwrap.dedent(processed_code) if self.dedent else processed_code
        )
        processed_code = processed_code.expandtabs(self.tab_size)
        return ends_on_nl, processed_code


def _get_code_index_for_syntax_position(
    newlines_offsets: Sequence[int], position: SyntaxPosition
) -> Optional[int]:
    """
    Returns the index of the code string for the given positions.

    Args:
        newlines_offsets (Sequence[int]): The offset of each newline character found in the code snippet.
        position (SyntaxPosition): The position to search for.

    Returns:
        Optional[int]: The index of the code string for this position, or `None`
            if the given position's line number is out of range (if it's the column that is out of range
            we silently clamp its value so that it reaches the end of the line)
    """
    lines_count = len(newlines_offsets)

    line_number, column_index = position
    if line_number > lines_count or len(newlines_offsets) < (line_number + 1):
        return None  # `line_number` is out of range
    line_index = line_number - 1
    line_length = newlines_offsets[line_index + 1] - newlines_offsets[line_index] - 1
    # If `column_index` is out of range: let's silently clamp it:
    column_index = min(line_length, column_index)
    return newlines_offsets[line_index] + column_index


if __name__ == "__main__":  # pragma: no cover
    import argparse
    import sys

    parser = argparse.ArgumentParser(
        description="Render syntax to the console with Rich"
    )
    parser.add_argument(
        "path",
        metavar="PATH",
        help="path to file, or - for stdin",
    )
    parser.add_argument(
        "-c",
        "--force-color",
        dest="force_color",
        action="store_true",
        default=None,
        help="force color for non-terminals",
    )
    parser.add_argument(
        "-i",
        "--indent-guides",
        dest="indent_guides",
        action="store_true",
        default=False,
        help="display indent guides",
    )
    parser.add_argument(
        "-l",
        "--line-numbers",
        dest="line_numbers",
        action="store_true",
        help="render line numbers",
    )
    parser.add_argument(
        "-w",
        "--width",
        type=int,
        dest="width",
        default=None,
        help="width of output (default will auto-detect)",
    )
    parser.add_argument(
        "-r",
        "--wrap",
        dest="word_wrap",
        action="store_true",
        default=False,
        help="word wrap long lines",
    )
    parser.add_argument(
        "-s",
        "--soft-wrap",
        action="store_true",
        dest="soft_wrap",
        default=False,
        help="enable soft wrapping mode",
    )
    parser.add_argument(
        "-t", "--theme", dest="theme", default="monokai", help="pygments theme"
    )
    parser.add_argument(
        "-b",
        "--background-color",
        dest="background_color",
        default=None,
        help="Override background color",
    )
    parser.add_argument(
        "-x",
        "--lexer",
        default=None,
        dest="lexer_name",
        help="Lexer name",
    )
    parser.add_argument(
        "-p", "--padding", type=int, default=0, dest="padding", help="Padding"
    )
    parser.add_argument(
        "--highlight-line",
        type=int,
        default=None,
        dest="highlight_line",
        help="The line number (not index!) to highlight",
    )
    args = parser.parse_args()

    from rich.console import Console

    console = Console(force_terminal=args.force_color, width=args.width)

    if args.path == "-":
        code = sys.stdin.read()
        syntax = Syntax(
            code=code,
            lexer=args.lexer_name,
            line_numbers=args.line_numbers,
            word_wrap=args.word_wrap,
            theme=args.theme,
            background_color=args.background_color,
            indent_guides=args.indent_guides,
            padding=args.padding,
            highlight_lines={args.highlight_line},
        )
    else:
        syntax = Syntax.from_path(
            args.path,
            lexer=args.lexer_name,
            line_numbers=args.line_numbers,
            word_wrap=args.word_wrap,
            theme=args.theme,
            background_color=args.background_color,
            indent_guides=args.indent_guides,
            padding=args.padding,
            highlight_lines={args.highlight_line},
        )
    console.print(syntax, soft_wrap=args.soft_wrap)

env_pyrecodes/lib/python3.9/site-packages/rich/syntax.py
NikolaBlagojevic/pyrecodes
import re
from functools import partial, reduce
from math import gcd
from operator import itemgetter
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    NamedTuple,
    Optional,
    Tuple,
    Union,
)

from ._loop import loop_last
from ._pick import pick_bool
from ._wrap import divide_line
from .align import AlignMethod
from .cells import cell_len, set_cell_size
from .containers import Lines
from .control import strip_control_codes
from .emoji import EmojiVariant
from .jupyter import JupyterMixin
from .measure import Measurement
from .segment import Segment
from .style import Style, StyleType

if TYPE_CHECKING:  # pragma: no cover
    from .console import Console, ConsoleOptions, JustifyMethod, OverflowMethod

DEFAULT_JUSTIFY: "JustifyMethod" = "default"
DEFAULT_OVERFLOW: "OverflowMethod" = "fold"


_re_whitespace = re.compile(r"\s+$")

TextType = Union[str, "Text"]
"""A plain string or a [Text][rich.text.Text] instance."""

GetStyleCallable = Callable[[str], Optional[StyleType]]


class Span(NamedTuple):
    """A marked up region in some text."""

    start: int
    """Span start index."""
    end: int
    """Span end index."""
    style: Union[str, Style]
    """Style associated with the span."""

    def __repr__(self) -> str:
        return f"Span({self.start}, {self.end}, {self.style!r})"

    def __bool__(self) -> bool:
        return self.end > self.start

    def split(self, offset: int) -> Tuple["Span", Optional["Span"]]:
        """Split a span in to 2 from a given offset."""

        if offset < self.start:
            return self, None
        if offset >= self.end:
            return self, None

        start, end, style = self
        span1 = Span(start, min(end, offset), style)
        span2 = Span(span1.end, end, style)
        return span1, span2

    def move(self, offset: int) -> "Span":
        """Move start and end by a given offset.

        Args:
            offset (int): Number of characters to add to start and end.

        Returns:
            TextSpan: A new TextSpan with adjusted position.
        """
        start, end, style = self
        return Span(start + offset, end + offset, style)

    def right_crop(self, offset: int) -> "Span":
        """Crop the span at the given offset.

        Args:
            offset (int): A value between start and end.

        Returns:
            Span: A new (possibly smaller) span.
        """
        start, end, style = self
        if offset >= end:
            return self
        return Span(start, min(offset, end), style)

    def extend(self, cells: int) -> "Span":
        """Extend the span by the given number of cells.

        Args:
            cells (int): Additional space to add to end of span.

        Returns:
            Span: A span.
        """
        if cells:
            start, end, style = self
            return Span(start, end + cells, style)
        else:
            return self


class Text(JupyterMixin):
    """Text with color / style.

    Args:
        text (str, optional): Default unstyled text. Defaults to "".
        style (Union[str, Style], optional): Base style for text. Defaults to "".
        justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
        overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
        no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None.
        end (str, optional): Character to end text with. Defaults to "\\\\n".
        tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to None.
        spans (List[Span], optional). A list of predefined style spans. Defaults to None.
    """

    __slots__ = [
        "_text",
        "style",
        "justify",
        "overflow",
        "no_wrap",
        "end",
        "tab_size",
        "_spans",
        "_length",
    ]

    def __init__(
        self,
        text: str = "",
        style: Union[str, Style] = "",
        *,
        justify: Optional["JustifyMethod"] = None,
        overflow: Optional["OverflowMethod"] = None,
        no_wrap: Optional[bool] = None,
        end: str = "\n",
        tab_size: Optional[int] = None,
        spans: Optional[List[Span]] = None,
    ) -> None:
        sanitized_text = strip_control_codes(text)
        self._text = [sanitized_text]
        self.style = style
        self.justify: Optional["JustifyMethod"] = justify
        self.overflow: Optional["OverflowMethod"] = overflow
        self.no_wrap = no_wrap
        self.end = end
        self.tab_size = tab_size
        self._spans: List[Span] = spans or []
        self._length: int = len(sanitized_text)

    def __len__(self) -> int:
        return self._length

    def __bool__(self) -> bool:
        return bool(self._length)

    def __str__(self) -> str:
        return self.plain

    def __repr__(self) -> str:
        return f"<text {self.plain!r} {self._spans!r}>"

    def __add__(self, other: Any) -> "Text":
        if isinstance(other, (str, Text)):
            result = self.copy()
            result.append(other)
            return result
        return NotImplemented

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, Text):
            return NotImplemented
        return self.plain == other.plain and self._spans == other._spans

    def __contains__(self, other: object) -> bool:
        if isinstance(other, str):
            return other in self.plain
        elif isinstance(other, Text):
            return other.plain in self.plain
        return False

    def __getitem__(self, slice: Union[int, slice]) -> "Text":
        def get_text_at(offset: int) -> "Text":
            _Span = Span
            text = Text(
                self.plain[offset],
                spans=[
                    _Span(0, 1, style)
                    for start, end, style in self._spans
                    if end > offset >= start
                ],
                end="",
            )
            return text

        if isinstance(slice, int):
            return get_text_at(slice)
        else:
            start, stop, step = slice.indices(len(self.plain))
            if step == 1:
                lines = self.divide([start, stop])
                return lines[1]
            else:
                # This would be a bit of work to implement efficiently
                # For now, its not required
                raise TypeError("slices with step!=1 are not supported")

    @property
    def cell_len(self) -> int:
        """Get the number of cells required to render this text."""
        return cell_len(self.plain)

    @property
    def markup(self) -> str:
        """Get console markup to render this Text.

        Returns:
            str: A string potentially creating markup tags.
        """
        from .markup import escape

        output: List[str] = []

        plain = self.plain
        markup_spans = [
            (0, False, self.style),
            *((span.start, False, span.style) for span in self._spans),
            *((span.end, True, span.style) for span in self._spans),
            (len(plain), True, self.style),
        ]
        markup_spans.sort(key=itemgetter(0, 1))
        position = 0
        append = output.append
        for offset, closing, style in markup_spans:
            if offset > position:
                append(escape(plain[position:offset]))
                position = offset
            if style:
                append(f"[/{style}]" if closing else f"[{style}]")
        markup = "".join(output)
        return markup

    @classmethod
    def from_markup(
        cls,
        text: str,
        *,
        style: Union[str, Style] = "",
        emoji: bool = True,
        emoji_variant: Optional[EmojiVariant] = None,
        justify: Optional["JustifyMethod"] = None,
        overflow: Optional["OverflowMethod"] = None,
        end: str = "\n",
    ) -> "Text":
        """Create Text instance from markup.

        Args:
            text (str): A string containing console markup.
            emoji (bool, optional): Also render emoji code. Defaults to True.
            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
            end (str, optional): Character to end text with. Defaults to "\\\\n".

        Returns:
            Text: A Text instance with markup rendered.
        """
        from .markup import render

        rendered_text = render(text, style, emoji=emoji, emoji_variant=emoji_variant)
        rendered_text.justify = justify
        rendered_text.overflow = overflow
        rendered_text.end = end
        return rendered_text

    @classmethod
    def from_ansi(
        cls,
        text: str,
        *,
        style: Union[str, Style] = "",
        justify: Optional["JustifyMethod"] = None,
        overflow: Optional["OverflowMethod"] = None,
        no_wrap: Optional[bool] = None,
        end: str = "\n",
        tab_size: Optional[int] = 8,
    ) -> "Text":
        """Create a Text object from a string containing ANSI escape codes.

        Args:
            text (str): A string containing escape codes.
            style (Union[str, Style], optional): Base style for text. Defaults to "".
            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
            no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None.
            end (str, optional): Character to end text with. Defaults to "\\\\n".
            tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to None.
        """
        from .ansi import AnsiDecoder

        joiner = Text(
            "\n",
            justify=justify,
            overflow=overflow,
            no_wrap=no_wrap,
            end=end,
            tab_size=tab_size,
            style=style,
        )
        decoder = AnsiDecoder()
        result = joiner.join(line for line in decoder.decode(text))
        return result

    @classmethod
    def styled(
        cls,
        text: str,
        style: StyleType = "",
        *,
        justify: Optional["JustifyMethod"] = None,
        overflow: Optional["OverflowMethod"] = None,
    ) -> "Text":
        """Construct a Text instance with a pre-applied styled. A style applied in this way won't be used
        to pad the text when it is justified.

        Args:
            text (str): A string containing console markup.
            style (Union[str, Style]): Style to apply to the text. Defaults to "".
            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.

        Returns:
            Text: A text instance with a style applied to the entire string.
        """
        styled_text = cls(text, justify=justify, overflow=overflow)
        styled_text.stylize(style)
        return styled_text

    @classmethod
    def assemble(
        cls,
        *parts: Union[str, "Text", Tuple[str, StyleType]],
        style: Union[str, Style] = "",
        justify: Optional["JustifyMethod"] = None,
        overflow: Optional["OverflowMethod"] = None,
        no_wrap: Optional[bool] = None,
        end: str = "\n",
        tab_size: int = 8,
        meta: Optional[Dict[str, Any]] = None,
    ) -> "Text":
        """Construct a text instance by combining a sequence of strings with optional styles.
        The positional arguments should be either strings, or a tuple of string + style.

        Args:
            style (Union[str, Style], optional): Base style for text. Defaults to "".
            justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None.
            overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None.
            end (str, optional): Character to end text with. Defaults to "\\\\n".
            tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to None.
            meta (Dict[str, Any], optional). Meta data to apply to text, or None for no meta data. Default to None

        Returns:
            Text: A new text instance.
        """
        text = cls(
            style=style,
            justify=justify,
            overflow=overflow,
            no_wrap=no_wrap,
            end=end,
            tab_size=tab_size,
        )
        append = text.append
        _Text = Text
        for part in parts:
            if isinstance(part, (_Text, str)):
                append(part)
            else:
                append(*part)
        if meta:
            text.apply_meta(meta)
        return text

    @property
    def plain(self) -> str:
        """Get the text as a single string."""
        if len(self._text) != 1:
            self._text[:] = ["".join(self._text)]
        return self._text[0]

    @plain.setter
    def plain(self, new_text: str) -> None:
        """Set the text to a new value."""
        if new_text != self.plain:
            sanitized_text = strip_control_codes(new_text)
            self._text[:] = [sanitized_text]
            old_length = self._length
            self._length = len(sanitized_text)
            if old_length > self._length:
                self._trim_spans()

    @property
    def spans(self) -> List[Span]:
        """Get a reference to the internal list of spans."""
        return self._spans

    @spans.setter
    def spans(self, spans: List[Span]) -> None:
        """Set spans."""
        self._spans = spans[:]

    def blank_copy(self, plain: str = "") -> "Text":
        """Return a new Text instance with copied meta data (but not the string or spans)."""
        copy_self = Text(
            plain,
            style=self.style,
            justify=self.justify,
            overflow=self.overflow,
            no_wrap=self.no_wrap,
            end=self.end,
            tab_size=self.tab_size,
        )
        return copy_self

    def copy(self) -> "Text":
        """Return a copy of this instance."""
        copy_self = Text(
            self.plain,
            style=self.style,
            justify=self.justify,
            overflow=self.overflow,
            no_wrap=self.no_wrap,
            end=self.end,
            tab_size=self.tab_size,
        )
        copy_self._spans[:] = self._spans
        return copy_self

    def stylize(
        self,
        style: Union[str, Style],
        start: int = 0,
        end: Optional[int] = None,
    ) -> None:
        """Apply a style to the text, or a portion of the text.

        Args:
            style (Union[str, Style]): Style instance or style definition to apply.
            start (int): Start offset (negative indexing is supported). Defaults to 0.
            end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None.
        """
        if style:
            length = len(self)
            if start < 0:
                start = length + start
            if end is None:
                end = length
            if end < 0:
                end = length + end
            if start >= length or end <= start:
                # Span not in text or not valid
                return
            self._spans.append(Span(start, min(length, end), style))

    def stylize_before(
        self,
        style: Union[str, Style],
        start: int = 0,
        end: Optional[int] = None,
    ) -> None:
        """Apply a style to the text, or a portion of the text. Styles will be applied before other styles already present.

        Args:
            style (Union[str, Style]): Style instance or style definition to apply.
            start (int): Start offset (negative indexing is supported). Defaults to 0.
            end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None.
        """
        if style:
            length = len(self)
            if start < 0:
                start = length + start
            if end is None:
                end = length
            if end < 0:
                end = length + end
            if start >= length or end <= start:
                # Span not in text or not valid
                return
            self._spans.insert(0, Span(start, min(length, end), style))

    def apply_meta(
        self, meta: Dict[str, Any], start: int = 0, end: Optional[int] = None
    ) -> None:
        """Apply meta data to the text, or a portion of the text.

        Args:
            meta (Dict[str, Any]): A dict of meta information.
            start (int): Start offset (negative indexing is supported). Defaults to 0.
            end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None.

        """
        style = Style.from_meta(meta)
        self.stylize(style, start=start, end=end)

    def on(self, meta: Optional[Dict[str, Any]] = None, **handlers: Any) -> "Text":
        """Apply event handlers (used by Textual project).

        Example:
            >>> from rich.text import Text
            >>> text = Text("hello world")
            >>> text.on(click="view.toggle('world')")

        Args:
            meta (Dict[str, Any]): Mapping of meta information.
            **handlers: Keyword args are prefixed with "@" to defined handlers.

        Returns:
            Text: Self is returned to method may be chained.
        """
        meta = {} if meta is None else meta
        meta.update({f"@{key}": value for key, value in handlers.items()})
        self.stylize(Style.from_meta(meta))
        return self

    def remove_suffix(self, suffix: str) -> None:
        """Remove a suffix if it exists.

        Args:
            suffix (str): Suffix to remove.
        """
        if self.plain.endswith(suffix):
            self.right_crop(len(suffix))

    def get_style_at_offset(self, console: "Console", offset: int) -> Style:
        """Get the style of a character at give offset.

        Args:
            console (~Console): Console where text will be rendered.
            offset (int): Offset in to text (negative indexing supported)

        Returns:
            Style: A Style instance.
        """
        # TODO: This is a little inefficient, it is only used by full justify
        if offset < 0:
            offset = len(self) + offset
        get_style = console.get_style
        style = get_style(self.style).copy()
        for start, end, span_style in self._spans:
            if end > offset >= start:
                style += get_style(span_style, default="")
        return style

    def extend_style(self, spaces: int) -> None:
        """Extend the Text given number of spaces where the spaces have the same style as the last character.

        Args:
            spaces (int): Number of spaces to add to the Text.
        """
        if spaces <= 0:
            return
        spans = self.spans
        new_spaces = " " * spaces
        if spans:
            end_offset = len(self)
            self._spans[:] = [
                span.extend(spaces) if span.end >= end_offset else span
                for span in spans
            ]
            self._text.append(new_spaces)
            self._length += spaces
        else:
            self.plain += new_spaces

    def highlight_regex(
        self,
        re_highlight: str,
        style: Optional[Union[GetStyleCallable, StyleType]] = None,
        *,
        style_prefix: str = "",
    ) -> int:
        """Highlight text with a regular expression, where group names are
        translated to styles.

        Args:
            re_highlight (str): A regular expression.
            style (Union[GetStyleCallable, StyleType]): Optional style to apply to whole match, or a callable
                which accepts the matched text and returns a style. Defaults to None.
            style_prefix (str, optional): Optional prefix to add to style group names.

        Returns:
            int: Number of regex matches
        """
        count = 0
        append_span = self._spans.append
        _Span = Span
        plain = self.plain
        for match in re.finditer(re_highlight, plain):
            get_span = match.span
            if style:
                start, end = get_span()
                match_style = style(plain[start:end]) if callable(style) else style
                if match_style is not None and end > start:
                    append_span(_Span(start, end, match_style))

            count += 1
            for name in match.groupdict().keys():
                start, end = get_span(name)
                if start != -1 and end > start:
                    append_span(_Span(start, end, f"{style_prefix}{name}"))
        return count

    def highlight_words(
        self,
        words: Iterable[str],
        style: Union[str, Style],
        *,
        case_sensitive: bool = True,
    ) -> int:
        """Highlight words with a style.

        Args:
            words (Iterable[str]): Worlds to highlight.
            style (Union[str, Style]): Style to apply.
            case_sensitive (bool, optional): Enable case sensitive matchings. Defaults to True.

        Returns:
            int: Number of words highlighted.
        """
        re_words = "|".join(re.escape(word) for word in words)
        add_span = self._spans.append
        count = 0
        _Span = Span
        for match in re.finditer(
            re_words, self.plain, flags=0 if case_sensitive else re.IGNORECASE
        ):
            start, end = match.span(0)
            add_span(_Span(start, end, style))
            count += 1
        return count

    def rstrip(self) -> None:
        """Strip whitespace from end of text."""
        self.plain = self.plain.rstrip()

    def rstrip_end(self, size: int) -> None:
        """Remove whitespace beyond a certain width at the end of the text.

        Args:
            size (int): The desired size of the text.
        """
        text_length = len(self)
        if text_length > size:
            excess = text_length - size
            whitespace_match = _re_whitespace.search(self.plain)
            if whitespace_match is not None:
                whitespace_count = len(whitespace_match.group(0))
                self.right_crop(min(whitespace_count, excess))

    def set_length(self, new_length: int) -> None:
        """Set new length of the text, clipping or padding is required."""
        length = len(self)
        if length != new_length:
            if length < new_length:
                self.pad_right(new_length - length)
            else:
                self.right_crop(length - new_length)

    def __rich_console__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> Iterable[Segment]:
        tab_size: int = console.tab_size if self.tab_size is None else self.tab_size
        justify = self.justify or options.justify or DEFAULT_JUSTIFY

        overflow = self.overflow or options.overflow or DEFAULT_OVERFLOW

        lines = self.wrap(
            console,
            options.max_width,
            justify=justify,
            overflow=overflow,
            tab_size=tab_size or 8,
            no_wrap=pick_bool(self.no_wrap, options.no_wrap, False),
        )
        all_lines = Text("\n").join(lines)
        yield from all_lines.render(console, end=self.end)

    def __rich_measure__(
        self, console: "Console", options: "ConsoleOptions"
    ) -> Measurement:
        text = self.plain
        lines = text.splitlines()
        max_text_width = max(cell_len(line) for line in lines) if lines else 0
        words = text.split()
        min_text_width = (
            max(cell_len(word) for word in words) if words else max_text_width
        )
        return Measurement(min_text_width, max_text_width)

    def render(self, console: "Console", end: str = "") -> Iterable["Segment"]:
        """Render the text as Segments.

        Args:
            console (Console): Console instance.
            end (Optional[str], optional): Optional end character.

        Returns:
            Iterable[Segment]: Result of render that may be written to the console.
        """
        _Segment = Segment
        text = self.plain
        if not self._spans:
            yield Segment(text)
            if end:
                yield _Segment(end)
            return
        get_style = partial(console.get_style, default=Style.null())

        enumerated_spans = list(enumerate(self._spans, 1))
        style_map = {index: get_style(span.style) for index, span in enumerated_spans}
        style_map[0] = get_style(self.style)

        spans = [
            (0, False, 0),
            *((span.start, False, index) for index, span in enumerated_spans),
            *((span.end, True, index) for index, span in enumerated_spans),
            (len(text), True, 0),
        ]
        spans.sort(key=itemgetter(0, 1))

        stack: List[int] = []
        stack_append = stack.append
        stack_pop = stack.remove

        style_cache: Dict[Tuple[Style, ...], Style] = {}
        style_cache_get = style_cache.get
        combine = Style.combine

        def get_current_style() -> Style:
            """Construct current style from stack."""
            styles = tuple(style_map[_style_id] for _style_id in sorted(stack))
            cached_style = style_cache_get(styles)
            if cached_style is not None:
                return cached_style
            current_style = combine(styles)
            style_cache[styles] = current_style
            return current_style

        for (offset, leaving, style_id), (next_offset, _, _) in zip(spans, spans[1:]):
            if leaving:
                stack_pop(style_id)
            else:
                stack_append(style_id)
            if next_offset > offset:
                yield _Segment(text[offset:next_offset], get_current_style())
        if end:
            yield _Segment(end)

    def join(self, lines: Iterable["Text"]) -> "Text":
        """Join text together with this instance as the separator.

        Args:
            lines (Iterable[Text]): An iterable of Text instances to join.

        Returns:
            Text: A new text instance containing join text.
        """

        new_text = self.blank_copy()

        def iter_text() -> Iterable["Text"]:
            if self.plain:
                for last, line in loop_last(lines):
                    yield line
                    if not last:
                        yield self
            else:
                yield from lines

        extend_text = new_text._text.extend
        append_span = new_text._spans.append
        extend_spans = new_text._spans.extend
        offset = 0
        _Span = Span

        for text in iter_text():
            extend_text(text._text)
            if text.style:
                append_span(_Span(offset, offset + len(text), text.style))
            extend_spans(
                _Span(offset + start, offset + end, style)
                for start, end, style in text._spans
            )
            offset += len(text)
        new_text._length = offset
        return new_text

    def expand_tabs(self, tab_size: Optional[int] = None) -> None:
        """Converts tabs to spaces.

        Args:
            tab_size (int, optional): Size of tabs. Defaults to 8.

        """
        if "\t" not in self.plain:
            return
        if tab_size is None:
            tab_size = self.tab_size
        if tab_size is None:
            tab_size = 8

        result = self.blank_copy()

        new_text: List[Text] = []
        append = new_text.append

        for line in self.split("\n", include_separator=True):
            if "\t" not in line.plain:
                append(line)
            else:
                cell_position = 0
                parts = line.split("\t", include_separator=True)
                for part in parts:
                    if part.plain.endswith("\t"):
                        part._text[-1] = part._text[-1][:-1] + " "
                        cell_position += part.cell_len
                        tab_remainder = cell_position % tab_size
                        if tab_remainder:
                            spaces = tab_size - tab_remainder
                            part.extend_style(spaces)
                            cell_position += spaces
                    else:
                        cell_position += part.cell_len
                    append(part)

        result = Text("").join(new_text)

        self._text = [result.plain]
        self._length = len(self.plain)
        self._spans[:] = result._spans

    def truncate(
        self,
        max_width: int,
        *,
        overflow: Optional["OverflowMethod"] = None,
        pad: bool = False,
    ) -> None:
        """Truncate text if it is longer that a given width.

        Args:
            max_width (int): Maximum number of characters in text.
            overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None, to use self.overflow.
            pad (bool, optional): Pad with spaces if the length is less than max_width. Defaults to False.
        """
        _overflow = overflow or self.overflow or DEFAULT_OVERFLOW
        if _overflow != "ignore":
            length = cell_len(self.plain)
            if length > max_width:
                if _overflow == "ellipsis":
                    self.plain = set_cell_size(self.plain, max_width - 1) + "…"
                else:
                    self.plain = set_cell_size(self.plain, max_width)
            if pad and length < max_width:
                spaces = max_width - length
                self._text = [f"{self.plain}{' ' * spaces}"]
                self._length = len(self.plain)

    def _trim_spans(self) -> None:
        """Remove or modify any spans that are over the end of the text."""
        max_offset = len(self.plain)
        _Span = Span
        self._spans[:] = [
            (
                span
                if span.end < max_offset
                else _Span(span.start, min(max_offset, span.end), span.style)
            )
            for span in self._spans
            if span.start < max_offset
        ]

    def pad(self, count: int, character: str = " ") -> None:
        """Pad left and right with a given number of characters.

        Args:
            count (int): Width of padding.
        """
        assert len(character) == 1, "Character must be a string of length 1"
        if count:
            pad_characters = character * count
            self.plain = f"{pad_characters}{self.plain}{pad_characters}"
            _Span = Span
            self._spans[:] = [
                _Span(start + count, end + count, style)
                for start, end, style in self._spans
            ]

    def pad_left(self, count: int, character: str = " ") -> None:
        """Pad the left with a given character.

        Args:
            count (int): Number of characters to pad.
            character (str, optional): Character to pad with. Defaults to " ".
        """
        assert len(character) == 1, "Character must be a string of length 1"
        if count:
            self.plain = f"{character * count}{self.plain}"
            _Span = Span
            self._spans[:] = [
                _Span(start + count, end + count, style)
                for start, end, style in self._spans
            ]

    def pad_right(self, count: int, character: str = " ") -> None:
        """Pad the right with a given character.

        Args:
            count (int): Number of characters to pad.
            character (str, optional): Character to pad with. Defaults to " ".
        """
        assert len(character) == 1, "Character must be a string of length 1"
        if count:
            self.plain = f"{self.plain}{character * count}"

    def align(self, align: AlignMethod, width: int, character: str = " ") -> None:
        """Align text to a given width.

        Args:
            align (AlignMethod): One of "left", "center", or "right".
            width (int): Desired width.
            character (str, optional): Character to pad with. Defaults to " ".
        """
        self.truncate(width)
        excess_space = width - cell_len(self.plain)
        if excess_space:
            if align == "left":
                self.pad_right(excess_space, character)
            elif align == "center":
                left = excess_space // 2
                self.pad_left(left, character)
                self.pad_right(excess_space - left, character)
            else:
                self.pad_left(excess_space, character)

    def append(
        self, text: Union["Text", str], style: Optional[Union[str, "Style"]] = None
    ) -> "Text":
        """Add text with an optional style.

        Args:
            text (Union[Text, str]): A str or Text to append.
            style (str, optional): A style name. Defaults to None.

        Returns:
            Text: Returns self for chaining.
        """

        if not isinstance(text, (str, Text)):
            raise TypeError("Only str or Text can be appended to Text")

        if len(text):
            if isinstance(text, str):
                sanitized_text = strip_control_codes(text)
                self._text.append(sanitized_text)
                offset = len(self)
                text_length = len(sanitized_text)
                if style:
                    self._spans.append(Span(offset, offset + text_length, style))
                self._length += text_length
            elif isinstance(text, Text):
                _Span = Span
                if style is not None:
                    raise ValueError(
                        "style must not be set when appending Text instance"
                    )
                text_length = self._length
                if text.style:
                    self._spans.append(
                        _Span(text_length, text_length + len(text), text.style)
                    )
                self._text.append(text.plain)
                self._spans.extend(
                    _Span(start + text_length, end + text_length, style)
                    for start, end, style in text._spans
                )
                self._length += len(text)
        return self

    def append_text(self, text: "Text") -> "Text":
        """Append another Text instance. This method is more performant that Text.append, but
        only works for Text.

        Returns:
            Text: Returns self for chaining.
        """
        _Span = Span
        text_length = self._length
        if text.style:
            self._spans.append(_Span(text_length, text_length + len(text), text.style))
        self._text.append(text.plain)
        self._spans.extend(
            _Span(start + text_length, end + text_length, style)
            for start, end, style in text._spans
        )
        self._length += len(text)
        return self

    def append_tokens(
        self, tokens: Iterable[Tuple[str, Optional[StyleType]]]
    ) -> "Text":
        """Append iterable of str and style. Style may be a Style instance or a str style definition.

        Args:
            pairs (Iterable[Tuple[str, Optional[StyleType]]]): An iterable of tuples containing str content and style.

        Returns:
            Text: Returns self for chaining.
        """
        append_text = self._text.append
        append_span = self._spans.append
        _Span = Span
        offset = len(self)
        for content, style in tokens:
            append_text(content)
            if style:
                append_span(_Span(offset, offset + len(content), style))
            offset += len(content)
        self._length = offset
        return self

    def copy_styles(self, text: "Text") -> None:
        """Copy styles from another Text instance.

        Args:
            text (Text): A Text instance to copy styles from, must be the same length.
        """
        self._spans.extend(text._spans)

    def split(
        self,
        separator: str = "\n",
        *,
        include_separator: bool = False,
        allow_blank: bool = False,
    ) -> Lines:
        """Split rich text in to lines, preserving styles.

        Args:
            separator (str, optional): String to split on. Defaults to "\\\\n".
            include_separator (bool, optional): Include the separator in the lines. Defaults to False.
            allow_blank (bool, optional): Return a blank line if the text ends with a separator. Defaults to False.

        Returns:
            List[RichText]: A list of rich text, one per line of the original.
        """
        assert separator, "separator must not be empty"

        text = self.plain
        if separator not in text:
            return Lines([self.copy()])

        if include_separator:
            lines = self.divide(
                match.end() for match in re.finditer(re.escape(separator), text)
            )
        else:

            def flatten_spans() -> Iterable[int]:
                for match in re.finditer(re.escape(separator), text):
                    start, end = match.span()
                    yield start
                    yield end

            lines = Lines(
                line for line in self.divide(flatten_spans()) if line.plain != separator
            )

        if not allow_blank and text.endswith(separator):
            lines.pop()

        return lines

    def divide(self, offsets: Iterable[int]) -> Lines:
        """Divide text in to a number of lines at given offsets.

        Args:
            offsets (Iterable[int]): Offsets used to divide text.

        Returns:
            Lines: New RichText instances between offsets.
        """
        _offsets = list(offsets)

        if not _offsets:
            return Lines([self.copy()])

        text = self.plain
        text_length = len(text)
        divide_offsets = [0, *_offsets, text_length]
        line_ranges = list(zip(divide_offsets, divide_offsets[1:]))

        style = self.style
        justify = self.justify
        overflow = self.overflow
        _Text = Text
        new_lines = Lines(
            _Text(
                text[start:end],
                style=style,
                justify=justify,
                overflow=overflow,
            )
            for start, end in line_ranges
        )
        if not self._spans:
            return new_lines

        _line_appends = [line._spans.append for line in new_lines._lines]
        line_count = len(line_ranges)
        _Span = Span

        for span_start, span_end, style in self._spans:
            lower_bound = 0
            upper_bound = line_count
            start_line_no = (lower_bound + upper_bound) // 2

            while True:
                line_start, line_end = line_ranges[start_line_no]
                if span_start < line_start:
                    upper_bound = start_line_no - 1
                elif span_start > line_end:
                    lower_bound = start_line_no + 1
                else:
                    break
                start_line_no = (lower_bound + upper_bound) // 2

            if span_end < line_end:
                end_line_no = start_line_no
            else:
                end_line_no = lower_bound = start_line_no
                upper_bound = line_count

                while True:
                    line_start, line_end = line_ranges[end_line_no]
                    if span_end < line_start:
                        upper_bound = end_line_no - 1
                    elif span_end > line_end:
                        lower_bound = end_line_no + 1
                    else:
                        break
                    end_line_no = (lower_bound + upper_bound) // 2

            for line_no in range(start_line_no, end_line_no + 1):
                line_start, line_end = line_ranges[line_no]
                new_start = max(0, span_start - line_start)
                new_end = min(span_end - line_start, line_end - line_start)
                if new_end > new_start:
                    _line_appends[line_no](_Span(new_start, new_end, style))

        return new_lines

    def right_crop(self, amount: int = 1) -> None:
        """Remove a number of characters from the end of the text."""
        max_offset = len(self.plain) - amount
        _Span = Span
        self._spans[:] = [
            (
                span
                if span.end < max_offset
                else _Span(span.start, min(max_offset, span.end), span.style)
            )
            for span in self._spans
            if span.start < max_offset
        ]
        self._text = [self.plain[:-amount]]
        self._length -= amount

    def wrap(
        self,
        console: "Console",
        width: int,
        *,
        justify: Optional["JustifyMethod"] = None,
        overflow: Optional["OverflowMethod"] = None,
        tab_size: int = 8,
        no_wrap: Optional[bool] = None,
    ) -> Lines:
        """Word wrap the text.

        Args:
            console (Console): Console instance.
            width (int): Number of characters per line.
            emoji (bool, optional): Also render emoji code. Defaults to True.
            justify (str, optional): Justify method: "default", "left", "center", "full", "right". Defaults to "default".
            overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None.
            tab_size (int, optional): Default tab size. Defaults to 8.
            no_wrap (bool, optional): Disable wrapping, Defaults to False.

        Returns:
            Lines: Number of lines.
        """
        wrap_justify = justify or self.justify or DEFAULT_JUSTIFY
        wrap_overflow = overflow or self.overflow or DEFAULT_OVERFLOW

        no_wrap = pick_bool(no_wrap, self.no_wrap, False) or overflow == "ignore"

        lines = Lines()
        for line in self.split(allow_blank=True):
            if "\t" in line:
                line.expand_tabs(tab_size)
            if no_wrap:
                new_lines = Lines([line])
            else:
                offsets = divide_line(str(line), width, fold=wrap_overflow == "fold")
                new_lines = line.divide(offsets)
            for line in new_lines:
                line.rstrip_end(width)
            if wrap_justify:
                new_lines.justify(
                    console, width, justify=wrap_justify, overflow=wrap_overflow
                )
            for line in new_lines:
                line.truncate(width, overflow=wrap_overflow)
            lines.extend(new_lines)
        return lines

    def fit(self, width: int) -> Lines:
        """Fit the text in to given width by chopping in to lines.

        Args:
            width (int): Maximum characters in a line.

        Returns:
            Lines: Lines container.
        """
        lines: Lines = Lines()
        append = lines.append
        for line in self.split():
            line.set_length(width)
            append(line)
        return lines

    def detect_indentation(self) -> int:
        """Auto-detect indentation of code.

        Returns:
            int: Number of spaces used to indent code.
        """

        _indentations = {
            len(match.group(1))
            for match in re.finditer(r"^( *)(.*)$", self.plain, flags=re.MULTILINE)
        }

        try:
            indentation = (
                reduce(gcd, [indent for indent in _indentations if not indent % 2]) or 1
            )
        except TypeError:
            indentation = 1

        return indentation

    def with_indent_guides(
        self,
        indent_size: Optional[int] = None,
        *,
        character: str = "│",
        style: StyleType = "dim green",
    ) -> "Text":
        """Adds indent guide lines to text.

        Args:
            indent_size (Optional[int]): Size of indentation, or None to auto detect. Defaults to None.
            character (str, optional): Character to use for indentation. Defaults to "│".
            style (Union[Style, str], optional): Style of indent guides.

        Returns:
            Text: New text with indentation guides.
        """

        _indent_size = self.detect_indentation() if indent_size is None else indent_size

        text = self.copy()
        text.expand_tabs()
        indent_line = f"{character}{' ' * (_indent_size - 1)}"

        re_indent = re.compile(r"^( *)(.*)$")
        new_lines: List[Text] = []
        add_line = new_lines.append
        blank_lines = 0
        for line in text.split(allow_blank=True):
            match = re_indent.match(line.plain)
            if not match or not match.group(2):
                blank_lines += 1
                continue
            indent = match.group(1)
            full_indents, remaining_space = divmod(len(indent), _indent_size)
            new_indent = f"{indent_line * full_indents}{' ' * remaining_space}"
            line.plain = new_indent + line.plain[len(new_indent) :]
            line.stylize(style, 0, len(new_indent))
            if blank_lines:
                new_lines.extend([Text(new_indent, style=style)] * blank_lines)
                blank_lines = 0
            add_line(line)
        if blank_lines:
            new_lines.extend([Text("", style=style)] * blank_lines)

        new_text = text.blank_copy("\n").join(new_lines)
        return new_text


if __name__ == "__main__":  # pragma: no cover
    from rich.console import Console

    text = Text(
        """\nLorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n"""
    )
    text.highlight_words(["Lorem"], "bold")
    text.highlight_words(["ipsum"], "italic")

    console = Console()

    console.rule("justify='left'")
    console.print(text, style="red")
    console.print()

    console.rule("justify='center'")
    console.print(text, style="green", justify="center")
    console.print()

    console.rule("justify='right'")
    console.print(text, style="blue", justify="right")
    console.print()

    console.rule("justify='full'")
    console.print(text, style="magenta", justify="full")
    console.print()

env_pyrecodes/lib/python3.9/site-packages/rich/text.py
NikolaBlagojevic/pyrecodes
from .rpds import *

__doc__ = rpds.__doc__
if hasattr(rpds, "__all__"):
    __all__ = rpds.__all__
env_pyrecodes/lib/python3.9/site-packages/rpds/__init__.py
NikolaBlagojevic/pyrecodes
from .ipymagic import *

VERSION = version = __version__ = '2.2.0'

env_pyrecodes/lib/python3.9/site-packages/snakeviz/__init__.py
NikolaBlagojevic/pyrecodes
"""
This module contains the command line interface for snakeviz.

"""
from __future__ import print_function

import argparse
import os
import random
import socket
import sys
import threading
import webbrowser
from pstats import Stats

try:
    from urllib.parse import quote
except ImportError:
    from urllib import quote

from snakeviz import VERSION


# As seen in IPython:
# https://github.com/ipython/ipython/blob/8be7f9abd97eafb493817371d70101d28640919c/IPython/html/notebookapp.py
# See the IPython license at:
# https://github.com/ipython/ipython/blob/master/COPYING.rst.
def random_ports(port, n):
    """Generate a list of n random ports near the given port.
    The first 5 ports will be sequential, and the remaining n-5 will be
    randomly selected in the range [port-2*n, port+2*n].
    """
    for i in range(min(5, n)):
        yield port + i
    for i in range(n-5):
        yield max(1, port + random.randint(-2*n, 2*n))


class SVArgumentParser(argparse.ArgumentParser):
    def error(self, message):
        message = message + '\n\n' + self.format_help()
        args = {'prog': self.prog, 'message': message}
        self.exit(2, '%(prog)s: error: %(message)s' % args)


def build_parser():
    parser = SVArgumentParser(
        description='Start SnakeViz to view a Python profile.')

    parser.add_argument('filename', help='Python profile to view')

    parser.add_argument('-v', '--version', action='version',
                        version=('%(prog)s ' + VERSION))

    parser.add_argument('-H', '--hostname', metavar='ADDR', default='127.0.0.1',
                        help='hostname to bind to (default: %(default)s)')

    parser.add_argument('-p', '--port', type=int, metavar='PORT', default=8080,
                        help='port to bind to; if this port is already in use a '
                             'free port will be selected automatically '
                             '(default: %(default)s)')

    parser.add_argument('-b', '--browser', metavar='BROWSER_PATH',
                        help='name of webbrowser to launch as described in '
                             'the documentation of Python\'s webbrowser module: '
                             'https://docs.python.org/3/library/webbrowser.html')

    parser.add_argument('-s', '--server', action="store_true", default=False,
                        help='start SnakeViz in server-only mode--'
                             'no attempt will be made to open a browser')

    return parser


def main(argv=None):
    parser = build_parser()
    args = parser.parse_args(argv)

    if args.browser and args.server:
        parser.error("options --browser and --server are mutually exclusive")

    filename = os.path.abspath(args.filename)
    if not os.path.exists(filename):
        parser.error('the path %s does not exist' % filename)

    if not os.path.isdir(filename):
        try:
            open(filename)
        except IOError as e:
            parser.error('the file %s could not be opened: %s'
                         % (filename, str(e)))

        try:
            Stats(filename)
        except Exception:
            parser.error(('The file %s is not a valid profile. ' % filename) +
                         'Generate profiles using: \n\n'
                         '\tpython -m cProfile -o my_program.prof my_program.py\n\n'
                         'Note that snakeviz must be run under the same '
                         'version of Python as was used to create the profile.\n')

    filename = quote(filename, safe='')

    hostname = args.hostname
    port = args.port

    if not 0 <= port <= 65535:
        parser.error('invalid port number %d: use a port between 0 and 65535'
                     % port)

    # Before starting tornado set the eventloop policy for windows and python 3.8 compatibility
    # https://github.com/tornadoweb/tornado/issues/2608
    if sys.platform == 'win32' and sys.version_info[:2] == (3, 8):
        import asyncio
        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

    # Go ahead and import the tornado app and start it; we do an inline import
    # here to avoid the extra overhead when just running the cli for --help and
    # the like

    from .main import app
    import tornado.ioloop

    # As seen in IPython:
    # https://github.com/ipython/ipython/blob/8be7f9abd97eafb493817371d70101d28640919c/IPython/html/notebookapp.py
    # See the IPython license at:
    # https://github.com/ipython/ipython/blob/master/COPYING.rst.
    for p in random_ports(port, 10):
        try:
            app.listen(p, address=hostname)
        except socket.error as e:
            print('Port {0} in use, trying another.'.format(p))
        else:
            port = p
            break
    else:
        print('No available port found.')
        return 1

    url = "http://{0}:{1}/snakeviz/{2}".format(hostname, port, filename)
    print(('snakeviz web server started on %s:%d; enter Ctrl-C to exit' %
           (hostname, port)))
    print(url)

    if not args.server:
        try:
            browser = webbrowser.get(args.browser)
        except webbrowser.Error as e:
            parser.error('no web browser found: %s' % e)

        # Launch the browser in a separate thread to avoid blocking the
        # ioloop from starting
        def bt():
            browser.open(url, new=2)
        threading.Thread(target=bt).start()

    try:
        tornado.ioloop.IOLoop.instance().start()
    except KeyboardInterrupt:
        # TODO: Cheap KeyboardInterrupt handler for now; iPython has some nicer
        # stuff for handling SIGINT and SIGTERM that might be worth borrowing
        tornado.ioloop.IOLoop.instance().stop()
        print('\nBye!')

    return 0


if __name__ == '__main__':
    sys.exit(main())

env_pyrecodes/lib/python3.9/site-packages/snakeviz/cli.py
NikolaBlagojevic/pyrecodes
from __future__ import print_function

import errno
import subprocess
import sys
import tempfile
import time
import uuid

try:
    from urllib.parse import quote
except ImportError:
    from urllib import quote

__all__ = ["load_ipython_extension"]


JUPYTER_HTML_TEMPLATE = """
<iframe id='snakeviz-{uuid}' frameborder=0 seamless width='100%' height='1000'></iframe>
<script>document.getElementById("snakeviz-{uuid}").setAttribute("src", "http://{host}:{port}{path}")</script>
"""
DEFAULT_HOST = "\" + document.location.hostname + \""

# Users may be using snakeviz in an environment where IPython is not
# installed, this try/except makes sure that snakeviz is operational
# in that case.
try:
    from IPython.core.magic import Magics, magics_class, line_cell_magic, line_magic
    from IPython.display import display, HTML
except ImportError:
    pass
else:

    @magics_class
    class SnakevizMagic(Magics):

        def __init__(self, shell=None, **kwargs):
            super().__init__(shell=shell, **kwargs)
            self._host = None
            self._port = None

        @line_cell_magic
        def snakeviz(self, line, cell=None):
            """
            Profile code and display the profile in Snakeviz.
            Works as a line or cell magic.

            Usage, in line mode:
            %snakeviz [options] statement

            Usage, in cell mode:
            %%snakeviz [options] [statement]
            code...
            code...

            Options:

            -t/--new-tab
            If running the snakeviz magic in the Jupyter Notebook,
            use this flag to open snakeviz visualization in a new tab
            instead of embedded within the notebook.

            Note that this will briefly open a server with host 0.0.0.0,
            which in some situations may present a slight security risk as
            0.0.0.0 means that the server will be available on all network
            interfaces (if they are not blocked by something like a firewall).

            """
            # get location for saved profile
            filename = tempfile.NamedTemporaryFile().name

            # parse options
            opts, line = self.parse_options(line, "t", "new-tab", posix=False)

            # call signature for prun
            line = "-q -D " + filename + " " + line

            # generate the stats file using IPython's prun magic
            ip = get_ipython()

            if cell:
                ip.run_cell_magic("prun", line, cell)
            else:
                ip.run_line_magic("prun", line)

            # start up a Snakeviz server
            if _check_ipynb() and not ("t" in opts or "new-tab" in opts):
                print("Embedding SnakeViz in this document...")
                sv = open_snakeviz_and_display_in_notebook(filename, self._host, self._port)
            else:
                print("Opening SnakeViz in a new tab...")
                sv = subprocess.Popen(
                    [sys.executable, "-m", "snakeviz", filename]
                )
            # give time for the Snakeviz page to load then shut down the server
            time.sleep(3)
            sv.terminate()

        @line_magic
        def snakeviz_config(self, line):
            """
            Configure the port and host name for snakeviz.

            This line magic takes two options, -h or -p (or alternatively the
            long forms --host and --post) for configuring the host and port,
            respectively, of the snakeviz server that is spun up when the
            snakeviz magic is called.

            The host is the url that will be used by the browser to connect
            to the server, and the port is the port used by the server and
            which will be supplied by the browser when it connects to the
            server.
            """
            opts, line = self.parse_options(line, "h:p:", "host=", "port=")
            for opt in opts:
                if opt in ("h", "host"):
                    self._host = opts[opt]
                elif opt in ("p", "port"):
                    self._port = opts[opt]
                else:
                    raise ValueError("Unsupported option {opt}.".format(opt))
            host = self._host or DEFAULT_HOST
            port = self._port or "dynamically chosen"
            print("Snakeviz configured with host {host} and port {port}".format(host=host,
                                                                                port=port))

def load_ipython_extension(ipython):
    """Called when user runs %load_ext snakeviz"""
    ipython.register_magics(SnakevizMagic)


def _check_ipynb():
    """
    Returns True if IPython is running as the backend for a
    Jupyter Notebook.

    """
    cfg = get_ipython().config
    return "connection_file" in cfg["IPKernelApp"]


def open_snakeviz_and_display_in_notebook(filename, override_host=None, override_port=None):
    def _find_free_port():
        import socket
        from contextlib import closing

        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
            # snakeviz frequently gets called many times in a short period.
            # This line tells the kernel it's okay to reuse TIME-WAIT sockets,
            # which means snakeviz will use the same socket on successive runs,
            # which makes life with snakeviz-over-SSH much easier.
            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

            # Try a default range of five ports, then use whatever's free.
            ports = list(range(8080, 8085)) + [0]
            for port in ports:
                try:
                    s.bind(("", port))
                except socket.error as e:
                    if e.errno == errno.EADDRINUSE:
                        pass
                    else:
                        raise
                else:
                    return s.getsockname()[1]

    port = override_port or str(_find_free_port())

    def _start_and_wait_when_ready():
        import os

        environ = os.environ.copy()
        environ["PYTHONUNBUFFERED"] = "TRUE"
        sv = subprocess.Popen(
            [
                sys.executable,
                "-m",
                "snakeviz",
                "-s",
                "-H",
                "0.0.0.0",
                "-p",
                port,
                filename,
            ],
            stdout=subprocess.PIPE,
            universal_newlines=True,
            env=environ,
        )
        while True:
            line = sv.stdout.readline()
            if line.strip().startswith("snakeviz web server started"):
                break
        return sv

    sv = _start_and_wait_when_ready()
    path = "/snakeviz/%s" % quote(filename, safe="")
    host = override_host or DEFAULT_HOST
    print(display)
    display(
        HTML(
            JUPYTER_HTML_TEMPLATE.format(
                port=port, path=path, uuid=uuid.uuid1(), host=host
            )
        )
    )
    return sv

env_pyrecodes/lib/python3.9/site-packages/snakeviz/ipymagic.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/python

from typing import Any, Dict, List

import sphinx.application
from docutils import nodes
from docutils.parsers import rst
from docutils.parsers.rst import directives
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import BashLexer, BatchLexer, PowerShellLexer, PythonLexer, ScalaLexer, TextLexer
from sphinx.util import logging

logger = logging.getLogger(__name__)


class PromptCache:
    """The cache of different prompt."""

    def __init__(self) -> None:
        """Initialize."""
        self.next_index = 1
        self.prompts: Dict[str, int] = {}

    def clear(self, *args: Any) -> None:
        """Clear all cache."""
        del args
        self.next_index = 1
        self.prompts = {}

    def register_prompt(self, prompt: str) -> str:
        """Initialize the prompts."""
        if prompt in self.prompts:
            return ""
        else:
            index = self.next_index
            self.next_index = index + 1
            self.prompts[prompt] = index
            return f"""span.prompt{index}:before {{
  content: "{prompt} ";
}}
"""

    def get_prompt_class(self, prompt: str) -> str:
        """Get the CSS class name."""
        return f"prompt{self.prompts[prompt]}"


_cache = PromptCache()
PROMPTS = {
    "bash": "$",
    "batch": r"C:\\>",
    "powershell": r"PS C:\\>",
}
LEXERS = {
    "bash": BashLexer,
    "batch": BatchLexer,
    "powershell": PowerShellLexer,
    "python": PythonLexer,
    "scala": ScalaLexer,
}


class PromptDirective(rst.Directive):
    """The prompt directive."""

    optional_arguments = 3
    option_spec = {
        "language": directives.unchanged_required,
        "prompts": directives.unchanged_required,
        "modifiers": directives.unchanged_required,
    }
    has_content = True

    def run(self) -> List[nodes.raw]:
        """Run the directive."""
        self.assert_has_content()

        arg_count = len(self.arguments)

        for idx, option_name in enumerate(("language", "prompts", "modifiers")):
            if arg_count > idx:
                if self.options.get(option_name):
                    logger.warning(
                        "%s is already passed as an option, ignoring the value passed"
                        " as positional argument and all arguments that come after it.",
                        option_name,
                        location=(self.state.document.settings.env.docname, self.lineno),
                    )
                    break
                else:
                    self.options[option_name] = self.arguments[idx]

        language: str = self.options.get("language") or "text"
        prompt: str = self.options.get("prompts") or PROMPTS.get(language, "")
        modifiers: List[str] = self.options.get("modifiers", "").split(",")
        if "auto" in modifiers:
            prompts: List[str] = prompt.split(",")

        html = '<div class="highlight-default notranslate"><div class="highlight"><pre>'
        styles = ""
        if "auto" in modifiers:
            for prompt in prompts:
                styles += _cache.register_prompt(prompt)
        else:
            if prompt is not None:
                styles += _cache.register_prompt(prompt)
        if styles:
            html += '<style type="text/css">\n' + styles + "</style>"
        latex = "\\begin{Verbatim}[commandchars=\\\\\\{\\}]"

        Lexer = LEXERS.get(language, TextLexer)  # noqa: N806, pylint: disable=invalid-name

        statement: List[str] = []
        if "auto" in modifiers:
            prompt_class = ""
            for line in self.content:
                latex += "\n" + line

                for prompt in prompts:
                    if line.startswith(prompt):
                        if len(statement) > 0:
                            highlighted_line = highlight(
                                "\n".join(statement), Lexer(), HtmlFormatter(nowrap=True)
                            ).strip("\r\n")
                            html += f'<span class="{prompt_class}">{highlighted_line}</span>\n'
                            statement = []
                        line = line[len(prompt) + 1 :].rstrip()
                        prompt_class = _cache.get_prompt_class(prompt)
                        break

                statement.append(line)
            # Add last prompt
            if len(statement) > 0:
                highlighted_line = highlight("\n".join(statement), Lexer(), HtmlFormatter(nowrap=True)).strip(
                    "\r\n"
                )
                html += f'<span class="{prompt_class}">{highlighted_line}</span>\n'
        elif language in ["bash", "python"]:
            for line in self.content:
                statement.append(line)
                highlighted_line = highlight("\n".join(statement), Lexer(), HtmlFormatter(nowrap=True)).strip(
                    "\r\n"
                )
                if len(line) == 0 or not line[-1] == "\\":
                    html += f'<span class="{_cache.get_prompt_class(prompt)}">{highlighted_line}</span>\n'
                    if prompt is not None:
                        statements = "\n".join(statement)
                        latex += f"\n{prompt} {statements}"
                    else:
                        latex += "\n" + "\n".join(statement)
                    statement = []
        else:
            for line in self.content:
                highlighted_line = highlight(line, Lexer(), HtmlFormatter(nowrap=True)).strip("\r\n")
                html += f'<span class="{_cache.get_prompt_class(prompt)}">{highlighted_line}</span>\n'
                if prompt is not None:
                    latex += f"\n{prompt} {line}"
                else:
                    latex += "\n" + line

        html += "</pre></div></div>"
        latex += "\n\\end{Verbatim}"

        return [
            nodes.raw("\n".join(self.content), html, format="html"),
            nodes.raw("\n".join(self.content), latex, format="latex"),
        ]


def setup(app: sphinx.application.Sphinx) -> Dict[str, bool]:
    """Register the plugin."""
    app.add_directive("prompt", PromptDirective)
    app.connect("env-purge-doc", _cache.clear)
    return {
        "parallel_read_safe": True,
        "parallel_write_safe": True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx-prompt/__init__.py
NikolaBlagojevic/pyrecodes
"""The Sphinx documentation toolchain."""

# Keep this file executable as-is in Python 3!
# (Otherwise getting the version out of it when packaging is impossible.)

import os
import warnings
from os import path

from .deprecation import RemovedInNextVersionWarning

# by default, all DeprecationWarning under sphinx package will be emit.
# Users can avoid this by using environment variable: PYTHONWARNINGS=
if 'PYTHONWARNINGS' not in os.environ:
    warnings.filterwarnings('default', category=RemovedInNextVersionWarning)
# docutils.io using mode='rU' for open
warnings.filterwarnings('ignore', "'U' mode is deprecated",
                        DeprecationWarning, module='docutils.io')
warnings.filterwarnings('ignore', 'The frontend.Option class .*',
                        DeprecationWarning, module='docutils.frontend')

__version__ = '7.2.5'
__display_version__ = __version__  # used for command line version

#: Version info for better programmatic use.
#:
#: A tuple of five elements; for Sphinx version 1.2.1 beta 3 this would be
#: ``(1, 2, 1, 'beta', 3)``. The fourth element can be one of: ``alpha``,
#: ``beta``, ``rc``, ``final``. ``final`` always has 0 as the last element.
#:
#: .. versionadded:: 1.2
#:    Before version 1.2, check the string ``sphinx.__version__``.
version_info = (7, 2, 5, 'final', 0)

package_dir = path.abspath(path.dirname(__file__))

_in_development = False
if _in_development:
    # Only import subprocess if needed
    import subprocess

    try:
        if ret := subprocess.run(
            ['git', 'rev-parse', '--short', 'HEAD'],
            cwd=package_dir,
            capture_output=True,
            encoding='ascii',
            errors='surrogateescape',
        ).stdout:
            __display_version__ += '+/' + ret.strip()
        del ret
    finally:
        del subprocess
del _in_development

env_pyrecodes/lib/python3.9/site-packages/sphinx/__init__.py
NikolaBlagojevic/pyrecodes
"""The Sphinx documentation toolchain."""

import sys

from sphinx.cmd.build import main

raise SystemExit(main(sys.argv[1:]))

env_pyrecodes/lib/python3.9/site-packages/sphinx/__main__.py
NikolaBlagojevic/pyrecodes
"""Sphinx application class and extensibility interface.

Gracefully adapted from the TextPress system by Armin.
"""

from __future__ import annotations

import contextlib
import os
import pickle
import sys
from collections import deque
from collections.abc import Sequence  # NoQA: TCH003
from io import StringIO
from os import path
from typing import IO, TYPE_CHECKING, Any, Callable

from docutils.nodes import TextElement  # NoQA: TCH002
from docutils.parsers.rst import Directive, roles
from docutils.transforms import Transform  # NoQA: TCH002
from pygments.lexer import Lexer  # NoQA: TCH002

import sphinx
from sphinx import locale, package_dir
from sphinx.config import Config
from sphinx.environment import BuildEnvironment
from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError
from sphinx.events import EventManager
from sphinx.highlighting import lexer_classes
from sphinx.locale import __
from sphinx.project import Project
from sphinx.registry import SphinxComponentRegistry
from sphinx.util import docutils, logging
from sphinx.util._pathlib import _StrPath
from sphinx.util.build_phase import BuildPhase
from sphinx.util.console import bold  # type: ignore[attr-defined]
from sphinx.util.display import progress_message
from sphinx.util.i18n import CatalogRepository
from sphinx.util.logging import prefixed_warnings
from sphinx.util.osutil import ensuredir, relpath
from sphinx.util.tags import Tags

if TYPE_CHECKING:
    from docutils import nodes
    from docutils.nodes import Element
    from docutils.parsers import Parser

    from sphinx.builders import Builder
    from sphinx.domains import Domain, Index
    from sphinx.environment.collectors import EnvironmentCollector
    from sphinx.extension import Extension
    from sphinx.roles import XRefRole
    from sphinx.theming import Theme
    from sphinx.util.typing import RoleFunction, TitleGetter


builtin_extensions: tuple[str, ...] = (
    'sphinx.addnodes',
    'sphinx.builders.changes',
    'sphinx.builders.epub3',
    'sphinx.builders.dirhtml',
    'sphinx.builders.dummy',
    'sphinx.builders.gettext',
    'sphinx.builders.html',
    'sphinx.builders.latex',
    'sphinx.builders.linkcheck',
    'sphinx.builders.manpage',
    'sphinx.builders.singlehtml',
    'sphinx.builders.texinfo',
    'sphinx.builders.text',
    'sphinx.builders.xml',
    'sphinx.config',
    'sphinx.domains.c',
    'sphinx.domains.changeset',
    'sphinx.domains.citation',
    'sphinx.domains.cpp',
    'sphinx.domains.index',
    'sphinx.domains.javascript',
    'sphinx.domains.math',
    'sphinx.domains.python',
    'sphinx.domains.rst',
    'sphinx.domains.std',
    'sphinx.directives',
    'sphinx.directives.code',
    'sphinx.directives.other',
    'sphinx.directives.patches',
    'sphinx.extension',
    'sphinx.parsers',
    'sphinx.registry',
    'sphinx.roles',
    'sphinx.transforms',
    'sphinx.transforms.compact_bullet_list',
    'sphinx.transforms.i18n',
    'sphinx.transforms.references',
    'sphinx.transforms.post_transforms',
    'sphinx.transforms.post_transforms.code',
    'sphinx.transforms.post_transforms.images',
    'sphinx.versioning',
    # collectors should be loaded by specific order
    'sphinx.environment.collectors.dependencies',
    'sphinx.environment.collectors.asset',
    'sphinx.environment.collectors.metadata',
    'sphinx.environment.collectors.title',
    'sphinx.environment.collectors.toctree',
)
_first_party_extensions = (
    # 1st party extensions
    'sphinxcontrib.applehelp',
    'sphinxcontrib.devhelp',
    'sphinxcontrib.htmlhelp',
    'sphinxcontrib.serializinghtml',
    'sphinxcontrib.qthelp',
)
_first_party_themes = (
    # Alabaster is loaded automatically to be used as the default theme
    'alabaster',
)
builtin_extensions += _first_party_themes
builtin_extensions += _first_party_extensions

ENV_PICKLE_FILENAME = 'environment.pickle'

logger = logging.getLogger(__name__)


class Sphinx:
    """The main application class and extensibility interface.

    :ivar srcdir: Directory containing source.
    :ivar confdir: Directory containing ``conf.py``.
    :ivar doctreedir: Directory for storing pickled doctrees.
    :ivar outdir: Directory for storing build documents.
    """

    warningiserror: bool
    _warncount: int

    def __init__(self, srcdir: str | os.PathLike[str], confdir: str | os.PathLike[str] | None,
                 outdir: str | os.PathLike[str], doctreedir: str | os.PathLike[str],
                 buildername: str, confoverrides: dict | None = None,
                 status: IO | None = sys.stdout, warning: IO | None = sys.stderr,
                 freshenv: bool = False, warningiserror: bool = False,
                 tags: list[str] | None = None,
                 verbosity: int = 0, parallel: int = 0, keep_going: bool = False,
                 pdb: bool = False) -> None:
        self.phase = BuildPhase.INITIALIZATION
        self.verbosity = verbosity
        self.extensions: dict[str, Extension] = {}
        self.registry = SphinxComponentRegistry()

        # validate provided directories
        self.srcdir = _StrPath(srcdir).resolve()
        self.outdir = _StrPath(outdir).resolve()
        self.doctreedir = _StrPath(doctreedir).resolve()

        if not path.isdir(self.srcdir):
            raise ApplicationError(__('Cannot find source directory (%s)') %
                                   self.srcdir)

        if path.exists(self.outdir) and not path.isdir(self.outdir):
            raise ApplicationError(__('Output directory (%s) is not a directory') %
                                   self.outdir)

        if self.srcdir == self.outdir:
            raise ApplicationError(__('Source directory and destination '
                                      'directory cannot be identical'))

        self.parallel = parallel

        if status is None:
            self._status: IO = StringIO()
            self.quiet: bool = True
        else:
            self._status = status
            self.quiet = False

        if warning is None:
            self._warning: IO = StringIO()
        else:
            self._warning = warning
        self._warncount = 0
        self.keep_going = warningiserror and keep_going
        if self.keep_going:
            self.warningiserror = False
        else:
            self.warningiserror = warningiserror
        self.pdb = pdb
        logging.setup(self, self._status, self._warning)

        self.events = EventManager(self)

        # keep last few messages for traceback
        # This will be filled by sphinx.util.logging.LastMessagesWriter
        self.messagelog: deque = deque(maxlen=10)

        # say hello to the world
        logger.info(bold(__('Running Sphinx v%s') % sphinx.__display_version__))

        # status code for command-line application
        self.statuscode = 0

        # read config
        self.tags = Tags(tags)
        if confdir is None:
            # set confdir to srcdir if -C given (!= no confdir); a few pieces
            # of code expect a confdir to be set
            self.confdir = self.srcdir
            self.config = Config({}, confoverrides or {})
        else:
            self.confdir = _StrPath(confdir).resolve()
            self.config = Config.read(self.confdir, confoverrides or {}, self.tags)

        # initialize some limited config variables before initialize i18n and loading
        # extensions
        self.config.pre_init_values()

        # set up translation infrastructure
        self._init_i18n()

        # check the Sphinx version if requested
        if self.config.needs_sphinx and self.config.needs_sphinx > sphinx.__display_version__:
            raise VersionRequirementError(
                __('This project needs at least Sphinx v%s and therefore cannot '
                   'be built with this version.') % self.config.needs_sphinx)

        # load all built-in extension modules, first-party extension modules,
        # and first-party themes
        for extension in builtin_extensions:
            self.setup_extension(extension)

        # load all user-given extension modules
        for extension in self.config.extensions:
            self.setup_extension(extension)

        # preload builder module (before init config values)
        self.preload_builder(buildername)

        if not path.isdir(outdir):
            with progress_message(__('making output directory')):
                ensuredir(outdir)

        # the config file itself can be an extension
        if self.config.setup:
            prefix = __('while setting up extension %s:') % "conf.py"
            with prefixed_warnings(prefix):
                if callable(self.config.setup):
                    self.config.setup(self)
                else:
                    raise ConfigError(
                        __("'setup' as currently defined in conf.py isn't a Python callable. "
                           "Please modify its definition to make it a callable function. "
                           "This is needed for conf.py to behave as a Sphinx extension."),
                    )

        # now that we know all config values, collect them from conf.py
        self.config.init_values()
        self.events.emit('config-inited', self.config)

        # create the project
        self.project = Project(self.srcdir, self.config.source_suffix)

        # set up the build environment
        self.env = self._init_env(freshenv)

        # create the builder
        self.builder = self.create_builder(buildername)

        # build environment post-initialisation, after creating the builder
        self._post_init_env()

        # set up the builder
        self._init_builder()

    def _init_i18n(self) -> None:
        """Load translated strings from the configured localedirs if enabled in
        the configuration.
        """
        if self.config.language == 'en':
            self.translator, _ = locale.init([], None)
        else:
            logger.info(bold(__('loading translations [%s]... ') % self.config.language),
                        nonl=True)

            # compile mo files if sphinx.po file in user locale directories are updated
            repo = CatalogRepository(self.srcdir, self.config.locale_dirs,
                                     self.config.language, self.config.source_encoding)
            for catalog in repo.catalogs:
                if catalog.domain == 'sphinx' and catalog.is_outdated():
                    catalog.write_mo(self.config.language,
                                     self.config.gettext_allow_fuzzy_translations)

            locale_dirs: list[str | None] = list(repo.locale_dirs)
            locale_dirs += [None]
            locale_dirs += [path.join(package_dir, 'locale')]

            self.translator, has_translation = locale.init(locale_dirs, self.config.language)
            if has_translation:
                logger.info(__('done'))
            else:
                logger.info(__('not available for built-in messages'))

    def _init_env(self, freshenv: bool) -> BuildEnvironment:
        filename = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
        if freshenv or not os.path.exists(filename):
            return self._create_fresh_env()
        else:
            return self._load_existing_env(filename)

    def _create_fresh_env(self) -> BuildEnvironment:
        env = BuildEnvironment(self)
        self._fresh_env_used = True
        return env

    @progress_message(__('loading pickled environment'))
    def _load_existing_env(self, filename: str) -> BuildEnvironment:
        try:
            with open(filename, 'rb') as f:
                env = pickle.load(f)
                env.setup(self)
                self._fresh_env_used = False
        except Exception as err:
            logger.info(__('failed: %s'), err)
            env = self._create_fresh_env()
        return env

    def _post_init_env(self) -> None:
        if self._fresh_env_used:
            self.env.find_files(self.config, self.builder)
        del self._fresh_env_used

    def preload_builder(self, name: str) -> None:
        self.registry.preload_builder(self, name)

    def create_builder(self, name: str) -> Builder:
        if name is None:
            logger.info(__('No builder selected, using default: html'))
            name = 'html'

        return self.registry.create_builder(self, name, self.env)

    def _init_builder(self) -> None:
        self.builder.init()
        self.events.emit('builder-inited')

    # ---- main "build" method -------------------------------------------------

    def build(self, force_all: bool = False, filenames: list[str] | None = None) -> None:
        self.phase = BuildPhase.READING
        try:
            if force_all:
                self.builder.build_all()
            elif filenames:
                self.builder.build_specific(filenames)
            else:
                self.builder.build_update()

            self.events.emit('build-finished', None)
        except Exception as err:
            # delete the saved env to force a fresh build next time
            envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME)
            if path.isfile(envfile):
                os.unlink(envfile)
            self.events.emit('build-finished', err)
            raise

        if self._warncount and self.keep_going:
            self.statuscode = 1

        status = (__('succeeded') if self.statuscode == 0
                  else __('finished with problems'))
        if self._warncount:
            if self.warningiserror:
                if self._warncount == 1:
                    msg = __('build %s, %s warning (with warnings treated as errors).')
                else:
                    msg = __('build %s, %s warnings (with warnings treated as errors).')
            else:
                if self._warncount == 1:
                    msg = __('build %s, %s warning.')
                else:
                    msg = __('build %s, %s warnings.')

            logger.info(bold(msg % (status, self._warncount)))
        else:
            logger.info(bold(__('build %s.') % status))

        if self.statuscode == 0 and self.builder.epilog:
            logger.info('')
            logger.info(self.builder.epilog % {
                'outdir': relpath(self.outdir),
                'project': self.config.project,
            })

        self.builder.cleanup()

    # ---- general extensibility interface -------------------------------------

    def setup_extension(self, extname: str) -> None:
        """Import and setup a Sphinx extension module.

        Load the extension given by the module *name*.  Use this if your
        extension needs the features provided by another extension.  No-op if
        called twice.
        """
        logger.debug('[app] setting up extension: %r', extname)
        self.registry.load_extension(self, extname)

    @staticmethod
    def require_sphinx(version: tuple[int, int] | str) -> None:
        """Check the Sphinx version if requested.

        Compare *version* with the version of the running Sphinx, and abort the
        build when it is too old.

        :param version: The required version in the form of ``major.minor`` or
                        ``(major, minor)``.

        .. versionadded:: 1.0
        .. versionchanged:: 7.1
           Type of *version* now allows ``(major, minor)`` form.
        """
        if isinstance(version, tuple):
            major, minor = version
        else:
            major, minor = map(int, version.split('.')[:2])
        if (major, minor) > sphinx.version_info[:2]:
            req = f'{major}.{minor}'
            raise VersionRequirementError(req)

    # event interface
    def connect(self, event: str, callback: Callable, priority: int = 500) -> int:
        """Register *callback* to be called when *event* is emitted.

        For details on available core events and the arguments of callback
        functions, please see :ref:`events`.

        :param event: The name of target event
        :param callback: Callback function for the event
        :param priority: The priority of the callback.  The callbacks will be invoked
                         in order of *priority* (ascending).
        :return: A listener ID.  It can be used for :meth:`disconnect`.

        .. versionchanged:: 3.0

           Support *priority*
        """
        listener_id = self.events.connect(event, callback, priority)
        logger.debug('[app] connecting event %r (%d): %r [id=%s]',
                     event, priority, callback, listener_id)
        return listener_id

    def disconnect(self, listener_id: int) -> None:
        """Unregister callback by *listener_id*.

        :param listener_id: A listener_id that :meth:`connect` returns
        """
        logger.debug('[app] disconnecting event: [id=%s]', listener_id)
        self.events.disconnect(listener_id)

    def emit(self, event: str, *args: Any,
             allowed_exceptions: tuple[type[Exception], ...] = ()) -> list:
        """Emit *event* and pass *arguments* to the callback functions.

        Return the return values of all callbacks as a list.  Do not emit core
        Sphinx events in extensions!

        :param event: The name of event that will be emitted
        :param args: The arguments for the event
        :param allowed_exceptions: The list of exceptions that are allowed in the callbacks

        .. versionchanged:: 3.1

           Added *allowed_exceptions* to specify path-through exceptions
        """
        return self.events.emit(event, *args, allowed_exceptions=allowed_exceptions)

    def emit_firstresult(self, event: str, *args: Any,
                         allowed_exceptions: tuple[type[Exception], ...] = ()) -> Any:
        """Emit *event* and pass *arguments* to the callback functions.

        Return the result of the first callback that doesn't return ``None``.

        :param event: The name of event that will be emitted
        :param args: The arguments for the event
        :param allowed_exceptions: The list of exceptions that are allowed in the callbacks

        .. versionadded:: 0.5
        .. versionchanged:: 3.1

           Added *allowed_exceptions* to specify path-through exceptions
        """
        return self.events.emit_firstresult(event, *args,
                                            allowed_exceptions=allowed_exceptions)

    # registering addon parts

    def add_builder(self, builder: type[Builder], override: bool = False) -> None:
        """Register a new builder.

        :param builder: A builder class
        :param override: If true, install the builder forcedly even if another builder
                         is already installed as the same name

        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_builder(builder, override=override)

    # TODO(stephenfin): Describe 'types' parameter
    def add_config_value(self, name: str, default: Any, rebuild: bool | str,
                         types: Any = ()) -> None:
        """Register a configuration value.

        This is necessary for Sphinx to recognize new values and set default
        values accordingly.


        :param name: The name of the configuration value.  It is recommended to be prefixed
                     with the extension name (ex. ``html_logo``, ``epub_title``)
        :param default: The default value of the configuration.
        :param rebuild: The condition of rebuild.  It must be one of those values:

                        * ``'env'`` if a change in the setting only takes effect when a
                          document is parsed -- this means that the whole environment must be
                          rebuilt.
                        * ``'html'`` if a change in the setting needs a full rebuild of HTML
                          documents.
                        * ``''`` if a change in the setting will not need any special rebuild.
        :param types: The type of configuration value.  A list of types can be specified.  For
                      example, ``[str]`` is used to describe a configuration that takes string
                      value.

        .. versionchanged:: 0.4
           If the *default* value is a callable, it will be called with the
           config object as its argument in order to get the default value.
           This can be used to implement config values whose default depends on
           other values.

        .. versionchanged:: 0.6
           Changed *rebuild* from a simple boolean (equivalent to ``''`` or
           ``'env'``) to a string.  However, booleans are still accepted and
           converted internally.
        """
        logger.debug('[app] adding config value: %r', (name, default, rebuild, types))
        if rebuild in (False, True):
            rebuild = 'env' if rebuild else ''
        self.config.add(name, default, rebuild, types)

    def add_event(self, name: str) -> None:
        """Register an event called *name*.

        This is needed to be able to emit it.

        :param name: The name of the event
        """
        logger.debug('[app] adding event: %r', name)
        self.events.add(name)

    def set_translator(self, name: str, translator_class: type[nodes.NodeVisitor],
                       override: bool = False) -> None:
        """Register or override a Docutils translator class.

        This is used to register a custom output translator or to replace a
        builtin translator.  This allows extensions to use a custom translator
        and define custom nodes for the translator (see :meth:`add_node`).

        :param name: The name of the builder for the translator
        :param translator_class: A translator class
        :param override: If true, install the translator forcedly even if another translator
                         is already installed as the same name

        .. versionadded:: 1.3
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_translator(name, translator_class, override=override)

    def add_node(self, node: type[Element], override: bool = False,
                 **kwargs: tuple[Callable, Callable | None]) -> None:
        """Register a Docutils node class.

        This is necessary for Docutils internals.  It may also be used in the
        future to validate nodes in the parsed documents.

        :param node: A node class
        :param kwargs: Visitor functions for each builder (see below)
        :param override: If true, install the node forcedly even if another node is already
                         installed as the same name

        Node visitor functions for the Sphinx HTML, LaTeX, text and manpage
        writers can be given as keyword arguments: the keyword should be one or
        more of ``'html'``, ``'latex'``, ``'text'``, ``'man'``, ``'texinfo'``
        or any other supported translators, the value a 2-tuple of ``(visit,
        depart)`` methods.  ``depart`` can be ``None`` if the ``visit``
        function raises :exc:`docutils.nodes.SkipNode`.  Example:

        .. code-block:: python

           class math(docutils.nodes.Element): pass

           def visit_math_html(self, node):
               self.body.append(self.starttag(node, 'math'))
           def depart_math_html(self, node):
               self.body.append('</math>')

           app.add_node(math, html=(visit_math_html, depart_math_html))

        Obviously, translators for which you don't specify visitor methods will
        choke on the node when encountered in a document to translate.

        .. versionchanged:: 0.5
           Added the support for keyword arguments giving visit functions.
        """
        logger.debug('[app] adding node: %r', (node, kwargs))
        if not override and docutils.is_node_registered(node):
            logger.warning(__('node class %r is already registered, '
                              'its visitors will be overridden'),
                           node.__name__, type='app', subtype='add_node')
        docutils.register_node(node)
        self.registry.add_translation_handlers(node, **kwargs)

    def add_enumerable_node(self, node: type[Element], figtype: str,
                            title_getter: TitleGetter | None = None, override: bool = False,
                            **kwargs: tuple[Callable, Callable]) -> None:
        """Register a Docutils node class as a numfig target.

        Sphinx numbers the node automatically. And then the users can refer it
        using :rst:role:`numref`.

        :param node: A node class
        :param figtype: The type of enumerable nodes.  Each figtype has individual numbering
                        sequences.  As system figtypes, ``figure``, ``table`` and
                        ``code-block`` are defined.  It is possible to add custom nodes to
                        these default figtypes.  It is also possible to define new custom
                        figtype if a new figtype is given.
        :param title_getter: A getter function to obtain the title of node.  It takes an
                             instance of the enumerable node, and it must return its title as
                             string.  The title is used to the default title of references for
                             :rst:role:`ref`.  By default, Sphinx searches
                             ``docutils.nodes.caption`` or ``docutils.nodes.title`` from the
                             node as a title.
        :param kwargs: Visitor functions for each builder (same as :meth:`add_node`)
        :param override: If true, install the node forcedly even if another node is already
                         installed as the same name

        .. versionadded:: 1.4
        """
        self.registry.add_enumerable_node(node, figtype, title_getter, override=override)
        self.add_node(node, override=override, **kwargs)

    def add_directive(self, name: str, cls: type[Directive], override: bool = False) -> None:
        """Register a Docutils directive.

        :param name: The name of the directive
        :param cls: A directive class
        :param override: If false, do not install it if another directive
                         is already installed as the same name
                         If true, unconditionally install the directive.

        For example, a custom directive named ``my-directive`` would be added
        like this:

        .. code-block:: python

           from docutils.parsers.rst import Directive, directives

           class MyDirective(Directive):
               has_content = True
               required_arguments = 1
               optional_arguments = 0
               final_argument_whitespace = True
               option_spec = {
                   'class': directives.class_option,
                   'name': directives.unchanged,
               }

               def run(self):
                   ...

           def setup(app):
               app.add_directive('my-directive', MyDirective)

        For more details, see `the Docutils docs
        <https://docutils.sourceforge.io/docs/howto/rst-directives.html>`__ .

        .. versionchanged:: 0.6
           Docutils 0.5-style directive classes are now supported.
        .. deprecated:: 1.8
           Docutils 0.4-style (function based) directives support is deprecated.
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        logger.debug('[app] adding directive: %r', (name, cls))
        if not override and docutils.is_directive_registered(name):
            logger.warning(__('directive %r is already registered, it will be overridden'),
                           name, type='app', subtype='add_directive')

        docutils.register_directive(name, cls)

    def add_role(self, name: str, role: Any, override: bool = False) -> None:
        """Register a Docutils role.

        :param name: The name of role
        :param role: A role function
        :param override: If false, do not install it if another role
                         is already installed as the same name
                         If true, unconditionally install the role.

        For more details about role functions, see `the Docutils docs
        <https://docutils.sourceforge.io/docs/howto/rst-roles.html>`__ .

        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        logger.debug('[app] adding role: %r', (name, role))
        if not override and docutils.is_role_registered(name):
            logger.warning(__('role %r is already registered, it will be overridden'),
                           name, type='app', subtype='add_role')
        docutils.register_role(name, role)

    def add_generic_role(self, name: str, nodeclass: Any, override: bool = False) -> None:
        """Register a generic Docutils role.

        Register a Docutils role that does nothing but wrap its contents in the
        node given by *nodeclass*.

        :param override: If false, do not install it if another role
                         is already installed as the same name
                         If true, unconditionally install the role.

        .. versionadded:: 0.6
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        # Don't use ``roles.register_generic_role`` because it uses
        # ``register_canonical_role``.
        logger.debug('[app] adding generic role: %r', (name, nodeclass))
        if not override and docutils.is_role_registered(name):
            logger.warning(__('role %r is already registered, it will be overridden'),
                           name, type='app', subtype='add_generic_role')
        role = roles.GenericRole(name, nodeclass)
        docutils.register_role(name, role)  # type: ignore[arg-type]

    def add_domain(self, domain: type[Domain], override: bool = False) -> None:
        """Register a domain.

        :param domain: A domain class
        :param override: If false, do not install it if another domain
                         is already installed as the same name
                         If true, unconditionally install the domain.

        .. versionadded:: 1.0
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_domain(domain, override=override)

    def add_directive_to_domain(self, domain: str, name: str,
                                cls: type[Directive], override: bool = False) -> None:
        """Register a Docutils directive in a domain.

        Like :meth:`add_directive`, but the directive is added to the domain
        named *domain*.

        :param domain: The name of target domain
        :param name: A name of directive
        :param cls: A directive class
        :param override: If false, do not install it if another directive
                         is already installed as the same name
                         If true, unconditionally install the directive.

        .. versionadded:: 1.0
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_directive_to_domain(domain, name, cls, override=override)

    def add_role_to_domain(self, domain: str, name: str, role: RoleFunction | XRefRole,
                           override: bool = False) -> None:
        """Register a Docutils role in a domain.

        Like :meth:`add_role`, but the role is added to the domain named
        *domain*.

        :param domain: The name of the target domain
        :param name: The name of the role
        :param role: The role function
        :param override: If false, do not install it if another role
                         is already installed as the same name
                         If true, unconditionally install the role.

        .. versionadded:: 1.0
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_role_to_domain(domain, name, role, override=override)

    def add_index_to_domain(self, domain: str, index: type[Index], override: bool = False,
                            ) -> None:
        """Register a custom index for a domain.

        Add a custom *index* class to the domain named *domain*.

        :param domain: The name of the target domain
        :param index: The index class
        :param override: If false, do not install it if another index
                         is already installed as the same name
                         If true, unconditionally install the index.

        .. versionadded:: 1.0
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_index_to_domain(domain, index)

    def add_object_type(self, directivename: str, rolename: str, indextemplate: str = '',
                        parse_node: Callable | None = None,
                        ref_nodeclass: type[TextElement] | None = None,
                        objname: str = '', doc_field_types: Sequence = (),
                        override: bool = False,
                        ) -> None:
        """Register a new object type.

        This method is a very convenient way to add a new :term:`object` type
        that can be cross-referenced.  It will do this:

        - Create a new directive (called *directivename*) for documenting an
          object.  It will automatically add index entries if *indextemplate*
          is nonempty; if given, it must contain exactly one instance of
          ``%s``.  See the example below for how the template will be
          interpreted.
        - Create a new role (called *rolename*) to cross-reference to these
          object descriptions.
        - If you provide *parse_node*, it must be a function that takes a
          string and a docutils node, and it must populate the node with
          children parsed from the string.  It must then return the name of the
          item to be used in cross-referencing and index entries.  See the
          :file:`conf.py` file in the source for this documentation for an
          example.
        - The *objname* (if not given, will default to *directivename*) names
          the type of object.  It is used when listing objects, e.g. in search
          results.

        For example, if you have this call in a custom Sphinx extension::

           app.add_object_type('directive', 'dir', 'pair: %s; directive')

        you can use this markup in your documents::

           .. rst:directive:: function

              Document a function.

           <...>

           See also the :rst:dir:`function` directive.

        For the directive, an index entry will be generated as if you had prepended ::

           .. index:: pair: function; directive

        The reference node will be of class ``literal`` (so it will be rendered
        in a proportional font, as appropriate for code) unless you give the
        *ref_nodeclass* argument, which must be a docutils node class.  Most
        useful are ``docutils.nodes.emphasis`` or ``docutils.nodes.strong`` --
        you can also use ``docutils.nodes.generated`` if you want no further
        text decoration.  If the text should be treated as literal (e.g. no
        smart quote replacement), but not have typewriter styling, use
        ``sphinx.addnodes.literal_emphasis`` or
        ``sphinx.addnodes.literal_strong``.

        For the role content, you have the same syntactical possibilities as
        for standard Sphinx roles (see :ref:`xref-syntax`).

        If *override* is True, the given object_type is forcedly installed even if
        an object_type having the same name is already installed.

        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_object_type(directivename, rolename, indextemplate, parse_node,
                                      ref_nodeclass, objname, doc_field_types,
                                      override=override)

    def add_crossref_type(self, directivename: str, rolename: str, indextemplate: str = '',
                          ref_nodeclass: type[TextElement] | None = None, objname: str = '',
                          override: bool = False) -> None:
        """Register a new crossref object type.

        This method is very similar to :meth:`~Sphinx.add_object_type` except that the
        directive it generates must be empty, and will produce no output.

        That means that you can add semantic targets to your sources, and refer
        to them using custom roles instead of generic ones (like
        :rst:role:`ref`).  Example call::

           app.add_crossref_type('topic', 'topic', 'single: %s',
                                 docutils.nodes.emphasis)

        Example usage::

           .. topic:: application API

           The application API
           -------------------

           Some random text here.

           See also :topic:`this section <application API>`.

        (Of course, the element following the ``topic`` directive needn't be a
        section.)


        :param override: If false, do not install it if another cross-reference type
                         is already installed as the same name
                         If true, unconditionally install the cross-reference type.

        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_crossref_type(directivename, rolename,
                                        indextemplate, ref_nodeclass, objname,
                                        override=override)

    def add_transform(self, transform: type[Transform]) -> None:
        """Register a Docutils transform to be applied after parsing.

        Add the standard docutils :class:`~docutils.transforms.Transform`
        subclass *transform* to the list of transforms that are applied after
        Sphinx parses a reST document.

        :param transform: A transform class

        .. list-table:: priority range categories for Sphinx transforms
           :widths: 20,80

           * - Priority
             - Main purpose in Sphinx
           * - 0-99
             - Fix invalid nodes by docutils. Translate a doctree.
           * - 100-299
             - Preparation
           * - 300-399
             - early
           * - 400-699
             - main
           * - 700-799
             - Post processing. Deadline to modify text and referencing.
           * - 800-899
             - Collect referencing and referenced nodes. Domain processing.
           * - 900-999
             - Finalize and clean up.

        refs: `Transform Priority Range Categories`__

        __ https://docutils.sourceforge.io/docs/ref/transforms.html#transform-priority-range-categories
        """  # NoQA: E501,RUF100  # Flake8 thinks the URL is too long, Ruff special cases URLs.
        self.registry.add_transform(transform)

    def add_post_transform(self, transform: type[Transform]) -> None:
        """Register a Docutils transform to be applied before writing.

        Add the standard docutils :class:`~docutils.transforms.Transform`
        subclass *transform* to the list of transforms that are applied before
        Sphinx writes a document.

        :param transform: A transform class
        """
        self.registry.add_post_transform(transform)

    def add_js_file(self, filename: str | None, priority: int = 500,
                    loading_method: str | None = None, **kwargs: Any) -> None:
        """Register a JavaScript file to include in the HTML output.

        :param filename: The name of a JavaScript file that the default HTML
                         template will include. It must be relative to the HTML
                         static path, or a full URI with scheme, or ``None`` .
                         The ``None`` value is used to create an inline
                         ``<script>`` tag.  See the description of *kwargs*
                         below.
        :param priority: Files are included in ascending order of priority. If
                         multiple JavaScript files have the same priority,
                         those files will be included in order of registration.
                         See list of "priority range for JavaScript files" below.
        :param loading_method: The loading method for the JavaScript file.
                               Either ``'async'`` or ``'defer'`` are allowed.
        :param kwargs: Extra keyword arguments are included as attributes of the
                       ``<script>`` tag.  If the special keyword argument
                       ``body`` is given, its value will be added as the content
                       of the  ``<script>`` tag.

        Example::

            app.add_js_file('example.js')
            # => <script src="_static/example.js"></script>

            app.add_js_file('example.js', loading_method="async")
            # => <script src="_static/example.js" async="async"></script>

            app.add_js_file(None, body="var myVariable = 'foo';")
            # => <script>var myVariable = 'foo';</script>

        .. list-table:: priority range for JavaScript files
           :widths: 20,80

           * - Priority
             - Main purpose in Sphinx
           * - 200
             - default priority for built-in JavaScript files
           * - 500
             - default priority for extensions
           * - 800
             - default priority for :confval:`html_js_files`

        A JavaScript file can be added to the specific HTML page when an extension
        calls this method on :event:`html-page-context` event.

        .. versionadded:: 0.5

        .. versionchanged:: 1.8
           Renamed from ``app.add_javascript()``.
           And it allows keyword arguments as attributes of script tag.

        .. versionchanged:: 3.5
           Take priority argument.  Allow to add a JavaScript file to the specific page.
        .. versionchanged:: 4.4
           Take loading_method argument.  Allow to change the loading method of the
           JavaScript file.
        """
        if loading_method == 'async':
            kwargs['async'] = 'async'
        elif loading_method == 'defer':
            kwargs['defer'] = 'defer'

        self.registry.add_js_file(filename, priority=priority, **kwargs)
        with contextlib.suppress(AttributeError):
            self.builder.add_js_file(  # type: ignore[attr-defined]
                filename, priority=priority, **kwargs,
            )

    def add_css_file(self, filename: str, priority: int = 500, **kwargs: Any) -> None:
        """Register a stylesheet to include in the HTML output.

        :param filename: The name of a CSS file that the default HTML
                         template will include. It must be relative to the HTML
                         static path, or a full URI with scheme.
        :param priority: Files are included in ascending order of priority. If
                         multiple CSS files have the same priority,
                         those files will be included in order of registration.
                         See list of "priority range for CSS files" below.
        :param kwargs: Extra keyword arguments are included as attributes of the
                       ``<link>`` tag.

        Example::

            app.add_css_file('custom.css')
            # => <link rel="stylesheet" href="_static/custom.css" type="text/css" />

            app.add_css_file('print.css', media='print')
            # => <link rel="stylesheet" href="_static/print.css"
            #          type="text/css" media="print" />

            app.add_css_file('fancy.css', rel='alternate stylesheet', title='fancy')
            # => <link rel="alternate stylesheet" href="_static/fancy.css"
            #          type="text/css" title="fancy" />

        .. list-table:: priority range for CSS files
           :widths: 20,80

           * - Priority
             - Main purpose in Sphinx
           * - 200
             - default priority for built-in CSS files
           * - 500
             - default priority for extensions
           * - 800
             - default priority for :confval:`html_css_files`

        A CSS file can be added to the specific HTML page when an extension calls
        this method on :event:`html-page-context` event.

        .. versionadded:: 1.0

        .. versionchanged:: 1.6
           Optional ``alternate`` and/or ``title`` attributes can be supplied
           with the arguments *alternate* (a Boolean) and *title* (a string).
           The default is no title and *alternate* = ``False``. For
           more information, refer to the `documentation
           <https://mdn.io/Web/CSS/Alternative_style_sheets>`__.

        .. versionchanged:: 1.8
           Renamed from ``app.add_stylesheet()``.
           And it allows keyword arguments as attributes of link tag.

        .. versionchanged:: 3.5
           Take priority argument.  Allow to add a CSS file to the specific page.
        """
        logger.debug('[app] adding stylesheet: %r', filename)
        self.registry.add_css_files(filename, priority=priority, **kwargs)
        with contextlib.suppress(AttributeError):
            self.builder.add_css_file(  # type: ignore[attr-defined]
                filename, priority=priority, **kwargs,
            )

    def add_latex_package(self, packagename: str, options: str | None = None,
                          after_hyperref: bool = False) -> None:
        r"""Register a package to include in the LaTeX source code.

        Add *packagename* to the list of packages that LaTeX source code will
        include.  If you provide *options*, it will be taken to the `\usepackage`
        declaration.  If you set *after_hyperref* truthy, the package will be
        loaded after ``hyperref`` package.

        .. code-block:: python

           app.add_latex_package('mypackage')
           # => \usepackage{mypackage}
           app.add_latex_package('mypackage', 'foo,bar')
           # => \usepackage[foo,bar]{mypackage}

        .. versionadded:: 1.3
        .. versionadded:: 3.1

           *after_hyperref* option.
        """
        self.registry.add_latex_package(packagename, options, after_hyperref)

    def add_lexer(self, alias: str, lexer: type[Lexer]) -> None:
        """Register a new lexer for source code.

        Use *lexer* to highlight code blocks with the given language *alias*.

        .. versionadded:: 0.6
        .. versionchanged:: 2.1
           Take a lexer class as an argument.
        .. versionchanged:: 4.0
           Removed support for lexer instances as an argument.
        """
        logger.debug('[app] adding lexer: %r', (alias, lexer))
        lexer_classes[alias] = lexer

    def add_autodocumenter(self, cls: Any, override: bool = False) -> None:
        """Register a new documenter class for the autodoc extension.

        Add *cls* as a new documenter class for the :mod:`sphinx.ext.autodoc`
        extension.  It must be a subclass of
        :class:`sphinx.ext.autodoc.Documenter`.  This allows auto-documenting
        new types of objects.  See the source of the autodoc module for
        examples on how to subclass :class:`~sphinx.ext.autodoc.Documenter`.

        If *override* is True, the given *cls* is forcedly installed even if
        a documenter having the same name is already installed.

        See :ref:`autodoc_ext_tutorial`.

        .. versionadded:: 0.6
        .. versionchanged:: 2.2
           Add *override* keyword.
        """
        logger.debug('[app] adding autodocumenter: %r', cls)
        from sphinx.ext.autodoc.directive import AutodocDirective
        self.registry.add_documenter(cls.objtype, cls)
        self.add_directive('auto' + cls.objtype, AutodocDirective, override=override)

    def add_autodoc_attrgetter(self, typ: type, getter: Callable[[Any, str, Any], Any],
                               ) -> None:
        """Register a new ``getattr``-like function for the autodoc extension.

        Add *getter*, which must be a function with an interface compatible to
        the :func:`getattr` builtin, as the autodoc attribute getter for
        objects that are instances of *typ*.  All cases where autodoc needs to
        get an attribute of a type are then handled by this function instead of
        :func:`getattr`.

        .. versionadded:: 0.6
        """
        logger.debug('[app] adding autodoc attrgetter: %r', (typ, getter))
        self.registry.add_autodoc_attrgetter(typ, getter)

    def add_search_language(self, cls: Any) -> None:
        """Register a new language for the HTML search index.

        Add *cls*, which must be a subclass of
        :class:`sphinx.search.SearchLanguage`, as a support language for
        building the HTML full-text search index.  The class must have a *lang*
        attribute that indicates the language it should be used for.  See
        :confval:`html_search_language`.

        .. versionadded:: 1.1
        """
        logger.debug('[app] adding search language: %r', cls)
        from sphinx.search import SearchLanguage, languages
        assert issubclass(cls, SearchLanguage)
        languages[cls.lang] = cls

    def add_source_suffix(self, suffix: str, filetype: str, override: bool = False) -> None:
        """Register a suffix of source files.

        Same as :confval:`source_suffix`.  The users can override this
        using the config setting.

        :param override: If false, do not install it the same suffix
                         is already installed.
                         If true, unconditionally install the suffix.

        .. versionadded:: 1.8
        """
        self.registry.add_source_suffix(suffix, filetype, override=override)

    def add_source_parser(self, parser: type[Parser], override: bool = False) -> None:
        """Register a parser class.

        :param override: If false, do not install it if another parser
                         is already installed for the same suffix.
                         If true, unconditionally install the parser.

        .. versionadded:: 1.4
        .. versionchanged:: 1.8
           *suffix* argument is deprecated.  It only accepts *parser* argument.
           Use :meth:`add_source_suffix` API to register suffix instead.
        .. versionchanged:: 1.8
           Add *override* keyword.
        """
        self.registry.add_source_parser(parser, override=override)

    def add_env_collector(self, collector: type[EnvironmentCollector]) -> None:
        """Register an environment collector class.

        Refer to :ref:`collector-api`.

        .. versionadded:: 1.6
        """
        logger.debug('[app] adding environment collector: %r', collector)
        collector().enable(self)

    def add_html_theme(self, name: str, theme_path: str) -> None:
        """Register a HTML Theme.

        The *name* is a name of theme, and *theme_path* is a full path to the
        theme (refs: :ref:`distribute-your-theme`).

        .. versionadded:: 1.6
        """
        logger.debug('[app] adding HTML theme: %r, %r', name, theme_path)
        self.registry.add_html_theme(name, theme_path)

    def add_html_math_renderer(
        self,
        name: str,
        inline_renderers: tuple[Callable, Callable | None] | None = None,
        block_renderers: tuple[Callable, Callable | None] | None = None,
    ) -> None:
        """Register a math renderer for HTML.

        The *name* is a name of math renderer.  Both *inline_renderers* and
        *block_renderers* are used as visitor functions for the HTML writer:
        the former for inline math node (``nodes.math``), the latter for
        block math node (``nodes.math_block``).  Regarding visitor functions,
        see :meth:`add_node` for details.

        .. versionadded:: 1.8

        """
        self.registry.add_html_math_renderer(name, inline_renderers, block_renderers)

    def add_message_catalog(self, catalog: str, locale_dir: str) -> None:
        """Register a message catalog.

        :param catalog: The name of the catalog
        :param locale_dir: The base path of the message catalog

        For more details, see :func:`sphinx.locale.get_translation()`.

        .. versionadded:: 1.8
        """
        locale.init([locale_dir], self.config.language, catalog)
        locale.init_console(locale_dir, catalog)

    # ---- other methods -------------------------------------------------
    def is_parallel_allowed(self, typ: str) -> bool:
        """Check whether parallel processing is allowed or not.

        :param typ: A type of processing; ``'read'`` or ``'write'``.
        """
        if typ == 'read':
            attrname = 'parallel_read_safe'
            message_not_declared = __("the %s extension does not declare if it "
                                      "is safe for parallel reading, assuming "
                                      "it isn't - please ask the extension author "
                                      "to check and make it explicit")
            message_not_safe = __("the %s extension is not safe for parallel reading")
        elif typ == 'write':
            attrname = 'parallel_write_safe'
            message_not_declared = __("the %s extension does not declare if it "
                                      "is safe for parallel writing, assuming "
                                      "it isn't - please ask the extension author "
                                      "to check and make it explicit")
            message_not_safe = __("the %s extension is not safe for parallel writing")
        else:
            raise ValueError('parallel type %s is not supported' % typ)

        for ext in self.extensions.values():
            allowed = getattr(ext, attrname, None)
            if allowed is None:
                logger.warning(message_not_declared, ext.name)
                logger.warning(__('doing serial %s'), typ)
                return False
            elif not allowed:
                logger.warning(message_not_safe, ext.name)
                logger.warning(__('doing serial %s'), typ)
                return False

        return True

    def set_html_assets_policy(self, policy):
        """Set the policy to include assets in HTML pages.

        - always: include the assets in all the pages
        - per_page: include the assets only in pages where they are used

        .. versionadded: 4.1
        """
        if policy not in ('always', 'per_page'):
            raise ValueError('policy %s is not supported' % policy)
        self.registry.html_assets_policy = policy


class TemplateBridge:
    """
    This class defines the interface for a "template bridge", that is, a class
    that renders templates given a template name and a context.
    """

    def init(
        self,
        builder: Builder,
        theme: Theme | None = None,
        dirs: list[str] | None = None,
    ) -> None:
        """Called by the builder to initialize the template system.

        *builder* is the builder object; you'll probably want to look at the
        value of ``builder.config.templates_path``.

        *theme* is a :class:`sphinx.theming.Theme` object or None; in the latter
        case, *dirs* can be list of fixed directories to look for templates.
        """
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

    def newest_template_mtime(self) -> float:
        """Called by the builder to determine if output files are outdated
        because of template changes.  Return the mtime of the newest template
        file that was changed.  The default implementation returns ``0``.
        """
        return 0

    def render(self, template: str, context: dict) -> None:
        """Called by the builder to render a template given as a filename with
        a specified context (a Python dictionary).
        """
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

    def render_string(self, template: str, context: dict) -> str:
        """Called by the builder to render a template given as a string with a
        specified context (a Python dictionary).
        """
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

env_pyrecodes/lib/python3.9/site-packages/sphinx/application.py
NikolaBlagojevic/pyrecodes
"""Several HTML builders."""

from __future__ import annotations

import contextlib
import hashlib
import html
import os
import posixpath
import re
import sys
import time
import warnings
from os import path
from typing import IO, TYPE_CHECKING, Any
from urllib.parse import quote

import docutils.readers.doctree
from docutils import nodes
from docutils.core import Publisher
from docutils.frontend import OptionParser
from docutils.io import DocTreeInput, StringOutput
from docutils.utils import relative_path

from sphinx import __display_version__, package_dir
from sphinx import version_info as sphinx_version
from sphinx.builders import Builder
from sphinx.builders.html._assets import _CascadingStyleSheet, _file_checksum, _JavaScript
from sphinx.config import ENUM, Config
from sphinx.deprecation import _deprecation_warning
from sphinx.domains import Domain, Index, IndexEntry
from sphinx.environment.adapters.asset import ImageAdapter
from sphinx.environment.adapters.indexentries import IndexEntries
from sphinx.environment.adapters.toctree import document_toc, global_toctree_for_doc
from sphinx.errors import ConfigError, ThemeError
from sphinx.highlighting import PygmentsBridge
from sphinx.locale import _, __
from sphinx.search import js_index
from sphinx.theming import HTMLThemeFactory
from sphinx.util import isurl, logging
from sphinx.util.display import progress_message, status_iterator
from sphinx.util.docutils import new_document
from sphinx.util.fileutil import copy_asset
from sphinx.util.i18n import format_date
from sphinx.util.inventory import InventoryFile
from sphinx.util.matching import DOTFILES, Matcher, patmatch
from sphinx.util.osutil import SEP, copyfile, ensuredir, os_path, relative_uri
from sphinx.writers.html import HTMLWriter
from sphinx.writers.html5 import HTML5Translator

if TYPE_CHECKING:
    from collections.abc import Iterable, Iterator, Sequence

    from docutils.nodes import Node

    from sphinx.application import Sphinx
    from sphinx.environment import BuildEnvironment
    from sphinx.util.tags import Tags

#: the filename for the inventory of objects
INVENTORY_FILENAME = 'objects.inv'

logger = logging.getLogger(__name__)
return_codes_re = re.compile('[\r\n]+')

DOMAIN_INDEX_TYPE = tuple[
    # Index name (e.g. py-modindex)
    str,
    # Index class
    type[Index],
    # list of (heading string, list of index entries) pairs.
    list[tuple[str, list[IndexEntry]]],
    # whether sub-entries should start collapsed
    bool,
]


def get_stable_hash(obj: Any) -> str:
    """
    Return a stable hash for a Python data structure.  We can't just use
    the md5 of str(obj) since for example dictionary items are enumerated
    in unpredictable order due to hash randomization in newer Pythons.
    """
    if isinstance(obj, dict):
        return get_stable_hash(list(obj.items()))
    elif isinstance(obj, (list, tuple)):
        obj = sorted(get_stable_hash(o) for o in obj)
    return hashlib.md5(str(obj).encode(), usedforsecurity=False).hexdigest()


def convert_locale_to_language_tag(locale: str | None) -> str | None:
    """Convert a locale string to a language tag (ex. en_US -> en-US).

    refs: BCP 47 (:rfc:`5646`)
    """
    if locale:
        return locale.replace('_', '-')
    else:
        return None


class BuildInfo:
    """buildinfo file manipulator.

    HTMLBuilder and its family are storing their own envdata to ``.buildinfo``.
    This class is a manipulator for the file.
    """

    @classmethod
    def load(cls, f: IO) -> BuildInfo:
        try:
            lines = f.readlines()
            assert lines[0].rstrip() == '# Sphinx build info version 1'
            assert lines[2].startswith('config: ')
            assert lines[3].startswith('tags: ')

            build_info = BuildInfo()
            build_info.config_hash = lines[2].split()[1].strip()
            build_info.tags_hash = lines[3].split()[1].strip()
            return build_info
        except Exception as exc:
            raise ValueError(__('build info file is broken: %r') % exc) from exc

    def __init__(
        self,
        config: Config | None = None,
        tags: Tags | None = None,
        config_categories: Sequence[str] = (),
    ) -> None:
        self.config_hash = ''
        self.tags_hash = ''

        if config:
            values = {c.name: c.value for c in config.filter(config_categories)}
            self.config_hash = get_stable_hash(values)

        if tags:
            self.tags_hash = get_stable_hash(sorted(tags))

    def __eq__(self, other: BuildInfo) -> bool:  # type: ignore[override]
        return (self.config_hash == other.config_hash and
                self.tags_hash == other.tags_hash)

    def dump(self, f: IO) -> None:
        f.write('# Sphinx build info version 1\n'
                '# This file hashes the configuration used when building these files.'
                ' When it is not found, a full rebuild will be done.\n'
                'config: %s\n'
                'tags: %s\n' %
                (self.config_hash, self.tags_hash))


class StandaloneHTMLBuilder(Builder):
    """
    Builds standalone HTML docs.
    """
    name = 'html'
    format = 'html'
    epilog = __('The HTML pages are in %(outdir)s.')

    default_translator_class = HTML5Translator
    copysource = True
    allow_parallel = True
    out_suffix = '.html'
    link_suffix = '.html'  # defaults to matching out_suffix
    indexer_format: Any = js_index
    indexer_dumps_unicode = True
    # create links to original images from images [True/False]
    html_scaled_image_link = True
    supported_image_types = ['image/svg+xml', 'image/png',
                             'image/gif', 'image/jpeg']
    supported_remote_images = True
    supported_data_uri_images = True
    searchindex_filename = 'searchindex.js'
    add_permalinks = True
    allow_sharp_as_current_path = True
    embedded = False  # for things like HTML help or Qt help: suppresses sidebar
    search = True  # for things like HTML help and Apple help: suppress search
    use_index = False
    download_support = True  # enable download role

    imgpath: str = ''
    domain_indices: list[DOMAIN_INDEX_TYPE] = []

    def __init__(self, app: Sphinx, env: BuildEnvironment) -> None:
        super().__init__(app, env)

        # CSS files
        self._css_files: list[_CascadingStyleSheet] = []

        # JS files
        self._js_files: list[_JavaScript] = []

        # Cached Publisher for writing doctrees to HTML
        reader = docutils.readers.doctree.Reader(parser_name='restructuredtext')
        pub = Publisher(
            reader=reader,
            parser=reader.parser,
            writer=HTMLWriter(self),
            source_class=DocTreeInput,
            destination=StringOutput(encoding='unicode'),
        )
        if docutils.__version_info__[:2] >= (0, 19):
            pub.get_settings(output_encoding='unicode', traceback=True)
        else:
            op = pub.setup_option_parser(output_encoding='unicode', traceback=True)
            pub.settings = op.get_default_values()
        self._publisher = pub

    def init(self) -> None:
        self.build_info = self.create_build_info()
        # basename of images directory
        self.imagedir = '_images'
        # section numbers for headings in the currently visited document
        self.secnumbers: dict[str, tuple[int, ...]] = {}
        # currently written docname
        self.current_docname: str = ''

        self.init_templates()
        self.init_highlighter()
        self.init_css_files()
        self.init_js_files()

        html_file_suffix = self.get_builder_config('file_suffix', 'html')
        if html_file_suffix is not None:
            self.out_suffix = html_file_suffix

        html_link_suffix = self.get_builder_config('link_suffix', 'html')
        if html_link_suffix is not None:
            self.link_suffix = html_link_suffix
        else:
            self.link_suffix = self.out_suffix

        self.use_index = self.get_builder_config('use_index', 'html')

    def create_build_info(self) -> BuildInfo:
        return BuildInfo(self.config, self.tags, ['html'])

    def _get_translations_js(self) -> str:
        candidates = [path.join(dir, self.config.language,
                                'LC_MESSAGES', 'sphinx.js')
                      for dir in self.config.locale_dirs] + \
                     [path.join(package_dir, 'locale', self.config.language,
                                'LC_MESSAGES', 'sphinx.js'),
                      path.join(sys.prefix, 'share/sphinx/locale',
                                self.config.language, 'sphinx.js')]

        for jsfile in candidates:
            if path.isfile(jsfile):
                return jsfile
        return ''

    def _get_style_filenames(self) -> Iterator[str]:
        if isinstance(self.config.html_style, str):
            yield self.config.html_style
        elif self.config.html_style is not None:
            yield from self.config.html_style
        elif self.theme:
            stylesheet = self.theme.get_config('theme', 'stylesheet')
            yield from map(str.strip, stylesheet.split(','))
        else:
            yield 'default.css'

    def get_theme_config(self) -> tuple[str, dict]:
        return self.config.html_theme, self.config.html_theme_options

    def init_templates(self) -> None:
        theme_factory = HTMLThemeFactory(self.app)
        themename, themeoptions = self.get_theme_config()
        self.theme = theme_factory.create(themename)
        self.theme_options = themeoptions.copy()
        self.create_template_bridge()
        self.templates.init(self, self.theme)

    def init_highlighter(self) -> None:
        # determine Pygments style and create the highlighter
        if self.config.pygments_style is not None:
            style = self.config.pygments_style
        elif self.theme:
            style = self.theme.get_config('theme', 'pygments_style', 'none')
        else:
            style = 'sphinx'
        self.highlighter = PygmentsBridge('html', style)

        if self.theme:
            dark_style = self.theme.get_config('theme', 'pygments_dark_style', None)
        else:
            dark_style = None

        self.dark_highlighter: PygmentsBridge | None
        if dark_style is not None:
            self.dark_highlighter = PygmentsBridge('html', dark_style)
            self.app.add_css_file('pygments_dark.css',
                                  media='(prefers-color-scheme: dark)',
                                  id='pygments_dark_css')
        else:
            self.dark_highlighter = None

    @property
    def css_files(self) -> list[_CascadingStyleSheet]:
        _deprecation_warning(__name__, f'{self.__class__.__name__}.css_files', '',
                             remove=(9, 0))
        return self._css_files

    def init_css_files(self) -> None:
        self._css_files = []
        self.add_css_file('pygments.css', priority=200)

        for filename in self._get_style_filenames():
            self.add_css_file(filename, priority=200)

        for filename, attrs in self.app.registry.css_files:
            self.add_css_file(filename, **attrs)

        for filename, attrs in self.get_builder_config('css_files', 'html'):
            attrs.setdefault('priority', 800)  # User's CSSs are loaded after extensions'
            self.add_css_file(filename, **attrs)

    def add_css_file(self, filename: str, **kwargs: Any) -> None:
        if '://' not in filename:
            filename = posixpath.join('_static', filename)

        if (asset := _CascadingStyleSheet(filename, **kwargs)) not in self._css_files:
            self._css_files.append(asset)

    @property
    def script_files(self) -> list[_JavaScript]:
        _deprecation_warning(__name__, f'{self.__class__.__name__}.script_files', '',
                             remove=(9, 0))
        return self._js_files

    def init_js_files(self) -> None:
        self._js_files = []
        self.add_js_file('documentation_options.js', priority=200)
        self.add_js_file('doctools.js', priority=200)
        self.add_js_file('sphinx_highlight.js', priority=200)

        for filename, attrs in self.app.registry.js_files:
            self.add_js_file(filename or '', **attrs)

        for filename, attrs in self.get_builder_config('js_files', 'html'):
            attrs.setdefault('priority', 800)  # User's JSs are loaded after extensions'
            self.add_js_file(filename or '', **attrs)

        if self._get_translations_js():
            self.add_js_file('translations.js')

    def add_js_file(self, filename: str, **kwargs: Any) -> None:
        if filename and '://' not in filename:
            filename = posixpath.join('_static', filename)

        if (asset := _JavaScript(filename, **kwargs)) not in self._js_files:
            self._js_files.append(asset)

    @property
    def math_renderer_name(self) -> str | None:
        name = self.get_builder_config('math_renderer', 'html')
        if name is not None:
            # use given name
            return name
        else:
            # not given: choose a math_renderer from registered ones as possible
            renderers = list(self.app.registry.html_inline_math_renderers)
            if len(renderers) == 1:
                # only default math_renderer (mathjax) is registered
                return renderers[0]
            elif len(renderers) == 2:
                # default and another math_renderer are registered; prior the another
                renderers.remove('mathjax')
                return renderers[0]
            else:
                # many math_renderers are registered. can't choose automatically!
                return None

    def get_outdated_docs(self) -> Iterator[str]:
        try:
            with open(path.join(self.outdir, '.buildinfo'), encoding="utf-8") as fp:
                buildinfo = BuildInfo.load(fp)

            if self.build_info != buildinfo:
                logger.debug('[build target] did not match: build_info ')
                yield from self.env.found_docs
                return
        except ValueError as exc:
            logger.warning(__('Failed to read build info file: %r'), exc)
        except OSError:
            # ignore errors on reading
            pass

        if self.templates:
            template_mtime = self.templates.newest_template_mtime()
        else:
            template_mtime = 0
        for docname in self.env.found_docs:
            if docname not in self.env.all_docs:
                logger.debug('[build target] did not in env: %r', docname)
                yield docname
                continue
            targetname = self.get_outfilename(docname)
            try:
                targetmtime = path.getmtime(targetname)
            except Exception:
                targetmtime = 0
            try:
                srcmtime = max(path.getmtime(self.env.doc2path(docname)), template_mtime)
                if srcmtime > targetmtime:
                    logger.debug(
                        '[build target] targetname %r(%s), template(%s), docname %r(%s)',
                        targetname,
                        _format_modified_time(targetmtime),
                        _format_modified_time(template_mtime),
                        docname,
                        _format_modified_time(path.getmtime(self.env.doc2path(docname))),
                    )
                    yield docname
            except OSError:
                # source doesn't exist anymore
                pass

    def get_asset_paths(self) -> list[str]:
        return self.config.html_extra_path + self.config.html_static_path

    def render_partial(self, node: Node | None) -> dict[str, str]:
        """Utility: Render a lone doctree node."""
        if node is None:
            return {'fragment': ''}

        doc = new_document('<partial node>')
        doc.append(node)
        self._publisher.set_source(doc)
        self._publisher.publish()
        return self._publisher.writer.parts  # type: ignore[union-attr]

    def prepare_writing(self, docnames: set[str]) -> None:
        # create the search indexer
        self.indexer = None
        if self.search:
            from sphinx.search import IndexBuilder
            lang = self.config.html_search_language or self.config.language
            self.indexer = IndexBuilder(self.env, lang,
                                        self.config.html_search_options,
                                        self.config.html_search_scorer)
            self.load_indexer(docnames)

        self.docwriter = HTMLWriter(self)
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DeprecationWarning)
            # DeprecationWarning: The frontend.OptionParser class will be replaced
            # by a subclass of argparse.ArgumentParser in Docutils 0.21 or later.
            self.docsettings: Any = OptionParser(
                defaults=self.env.settings,
                components=(self.docwriter,),
                read_config_files=True).get_default_values()
        self.docsettings.compact_lists = bool(self.config.html_compact_lists)

        # determine the additional indices to include
        self.domain_indices = []
        # html_domain_indices can be False/True or a list of index names
        indices_config = self.config.html_domain_indices
        if indices_config:
            for domain_name in sorted(self.env.domains):
                domain: Domain = self.env.domains[domain_name]
                for indexcls in domain.indices:
                    indexname = f'{domain.name}-{indexcls.name}'
                    if isinstance(indices_config, list):
                        if indexname not in indices_config:
                            continue
                    content, collapse = indexcls(domain).generate()
                    if content:
                        self.domain_indices.append(
                            (indexname, indexcls, content, collapse))

        # format the "last updated on" string, only once is enough since it
        # typically doesn't include the time of day
        self.last_updated: str | None
        lufmt = self.config.html_last_updated_fmt
        if lufmt is not None:
            self.last_updated = format_date(lufmt or _('%b %d, %Y'),
                                            language=self.config.language)
        else:
            self.last_updated = None

        # If the logo or favicon are urls, keep them as-is, otherwise
        # strip the relative path as the files will be copied into _static.
        logo = self.config.html_logo or ''
        favicon = self.config.html_favicon or ''

        if not isurl(logo):
            logo = path.basename(logo)
        if not isurl(favicon):
            favicon = path.basename(favicon)

        self.relations = self.env.collect_relations()

        rellinks: list[tuple[str, str, str, str]] = []
        if self.use_index:
            rellinks.append(('genindex', _('General Index'), 'I', _('index')))
        for indexname, indexcls, _content, _collapse in self.domain_indices:
            # if it has a short name
            if indexcls.shortname:
                rellinks.append((indexname, indexcls.localname,
                                 '', indexcls.shortname))

        # add assets registered after ``Builder.init()``.
        for css_filename, attrs in self.app.registry.css_files:
            self.add_css_file(css_filename, **attrs)
        for js_filename, attrs in self.app.registry.js_files:
            self.add_js_file(js_filename or '', **attrs)

        # back up _css_files and _js_files to allow adding CSS/JS files to a specific page.
        self._orig_css_files = list(dict.fromkeys(self._css_files))
        self._orig_js_files = list(dict.fromkeys(self._js_files))
        styles = list(self._get_style_filenames())

        self.globalcontext = {
            'embedded': self.embedded,
            'project': self.config.project,
            'release': return_codes_re.sub('', self.config.release),
            'version': self.config.version,
            'last_updated': self.last_updated,
            'copyright': self.config.copyright,
            'master_doc': self.config.root_doc,
            'root_doc': self.config.root_doc,
            'use_opensearch': self.config.html_use_opensearch,
            'docstitle': self.config.html_title,
            'shorttitle': self.config.html_short_title,
            'show_copyright': self.config.html_show_copyright,
            'show_search_summary': self.config.html_show_search_summary,
            'show_sphinx': self.config.html_show_sphinx,
            'has_source': self.config.html_copy_source,
            'show_source': self.config.html_show_sourcelink,
            'sourcelink_suffix': self.config.html_sourcelink_suffix,
            'file_suffix': self.out_suffix,
            'link_suffix': self.link_suffix,
            'script_files': self._js_files,
            'language': convert_locale_to_language_tag(self.config.language),
            'css_files': self._css_files,
            'sphinx_version': __display_version__,
            'sphinx_version_tuple': sphinx_version,
            'docutils_version_info': docutils.__version_info__[:5],
            'styles': styles,
            'rellinks': rellinks,
            'builder': self.name,
            'parents': [],
            'logo_url': logo,
            'favicon_url': favicon,
            'html5_doctype': True,
        }
        if self.theme:
            self.globalcontext.update(
                ('theme_' + key, val) for (key, val) in
                self.theme.get_options(self.theme_options).items())
        self.globalcontext.update(self.config.html_context)

    def get_doc_context(self, docname: str, body: str, metatags: str) -> dict[str, Any]:
        """Collect items for the template context of a page."""
        # find out relations
        prev = next = None
        parents = []
        rellinks = self.globalcontext['rellinks'][:]
        related = self.relations.get(docname)
        titles = self.env.titles
        if related and related[2]:
            try:
                next = {
                    'link': self.get_relative_uri(docname, related[2]),
                    'title': self.render_partial(titles[related[2]])['title'],
                }
                rellinks.append((related[2], next['title'], 'N', _('next')))
            except KeyError:
                next = None
        if related and related[1]:
            try:
                prev = {
                    'link': self.get_relative_uri(docname, related[1]),
                    'title': self.render_partial(titles[related[1]])['title'],
                }
                rellinks.append((related[1], prev['title'], 'P', _('previous')))
            except KeyError:
                # the relation is (somehow) not in the TOC tree, handle
                # that gracefully
                prev = None
        while related and related[0]:
            with contextlib.suppress(KeyError):
                parents.append(
                    {'link': self.get_relative_uri(docname, related[0]),
                     'title': self.render_partial(titles[related[0]])['title']})

            related = self.relations.get(related[0])
        if parents:
            # remove link to the master file; we have a generic
            # "back to index" link already
            parents.pop()
        parents.reverse()

        # title rendered as HTML
        title_node = self.env.longtitles.get(docname)
        title = self.render_partial(title_node)['title'] if title_node else ''

        # Suffix for the document
        source_suffix = self.env.doc2path(docname, False)[len(docname):]

        # the name for the copied source
        if self.config.html_copy_source:
            sourcename = docname + source_suffix
            if source_suffix != self.config.html_sourcelink_suffix:
                sourcename += self.config.html_sourcelink_suffix
        else:
            sourcename = ''

        # metadata for the document
        meta = self.env.metadata.get(docname)

        # local TOC and global TOC tree
        self_toc = document_toc(self.env, docname, self.tags)
        toc = self.render_partial(self_toc)['fragment']

        return {
            'parents': parents,
            'prev': prev,
            'next': next,
            'title': title,
            'meta': meta,
            'body': body,
            'metatags': metatags,
            'rellinks': rellinks,
            'sourcename': sourcename,
            'toc': toc,
            # only display a TOC if there's more than one item to show
            'display_toc': (self.env.toc_num_entries[docname] > 1),
            'page_source_suffix': source_suffix,
        }

    def copy_assets(self) -> None:
        self.finish_tasks.add_task(self.copy_download_files)
        self.finish_tasks.add_task(self.copy_static_files)
        self.finish_tasks.add_task(self.copy_extra_files)
        self.finish_tasks.join()

    def write_doc(self, docname: str, doctree: nodes.document) -> None:
        destination = StringOutput(encoding='utf-8')
        doctree.settings = self.docsettings

        self.secnumbers = self.env.toc_secnumbers.get(docname, {})
        self.fignumbers = self.env.toc_fignumbers.get(docname, {})
        self.imgpath = relative_uri(self.get_target_uri(docname), '_images')
        self.dlpath = relative_uri(self.get_target_uri(docname), '_downloads')
        self.current_docname = docname
        self.docwriter.write(doctree, destination)
        self.docwriter.assemble_parts()
        body = self.docwriter.parts['fragment']
        metatags = self.docwriter.clean_meta

        ctx = self.get_doc_context(docname, body, metatags)
        self.handle_page(docname, ctx, event_arg=doctree)

    def write_doc_serialized(self, docname: str, doctree: nodes.document) -> None:
        self.imgpath = relative_uri(self.get_target_uri(docname), self.imagedir)
        self.post_process_images(doctree)
        title_node = self.env.longtitles.get(docname)
        title = self.render_partial(title_node)['title'] if title_node else ''
        self.index_page(docname, doctree, title)

    def finish(self) -> None:
        self.finish_tasks.add_task(self.gen_indices)
        self.finish_tasks.add_task(self.gen_pages_from_extensions)
        self.finish_tasks.add_task(self.gen_additional_pages)
        self.finish_tasks.add_task(self.copy_image_files)
        self.finish_tasks.add_task(self.write_buildinfo)

        # dump the search index
        self.handle_finish()

    @progress_message(__('generating indices'))
    def gen_indices(self) -> None:
        # the global general index
        if self.use_index:
            self.write_genindex()

        # the global domain-specific indices
        self.write_domain_indices()

    def gen_pages_from_extensions(self) -> None:
        # pages from extensions
        for pagelist in self.events.emit('html-collect-pages'):
            for pagename, context, template in pagelist:
                self.handle_page(pagename, context, template)

    @progress_message(__('writing additional pages'))
    def gen_additional_pages(self) -> None:
        # additional pages from conf.py
        for pagename, template in self.config.html_additional_pages.items():
            logger.info(pagename + ' ', nonl=True)
            self.handle_page(pagename, {}, template)

        # the search page
        if self.search:
            logger.info('search ', nonl=True)
            self.handle_page('search', {}, 'search.html')

        # the opensearch xml file
        if self.config.html_use_opensearch and self.search:
            logger.info('opensearch ', nonl=True)
            fn = path.join(self.outdir, '_static', 'opensearch.xml')
            self.handle_page('opensearch', {}, 'opensearch.xml', outfilename=fn)

    def write_genindex(self) -> None:
        # the total count of lines for each index letter, used to distribute
        # the entries into two columns
        genindex = IndexEntries(self.env).create_index(self)
        indexcounts = []
        for _k, entries in genindex:
            indexcounts.append(sum(1 + len(subitems)
                                   for _, (_, subitems, _) in entries))

        genindexcontext = {
            'genindexentries': genindex,
            'genindexcounts': indexcounts,
            'split_index': self.config.html_split_index,
        }
        logger.info('genindex ', nonl=True)

        if self.config.html_split_index:
            self.handle_page('genindex', genindexcontext,
                             'genindex-split.html')
            self.handle_page('genindex-all', genindexcontext,
                             'genindex.html')
            for (key, entries), count in zip(genindex, indexcounts):
                ctx = {'key': key, 'entries': entries, 'count': count,
                       'genindexentries': genindex}
                self.handle_page('genindex-' + key, ctx,
                                 'genindex-single.html')
        else:
            self.handle_page('genindex', genindexcontext, 'genindex.html')

    def write_domain_indices(self) -> None:
        for indexname, indexcls, content, collapse in self.domain_indices:
            indexcontext = {
                'indextitle': indexcls.localname,
                'content': content,
                'collapse_index': collapse,
            }
            logger.info(indexname + ' ', nonl=True)
            self.handle_page(indexname, indexcontext, 'domainindex.html')

    def copy_image_files(self) -> None:
        if self.images:
            stringify_func = ImageAdapter(self.app.env).get_original_image_uri
            ensuredir(path.join(self.outdir, self.imagedir))
            for src in status_iterator(self.images, __('copying images... '), "brown",
                                       len(self.images), self.app.verbosity,
                                       stringify_func=stringify_func):
                dest = self.images[src]
                try:
                    copyfile(path.join(self.srcdir, src),
                             path.join(self.outdir, self.imagedir, dest))
                except Exception as err:
                    logger.warning(__('cannot copy image file %r: %s'),
                                   path.join(self.srcdir, src), err)

    def copy_download_files(self) -> None:
        def to_relpath(f: str) -> str:
            return relative_path(self.srcdir, f)  # type: ignore[arg-type]

        # copy downloadable files
        if self.env.dlfiles:
            ensuredir(path.join(self.outdir, '_downloads'))
            for src in status_iterator(self.env.dlfiles, __('copying downloadable files... '),
                                       "brown", len(self.env.dlfiles), self.app.verbosity,
                                       stringify_func=to_relpath):
                try:
                    dest = path.join(self.outdir, '_downloads', self.env.dlfiles[src][1])
                    ensuredir(path.dirname(dest))
                    copyfile(path.join(self.srcdir, src), dest)
                except OSError as err:
                    logger.warning(__('cannot copy downloadable file %r: %s'),
                                   path.join(self.srcdir, src), err)

    def create_pygments_style_file(self) -> None:
        """create a style file for pygments."""
        with open(path.join(self.outdir, '_static', 'pygments.css'), 'w',
                  encoding="utf-8") as f:
            f.write(self.highlighter.get_stylesheet())

        if self.dark_highlighter:
            with open(path.join(self.outdir, '_static', 'pygments_dark.css'), 'w',
                      encoding="utf-8") as f:
                f.write(self.dark_highlighter.get_stylesheet())

    def copy_translation_js(self) -> None:
        """Copy a JavaScript file for translations."""
        jsfile = self._get_translations_js()
        if jsfile:
            copyfile(jsfile, path.join(self.outdir, '_static', 'translations.js'))

    def copy_stemmer_js(self) -> None:
        """Copy a JavaScript file for stemmer."""
        if self.indexer is not None:
            if hasattr(self.indexer, 'get_js_stemmer_rawcodes'):
                for jsfile in self.indexer.get_js_stemmer_rawcodes():
                    copyfile(jsfile, path.join(self.outdir, '_static', path.basename(jsfile)))
            else:
                if js_stemmer_rawcode := self.indexer.get_js_stemmer_rawcode():
                    copyfile(js_stemmer_rawcode,
                             path.join(self.outdir, '_static', '_stemmer.js'))

    def copy_theme_static_files(self, context: dict[str, Any]) -> None:
        def onerror(filename: str, error: Exception) -> None:
            logger.warning(__('Failed to copy a file in html_static_file: %s: %r'),
                           filename, error)

        if self.theme:
            for entry in self.theme.get_theme_dirs()[::-1]:
                copy_asset(path.join(entry, 'static'),
                           path.join(self.outdir, '_static'),
                           excluded=DOTFILES, context=context,
                           renderer=self.templates, onerror=onerror)

    def copy_html_static_files(self, context: dict) -> None:
        def onerror(filename: str, error: Exception) -> None:
            logger.warning(__('Failed to copy a file in html_static_file: %s: %r'),
                           filename, error)

        excluded = Matcher(self.config.exclude_patterns + ["**/.*"])
        for entry in self.config.html_static_path:
            copy_asset(path.join(self.confdir, entry),
                       path.join(self.outdir, '_static'),
                       excluded, context=context, renderer=self.templates, onerror=onerror)

    def copy_html_logo(self) -> None:
        if self.config.html_logo and not isurl(self.config.html_logo):
            copy_asset(path.join(self.confdir, self.config.html_logo),
                       path.join(self.outdir, '_static'))

    def copy_html_favicon(self) -> None:
        if self.config.html_favicon and not isurl(self.config.html_favicon):
            copy_asset(path.join(self.confdir, self.config.html_favicon),
                       path.join(self.outdir, '_static'))

    def copy_static_files(self) -> None:
        try:
            with progress_message(__('copying static files')):
                ensuredir(path.join(self.outdir, '_static'))

                # prepare context for templates
                context = self.globalcontext.copy()
                if self.indexer is not None:
                    context.update(self.indexer.context_for_searchtool())

                self.create_pygments_style_file()
                self.copy_translation_js()
                self.copy_stemmer_js()
                self.copy_theme_static_files(context)
                self.copy_html_static_files(context)
                self.copy_html_logo()
                self.copy_html_favicon()
        except OSError as err:
            logger.warning(__('cannot copy static file %r'), err)

    def copy_extra_files(self) -> None:
        """copy html_extra_path files."""
        try:
            with progress_message(__('copying extra files')):
                excluded = Matcher(self.config.exclude_patterns)
                for extra_path in self.config.html_extra_path:
                    entry = path.join(self.confdir, extra_path)
                    copy_asset(entry, self.outdir, excluded)
        except OSError as err:
            logger.warning(__('cannot copy extra file %r'), err)

    def write_buildinfo(self) -> None:
        try:
            with open(path.join(self.outdir, '.buildinfo'), 'w', encoding="utf-8") as fp:
                self.build_info.dump(fp)
        except OSError as exc:
            logger.warning(__('Failed to write build info file: %r'), exc)

    def cleanup(self) -> None:
        # clean up theme stuff
        if self.theme:
            self.theme.cleanup()

    def post_process_images(self, doctree: Node) -> None:
        """Pick the best candidate for an image and link down-scaled images to
        their high res version.
        """
        super().post_process_images(doctree)

        if self.config.html_scaled_image_link and self.html_scaled_image_link:
            for node in doctree.findall(nodes.image):
                if not any((key in node) for key in ['scale', 'width', 'height']):
                    # resizing options are not given. scaled image link is available
                    # only for resized images.
                    continue
                if isinstance(node.parent, nodes.reference):
                    # A image having hyperlink target
                    continue
                if 'no-scaled-link' in node['classes']:
                    # scaled image link is disabled for this node
                    continue

                uri = node['uri']
                reference = nodes.reference('', '', internal=True)
                if uri in self.images:
                    reference['refuri'] = posixpath.join(self.imgpath,
                                                         self.images[uri])
                else:
                    reference['refuri'] = uri
                node.replace_self(reference)
                reference.append(node)

    def load_indexer(self, docnames: Iterable[str]) -> None:
        assert self.indexer is not None
        keep = set(self.env.all_docs) - set(docnames)
        try:
            searchindexfn = path.join(self.outdir, self.searchindex_filename)
            if self.indexer_dumps_unicode:
                with open(searchindexfn, encoding='utf-8') as ft:
                    self.indexer.load(ft, self.indexer_format)
            else:
                with open(searchindexfn, 'rb') as fb:
                    self.indexer.load(fb, self.indexer_format)
        except (OSError, ValueError):
            if keep:
                logger.warning(__("search index couldn't be loaded, but not all "
                                  'documents will be built: the index will be '
                                  'incomplete.'))
        # delete all entries for files that will be rebuilt
        self.indexer.prune(keep)

    def index_page(self, pagename: str, doctree: nodes.document, title: str) -> None:
        # only index pages with title
        if self.indexer is not None and title:
            filename = self.env.doc2path(pagename, base=False)
            metadata = self.env.metadata.get(pagename, {})
            if 'nosearch' in metadata:
                self.indexer.feed(pagename, filename, '', new_document(''))
            else:
                self.indexer.feed(pagename, filename, title, doctree)

    def _get_local_toctree(self, docname: str, collapse: bool = True, **kwargs: Any) -> str:
        if 'includehidden' not in kwargs:
            kwargs['includehidden'] = False
        if kwargs.get('maxdepth') == '':
            kwargs.pop('maxdepth')
        toctree = global_toctree_for_doc(self.env, docname, self, collapse=collapse, **kwargs)
        return self.render_partial(toctree)['fragment']

    def get_outfilename(self, pagename: str) -> str:
        return path.join(self.outdir, os_path(pagename) + self.out_suffix)

    def add_sidebars(self, pagename: str, ctx: dict) -> None:
        def has_wildcard(pattern: str) -> bool:
            return any(char in pattern for char in '*?[')

        sidebars = None
        matched = None
        customsidebar = None

        # default sidebars settings for selected theme
        if self.theme.name == 'alabaster':
            # provide default settings for alabaster (for compatibility)
            # Note: this will be removed before Sphinx-2.0
            try:
                # get default sidebars settings from alabaster (if defined)
                theme_default_sidebars = self.theme.config.get('theme', 'sidebars')
                if theme_default_sidebars:
                    sidebars = [name.strip() for name in theme_default_sidebars.split(',')]
            except Exception:
                # fallback to better default settings
                sidebars = ['about.html', 'navigation.html', 'relations.html',
                            'searchbox.html', 'donate.html']
        else:
            theme_default_sidebars = self.theme.get_config('theme', 'sidebars', None)
            if theme_default_sidebars:
                sidebars = [name.strip() for name in theme_default_sidebars.split(',')]

        # user sidebar settings
        html_sidebars = self.get_builder_config('sidebars', 'html')
        for pattern, patsidebars in html_sidebars.items():
            if patmatch(pagename, pattern):
                if matched:
                    if has_wildcard(pattern):
                        # warn if both patterns contain wildcards
                        if has_wildcard(matched):
                            logger.warning(__('page %s matches two patterns in '
                                              'html_sidebars: %r and %r'),
                                           pagename, matched, pattern)
                        # else the already matched pattern is more specific
                        # than the present one, because it contains no wildcard
                        continue
                matched = pattern
                sidebars = patsidebars

        if sidebars is None:
            # keep defaults
            pass

        ctx['sidebars'] = sidebars
        ctx['customsidebar'] = customsidebar

    # --------- these are overwritten by the serialization builder

    def get_target_uri(self, docname: str, typ: str | None = None) -> str:
        return quote(docname) + self.link_suffix

    def handle_page(self, pagename: str, addctx: dict, templatename: str = 'page.html',
                    outfilename: str | None = None, event_arg: Any = None) -> None:
        ctx = self.globalcontext.copy()
        # current_page_name is backwards compatibility
        ctx['pagename'] = ctx['current_page_name'] = pagename
        ctx['encoding'] = self.config.html_output_encoding
        default_baseuri = self.get_target_uri(pagename)
        # in the singlehtml builder, default_baseuri still contains an #anchor
        # part, which relative_uri doesn't really like...
        default_baseuri = default_baseuri.rsplit('#', 1)[0]

        if self.config.html_baseurl:
            ctx['pageurl'] = posixpath.join(self.config.html_baseurl,
                                            pagename + self.out_suffix)
        else:
            ctx['pageurl'] = None

        def pathto(
            otheruri: str, resource: bool = False, baseuri: str = default_baseuri,
        ) -> str:
            if resource and '://' in otheruri:
                # allow non-local resources given by scheme
                return otheruri
            elif not resource:
                otheruri = self.get_target_uri(otheruri)
            uri = relative_uri(baseuri, otheruri) or '#'
            if uri == '#' and not self.allow_sharp_as_current_path:
                uri = baseuri
            return uri
        ctx['pathto'] = pathto

        def hasdoc(name: str) -> bool:
            if name in self.env.all_docs:
                return True
            if name == 'search' and self.search:
                return True
            if name == 'genindex' and self.get_builder_config('use_index', 'html'):
                return True
            return False
        ctx['hasdoc'] = hasdoc

        ctx['toctree'] = lambda **kwargs: self._get_local_toctree(pagename, **kwargs)
        self.add_sidebars(pagename, ctx)
        ctx.update(addctx)

        # 'blah.html' should have content_root = './' not ''.
        ctx['content_root'] = (f'..{SEP}' * default_baseuri.count(SEP)) or f'.{SEP}'

        outdir = self.app.outdir

        def css_tag(css: _CascadingStyleSheet) -> str:
            attrs = []
            for key, value in css.attributes.items():
                if value is not None:
                    attrs.append(f'{key}="{html.escape(value, quote=True)}"')
            uri = pathto(os.fspath(css.filename), resource=True)
            if checksum := _file_checksum(outdir, css.filename):
                uri += f'?v={checksum}'
            return f'<link {" ".join(sorted(attrs))} href="{uri}" />'

        ctx['css_tag'] = css_tag

        def js_tag(js: _JavaScript | str) -> str:
            if not isinstance(js, _JavaScript):
                # str value (old styled)
                return f'<script src="{pathto(js, resource=True)}"></script>'

            attrs = []
            body = js.attributes.get('body', '')
            for key, value in js.attributes.items():
                if key == 'body':
                    continue
                if value is not None:
                    attrs.append(f'{key}="{html.escape(value, quote=True)}"')

            if not js.filename:
                if attrs:
                    return f'<script {" ".join(sorted(attrs))}>{body}</script>'
                return f'<script>{body}</script>'

            uri = pathto(os.fspath(js.filename), resource=True)
            if 'MathJax.js?' in os.fspath(js.filename):
                # MathJax v2 reads a ``?config=...`` query parameter,
                # special case this and just skip adding the checksum.
                # https://docs.mathjax.org/en/v2.7-latest/configuration.html#considerations-for-using-combined-configuration-files
                # https://github.com/sphinx-doc/sphinx/issues/11658
                pass
            elif checksum := _file_checksum(outdir, js.filename):
                uri += f'?v={checksum}'
            if attrs:
                return f'<script {" ".join(sorted(attrs))} src="{uri}"></script>'
            return f'<script src="{uri}"></script>'

        ctx['js_tag'] = js_tag

        # revert _css_files and _js_files
        self._css_files[:] = self._orig_css_files
        self._js_files[:] = self._orig_js_files

        self.update_page_context(pagename, templatename, ctx, event_arg)
        newtmpl = self.app.emit_firstresult('html-page-context', pagename,
                                            templatename, ctx, event_arg)
        if newtmpl:
            templatename = newtmpl

        # sort JS/CSS before rendering HTML
        try:  # NoQA: SIM105
            # Convert script_files to list to support non-list script_files (refs: #8889)
            ctx['script_files'] = sorted(ctx['script_files'], key=lambda js: js.priority)
        except AttributeError:
            # Skip sorting if users modifies script_files directly (maybe via `html_context`).
            # refs: #8885
            #
            # Note: priority sorting feature will not work in this case.
            pass

        with contextlib.suppress(AttributeError):
            ctx['css_files'] = sorted(ctx['css_files'], key=lambda css: css.priority)

        try:
            output = self.templates.render(templatename, ctx)
        except UnicodeError:
            logger.warning(__("a Unicode error occurred when rendering the page %s. "
                              "Please make sure all config values that contain "
                              "non-ASCII content are Unicode strings."), pagename)
            return
        except Exception as exc:
            raise ThemeError(__("An error happened in rendering the page %s.\nReason: %r") %
                             (pagename, exc)) from exc

        if not outfilename:
            outfilename = self.get_outfilename(pagename)
        # outfilename's path is in general different from self.outdir
        ensuredir(path.dirname(outfilename))
        try:
            with open(outfilename, 'w', encoding=ctx['encoding'],
                      errors='xmlcharrefreplace') as f:
                f.write(output)
        except OSError as err:
            logger.warning(__("error writing file %s: %s"), outfilename, err)
        if self.copysource and ctx.get('sourcename'):
            # copy the source file for the "show source" link
            source_name = path.join(self.outdir, '_sources',
                                    os_path(ctx['sourcename']))
            ensuredir(path.dirname(source_name))
            copyfile(self.env.doc2path(pagename), source_name)

    def update_page_context(self, pagename: str, templatename: str,
                            ctx: dict, event_arg: Any) -> None:
        pass

    def handle_finish(self) -> None:
        self.finish_tasks.add_task(self.dump_search_index)
        self.finish_tasks.add_task(self.dump_inventory)

    @progress_message(__('dumping object inventory'))
    def dump_inventory(self) -> None:
        InventoryFile.dump(path.join(self.outdir, INVENTORY_FILENAME), self.env, self)

    def dump_search_index(self) -> None:
        if self.indexer is None:
            return

        with progress_message(__('dumping search index in %s') % self.indexer.label()):
            self.indexer.prune(self.env.all_docs)
            searchindexfn = path.join(self.outdir, self.searchindex_filename)
            # first write to a temporary file, so that if dumping fails,
            # the existing index won't be overwritten
            if self.indexer_dumps_unicode:
                with open(searchindexfn + '.tmp', 'w', encoding='utf-8') as ft:
                    self.indexer.dump(ft, self.indexer_format)
            else:
                with open(searchindexfn + '.tmp', 'wb') as fb:
                    self.indexer.dump(fb, self.indexer_format)
            os.replace(searchindexfn + '.tmp', searchindexfn)


def convert_html_css_files(app: Sphinx, config: Config) -> None:
    """This converts string styled html_css_files to tuple styled one."""
    html_css_files: list[tuple[str, dict]] = []
    for entry in config.html_css_files:
        if isinstance(entry, str):
            html_css_files.append((entry, {}))
        else:
            try:
                filename, attrs = entry
                html_css_files.append((filename, attrs))
            except Exception:
                logger.warning(__('invalid css_file: %r, ignored'), entry)
                continue

    config.html_css_files = html_css_files  # type: ignore[attr-defined]


def _format_modified_time(timestamp: float) -> str:
    """Return an RFC 3339 formatted string representing the given timestamp."""
    seconds, fraction = divmod(timestamp, 1)
    return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(seconds)) + f'.{fraction:.3f}'


def convert_html_js_files(app: Sphinx, config: Config) -> None:
    """This converts string styled html_js_files to tuple styled one."""
    html_js_files: list[tuple[str, dict]] = []
    for entry in config.html_js_files:
        if isinstance(entry, str):
            html_js_files.append((entry, {}))
        else:
            try:
                filename, attrs = entry
                html_js_files.append((filename, attrs))
            except Exception:
                logger.warning(__('invalid js_file: %r, ignored'), entry)
                continue

    config.html_js_files = html_js_files  # type: ignore[attr-defined]


def setup_resource_paths(app: Sphinx, pagename: str, templatename: str,
                         context: dict, doctree: Node) -> None:
    """Set up relative resource paths."""
    pathto = context['pathto']

    # favicon_url
    favicon_url = context.get('favicon_url')
    if favicon_url and not isurl(favicon_url):
        context['favicon_url'] = pathto('_static/' + favicon_url, resource=True)

    # logo_url
    logo_url = context.get('logo_url')
    if logo_url and not isurl(logo_url):
        context['logo_url'] = pathto('_static/' + logo_url, resource=True)


def validate_math_renderer(app: Sphinx) -> None:
    if app.builder.format != 'html':
        return

    name = app.builder.math_renderer_name  # type: ignore[attr-defined]
    if name is None:
        raise ConfigError(__('Many math_renderers are registered. '
                             'But no math_renderer is selected.'))
    if name not in app.registry.html_inline_math_renderers:
        raise ConfigError(__('Unknown math_renderer %r is given.') % name)


def validate_html_extra_path(app: Sphinx, config: Config) -> None:
    """Check html_extra_paths setting."""
    for entry in config.html_extra_path[:]:
        extra_path = path.normpath(path.join(app.confdir, entry))
        if not path.exists(extra_path):
            logger.warning(__('html_extra_path entry %r does not exist'), entry)
            config.html_extra_path.remove(entry)
        elif (path.splitdrive(app.outdir)[0] == path.splitdrive(extra_path)[0] and
              path.commonpath((app.outdir, extra_path)) == path.normpath(app.outdir)):
            logger.warning(__('html_extra_path entry %r is placed inside outdir'), entry)
            config.html_extra_path.remove(entry)


def validate_html_static_path(app: Sphinx, config: Config) -> None:
    """Check html_static_paths setting."""
    for entry in config.html_static_path[:]:
        static_path = path.normpath(path.join(app.confdir, entry))
        if not path.exists(static_path):
            logger.warning(__('html_static_path entry %r does not exist'), entry)
            config.html_static_path.remove(entry)
        elif (path.splitdrive(app.outdir)[0] == path.splitdrive(static_path)[0] and
              path.commonpath((app.outdir, static_path)) == path.normpath(app.outdir)):
            logger.warning(__('html_static_path entry %r is placed inside outdir'), entry)
            config.html_static_path.remove(entry)


def validate_html_logo(app: Sphinx, config: Config) -> None:
    """Check html_logo setting."""
    if (config.html_logo and
            not path.isfile(path.join(app.confdir, config.html_logo)) and
            not isurl(config.html_logo)):
        logger.warning(__('logo file %r does not exist'), config.html_logo)
        config.html_logo = None  # type: ignore[attr-defined]


def validate_html_favicon(app: Sphinx, config: Config) -> None:
    """Check html_favicon setting."""
    if (config.html_favicon and
            not path.isfile(path.join(app.confdir, config.html_favicon)) and
            not isurl(config.html_favicon)):
        logger.warning(__('favicon file %r does not exist'), config.html_favicon)
        config.html_favicon = None  # type: ignore[attr-defined]


def error_on_html_4(_app: Sphinx, config: Config) -> None:
    """Error on HTML 4."""
    if config.html4_writer:
        raise ConfigError(_(
            'HTML 4 is no longer supported by Sphinx. '
            '("html4_writer=True" detected in configuration options)',
        ))


def setup(app: Sphinx) -> dict[str, Any]:
    # builders
    app.add_builder(StandaloneHTMLBuilder)

    # config values
    app.add_config_value('html_theme', 'alabaster', 'html')
    app.add_config_value('html_theme_path', [], 'html')
    app.add_config_value('html_theme_options', {}, 'html')
    app.add_config_value('html_title',
                         lambda self: _('%s %s documentation') % (self.project, self.release),
                         'html', [str])
    app.add_config_value('html_short_title', lambda self: self.html_title, 'html')
    app.add_config_value('html_style', None, 'html', [list, str])
    app.add_config_value('html_logo', None, 'html', [str])
    app.add_config_value('html_favicon', None, 'html', [str])
    app.add_config_value('html_css_files', [], 'html')
    app.add_config_value('html_js_files', [], 'html')
    app.add_config_value('html_static_path', [], 'html')
    app.add_config_value('html_extra_path', [], 'html')
    app.add_config_value('html_last_updated_fmt', None, 'html', [str])
    app.add_config_value('html_sidebars', {}, 'html')
    app.add_config_value('html_additional_pages', {}, 'html')
    app.add_config_value('html_domain_indices', True, 'html', [list])
    app.add_config_value('html_permalinks', True, 'html')
    app.add_config_value('html_permalinks_icon', '¶', 'html')
    app.add_config_value('html_use_index', True, 'html')
    app.add_config_value('html_split_index', False, 'html')
    app.add_config_value('html_copy_source', True, 'html')
    app.add_config_value('html_show_sourcelink', True, 'html')
    app.add_config_value('html_sourcelink_suffix', '.txt', 'html')
    app.add_config_value('html_use_opensearch', '', 'html')
    app.add_config_value('html_file_suffix', None, 'html', [str])
    app.add_config_value('html_link_suffix', None, 'html', [str])
    app.add_config_value('html_show_copyright', True, 'html')
    app.add_config_value('html_show_search_summary', True, 'html')
    app.add_config_value('html_show_sphinx', True, 'html')
    app.add_config_value('html_context', {}, 'html')
    app.add_config_value('html_output_encoding', 'utf-8', 'html')
    app.add_config_value('html_compact_lists', True, 'html')
    app.add_config_value('html_secnumber_suffix', '. ', 'html')
    app.add_config_value('html_search_language', None, 'html', [str])
    app.add_config_value('html_search_options', {}, 'html')
    app.add_config_value('html_search_scorer', '', '')
    app.add_config_value('html_scaled_image_link', True, 'html')
    app.add_config_value('html_baseurl', '', 'html')
    app.add_config_value('html_codeblock_linenos_style', 'inline', 'html',  # RemovedInSphinx70Warning  # noqa: E501
                         ENUM('table', 'inline'))
    app.add_config_value('html_math_renderer', None, 'env')
    app.add_config_value('html4_writer', False, 'html')

    # events
    app.add_event('html-collect-pages')
    app.add_event('html-page-context')

    # event handlers
    app.connect('config-inited', convert_html_css_files, priority=800)
    app.connect('config-inited', convert_html_js_files, priority=800)
    app.connect('config-inited', validate_html_extra_path, priority=800)
    app.connect('config-inited', validate_html_static_path, priority=800)
    app.connect('config-inited', validate_html_logo, priority=800)
    app.connect('config-inited', validate_html_favicon, priority=800)
    app.connect('config-inited', error_on_html_4, priority=800)
    app.connect('builder-inited', validate_math_renderer)
    app.connect('html-page-context', setup_resource_paths)

    # load default math renderer
    app.setup_extension('sphinx.ext.mathjax')

    # load transforms for HTML builder
    app.setup_extension('sphinx.builders.html.transforms')

    return {
        'version': 'builtin',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }


# deprecated name -> (object to return, canonical path or empty string)
_DEPRECATED_OBJECTS = {
    'Stylesheet': (_CascadingStyleSheet, 'sphinx.builders.html._assets._CascadingStyleSheet', (9, 0)),  # NoQA: E501
    'JavaScript': (_JavaScript, 'sphinx.builders.html._assets._JavaScript', (9, 0)),
}


def __getattr__(name):
    if name not in _DEPRECATED_OBJECTS:
        msg = f'module {__name__!r} has no attribute {name!r}'
        raise AttributeError(msg)

    from sphinx.deprecation import _deprecation_warning

    deprecated_object, canonical_name, remove = _DEPRECATED_OBJECTS[name]
    _deprecation_warning(__name__, name, canonical_name, remove=remove)
    return deprecated_object

env_pyrecodes/lib/python3.9/site-packages/sphinx/builders/html/__init__.py
NikolaBlagojevic/pyrecodes
from __future__ import annotations

import os
import warnings
import zlib
from typing import TYPE_CHECKING

from sphinx.deprecation import RemovedInSphinx90Warning
from sphinx.errors import ThemeError

if TYPE_CHECKING:
    from pathlib import Path


class _CascadingStyleSheet:
    filename: str | os.PathLike[str]
    priority: int
    attributes: dict[str, str]

    def __init__(
        self,
        filename: str | os.PathLike[str], /, *,
        priority: int = 500,
        rel: str = 'stylesheet',
        type: str = 'text/css',
        **attributes: str,
    ) -> None:
        object.__setattr__(self, 'filename', filename)
        object.__setattr__(self, 'priority', priority)
        object.__setattr__(self, 'attributes', {'rel': rel, 'type': type, **attributes})

    def __str__(self):
        attr = ', '.join(f'{k}={v!r}' for k, v in self.attributes.items())
        return (f'{self.__class__.__name__}({self.filename!r}, '
                f'priority={self.priority}, '
                f'{attr})')

    def __eq__(self, other):
        if isinstance(other, str):
            warnings.warn('The str interface for _CascadingStyleSheet objects is deprecated. '
                          'Use css.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
            return self.filename == other
        if not isinstance(other, _CascadingStyleSheet):
            return NotImplemented
        return (self.filename == other.filename
                and self.priority == other.priority
                and self.attributes == other.attributes)

    def __hash__(self):
        return hash((self.filename, self.priority, *sorted(self.attributes.items())))

    def __setattr__(self, key, value):
        msg = f'{self.__class__.__name__} is immutable'
        raise AttributeError(msg)

    def __delattr__(self, key):
        msg = f'{self.__class__.__name__} is immutable'
        raise AttributeError(msg)

    def __getattr__(self, key):
        warnings.warn('The str interface for _CascadingStyleSheet objects is deprecated. '
                      'Use css.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
        return getattr(os.fspath(self.filename), key)

    def __getitem__(self, key):
        warnings.warn('The str interface for _CascadingStyleSheet objects is deprecated. '
                      'Use css.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
        return os.fspath(self.filename)[key]


class _JavaScript:
    filename: str | os.PathLike[str]
    priority: int
    attributes: dict[str, str]

    def __init__(
        self,
        filename: str | os.PathLike[str], /, *,
        priority: int = 500,
        **attributes: str,
    ) -> None:
        object.__setattr__(self, 'filename', filename)
        object.__setattr__(self, 'priority', priority)
        object.__setattr__(self, 'attributes', attributes)

    def __str__(self):
        attr = ''
        if self.attributes:
            attr = ', ' + ', '.join(f'{k}={v!r}' for k, v in self.attributes.items())
        return (f'{self.__class__.__name__}({self.filename!r}, '
                f'priority={self.priority}'
                f'{attr})')

    def __eq__(self, other):
        if isinstance(other, str):
            warnings.warn('The str interface for _JavaScript objects is deprecated. '
                          'Use js.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
            return self.filename == other
        if not isinstance(other, _JavaScript):
            return NotImplemented
        return (self.filename == other.filename
                and self.priority == other.priority
                and self.attributes == other.attributes)

    def __hash__(self):
        return hash((self.filename, self.priority, *sorted(self.attributes.items())))

    def __setattr__(self, key, value):
        msg = f'{self.__class__.__name__} is immutable'
        raise AttributeError(msg)

    def __delattr__(self, key):
        msg = f'{self.__class__.__name__} is immutable'
        raise AttributeError(msg)

    def __getattr__(self, key):
        warnings.warn('The str interface for _JavaScript objects is deprecated. '
                      'Use js.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
        return getattr(os.fspath(self.filename), key)

    def __getitem__(self, key):
        warnings.warn('The str interface for _JavaScript objects is deprecated. '
                      'Use js.filename instead.', RemovedInSphinx90Warning, stacklevel=2)
        return os.fspath(self.filename)[key]


def _file_checksum(outdir: Path, filename: str | os.PathLike[str]) -> str:
    filename = os.fspath(filename)
    # Don't generate checksums for HTTP URIs
    if '://' in filename:
        return ''
    # Some themes and extensions have used query strings
    # for a similar asset checksum feature.
    # As we cannot safely strip the query string,
    # raise an error to the user.
    if '?' in filename:
        msg = f'Local asset file paths must not contain query strings: {filename!r}'
        raise ThemeError(msg)
    try:
        # Remove all carriage returns to avoid checksum differences
        content = outdir.joinpath(filename).read_bytes().translate(None, b'\r')
    except FileNotFoundError:
        return ''
    if not content:
        return ''
    return f'{zlib.crc32(content):08x}'

env_pyrecodes/lib/python3.9/site-packages/sphinx/builders/html/_assets.py
NikolaBlagojevic/pyrecodes
"""The CheckExternalLinksBuilder class."""

from __future__ import annotations

import contextlib
import json
import re
import socket
import time
from html.parser import HTMLParser
from os import path
from queue import PriorityQueue, Queue
from threading import Thread
from typing import TYPE_CHECKING, NamedTuple, cast
from urllib.parse import unquote, urlparse, urlsplit, urlunparse

from docutils import nodes
from requests.exceptions import ConnectionError, HTTPError, SSLError, TooManyRedirects

from sphinx.builders.dummy import DummyBuilder
from sphinx.locale import __
from sphinx.transforms.post_transforms import SphinxPostTransform
from sphinx.util import encode_uri, logging, requests
from sphinx.util.console import (  # type: ignore[attr-defined]
    darkgray,
    darkgreen,
    purple,
    red,
    turquoise,
)
from sphinx.util.http_date import rfc1123_to_epoch
from sphinx.util.nodes import get_node_line

if TYPE_CHECKING:
    from collections.abc import Generator, Iterator
    from typing import Any, Callable

    from requests import Response

    from sphinx.application import Sphinx
    from sphinx.config import Config

logger = logging.getLogger(__name__)

uri_re = re.compile('([a-z]+:)?//')  # matches to foo:// and // (a protocol relative URL)

DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8',
}
CHECK_IMMEDIATELY = 0
QUEUE_POLL_SECS = 1
DEFAULT_DELAY = 60.0


class CheckExternalLinksBuilder(DummyBuilder):
    """
    Checks for broken external links.
    """
    name = 'linkcheck'
    epilog = __('Look for any errors in the above output or in '
                '%(outdir)s/output.txt')

    def init(self) -> None:
        self.broken_hyperlinks = 0
        self.hyperlinks: dict[str, Hyperlink] = {}
        # set a timeout for non-responding servers
        socket.setdefaulttimeout(5.0)

    def finish(self) -> None:
        checker = HyperlinkAvailabilityChecker(self.config)
        logger.info('')

        output_text = path.join(self.outdir, 'output.txt')
        output_json = path.join(self.outdir, 'output.json')
        with open(output_text, 'w', encoding='utf-8') as self.txt_outfile, \
             open(output_json, 'w', encoding='utf-8') as self.json_outfile:
            for result in checker.check(self.hyperlinks):
                self.process_result(result)

        if self.broken_hyperlinks:
            self.app.statuscode = 1

    def process_result(self, result: CheckResult) -> None:
        filename = self.env.doc2path(result.docname, False)

        linkstat = {'filename': filename, 'lineno': result.lineno,
                    'status': result.status, 'code': result.code, 'uri': result.uri,
                    'info': result.message}
        self.write_linkstat(linkstat)

        if result.status == 'unchecked':
            return
        if result.status == 'working' and result.message == 'old':
            return
        if result.lineno:
            logger.info('(%16s: line %4d) ', result.docname, result.lineno, nonl=True)
        if result.status == 'ignored':
            if result.message:
                logger.info(darkgray('-ignored- ') + result.uri + ': ' + result.message)
            else:
                logger.info(darkgray('-ignored- ') + result.uri)
        elif result.status == 'local':
            logger.info(darkgray('-local-   ') + result.uri)
            self.write_entry('local', result.docname, filename, result.lineno, result.uri)
        elif result.status == 'working':
            logger.info(darkgreen('ok        ') + result.uri + result.message)
        elif result.status == 'broken':
            if self.app.quiet or self.app.warningiserror:
                logger.warning(__('broken link: %s (%s)'), result.uri, result.message,
                               location=(result.docname, result.lineno))
            else:
                logger.info(red('broken    ') + result.uri + red(' - ' + result.message))
            self.write_entry('broken', result.docname, filename, result.lineno,
                             result.uri + ': ' + result.message)
            self.broken_hyperlinks += 1
        elif result.status == 'redirected':
            try:
                text, color = {
                    301: ('permanently', purple),
                    302: ('with Found', purple),
                    303: ('with See Other', purple),
                    307: ('temporarily', turquoise),
                    308: ('permanently', purple),
                }[result.code]
            except KeyError:
                text, color = ('with unknown code', purple)
            linkstat['text'] = text
            if self.config.linkcheck_allowed_redirects:
                logger.warning('redirect  ' + result.uri + ' - ' + text + ' to ' +
                               result.message, location=(result.docname, result.lineno))
            else:
                logger.info(color('redirect  ') + result.uri +
                            color(' - ' + text + ' to ' + result.message))
            self.write_entry('redirected ' + text, result.docname, filename,
                             result.lineno, result.uri + ' to ' + result.message)
        else:
            raise ValueError('Unknown status %s.' % result.status)

    def write_linkstat(self, data: dict) -> None:
        self.json_outfile.write(json.dumps(data))
        self.json_outfile.write('\n')

    def write_entry(self, what: str, docname: str, filename: str, line: int,
                    uri: str) -> None:
        self.txt_outfile.write(f'{filename}:{line}: [{what}] {uri}\n')


class HyperlinkCollector(SphinxPostTransform):
    builders = ('linkcheck',)
    default_priority = 800

    def run(self, **kwargs: Any) -> None:
        builder = cast(CheckExternalLinksBuilder, self.app.builder)
        hyperlinks = builder.hyperlinks
        docname = self.env.docname

        # reference nodes
        for refnode in self.document.findall(nodes.reference):
            if 'refuri' in refnode:
                uri = refnode['refuri']
                _add_uri(self.app, uri, refnode, hyperlinks, docname)

        # image nodes
        for imgnode in self.document.findall(nodes.image):
            uri = imgnode['candidates'].get('?')
            if uri and '://' in uri:
                _add_uri(self.app, uri, imgnode, hyperlinks, docname)

        # raw nodes
        for rawnode in self.document.findall(nodes.raw):
            uri = rawnode.get('source')
            if uri and '://' in uri:
                _add_uri(self.app, uri, rawnode, hyperlinks, docname)


def _add_uri(app: Sphinx, uri: str, node: nodes.Element,
             hyperlinks: dict[str, Hyperlink], docname: str) -> None:
    if newuri := app.emit_firstresult('linkcheck-process-uri', uri):
        uri = newuri

    try:
        lineno = get_node_line(node)
    except ValueError:
        lineno = -1

    if uri not in hyperlinks:
        hyperlinks[uri] = Hyperlink(uri, docname, app.env.doc2path(docname), lineno)


class Hyperlink(NamedTuple):
    uri: str
    docname: str
    docpath: str
    lineno: int


class HyperlinkAvailabilityChecker:
    def __init__(self, config: Config) -> None:
        self.config = config
        self.rate_limits: dict[str, RateLimit] = {}
        self.rqueue: Queue[CheckResult] = Queue()
        self.workers: list[Thread] = []
        self.wqueue: PriorityQueue[CheckRequest] = PriorityQueue()
        self.num_workers: int = config.linkcheck_workers

        self.to_ignore: list[re.Pattern[str]] = list(map(re.compile,
                                                         self.config.linkcheck_ignore))

    def check(self, hyperlinks: dict[str, Hyperlink]) -> Generator[CheckResult, None, None]:
        self.invoke_threads()

        total_links = 0
        for hyperlink in hyperlinks.values():
            if self.is_ignored_uri(hyperlink.uri):
                yield CheckResult(hyperlink.uri, hyperlink.docname, hyperlink.lineno,
                                  'ignored', '', 0)
            else:
                self.wqueue.put(CheckRequest(CHECK_IMMEDIATELY, hyperlink), False)
                total_links += 1

        done = 0
        while done < total_links:
            yield self.rqueue.get()
            done += 1

        self.shutdown_threads()

    def invoke_threads(self) -> None:
        for _i in range(self.num_workers):
            thread = HyperlinkAvailabilityCheckWorker(self.config,
                                                      self.rqueue, self.wqueue,
                                                      self.rate_limits)
            thread.start()
            self.workers.append(thread)

    def shutdown_threads(self) -> None:
        self.wqueue.join()
        for _worker in self.workers:
            self.wqueue.put(CheckRequest(CHECK_IMMEDIATELY, None), False)

    def is_ignored_uri(self, uri: str) -> bool:
        return any(pat.match(uri) for pat in self.to_ignore)


class CheckRequest(NamedTuple):
    next_check: float
    hyperlink: Hyperlink | None


class CheckResult(NamedTuple):
    uri: str
    docname: str
    lineno: int
    status: str
    message: str
    code: int


class HyperlinkAvailabilityCheckWorker(Thread):
    """A worker class for checking the availability of hyperlinks."""

    def __init__(self, config: Config,
                 rqueue: Queue[CheckResult],
                 wqueue: Queue[CheckRequest],
                 rate_limits: dict[str, RateLimit]) -> None:
        self.rate_limits = rate_limits
        self.rqueue = rqueue
        self.wqueue = wqueue

        self.anchors_ignore: list[re.Pattern[str]] = list(
            map(re.compile, config.linkcheck_anchors_ignore))
        self.anchors_ignore_for_url: list[re.Pattern[str]] = list(
            map(re.compile, config.linkcheck_anchors_ignore_for_url))
        self.documents_exclude: list[re.Pattern[str]] = list(
            map(re.compile, config.linkcheck_exclude_documents))
        self.auth = [(re.compile(pattern), auth_info) for pattern, auth_info
                     in config.linkcheck_auth]

        self.timeout: int | float | None = config.linkcheck_timeout
        self.request_headers: dict[str, dict[str, str]] = config.linkcheck_request_headers
        self.check_anchors: bool = config.linkcheck_anchors
        self.allowed_redirects: dict[re.Pattern[str], re.Pattern[str]]
        self.allowed_redirects = config.linkcheck_allowed_redirects
        self.retries: int = config.linkcheck_retries
        self.rate_limit_timeout = config.linkcheck_rate_limit_timeout

        self.user_agent = config.user_agent
        self.tls_verify = config.tls_verify
        self.tls_cacerts = config.tls_cacerts

        self._session = requests._Session()

        super().__init__(daemon=True)

    def run(self) -> None:
        while True:
            next_check, hyperlink = self.wqueue.get()
            if hyperlink is None:
                # An empty hyperlink is a signal to shutdown the worker; cleanup resources here
                self._session.close()
                break

            uri, docname, _docpath, lineno = hyperlink
            if uri is None:
                break

            netloc = urlsplit(uri).netloc
            with contextlib.suppress(KeyError):
                # Refresh rate limit.
                # When there are many links in the queue, workers are all stuck waiting
                # for responses, but the builder keeps queuing. Links in the queue may
                # have been queued before rate limits were discovered.
                next_check = self.rate_limits[netloc].next_check
            if next_check > time.time():
                # Sleep before putting message back in the queue to avoid
                # waking up other threads.
                time.sleep(QUEUE_POLL_SECS)
                self.wqueue.put(CheckRequest(next_check, hyperlink), False)
                self.wqueue.task_done()
                continue
            status, info, code = self._check(docname, uri, hyperlink)
            if status == 'rate-limited':
                logger.info(darkgray('-rate limited-   ') + uri + darkgray(' | sleeping...'))
            else:
                self.rqueue.put(CheckResult(uri, docname, lineno, status, info, code))
            self.wqueue.task_done()

    def _check(self, docname: str, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
        # check for various conditions without bothering the network

        for doc_matcher in self.documents_exclude:
            if doc_matcher.match(docname):
                info = (
                    f'{docname} matched {doc_matcher.pattern} from '
                    'linkcheck_exclude_documents'
                )
                return 'ignored', info, 0

        if len(uri) == 0 or uri.startswith(('#', 'mailto:', 'tel:')):
            return 'unchecked', '', 0
        if not uri.startswith(('http:', 'https:')):
            if uri_re.match(uri):
                # Non-supported URI schemes (ex. ftp)
                return 'unchecked', '', 0

            src_dir = path.dirname(hyperlink.docpath)
            if path.exists(path.join(src_dir, uri)):
                return 'working', '', 0
            return 'broken', '', 0

        # need to actually check the URI
        status, info, code = '', '', 0
        for _ in range(self.retries):
            status, info, code = self._check_uri(uri, hyperlink)
            if status != 'broken':
                break

        return status, info, code

    def _retrieval_methods(self,
                           check_anchors: bool,
                           anchor: str) -> Iterator[tuple[Callable, dict]]:
        if not check_anchors or not anchor:
            yield self._session.head, {'allow_redirects': True}
        yield self._session.get, {'stream': True}

    def _check_uri(self, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
        req_url, delimiter, anchor = uri.partition('#')
        if delimiter and anchor:
            for rex in self.anchors_ignore:
                if rex.match(anchor):
                    anchor = ''
                    break
            else:
                for rex in self.anchors_ignore_for_url:
                    if rex.match(req_url):
                        anchor = ''
                        break

        # handle non-ASCII URIs
        try:
            req_url.encode('ascii')
        except UnicodeError:
            req_url = encode_uri(req_url)

        # Get auth info, if any
        for pattern, auth_info in self.auth:  # noqa: B007 (false positive)
            if pattern.match(uri):
                break
        else:
            auth_info = None

        # update request headers for the URL
        headers = _get_request_headers(uri, self.request_headers)

        # Linkcheck HTTP request logic:
        #
        # - Attempt HTTP HEAD before HTTP GET unless page content is required.
        # - Follow server-issued HTTP redirects.
        # - Respect server-issued HTTP 429 back-offs.
        error_message = ''
        status_code = -1
        response_url = retry_after = ''
        for retrieval_method, kwargs in self._retrieval_methods(self.check_anchors, anchor):
            try:
                with retrieval_method(
                    url=req_url, auth=auth_info,
                    headers=headers,
                    timeout=self.timeout,
                    **kwargs,
                    _user_agent=self.user_agent,
                    _tls_info=(self.tls_verify, self.tls_cacerts),
                ) as response:
                    if (self.check_anchors and response.ok and anchor
                            and not contains_anchor(response, anchor)):
                        raise Exception(__(f'Anchor {anchor!r} not found'))

                # Copy data we need from the (closed) response
                status_code = response.status_code
                redirect_status_code = response.history[-1].status_code if response.history else None  # NoQA: E501
                retry_after = response.headers.get('Retry-After')
                response_url = f'{response.url}'
                response.raise_for_status()
                del response
                break

            except SSLError as err:
                # SSL failure; report that the link is broken.
                return 'broken', str(err), 0

            except (ConnectionError, TooManyRedirects) as err:
                # Servers drop the connection on HEAD requests, causing
                # ConnectionError.
                error_message = str(err)
                continue

            except HTTPError as err:
                error_message = str(err)

                # Unauthorised: the reference probably exists
                if status_code == 401:
                    return 'working', 'unauthorized', 0

                # Rate limiting; back-off if allowed, or report failure otherwise
                if status_code == 429:
                    if next_check := self.limit_rate(response_url, retry_after):
                        self.wqueue.put(CheckRequest(next_check, hyperlink), False)
                        return 'rate-limited', '', 0
                    return 'broken', error_message, 0

                # Don't claim success/failure during server-side outages
                if status_code == 503:
                    return 'ignored', 'service unavailable', 0

                # For most HTTP failures, continue attempting alternate retrieval methods
                continue

            except Exception as err:
                # Unhandled exception (intermittent or permanent); report that
                # the link is broken.
                return 'broken', str(err), 0

        else:
            # All available retrieval methods have been exhausted; report
            # that the link is broken.
            return 'broken', error_message, 0

        # Success; clear rate limits for the origin
        netloc = urlsplit(req_url).netloc
        self.rate_limits.pop(netloc, None)

        if ((response_url.rstrip('/') == req_url.rstrip('/'))
                or _allowed_redirect(req_url, response_url,
                                     self.allowed_redirects)):
            return 'working', '', 0
        elif redirect_status_code is not None:
            return 'redirected', response_url, redirect_status_code
        else:
            return 'redirected', response_url, 0

    def limit_rate(self, response_url: str, retry_after: str) -> float | None:
        delay = DEFAULT_DELAY
        next_check = None
        if retry_after:
            try:
                # Integer: time to wait before next attempt.
                delay = float(retry_after)
            except ValueError:
                try:
                    # An HTTP-date: time of next attempt.
                    next_check = rfc1123_to_epoch(retry_after)
                except (ValueError, TypeError):
                    # TypeError: Invalid date format.
                    # ValueError: Invalid date, e.g. Oct 52th.
                    pass
                else:
                    delay = next_check - time.time()
            else:
                next_check = time.time() + delay
        netloc = urlsplit(response_url).netloc
        if next_check is None:
            max_delay = self.rate_limit_timeout
            try:
                rate_limit = self.rate_limits[netloc]
            except KeyError:
                delay = DEFAULT_DELAY
            else:
                last_wait_time = rate_limit.delay
                delay = 2.0 * last_wait_time
                if delay > max_delay > last_wait_time:
                    delay = max_delay
            if delay > max_delay:
                return None
            next_check = time.time() + delay
        self.rate_limits[netloc] = RateLimit(delay, next_check)
        return next_check


def _get_request_headers(
    uri: str,
    request_headers: dict[str, dict[str, str]],
) -> dict[str, str]:
    url = urlsplit(uri)
    candidates = (f'{url.scheme}://{url.netloc}',
                  f'{url.scheme}://{url.netloc}/',
                  uri,
                  '*')

    for u in candidates:
        if u in request_headers:
            return {**DEFAULT_REQUEST_HEADERS, **request_headers[u]}
    return {}


def contains_anchor(response: Response, anchor: str) -> bool:
    """Determine if an anchor is contained within an HTTP response."""

    parser = AnchorCheckParser(unquote(anchor))
    # Read file in chunks. If we find a matching anchor, we break
    # the loop early in hopes not to have to download the whole thing.
    for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
        if isinstance(chunk, bytes):    # requests failed to decode
            chunk = chunk.decode()      # manually try to decode it

        parser.feed(chunk)
        if parser.found:
            break
    parser.close()
    return parser.found


class AnchorCheckParser(HTMLParser):
    """Specialised HTML parser that looks for a specific anchor."""

    def __init__(self, search_anchor: str) -> None:
        super().__init__()

        self.search_anchor = search_anchor
        self.found = False

    def handle_starttag(self, tag: Any, attrs: Any) -> None:
        for key, value in attrs:
            if key in ('id', 'name') and value == self.search_anchor:
                self.found = True
                break


def _allowed_redirect(url: str, new_url: str,
                      allowed_redirects: dict[re.Pattern[str], re.Pattern[str]]) -> bool:
    return any(
        from_url.match(url) and to_url.match(new_url)
        for from_url, to_url
        in allowed_redirects.items()
    )


class RateLimit(NamedTuple):
    delay: float
    next_check: float


def rewrite_github_anchor(app: Sphinx, uri: str) -> str | None:
    """Rewrite anchor name of the hyperlink to github.com

    The hyperlink anchors in github.com are dynamically generated.  This rewrites
    them before checking and makes them comparable.
    """
    parsed = urlparse(uri)
    if parsed.hostname == 'github.com' and parsed.fragment:
        prefixed = parsed.fragment.startswith('user-content-')
        if not prefixed:
            fragment = f'user-content-{parsed.fragment}'
            return urlunparse(parsed._replace(fragment=fragment))
    return None


def compile_linkcheck_allowed_redirects(app: Sphinx, config: Config) -> None:
    """Compile patterns in linkcheck_allowed_redirects to the regexp objects."""
    for url, pattern in list(app.config.linkcheck_allowed_redirects.items()):
        try:
            app.config.linkcheck_allowed_redirects[re.compile(url)] = re.compile(pattern)
        except re.error as exc:
            logger.warning(__('Failed to compile regex in linkcheck_allowed_redirects: %r %s'),
                           exc.pattern, exc.msg)
        finally:
            # Remove the original regexp-string
            app.config.linkcheck_allowed_redirects.pop(url)


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_builder(CheckExternalLinksBuilder)
    app.add_post_transform(HyperlinkCollector)

    app.add_config_value('linkcheck_ignore', [], False)
    app.add_config_value('linkcheck_exclude_documents', [], False)
    app.add_config_value('linkcheck_allowed_redirects', {}, False)
    app.add_config_value('linkcheck_auth', [], False)
    app.add_config_value('linkcheck_request_headers', {}, False)
    app.add_config_value('linkcheck_retries', 1, False)
    app.add_config_value('linkcheck_timeout', None, False, [int, float])
    app.add_config_value('linkcheck_workers', 5, False)
    app.add_config_value('linkcheck_anchors', True, False)
    # Anchors starting with ! are ignored since they are
    # commonly used for dynamic pages
    app.add_config_value('linkcheck_anchors_ignore', ['^!'], False)
    app.add_config_value('linkcheck_anchors_ignore_for_url', (), False, (tuple, list))
    app.add_config_value('linkcheck_rate_limit_timeout', 300.0, False)

    app.add_event('linkcheck-process-uri')

    app.connect('config-inited', compile_linkcheck_allowed_redirects, priority=800)

    # FIXME: Disable URL rewrite handler for github.com temporarily.
    # ref: https://github.com/sphinx-doc/sphinx/issues/9435
    # app.connect('linkcheck-process-uri', rewrite_github_anchor)

    return {
        'version': 'builtin',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/builders/linkcheck.py
NikolaBlagojevic/pyrecodes
"""Single HTML builders."""

from __future__ import annotations

from os import path
from typing import TYPE_CHECKING, Any

from docutils import nodes

from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.environment.adapters.toctree import global_toctree_for_doc
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.console import darkgreen  # type: ignore[attr-defined]
from sphinx.util.display import progress_message
from sphinx.util.nodes import inline_all_toctrees

if TYPE_CHECKING:
    from docutils.nodes import Node

    from sphinx.application import Sphinx

logger = logging.getLogger(__name__)


class SingleFileHTMLBuilder(StandaloneHTMLBuilder):
    """
    A StandaloneHTMLBuilder subclass that puts the whole document tree on one
    HTML page.
    """
    name = 'singlehtml'
    epilog = __('The HTML page is in %(outdir)s.')

    copysource = False

    def get_outdated_docs(self) -> str | list[str]:  # type: ignore[override]
        return 'all documents'

    def get_target_uri(self, docname: str, typ: str | None = None) -> str:
        if docname in self.env.all_docs:
            # all references are on the same page...
            return self.config.root_doc + self.out_suffix + \
                '#document-' + docname
        else:
            # chances are this is a html_additional_page
            return docname + self.out_suffix

    def get_relative_uri(self, from_: str, to: str, typ: str | None = None) -> str:
        # ignore source
        return self.get_target_uri(to, typ)

    def fix_refuris(self, tree: Node) -> None:
        # fix refuris with double anchor
        fname = self.config.root_doc + self.out_suffix
        for refnode in tree.findall(nodes.reference):
            if 'refuri' not in refnode:
                continue
            refuri = refnode['refuri']
            hashindex = refuri.find('#')
            if hashindex < 0:
                continue
            hashindex = refuri.find('#', hashindex + 1)
            if hashindex >= 0:
                refnode['refuri'] = fname + refuri[hashindex:]

    def _get_local_toctree(self, docname: str, collapse: bool = True, **kwargs: Any) -> str:
        if isinstance(includehidden := kwargs.get('includehidden'), str):
            if includehidden.lower() == 'false':
                kwargs['includehidden'] = False
            elif includehidden.lower() == 'true':
                kwargs['includehidden'] = True
        if kwargs.get('maxdepth') == '':
            kwargs.pop('maxdepth')
        toctree = global_toctree_for_doc(self.env, docname, self, collapse=collapse, **kwargs)
        if toctree is not None:
            self.fix_refuris(toctree)
        return self.render_partial(toctree)['fragment']

    def assemble_doctree(self) -> nodes.document:
        master = self.config.root_doc
        tree = self.env.get_doctree(master)
        tree = inline_all_toctrees(self, set(), master, tree, darkgreen, [master])
        tree['docname'] = master
        self.env.resolve_references(tree, master, self)
        self.fix_refuris(tree)
        return tree

    def assemble_toc_secnumbers(self) -> dict[str, dict[str, tuple[int, ...]]]:
        # Assemble toc_secnumbers to resolve section numbers on SingleHTML.
        # Merge all secnumbers to single secnumber.
        #
        # Note: current Sphinx has refid confliction in singlehtml mode.
        #       To avoid the problem, it replaces key of secnumbers to
        #       tuple of docname and refid.
        #
        #       There are related codes in inline_all_toctres() and
        #       HTMLTranslter#add_secnumber().
        new_secnumbers: dict[str, tuple[int, ...]] = {}
        for docname, secnums in self.env.toc_secnumbers.items():
            for id, secnum in secnums.items():
                alias = f"{docname}/{id}"
                new_secnumbers[alias] = secnum

        return {self.config.root_doc: new_secnumbers}

    def assemble_toc_fignumbers(self) -> dict[str, dict[str, dict[str, tuple[int, ...]]]]:
        # Assemble toc_fignumbers to resolve figure numbers on SingleHTML.
        # Merge all fignumbers to single fignumber.
        #
        # Note: current Sphinx has refid confliction in singlehtml mode.
        #       To avoid the problem, it replaces key of secnumbers to
        #       tuple of docname and refid.
        #
        #       There are related codes in inline_all_toctres() and
        #       HTMLTranslter#add_fignumber().
        new_fignumbers: dict[str, dict[str, tuple[int, ...]]] = {}
        # {'foo': {'figure': {'id2': (2,), 'id1': (1,)}}, 'bar': {'figure': {'id1': (3,)}}}
        for docname, fignumlist in self.env.toc_fignumbers.items():
            for figtype, fignums in fignumlist.items():
                alias = f"{docname}/{figtype}"
                new_fignumbers.setdefault(alias, {})
                for id, fignum in fignums.items():
                    new_fignumbers[alias][id] = fignum

        return {self.config.root_doc: new_fignumbers}

    def get_doc_context(self, docname: str, body: str, metatags: str) -> dict[str, Any]:
        # no relation links...
        toctree = global_toctree_for_doc(self.env, self.config.root_doc, self, collapse=False)
        # if there is no toctree, toc is None
        if toctree:
            self.fix_refuris(toctree)
            toc = self.render_partial(toctree)['fragment']
            display_toc = True
        else:
            toc = ''
            display_toc = False
        return {
            'parents': [],
            'prev': None,
            'next': None,
            'docstitle': None,
            'title': self.config.html_title,
            'meta': None,
            'body': body,
            'metatags': metatags,
            'rellinks': [],
            'sourcename': '',
            'toc': toc,
            'display_toc': display_toc,
        }

    def write(self, *ignored: Any) -> None:
        docnames = self.env.all_docs

        with progress_message(__('preparing documents')):
            self.prepare_writing(docnames)  # type: ignore[arg-type]

        with progress_message(__('assembling single document')):
            doctree = self.assemble_doctree()
            self.env.toc_secnumbers = self.assemble_toc_secnumbers()
            self.env.toc_fignumbers = self.assemble_toc_fignumbers()

        with progress_message(__('writing')):
            self.write_doc_serialized(self.config.root_doc, doctree)
            self.write_doc(self.config.root_doc, doctree)

    def finish(self) -> None:
        self.write_additional_files()
        self.copy_image_files()
        self.copy_download_files()
        self.copy_static_files()
        self.copy_extra_files()
        self.write_buildinfo()
        self.dump_inventory()

    @progress_message(__('writing additional files'))
    def write_additional_files(self) -> None:
        # no indices or search pages are supported

        # additional pages from conf.py
        for pagename, template in self.config.html_additional_pages.items():
            logger.info(' ' + pagename, nonl=True)
            self.handle_page(pagename, {}, template)

        if self.config.html_use_opensearch:
            logger.info(' opensearch', nonl=True)
            fn = path.join(self.outdir, '_static', 'opensearch.xml')
            self.handle_page('opensearch', {}, 'opensearch.xml', outfilename=fn)


def setup(app: Sphinx) -> dict[str, Any]:
    app.setup_extension('sphinx.builders.html')

    app.add_builder(SingleFileHTMLBuilder)
    app.add_config_value('singlehtml_sidebars', lambda self: self.html_sidebars, 'html')

    return {
        'version': 'builtin',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/builders/singlehtml.py
NikolaBlagojevic/pyrecodes
"""Build documentation from a provided source."""

from __future__ import annotations

import argparse
import bdb
import contextlib
import locale
import multiprocessing
import os
import pdb  # NoQA: T100
import sys
import traceback
from os import path
from typing import TYPE_CHECKING, Any, TextIO

from docutils.utils import SystemMessage

import sphinx.locale
from sphinx import __display_version__
from sphinx.application import Sphinx
from sphinx.errors import SphinxError, SphinxParallelError
from sphinx.locale import __
from sphinx.util import Tee
from sphinx.util.console import (  # type: ignore[attr-defined]
    color_terminal,
    nocolor,
    red,
    terminal_safe,
)
from sphinx.util.docutils import docutils_namespace, patch_docutils
from sphinx.util.exceptions import format_exception_cut_frames, save_traceback
from sphinx.util.osutil import ensuredir

if TYPE_CHECKING:
    from collections.abc import Sequence


def handle_exception(
    app: Sphinx | None, args: Any, exception: BaseException, stderr: TextIO = sys.stderr,
) -> None:
    if isinstance(exception, bdb.BdbQuit):
        return

    if args.pdb:
        print(red(__('Exception occurred while building, starting debugger:')),
              file=stderr)
        traceback.print_exc()
        pdb.post_mortem(sys.exc_info()[2])
    else:
        print(file=stderr)
        if args.verbosity or args.traceback:
            exc = sys.exc_info()[1]
            if isinstance(exc, SphinxParallelError):
                exc_format = '(Error in parallel process)\n' + exc.traceback
                print(exc_format, file=stderr)
            else:
                traceback.print_exc(None, stderr)
                print(file=stderr)
        if isinstance(exception, KeyboardInterrupt):
            print(__('Interrupted!'), file=stderr)
        elif isinstance(exception, SystemMessage):
            print(red(__('reST markup error:')), file=stderr)
            print(terminal_safe(exception.args[0]), file=stderr)
        elif isinstance(exception, SphinxError):
            print(red('%s:' % exception.category), file=stderr)
            print(str(exception), file=stderr)
        elif isinstance(exception, UnicodeError):
            print(red(__('Encoding error:')), file=stderr)
            print(terminal_safe(str(exception)), file=stderr)
            tbpath = save_traceback(app, exception)
            print(red(__('The full traceback has been saved in %s, if you want '
                         'to report the issue to the developers.') % tbpath),
                  file=stderr)
        elif isinstance(exception, RuntimeError) and 'recursion depth' in str(exception):
            print(red(__('Recursion error:')), file=stderr)
            print(terminal_safe(str(exception)), file=stderr)
            print(file=stderr)
            print(__('This can happen with very large or deeply nested source '
                     'files. You can carefully increase the default Python '
                     'recursion limit of 1000 in conf.py with e.g.:'), file=stderr)
            print('    import sys; sys.setrecursionlimit(1500)', file=stderr)
        else:
            print(red(__('Exception occurred:')), file=stderr)
            print(format_exception_cut_frames().rstrip(), file=stderr)
            tbpath = save_traceback(app, exception)
            print(red(__('The full traceback has been saved in %s, if you '
                         'want to report the issue to the developers.') % tbpath),
                  file=stderr)
            print(__('Please also report this if it was a user error, so '
                     'that a better error message can be provided next time.'),
                  file=stderr)
            print(__('A bug report can be filed in the tracker at '
                     '<https://github.com/sphinx-doc/sphinx/issues>. Thanks!'),
                  file=stderr)


def jobs_argument(value: str) -> int:
    """
    Special type to handle 'auto' flags passed to 'sphinx-build' via -j flag. Can
    be expanded to handle other special scaling requests, such as setting job count
    to cpu_count.
    """
    if value == 'auto':
        return multiprocessing.cpu_count()
    else:
        jobs = int(value)
        if jobs <= 0:
            raise argparse.ArgumentTypeError(__('job number should be a positive number'))
        else:
            return jobs


def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        usage='%(prog)s [OPTIONS] SOURCEDIR OUTPUTDIR [FILENAMES...]',
        epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'),
        description=__("""
Generate documentation from source files.

sphinx-build generates documentation from the files in SOURCEDIR and places it
in OUTPUTDIR. It looks for 'conf.py' in SOURCEDIR for the configuration
settings. The 'sphinx-quickstart' tool may be used to generate template files,
including 'conf.py'

sphinx-build can create documentation in different formats. A format is
selected by specifying the builder name on the command line; it defaults to
HTML. Builders can also perform other tasks related to documentation
processing.

By default, everything that is outdated is built. Output only for selected
files can be built by specifying individual filenames.
"""))

    parser.add_argument('--version', action='version', dest='show_version',
                        version='%%(prog)s %s' % __display_version__)

    parser.add_argument('sourcedir',
                        help=__('path to documentation source files'))
    parser.add_argument('outputdir',
                        help=__('path to output directory'))
    parser.add_argument('filenames', nargs='*',
                        help=__('a list of specific files to rebuild. Ignored '
                                'if -a is specified'))

    group = parser.add_argument_group(__('general options'))
    group.add_argument('-b', metavar='BUILDER', dest='builder',
                       default='html',
                       help=__('builder to use (default: html)'))
    group.add_argument('-a', action='store_true', dest='force_all',
                       help=__('write all files (default: only write new and '
                               'changed files)'))
    group.add_argument('-E', action='store_true', dest='freshenv',
                       help=__("don't use a saved environment, always read "
                               'all files'))
    group.add_argument('-d', metavar='PATH', dest='doctreedir',
                       help=__('path for the cached environment and doctree '
                               'files (default: OUTPUTDIR/.doctrees)'))
    group.add_argument('-j', '--jobs', metavar='N', default=1, type=jobs_argument,
                       dest='jobs',
                       help=__('build in parallel with N processes where '
                               'possible (special value "auto" will set N to cpu-count)'))
    group = parser.add_argument_group('build configuration options')
    group.add_argument('-c', metavar='PATH', dest='confdir',
                       help=__('path where configuration file (conf.py) is '
                               'located (default: same as SOURCEDIR)'))
    group.add_argument('-C', action='store_true', dest='noconfig',
                       help=__('use no config file at all, only -D options'))
    group.add_argument('-D', metavar='setting=value', action='append',
                       dest='define', default=[],
                       help=__('override a setting in configuration file'))
    group.add_argument('-A', metavar='name=value', action='append',
                       dest='htmldefine', default=[],
                       help=__('pass a value into HTML templates'))
    group.add_argument('-t', metavar='TAG', action='append',
                       dest='tags', default=[],
                       help=__('define tag: include "only" blocks with TAG'))
    group.add_argument('-n', action='store_true', dest='nitpicky',
                       help=__('nit-picky mode, warn about all missing '
                               'references'))

    group = parser.add_argument_group(__('console output options'))
    group.add_argument('-v', action='count', dest='verbosity', default=0,
                       help=__('increase verbosity (can be repeated)'))
    group.add_argument('-q', action='store_true', dest='quiet',
                       help=__('no output on stdout, just warnings on stderr'))
    group.add_argument('-Q', action='store_true', dest='really_quiet',
                       help=__('no output at all, not even warnings'))
    group.add_argument('--color', action='store_const', const='yes',
                       default='auto',
                       help=__('do emit colored output (default: auto-detect)'))
    group.add_argument('-N', '--no-color', dest='color', action='store_const',
                       const='no',
                       help=__('do not emit colored output (default: '
                               'auto-detect)'))
    group.add_argument('-w', metavar='FILE', dest='warnfile',
                       help=__('write warnings (and errors) to given file'))
    group.add_argument('-W', action='store_true', dest='warningiserror',
                       help=__('turn warnings into errors'))
    group.add_argument('--keep-going', action='store_true', dest='keep_going',
                       help=__("with -W, keep going when getting warnings"))
    group.add_argument('-T', action='store_true', dest='traceback',
                       help=__('show full traceback on exception'))
    group.add_argument('-P', action='store_true', dest='pdb',
                       help=__('run Pdb on exception'))

    return parser


def make_main(argv: Sequence[str]) -> int:
    """Sphinx build "make mode" entry."""
    from sphinx.cmd import make_mode
    return make_mode.run_make_mode(argv[1:])


def _parse_arguments(argv: Sequence[str]) -> argparse.Namespace:
    parser = get_parser()
    args = parser.parse_args(argv)

    if args.noconfig:
        args.confdir = None
    elif not args.confdir:
        args.confdir = args.sourcedir

    if not args.doctreedir:
        args.doctreedir = os.path.join(args.outputdir, '.doctrees')

    if args.force_all and args.filenames:
        parser.error(__('cannot combine -a option and filenames'))

    if args.color == 'no' or (args.color == 'auto' and not color_terminal()):
        nocolor()

    status: TextIO | None = sys.stdout
    warning: TextIO | None = sys.stderr
    error = sys.stderr

    if args.quiet:
        status = None

    if args.really_quiet:
        status = warning = None

    if warning and args.warnfile:
        try:
            warnfile = path.abspath(args.warnfile)
            ensuredir(path.dirname(warnfile))
            warnfp = open(args.warnfile, 'w', encoding="utf-8")  # NoQA: SIM115
        except Exception as exc:
            parser.error(__('cannot open warning file %r: %s') % (
                args.warnfile, exc))
        warning = Tee(warning, warnfp)  # type: ignore[assignment]
        error = warning

    args.status = status
    args.warning = warning
    args.error = error

    confoverrides = {}
    for val in args.define:
        try:
            key, val = val.split('=', 1)
        except ValueError:
            parser.error(__('-D option argument must be in the form name=value'))
        confoverrides[key] = val

    for val in args.htmldefine:
        try:
            key, val = val.split('=')
        except ValueError:
            parser.error(__('-A option argument must be in the form name=value'))
        with contextlib.suppress(ValueError):
            val = int(val)

        confoverrides['html_context.%s' % key] = val

    if args.nitpicky:
        confoverrides['nitpicky'] = True

    args.confoverrides = confoverrides

    return args


def build_main(argv: Sequence[str]) -> int:
    """Sphinx build "main" command-line entry."""
    args = _parse_arguments(argv)

    app = None
    try:
        confdir = args.confdir or args.sourcedir
        with patch_docutils(confdir), docutils_namespace():
            app = Sphinx(args.sourcedir, args.confdir, args.outputdir,
                         args.doctreedir, args.builder, args.confoverrides, args.status,
                         args.warning, args.freshenv, args.warningiserror,
                         args.tags, args.verbosity, args.jobs, args.keep_going,
                         args.pdb)
            app.build(args.force_all, args.filenames)
            return app.statuscode
    except (Exception, KeyboardInterrupt) as exc:
        handle_exception(app, args, exc, args.error)
        return 2


def _bug_report_info() -> int:
    from platform import platform, python_implementation

    import docutils
    import jinja2
    import pygments

    print('Please paste all output below into the bug report template\n\n')
    print('```text')
    print(f'Platform:              {sys.platform}; ({platform()})')
    print(f'Python version:        {sys.version})')
    print(f'Python implementation: {python_implementation()}')
    print(f'Sphinx version:        {sphinx.__display_version__}')
    print(f'Docutils version:      {docutils.__version__}')
    print(f'Jinja2 version:        {jinja2.__version__}')
    print(f'Pygments version:      {pygments.__version__}')
    print('```')
    return 0


def main(argv: Sequence[str] = (), /) -> int:
    locale.setlocale(locale.LC_ALL, '')
    sphinx.locale.init_console()

    if not argv:
        argv = sys.argv[1:]

    # Allow calling as 'python -m sphinx build …'
    if argv[:1] == ['build']:
        argv = argv[1:]

    if argv[:1] == ['--bug-report']:
        return _bug_report_info()
    if argv[:1] == ['-M']:
        return make_main(argv)
    else:
        return build_main(argv)


if __name__ == '__main__':
    raise SystemExit(main(sys.argv[1:]))

env_pyrecodes/lib/python3.9/site-packages/sphinx/cmd/build.py
NikolaBlagojevic/pyrecodes
"""sphinx-build -M command-line handling.

This replaces the old, platform-dependent and once-generated content
of Makefile / make.bat.

This is in its own module so that importing it is fast.  It should not
import the main Sphinx modules (like sphinx.applications, sphinx.builders).
"""

from __future__ import annotations

import os
import subprocess
import sys
from os import path
from typing import TYPE_CHECKING

import sphinx
from sphinx.cmd.build import build_main
from sphinx.util.console import (  # type: ignore[attr-defined]
    blue,
    bold,
    color_terminal,
    nocolor,
)
from sphinx.util.osutil import rmtree

try:
    from contextlib import chdir  # type: ignore[attr-defined]
except ImportError:
    from sphinx.util.osutil import _chdir as chdir

if TYPE_CHECKING:
    from collections.abc import Sequence

BUILDERS = [
    ("",      "html",        "to make standalone HTML files"),
    ("",      "dirhtml",     "to make HTML files named index.html in directories"),
    ("",      "singlehtml",  "to make a single large HTML file"),
    ("",      "pickle",      "to make pickle files"),
    ("",      "json",        "to make JSON files"),
    ("",      "htmlhelp",    "to make HTML files and an HTML help project"),
    ("",      "qthelp",      "to make HTML files and a qthelp project"),
    ("",      "devhelp",     "to make HTML files and a Devhelp project"),
    ("",      "epub",        "to make an epub"),
    ("",      "latex",       "to make LaTeX files, you can set PAPER=a4 or PAPER=letter"),
    ("posix", "latexpdf",    "to make LaTeX and PDF files (default pdflatex)"),
    ("posix", "latexpdfja",  "to make LaTeX files and run them through platex/dvipdfmx"),
    ("",      "text",        "to make text files"),
    ("",      "man",         "to make manual pages"),
    ("",      "texinfo",     "to make Texinfo files"),
    ("posix", "info",        "to make Texinfo files and run them through makeinfo"),
    ("",      "gettext",     "to make PO message catalogs"),
    ("",      "changes",     "to make an overview of all changed/added/deprecated items"),
    ("",      "xml",         "to make Docutils-native XML files"),
    ("",      "pseudoxml",   "to make pseudoxml-XML files for display purposes"),
    ("",      "linkcheck",   "to check all external links for integrity"),
    ("",      "doctest",     "to run all doctests embedded in the documentation "
                             "(if enabled)"),
    ("",      "coverage",    "to run coverage check of the documentation (if enabled)"),
    ("",      "clean",       "to remove everything in the build directory"),
]


class Make:
    def __init__(self, srcdir: str, builddir: str, opts: Sequence[str]) -> None:
        self.srcdir = srcdir
        self.builddir = builddir
        self.opts = [*opts]
        self.makecmd = os.environ.get('MAKE', 'make')  # refer $MAKE to determine make command

    def builddir_join(self, *comps: str) -> str:
        return path.join(self.builddir, *comps)

    def build_clean(self) -> int:
        srcdir = path.abspath(self.srcdir)
        builddir = path.abspath(self.builddir)
        if not path.exists(self.builddir):
            return 0
        elif not path.isdir(self.builddir):
            print("Error: %r is not a directory!" % self.builddir)
            return 1
        elif srcdir == builddir:
            print("Error: %r is same as source directory!" % self.builddir)
            return 1
        elif path.commonpath([srcdir, builddir]) == builddir:
            print("Error: %r directory contains source directory!" % self.builddir)
            return 1
        print("Removing everything under %r..." % self.builddir)
        for item in os.listdir(self.builddir):
            rmtree(self.builddir_join(item))
        return 0

    def build_help(self) -> None:
        if not color_terminal():
            nocolor()

        print(bold("Sphinx v%s" % sphinx.__display_version__))
        print("Please use `make %s' where %s is one of" % ((blue('target'),) * 2))
        for osname, bname, description in BUILDERS:
            if not osname or os.name == osname:
                print(f'  {blue(bname.ljust(10))}  {description}')

    def build_latexpdf(self) -> int:
        if self.run_generic_build('latex') > 0:
            return 1

        if sys.platform == 'win32':
            makecmd = os.environ.get('MAKE', 'make.bat')
        else:
            makecmd = self.makecmd
        try:
            with chdir(self.builddir_join('latex')):
                return subprocess.call([makecmd, 'all-pdf'])
        except OSError:
            print('Error: Failed to run: %s' % makecmd)
            return 1

    def build_latexpdfja(self) -> int:
        if self.run_generic_build('latex') > 0:
            return 1

        if sys.platform == 'win32':
            makecmd = os.environ.get('MAKE', 'make.bat')
        else:
            makecmd = self.makecmd
        try:
            with chdir(self.builddir_join('latex')):
                return subprocess.call([makecmd, 'all-pdf'])
        except OSError:
            print('Error: Failed to run: %s' % makecmd)
            return 1

    def build_info(self) -> int:
        if self.run_generic_build('texinfo') > 0:
            return 1
        try:
            with chdir(self.builddir_join('texinfo')):
                return subprocess.call([self.makecmd, 'info'])
        except OSError:
            print('Error: Failed to run: %s' % self.makecmd)
            return 1

    def build_gettext(self) -> int:
        dtdir = self.builddir_join('gettext', '.doctrees')
        if self.run_generic_build('gettext', doctreedir=dtdir) > 0:
            return 1
        return 0

    def run_generic_build(self, builder: str, doctreedir: str | None = None) -> int:
        # compatibility with old Makefile
        papersize = os.getenv('PAPER', '')
        opts = self.opts
        if papersize in ('a4', 'letter'):
            opts.extend(['-D', 'latex_elements.papersize=' + papersize + 'paper'])
        if doctreedir is None:
            doctreedir = self.builddir_join('doctrees')

        args = ['-b', builder,
                '-d', doctreedir,
                self.srcdir,
                self.builddir_join(builder)]
        return build_main(args + opts)


def run_make_mode(args: Sequence[str]) -> int:
    if len(args) < 3:
        print('Error: at least 3 arguments (builder, source '
              'dir, build dir) are required.', file=sys.stderr)
        return 1
    make = Make(args[1], args[2], args[3:])
    run_method = 'build_' + args[0]
    if hasattr(make, run_method):
        return getattr(make, run_method)()
    return make.run_generic_build(args[0])

env_pyrecodes/lib/python3.9/site-packages/sphinx/cmd/make_mode.py
NikolaBlagojevic/pyrecodes
"""Quickly setup documentation source to work with Sphinx."""

from __future__ import annotations

import argparse
import locale
import os
import sys
import time
from os import path
from typing import TYPE_CHECKING, Any, Callable

# try to import readline, unix specific enhancement
try:
    import readline
    if TYPE_CHECKING and sys.platform == "win32":  # always false, for type checking
        raise ImportError
    READLINE_AVAILABLE = True
    if readline.__doc__ and 'libedit' in readline.__doc__:
        readline.parse_and_bind("bind ^I rl_complete")
        USE_LIBEDIT = True
    else:
        readline.parse_and_bind("tab: complete")
        USE_LIBEDIT = False
except ImportError:
    READLINE_AVAILABLE = False
    USE_LIBEDIT = False

from docutils.utils import column_width

import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.locale import __
from sphinx.util.console import (  # type: ignore[attr-defined]
    bold,
    color_terminal,
    colorize,
    nocolor,
    red,
)
from sphinx.util.osutil import ensuredir
from sphinx.util.template import SphinxRenderer

if TYPE_CHECKING:
    from collections.abc import Sequence

EXTENSIONS = {
    'autodoc': __('automatically insert docstrings from modules'),
    'doctest': __('automatically test code snippets in doctest blocks'),
    'intersphinx': __('link between Sphinx documentation of different projects'),
    'todo': __('write "todo" entries that can be shown or hidden on build'),
    'coverage': __('checks for documentation coverage'),
    'imgmath': __('include math, rendered as PNG or SVG images'),
    'mathjax': __('include math, rendered in the browser by MathJax'),
    'ifconfig': __('conditional inclusion of content based on config values'),
    'viewcode': __('include links to the source code of documented Python objects'),
    'githubpages': __('create .nojekyll file to publish the document on GitHub pages'),
}

DEFAULTS = {
    'path': '.',
    'sep': False,
    'dot': '_',
    'language': None,
    'suffix': '.rst',
    'master': 'index',
    'makefile': True,
    'batchfile': True,
}

PROMPT_PREFIX = '> '

if sys.platform == 'win32':
    # On Windows, show questions as bold because of color scheme of PowerShell (refs: #5294).
    COLOR_QUESTION = 'bold'
else:
    COLOR_QUESTION = 'purple'


# function to get input from terminal -- overridden by the test suite
def term_input(prompt: str) -> str:
    if sys.platform == 'win32':
        # Important: On windows, readline is not enabled by default.  In these
        #            environment, escape sequences have been broken.  To avoid the
        #            problem, quickstart uses ``print()`` to show prompt.
        print(prompt, end='')
        return input('')
    else:
        return input(prompt)


class ValidationError(Exception):
    """Raised for validation errors."""


def is_path(x: str) -> str:
    x = path.expanduser(x)
    if not path.isdir(x):
        raise ValidationError(__("Please enter a valid path name."))
    return x


def is_path_or_empty(x: str) -> str:
    if x == '':
        return x
    return is_path(x)


def allow_empty(x: str) -> str:
    return x


def nonempty(x: str) -> str:
    if not x:
        raise ValidationError(__("Please enter some text."))
    return x


def choice(*l: str) -> Callable[[str], str]:
    def val(x: str) -> str:
        if x not in l:
            raise ValidationError(__('Please enter one of %s.') % ', '.join(l))
        return x
    return val


def boolean(x: str) -> bool:
    if x.upper() not in ('Y', 'YES', 'N', 'NO'):
        raise ValidationError(__("Please enter either 'y' or 'n'."))
    return x.upper() in ('Y', 'YES')


def suffix(x: str) -> str:
    if not (x[0:1] == '.' and len(x) > 1):
        raise ValidationError(__("Please enter a file suffix, e.g. '.rst' or '.txt'."))
    return x


def ok(x: str) -> str:
    return x


def do_prompt(
    text: str, default: str | None = None, validator: Callable[[str], Any] = nonempty,
) -> str | bool:
    while True:
        if default is not None:
            prompt = PROMPT_PREFIX + f'{text} [{default}]: '
        else:
            prompt = PROMPT_PREFIX + text + ': '
        if USE_LIBEDIT:
            # Note: libedit has a problem for combination of ``input()`` and escape
            # sequence (see #5335).  To avoid the problem, all prompts are not colored
            # on libedit.
            pass
        elif READLINE_AVAILABLE:
            # pass input_mode=True if readline available
            prompt = colorize(COLOR_QUESTION, prompt, input_mode=True)
        else:
            prompt = colorize(COLOR_QUESTION, prompt, input_mode=False)
        x = term_input(prompt).strip()
        if default and not x:
            x = default
        try:
            x = validator(x)
        except ValidationError as err:
            print(red('* ' + str(err)))
            continue
        break
    return x


class QuickstartRenderer(SphinxRenderer):
    def __init__(self, templatedir: str = '') -> None:
        self.templatedir = templatedir
        super().__init__()

    def _has_custom_template(self, template_name: str) -> bool:
        """Check if custom template file exists.

        Note: Please don't use this function from extensions.
              It will be removed in the future without deprecation period.
        """
        template = path.join(self.templatedir, path.basename(template_name))
        return bool(self.templatedir) and path.exists(template)

    def render(self, template_name: str, context: dict[str, Any]) -> str:
        if self._has_custom_template(template_name):
            custom_template = path.join(self.templatedir, path.basename(template_name))
            return self.render_from_file(custom_template, context)
        else:
            return super().render(template_name, context)


def ask_user(d: dict[str, Any]) -> None:
    """Ask the user for quickstart values missing from *d*.

    Values are:

    * path:      root path
    * sep:       separate source and build dirs (bool)
    * dot:       replacement for dot in _templates etc.
    * project:   project name
    * author:    author names
    * version:   version of project
    * release:   release of project
    * language:  document language
    * suffix:    source file suffix
    * master:    master document name
    * extensions:  extensions to use (list)
    * makefile:  make Makefile
    * batchfile: make command file
    """

    print(bold(__('Welcome to the Sphinx %s quickstart utility.')) % __display_version__)
    print()
    print(__('Please enter values for the following settings (just press Enter to\n'
             'accept a default value, if one is given in brackets).'))

    if 'path' in d:
        print()
        print(bold(__('Selected root path: %s')) % d['path'])
    else:
        print()
        print(__('Enter the root path for documentation.'))
        d['path'] = do_prompt(__('Root path for the documentation'), '.', is_path)

    while path.isfile(path.join(d['path'], 'conf.py')) or \
            path.isfile(path.join(d['path'], 'source', 'conf.py')):
        print()
        print(bold(__('Error: an existing conf.py has been found in the '
                      'selected root path.')))
        print(__('sphinx-quickstart will not overwrite existing Sphinx projects.'))
        print()
        d['path'] = do_prompt(__('Please enter a new root path (or just Enter to exit)'),
                              '', is_path_or_empty)
        if not d['path']:
            raise SystemExit(1)

    if 'sep' not in d:
        print()
        print(__('You have two options for placing the build directory for Sphinx output.\n'
                 'Either, you use a directory "_build" within the root path, or you separate\n'
                 '"source" and "build" directories within the root path.'))
        d['sep'] = do_prompt(__('Separate source and build directories (y/n)'), 'n', boolean)

    if 'dot' not in d:
        print()
        print(__('Inside the root directory, two more directories will be created; "_templates"\n'      # noqa: E501
                 'for custom HTML templates and "_static" for custom stylesheets and other static\n'    # noqa: E501
                 'files. You can enter another prefix (such as ".") to replace the underscore.'))       # noqa: E501
        d['dot'] = do_prompt(__('Name prefix for templates and static dir'), '_', ok)

    if 'project' not in d:
        print()
        print(__('The project name will occur in several places in the built documentation.'))
        d['project'] = do_prompt(__('Project name'))
    if 'author' not in d:
        d['author'] = do_prompt(__('Author name(s)'))

    if 'version' not in d:
        print()
        print(__('Sphinx has the notion of a "version" and a "release" for the\n'
                 'software. Each version can have multiple releases. For example, for\n'
                 'Python the version is something like 2.5 or 3.0, while the release is\n'
                 "something like 2.5.1 or 3.0a1. If you don't need this dual structure,\n"
                 'just set both to the same value.'))
        d['version'] = do_prompt(__('Project version'), '', allow_empty)
    if 'release' not in d:
        d['release'] = do_prompt(__('Project release'), d['version'], allow_empty)

    if 'language' not in d:
        print()
        print(__(
            'If the documents are to be written in a language other than English,\n'
            'you can select a language here by its language code. Sphinx will then\n'
            'translate text that it generates into that language.\n'
            '\n'
            'For a list of supported codes, see\n'
            'https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-language.',
        ))
        d['language'] = do_prompt(__('Project language'), 'en')
        if d['language'] == 'en':
            d['language'] = None

    if 'suffix' not in d:
        print()
        print(__('The file name suffix for source files. Commonly, this is either ".txt"\n'
                 'or ".rst". Only files with this suffix are considered documents.'))
        d['suffix'] = do_prompt(__('Source file suffix'), '.rst', suffix)

    if 'master' not in d:
        print()
        print(__('One document is special in that it is considered the top node of the\n'
                 '"contents tree", that is, it is the root of the hierarchical structure\n'
                 'of the documents. Normally, this is "index", but if your "index"\n'
                 'document is a custom template, you can also set this to another filename.'))
        d['master'] = do_prompt(__('Name of your master document (without suffix)'), 'index')

    while path.isfile(path.join(d['path'], d['master'] + d['suffix'])) or \
            path.isfile(path.join(d['path'], 'source', d['master'] + d['suffix'])):
        print()
        print(bold(__('Error: the master file %s has already been found in the '
                      'selected root path.') % (d['master'] + d['suffix'])))
        print(__('sphinx-quickstart will not overwrite the existing file.'))
        print()
        d['master'] = do_prompt(__('Please enter a new file name, or rename the '
                                   'existing file and press Enter'), d['master'])

    if 'extensions' not in d:
        print(__('Indicate which of the following Sphinx extensions should be enabled:'))
        d['extensions'] = []
        for name, description in EXTENSIONS.items():
            if do_prompt(f'{name}: {description} (y/n)', 'n', boolean):
                d['extensions'].append('sphinx.ext.%s' % name)

        # Handle conflicting options
        if {'sphinx.ext.imgmath', 'sphinx.ext.mathjax'}.issubset(d['extensions']):
            print(__('Note: imgmath and mathjax cannot be enabled at the same time. '
                     'imgmath has been deselected.'))
            d['extensions'].remove('sphinx.ext.imgmath')

    if 'makefile' not in d:
        print()
        print(__('A Makefile and a Windows command file can be generated for you so that you\n'
                 "only have to run e.g. `make html' instead of invoking sphinx-build\n"
                 'directly.'))
        d['makefile'] = do_prompt(__('Create Makefile? (y/n)'), 'y', boolean)

    if 'batchfile' not in d:
        d['batchfile'] = do_prompt(__('Create Windows command file? (y/n)'), 'y', boolean)
    print()


def generate(
    d: dict, overwrite: bool = True, silent: bool = False, templatedir: str | None = None,
) -> None:
    """Generate project based on values in *d*."""
    template = QuickstartRenderer(templatedir or '')

    if 'mastertoctree' not in d:
        d['mastertoctree'] = ''
    if 'mastertocmaxdepth' not in d:
        d['mastertocmaxdepth'] = 2

    d['root_doc'] = d['master']
    d['now'] = time.asctime()
    d['project_underline'] = column_width(d['project']) * '='
    d.setdefault('extensions', [])
    d['copyright'] = time.strftime('%Y') + ', ' + d['author']

    d["path"] = os.path.abspath(d['path'])
    ensuredir(d['path'])

    srcdir = path.join(d['path'], 'source') if d['sep'] else d['path']

    ensuredir(srcdir)
    if d['sep']:
        builddir = path.join(d['path'], 'build')
        d['exclude_patterns'] = ''
    else:
        builddir = path.join(srcdir, d['dot'] + 'build')
        exclude_patterns = map(repr, [
            d['dot'] + 'build',
            'Thumbs.db', '.DS_Store',
        ])
        d['exclude_patterns'] = ', '.join(exclude_patterns)
    ensuredir(builddir)
    ensuredir(path.join(srcdir, d['dot'] + 'templates'))
    ensuredir(path.join(srcdir, d['dot'] + 'static'))

    def write_file(fpath: str, content: str, newline: str | None = None) -> None:
        if overwrite or not path.isfile(fpath):
            if 'quiet' not in d:
                print(__('Creating file %s.') % fpath)
            with open(fpath, 'w', encoding='utf-8', newline=newline) as f:
                f.write(content)
        else:
            if 'quiet' not in d:
                print(__('File %s already exists, skipping.') % fpath)

    conf_path = os.path.join(templatedir, 'conf.py_t') if templatedir else None
    if not conf_path or not path.isfile(conf_path):
        conf_path = os.path.join(package_dir, 'templates', 'quickstart', 'conf.py_t')
    with open(conf_path, encoding="utf-8") as f:
        conf_text = f.read()

    write_file(path.join(srcdir, 'conf.py'), template.render_string(conf_text, d))

    masterfile = path.join(srcdir, d['master'] + d['suffix'])
    if template._has_custom_template('quickstart/master_doc.rst_t'):
        msg = ('A custom template `master_doc.rst_t` found. It has been renamed to '
               '`root_doc.rst_t`.  Please rename it on your project too.')
        print(colorize('red', msg))
        write_file(masterfile, template.render('quickstart/master_doc.rst_t', d))
    else:
        write_file(masterfile, template.render('quickstart/root_doc.rst_t', d))

    if d.get('make_mode') is True:
        makefile_template = 'quickstart/Makefile.new_t'
        batchfile_template = 'quickstart/make.bat.new_t'
    else:
        makefile_template = 'quickstart/Makefile_t'
        batchfile_template = 'quickstart/make.bat_t'

    if d['makefile'] is True:
        d['rsrcdir'] = 'source' if d['sep'] else '.'
        d['rbuilddir'] = 'build' if d['sep'] else d['dot'] + 'build'
        # use binary mode, to avoid writing \r\n on Windows
        write_file(path.join(d['path'], 'Makefile'),
                   template.render(makefile_template, d), '\n')

    if d['batchfile'] is True:
        d['rsrcdir'] = 'source' if d['sep'] else '.'
        d['rbuilddir'] = 'build' if d['sep'] else d['dot'] + 'build'
        write_file(path.join(d['path'], 'make.bat'),
                   template.render(batchfile_template, d), '\r\n')

    if silent:
        return
    print()
    print(bold(__('Finished: An initial directory structure has been created.')))
    print()
    print(__('You should now populate your master file %s and create other documentation\n'
             'source files. ') % masterfile, end='')
    if d['makefile'] or d['batchfile']:
        print(__('Use the Makefile to build the docs, like so:\n'
                 '   make builder'))
    else:
        print(__('Use the sphinx-build command to build the docs, like so:\n'
                 '   sphinx-build -b builder %s %s') % (srcdir, builddir))
    print(__('where "builder" is one of the supported builders, '
             'e.g. html, latex or linkcheck.'))
    print()


def valid_dir(d: dict) -> bool:
    dir = d['path']
    if not path.exists(dir):
        return True
    if not path.isdir(dir):
        return False

    if {'Makefile', 'make.bat'} & set(os.listdir(dir)):
        return False

    if d['sep']:
        dir = os.path.join('source', dir)
        if not path.exists(dir):
            return True
        if not path.isdir(dir):
            return False

    reserved_names = [
        'conf.py',
        d['dot'] + 'static',
        d['dot'] + 'templates',
        d['master'] + d['suffix'],
    ]
    if set(reserved_names) & set(os.listdir(dir)):
        return False

    return True


def get_parser() -> argparse.ArgumentParser:
    description = __(
        "\n"
        "Generate required files for a Sphinx project.\n"
        "\n"
        "sphinx-quickstart is an interactive tool that asks some questions about your\n"
        "project and then generates a complete documentation directory and sample\n"
        "Makefile to be used with sphinx-build.\n",
    )
    parser = argparse.ArgumentParser(
        usage='%(prog)s [OPTIONS] <PROJECT_DIR>',
        epilog=__("For more information, visit <https://www.sphinx-doc.org/>."),
        description=description)

    parser.add_argument('-q', '--quiet', action='store_true', dest='quiet',
                        default=None,
                        help=__('quiet mode'))
    parser.add_argument('--version', action='version', dest='show_version',
                        version='%%(prog)s %s' % __display_version__)

    parser.add_argument('path', metavar='PROJECT_DIR', default='.', nargs='?',
                        help=__('project root'))

    group = parser.add_argument_group(__('Structure options'))
    group.add_argument('--sep', action='store_true', dest='sep', default=None,
                       help=__('if specified, separate source and build dirs'))
    group.add_argument('--no-sep', action='store_false', dest='sep',
                       help=__('if specified, create build dir under source dir'))
    group.add_argument('--dot', metavar='DOT', default='_',
                       help=__('replacement for dot in _templates etc.'))

    group = parser.add_argument_group(__('Project basic options'))
    group.add_argument('-p', '--project', metavar='PROJECT', dest='project',
                       help=__('project name'))
    group.add_argument('-a', '--author', metavar='AUTHOR', dest='author',
                       help=__('author names'))
    group.add_argument('-v', metavar='VERSION', dest='version', default='',
                       help=__('version of project'))
    group.add_argument('-r', '--release', metavar='RELEASE', dest='release',
                       help=__('release of project'))
    group.add_argument('-l', '--language', metavar='LANGUAGE', dest='language',
                       help=__('document language'))
    group.add_argument('--suffix', metavar='SUFFIX', default='.rst',
                       help=__('source file suffix'))
    group.add_argument('--master', metavar='MASTER', default='index',
                       help=__('master document name'))
    group.add_argument('--epub', action='store_true', default=False,
                       help=__('use epub'))

    group = parser.add_argument_group(__('Extension options'))
    for ext in EXTENSIONS:
        group.add_argument('--ext-%s' % ext, action='append_const',
                           const='sphinx.ext.%s' % ext, dest='extensions',
                           help=__('enable %s extension') % ext)
    group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
                       action='append', help=__('enable arbitrary extensions'))

    group = parser.add_argument_group(__('Makefile and Batchfile creation'))
    group.add_argument('--makefile', action='store_true', dest='makefile', default=True,
                       help=__('create makefile'))
    group.add_argument('--no-makefile', action='store_false', dest='makefile',
                       help=__('do not create makefile'))
    group.add_argument('--batchfile', action='store_true', dest='batchfile', default=True,
                       help=__('create batchfile'))
    group.add_argument('--no-batchfile', action='store_false',
                       dest='batchfile',
                       help=__('do not create batchfile'))
    group.add_argument('-m', '--use-make-mode', action='store_true',
                       dest='make_mode', default=True,
                       help=__('use make-mode for Makefile/make.bat'))
    group.add_argument('-M', '--no-use-make-mode', action='store_false',
                       dest='make_mode',
                       help=__('do not use make-mode for Makefile/make.bat'))

    group = parser.add_argument_group(__('Project templating'))
    group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
                       dest='templatedir',
                       help=__('template directory for template files'))
    group.add_argument('-d', metavar='NAME=VALUE', action='append',
                       dest='variables',
                       help=__('define a template variable'))

    return parser


def main(argv: Sequence[str] = (), /) -> int:
    locale.setlocale(locale.LC_ALL, '')
    sphinx.locale.init_console()

    if not color_terminal():
        nocolor()

    # parse options
    parser = get_parser()
    try:
        args = parser.parse_args(argv or sys.argv[1:])
    except SystemExit as err:
        return err.code  # type: ignore[return-value]

    d = vars(args)
    # delete None or False value
    d = {k: v for k, v in d.items() if v is not None}

    # handle use of CSV-style extension values
    d.setdefault('extensions', [])
    for ext in d['extensions'][:]:
        if ',' in ext:
            d['extensions'].remove(ext)
            d['extensions'].extend(ext.split(','))

    try:
        if 'quiet' in d:
            if not {'project', 'author'}.issubset(d):
                print(__('"quiet" is specified, but any of "project" or '
                         '"author" is not specified.'))
                return 1

        if {'quiet', 'project', 'author'}.issubset(d):
            # quiet mode with all required params satisfied, use default
            d.setdefault('version', '')
            d.setdefault('release', d['version'])
            d2 = DEFAULTS.copy()
            d2.update(d)
            d = d2

            if not valid_dir(d):
                print()
                print(bold(__('Error: specified path is not a directory, or sphinx'
                              ' files already exist.')))
                print(__('sphinx-quickstart only generate into a empty directory.'
                         ' Please specify a new root path.'))
                return 1
        else:
            ask_user(d)
    except (KeyboardInterrupt, EOFError):
        print()
        print('[Interrupted.]')
        return 130  # 128 + SIGINT

    for variable in d.get('variables', []):
        try:
            name, value = variable.split('=')
            d[name] = value
        except ValueError:
            print(__('Invalid template variable: %s') % variable)

    generate(d, overwrite=False, templatedir=args.templatedir)
    return 0


if __name__ == '__main__':
    raise SystemExit(main(sys.argv[1:]))

env_pyrecodes/lib/python3.9/site-packages/sphinx/cmd/quickstart.py
NikolaBlagojevic/pyrecodes
"""Build configuration file handling."""

from __future__ import annotations

import time
import traceback
import types
from os import getenv, path
from typing import TYPE_CHECKING, Any, Callable, NamedTuple

from sphinx.errors import ConfigError, ExtensionError
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.osutil import fs_encoding
from sphinx.util.typing import NoneType

try:
    from contextlib import chdir  # type: ignore[attr-defined]
except ImportError:
    from sphinx.util.osutil import _chdir as chdir

if TYPE_CHECKING:
    import os
    from collections.abc import Generator, Iterator, Sequence

    from sphinx.application import Sphinx
    from sphinx.environment import BuildEnvironment
    from sphinx.util.tags import Tags

logger = logging.getLogger(__name__)

CONFIG_FILENAME = 'conf.py'
UNSERIALIZABLE_TYPES = (type, types.ModuleType, types.FunctionType)


class ConfigValue(NamedTuple):
    name: str
    value: Any
    rebuild: bool | str


def is_serializable(obj: Any) -> bool:
    """Check if object is serializable or not."""
    if isinstance(obj, UNSERIALIZABLE_TYPES):
        return False
    elif isinstance(obj, dict):
        for key, value in obj.items():
            if not is_serializable(key) or not is_serializable(value):
                return False
    elif isinstance(obj, (list, tuple, set)):
        return all(is_serializable(i) for i in obj)

    return True


class ENUM:
    """Represents the candidates which a config value should be one of.

    Example:
        app.add_config_value('latex_show_urls', 'no', None, ENUM('no', 'footnote', 'inline'))
    """
    def __init__(self, *candidates: str | bool | None) -> None:
        self.candidates = candidates

    def match(self, value: str | list | tuple) -> bool:
        if isinstance(value, (list, tuple)):
            return all(item in self.candidates for item in value)
        else:
            return value in self.candidates


class Config:
    r"""Configuration file abstraction.

    The config object makes the values of all config values available as
    attributes.

    It is exposed via the :py:class:`~sphinx.application.Sphinx`\ ``.config``
    and :py:class:`sphinx.environment.BuildEnvironment`\ ``.config`` attributes.
    For example, to get the value of :confval:`language`, use either
    ``app.config.language`` or ``env.config.language``.
    """

    # the values are: (default, what needs to be rebuilt if changed)

    # If you add a value here, don't forget to include it in the
    # quickstart.py file template as well as in the docs!

    config_values: dict[str, tuple] = {
        # general options
        'project': ('Python', 'env', []),
        'author': ('unknown', 'env', []),
        'project_copyright': ('', 'html', [str, tuple, list]),
        'copyright': (lambda c: c.project_copyright, 'html', [str, tuple, list]),
        'version': ('', 'env', []),
        'release': ('', 'env', []),
        'today': ('', 'env', []),
        # the real default is locale-dependent
        'today_fmt': (None, 'env', [str]),

        'language': ('en', 'env', [str]),
        'locale_dirs': (['locales'], 'env', []),
        'figure_language_filename': ('{root}.{language}{ext}', 'env', [str]),
        'gettext_allow_fuzzy_translations': (False, 'gettext', []),
        'translation_progress_classes': (False, 'env',
                                         ENUM(True, False, 'translated', 'untranslated')),

        'master_doc': ('index', 'env', []),
        'root_doc': (lambda config: config.master_doc, 'env', []),
        'source_suffix': ({'.rst': 'restructuredtext'}, 'env', Any),
        'source_encoding': ('utf-8-sig', 'env', []),
        'exclude_patterns': ([], 'env', [str]),
        'include_patterns': (["**"], 'env', [str]),
        'default_role': (None, 'env', [str]),
        'add_function_parentheses': (True, 'env', []),
        'add_module_names': (True, 'env', []),
        'toc_object_entries': (True, 'env', [bool]),
        'toc_object_entries_show_parents': ('domain', 'env',
                                            ENUM('domain', 'all', 'hide')),
        'trim_footnote_reference_space': (False, 'env', []),
        'show_authors': (False, 'env', []),
        'pygments_style': (None, 'html', [str]),
        'highlight_language': ('default', 'env', []),
        'highlight_options': ({}, 'env', []),
        'templates_path': ([], 'html', []),
        'template_bridge': (None, 'html', [str]),
        'keep_warnings': (False, 'env', []),
        'suppress_warnings': ([], 'env', []),
        'modindex_common_prefix': ([], 'html', []),
        'rst_epilog': (None, 'env', [str]),
        'rst_prolog': (None, 'env', [str]),
        'trim_doctest_flags': (True, 'env', []),
        'primary_domain': ('py', 'env', [NoneType]),
        'needs_sphinx': (None, None, [str]),
        'needs_extensions': ({}, None, []),
        'manpages_url': (None, 'env', []),
        'nitpicky': (False, None, []),
        'nitpick_ignore': ([], None, [set, list, tuple]),
        'nitpick_ignore_regex': ([], None, [set, list, tuple]),
        'numfig': (False, 'env', []),
        'numfig_secnum_depth': (1, 'env', []),
        'numfig_format': ({}, 'env', []),  # will be initialized in init_numfig_format()
        'maximum_signature_line_length': (None, 'env', {int, None}),
        'math_number_all': (False, 'env', []),
        'math_eqref_format': (None, 'env', [str]),
        'math_numfig': (True, 'env', []),
        'tls_verify': (True, 'env', []),
        'tls_cacerts': (None, 'env', []),
        'user_agent': (None, 'env', [str]),
        'smartquotes': (True, 'env', []),
        'smartquotes_action': ('qDe', 'env', []),
        'smartquotes_excludes': ({'languages': ['ja'],
                                  'builders': ['man', 'text']},
                                 'env', []),
        'option_emphasise_placeholders': (False, 'env', []),
    }

    def __init__(self, config: dict[str, Any] | None = None,
                 overrides: dict[str, Any] | None = None) -> None:
        config = config or {}
        self.overrides = dict(overrides) if overrides is not None else {}
        self.values = Config.config_values.copy()
        self._raw_config = config
        self.setup: Callable | None = config.get('setup', None)

        if 'extensions' in self.overrides:
            if isinstance(self.overrides['extensions'], str):
                config['extensions'] = self.overrides.pop('extensions').split(',')
            else:
                config['extensions'] = self.overrides.pop('extensions')
        self.extensions: list[str] = config.get('extensions', [])

    @classmethod
    def read(cls, confdir: str | os.PathLike[str], overrides: dict | None = None,
             tags: Tags | None = None) -> Config:
        """Create a Config object from configuration file."""
        filename = path.join(confdir, CONFIG_FILENAME)
        if not path.isfile(filename):
            raise ConfigError(__("config directory doesn't contain a conf.py file (%s)") %
                              confdir)
        namespace = eval_config_file(filename, tags)

        # Note: Old sphinx projects have been configured as "language = None" because
        #       sphinx-quickstart previously generated this by default.
        #       To keep compatibility, they should be fallback to 'en' for a while
        #       (This conversion should not be removed before 2025-01-01).
        if namespace.get("language", ...) is None:
            logger.warning(__("Invalid configuration value found: 'language = None'. "
                              "Update your configuration to a valid language code. "
                              "Falling back to 'en' (English)."))
            namespace["language"] = "en"

        return cls(namespace, overrides or {})

    def convert_overrides(self, name: str, value: Any) -> Any:
        if not isinstance(value, str):
            return value
        else:
            defvalue = self.values[name][0]
            if self.values[name][2] == Any:
                return value
            elif self.values[name][2] == {bool, str}:
                if value == '0':
                    # given falsy string from command line option
                    return False
                elif value == '1':
                    return True
                else:
                    return value
            elif type(defvalue) is bool or self.values[name][2] == [bool]:
                if value == '0':
                    # given falsy string from command line option
                    return False
                else:
                    return bool(value)
            elif isinstance(defvalue, dict):
                raise ValueError(__('cannot override dictionary config setting %r, '
                                    'ignoring (use %r to set individual elements)') %
                                 (name, name + '.key=value'))
            elif isinstance(defvalue, list):
                return value.split(',')
            elif isinstance(defvalue, int):
                try:
                    return int(value)
                except ValueError as exc:
                    raise ValueError(__('invalid number %r for config value %r, ignoring') %
                                     (value, name)) from exc
            elif callable(defvalue):
                return value
            elif defvalue is not None and not isinstance(defvalue, str):
                raise ValueError(__('cannot override config setting %r with unsupported '
                                    'type, ignoring') % name)
            else:
                return value

    def pre_init_values(self) -> None:
        """
        Initialize some limited config variables before initializing i18n and loading
        extensions.
        """
        variables = ['needs_sphinx', 'suppress_warnings', 'language', 'locale_dirs']
        for name in variables:
            try:
                if name in self.overrides:
                    self.__dict__[name] = self.convert_overrides(name, self.overrides[name])
                elif name in self._raw_config:
                    self.__dict__[name] = self._raw_config[name]
            except ValueError as exc:
                logger.warning("%s", exc)

    def init_values(self) -> None:
        config = self._raw_config
        for valname, value in self.overrides.items():
            try:
                if '.' in valname:
                    realvalname, key = valname.split('.', 1)
                    config.setdefault(realvalname, {})[key] = value
                    continue
                if valname not in self.values:
                    logger.warning(__('unknown config value %r in override, ignoring'),
                                   valname)
                    continue
                if isinstance(value, str):
                    config[valname] = self.convert_overrides(valname, value)
                else:
                    config[valname] = value
            except ValueError as exc:
                logger.warning("%s", exc)
        for name in config:
            if name in self.values:
                self.__dict__[name] = config[name]

    def post_init_values(self) -> None:
        """
        Initialize additional config variables that are added after init_values() called.
        """
        config = self._raw_config
        for name in config:
            if name not in self.__dict__ and name in self.values:
                self.__dict__[name] = config[name]

        check_confval_types(None, self)

    def __getattr__(self, name: str) -> Any:
        if name.startswith('_'):
            raise AttributeError(name)
        if name not in self.values:
            raise AttributeError(__('No such config value: %s') % name)
        default = self.values[name][0]
        if callable(default):
            return default(self)
        return default

    def __getitem__(self, name: str) -> Any:
        return getattr(self, name)

    def __setitem__(self, name: str, value: Any) -> None:
        setattr(self, name, value)

    def __delitem__(self, name: str) -> None:
        delattr(self, name)

    def __contains__(self, name: str) -> bool:
        return name in self.values

    def __iter__(self) -> Generator[ConfigValue, None, None]:
        for name, value in self.values.items():
            yield ConfigValue(name, getattr(self, name), value[1])

    def add(self, name: str, default: Any, rebuild: bool | str, types: Any) -> None:
        if name in self.values:
            raise ExtensionError(__('Config value %r already present') % name)
        self.values[name] = (default, rebuild, types)

    def filter(self, rebuild: str | Sequence[str]) -> Iterator[ConfigValue]:
        if isinstance(rebuild, str):
            rebuild = [rebuild]
        return (value for value in self if value.rebuild in rebuild)

    def __getstate__(self) -> dict:
        """Obtains serializable data for pickling."""
        # remove potentially pickling-problematic values from config
        __dict__ = {}
        for key, value in self.__dict__.items():
            if key.startswith('_') or not is_serializable(value):
                pass
            else:
                __dict__[key] = value

        # create a picklable copy of values list
        __dict__['values'] = {}
        for key, value in self.values.items():
            real_value = getattr(self, key)
            if not is_serializable(real_value):
                # omit unserializable value
                real_value = None

            # types column is also omitted
            __dict__['values'][key] = (real_value, value[1], None)

        return __dict__

    def __setstate__(self, state: dict) -> None:
        self.__dict__.update(state)


def eval_config_file(filename: str, tags: Tags | None) -> dict[str, Any]:
    """Evaluate a config file."""
    namespace: dict[str, Any] = {}
    namespace['__file__'] = filename
    namespace['tags'] = tags

    with chdir(path.dirname(filename)):
        # during executing config file, current dir is changed to ``confdir``.
        try:
            with open(filename, 'rb') as f:
                code = compile(f.read(), filename.encode(fs_encoding), 'exec')
                exec(code, namespace)  # NoQA: S102
        except SyntaxError as err:
            msg = __("There is a syntax error in your configuration file: %s\n")
            raise ConfigError(msg % err) from err
        except SystemExit as exc:
            msg = __("The configuration file (or one of the modules it imports) "
                     "called sys.exit()")
            raise ConfigError(msg) from exc
        except ConfigError:
            # pass through ConfigError from conf.py as is.  It will be shown in console.
            raise
        except Exception as exc:
            msg = __("There is a programmable error in your configuration file:\n\n%s")
            raise ConfigError(msg % traceback.format_exc()) from exc

    return namespace


def convert_source_suffix(app: Sphinx, config: Config) -> None:
    """Convert old styled source_suffix to new styled one.

    * old style: str or list
    * new style: a dict which maps from fileext to filetype
    """
    source_suffix = config.source_suffix
    if isinstance(source_suffix, str):
        # if str, considers as default filetype (None)
        #
        # The default filetype is determined on later step.
        # By default, it is considered as restructuredtext.
        config.source_suffix = {source_suffix: None}  # type: ignore[attr-defined]
    elif isinstance(source_suffix, (list, tuple)):
        # if list, considers as all of them are default filetype
        config.source_suffix = {s: None for s in source_suffix}  # type: ignore[attr-defined]
    elif not isinstance(source_suffix, dict):
        logger.warning(__("The config value `source_suffix' expects "
                          "a string, list of strings, or dictionary. "
                          "But `%r' is given." % source_suffix))


def convert_highlight_options(app: Sphinx, config: Config) -> None:
    """Convert old styled highlight_options to new styled one.

    * old style: options
    * new style: a dict which maps from language name to options
    """
    options = config.highlight_options
    if options and not all(isinstance(v, dict) for v in options.values()):
        # old styled option detected because all values are not dictionary.
        config.highlight_options = {config.highlight_language:  # type: ignore[attr-defined]
                                    options}


def init_numfig_format(app: Sphinx, config: Config) -> None:
    """Initialize :confval:`numfig_format`."""
    numfig_format = {'section': _('Section %s'),
                     'figure': _('Fig. %s'),
                     'table': _('Table %s'),
                     'code-block': _('Listing %s')}

    # override default labels by configuration
    numfig_format.update(config.numfig_format)
    config.numfig_format = numfig_format  # type: ignore[attr-defined]


def correct_copyright_year(_app: Sphinx, config: Config) -> None:
    """Correct values of copyright year that are not coherent with
    the SOURCE_DATE_EPOCH environment variable (if set)

    See https://reproducible-builds.org/specs/source-date-epoch/
    """
    if (source_date_epoch := getenv('SOURCE_DATE_EPOCH')) is None:
        return

    source_date_epoch_year = str(time.gmtime(int(source_date_epoch)).tm_year)

    for k in ('copyright', 'epub_copyright'):
        if k in config:
            value: str | Sequence[str] = config[k]
            if isinstance(value, str):
                config[k] = _substitute_copyright_year(value, source_date_epoch_year)
            else:
                items = (_substitute_copyright_year(x, source_date_epoch_year) for x in value)
                config[k] = type(value)(items)  # type: ignore[call-arg]


def _substitute_copyright_year(copyright_line: str, replace_year: str) -> str:
    """Replace the year in a single copyright line.

    Legal formats are:

    * ``YYYY``
    * ``YYYY,``
    * ``YYYY ``
    * ``YYYY-YYYY,``
    * ``YYYY-YYYY ``

    The final year in the string is replaced with ``replace_year``.
    """
    if len(copyright_line) < 4 or not copyright_line[:4].isdigit():
        return copyright_line

    if copyright_line[4:5] in {'', ' ', ','}:
        return replace_year + copyright_line[4:]

    if copyright_line[4] != '-':
        return copyright_line

    if copyright_line[5:9].isdigit() and copyright_line[9] in ' ,':
        return copyright_line[:5] + replace_year + copyright_line[9:]

    return copyright_line


def check_confval_types(app: Sphinx | None, config: Config) -> None:
    """Check all values for deviation from the default value's type, since
    that can result in TypeErrors all over the place NB.
    """
    for confval in config:
        default, rebuild, annotations = config.values[confval.name]

        if callable(default):
            default = default(config)  # evaluate default value
        if default is None and not annotations:
            continue  # neither inferable nor expliclitly annotated types

        if annotations is Any:
            # any type of value is accepted
            pass
        elif isinstance(annotations, ENUM):
            if not annotations.match(confval.value):
                msg = __("The config value `{name}` has to be a one of {candidates}, "
                         "but `{current}` is given.")
                logger.warning(msg.format(name=confval.name,
                                          current=confval.value,
                                          candidates=annotations.candidates), once=True)
        else:
            if type(confval.value) is type(default):
                continue
            if type(confval.value) in annotations:
                continue

            common_bases = (set(type(confval.value).__bases__ + (type(confval.value),)) &
                            set(type(default).__bases__))
            common_bases.discard(object)
            if common_bases:
                continue  # at least we share a non-trivial base class

            if annotations:
                msg = __("The config value `{name}' has type `{current.__name__}'; "
                         "expected {permitted}.")
                wrapped_annotations = [f"`{c.__name__}'" for c in annotations]
                if len(wrapped_annotations) > 2:
                    permitted = (", ".join(wrapped_annotations[:-1])
                                 + f", or {wrapped_annotations[-1]}")
                else:
                    permitted = " or ".join(wrapped_annotations)
                logger.warning(msg.format(name=confval.name,
                                          current=type(confval.value),
                                          permitted=permitted), once=True)
            else:
                msg = __("The config value `{name}' has type `{current.__name__}', "
                         "defaults to `{default.__name__}'.")
                logger.warning(msg.format(name=confval.name,
                                          current=type(confval.value),
                                          default=type(default)), once=True)


def check_primary_domain(app: Sphinx, config: Config) -> None:
    primary_domain = config.primary_domain
    if primary_domain and not app.registry.has_domain(primary_domain):
        logger.warning(__('primary_domain %r not found, ignored.'), primary_domain)
        config.primary_domain = None  # type: ignore[attr-defined]


def check_root_doc(app: Sphinx, env: BuildEnvironment, added: set[str],
                   changed: set[str], removed: set[str]) -> set[str]:
    """Adjust root_doc to 'contents' to support an old project which does not have
    any root_doc setting.
    """
    if (app.config.root_doc == 'index' and
            'index' not in app.project.docnames and
            'contents' in app.project.docnames):
        logger.warning(__('Since v2.0, Sphinx uses "index" as root_doc by default. '
                          'Please add "root_doc = \'contents\'" to your conf.py.'))
        app.config.root_doc = "contents"  # type: ignore[attr-defined]

    return changed


def setup(app: Sphinx) -> dict[str, Any]:
    app.connect('config-inited', convert_source_suffix, priority=800)
    app.connect('config-inited', convert_highlight_options, priority=800)
    app.connect('config-inited', init_numfig_format, priority=800)
    app.connect('config-inited', correct_copyright_year, priority=800)
    app.connect('config-inited', check_confval_types, priority=800)
    app.connect('config-inited', check_primary_domain, priority=800)
    app.connect('env-get-outdated', check_root_doc)

    return {
        'version': 'builtin',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/config.py
NikolaBlagojevic/pyrecodes
from __future__ import annotations

import re
from os.path import abspath, relpath
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast

from docutils import nodes
from docutils.parsers.rst import directives
from docutils.parsers.rst.directives.admonitions import BaseAdmonition
from docutils.parsers.rst.directives.misc import Class
from docutils.parsers.rst.directives.misc import Include as BaseInclude
from docutils.statemachine import StateMachine

from sphinx import addnodes
from sphinx.domains.changeset import VersionChange  # noqa: F401  # for compatibility
from sphinx.domains.std import StandardDomain
from sphinx.locale import _, __
from sphinx.util import docname_join, logging, url_re
from sphinx.util.docutils import SphinxDirective
from sphinx.util.matching import Matcher, patfilter
from sphinx.util.nodes import explicit_title_re

if TYPE_CHECKING:
    from docutils.nodes import Element, Node

    from sphinx.application import Sphinx
    from sphinx.util.typing import OptionSpec


glob_re = re.compile(r'.*[*?\[].*')
logger = logging.getLogger(__name__)


def int_or_nothing(argument: str) -> int:
    if not argument:
        return 999
    return int(argument)


class TocTree(SphinxDirective):
    """
    Directive to notify Sphinx about the hierarchical structure of the docs,
    and to include a table-of-contents like tree in the current document.
    """
    has_content = True
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec = {
        'maxdepth': int,
        'name': directives.unchanged,
        'caption': directives.unchanged_required,
        'glob': directives.flag,
        'hidden': directives.flag,
        'includehidden': directives.flag,
        'numbered': int_or_nothing,
        'titlesonly': directives.flag,
        'reversed': directives.flag,
    }

    def run(self) -> list[Node]:
        subnode = addnodes.toctree()
        subnode['parent'] = self.env.docname

        # (title, ref) pairs, where ref may be a document, or an external link,
        # and title may be None if the document's title is to be used
        subnode['entries'] = []
        subnode['includefiles'] = []
        subnode['maxdepth'] = self.options.get('maxdepth', -1)
        subnode['caption'] = self.options.get('caption')
        subnode['glob'] = 'glob' in self.options
        subnode['hidden'] = 'hidden' in self.options
        subnode['includehidden'] = 'includehidden' in self.options
        subnode['numbered'] = self.options.get('numbered', 0)
        subnode['titlesonly'] = 'titlesonly' in self.options
        self.set_source_info(subnode)
        wrappernode = nodes.compound(classes=['toctree-wrapper'])
        wrappernode.append(subnode)
        self.add_name(wrappernode)

        ret = self.parse_content(subnode)
        ret.append(wrappernode)
        return ret

    def parse_content(self, toctree: addnodes.toctree) -> list[Node]:
        generated_docnames = frozenset(StandardDomain._virtual_doc_names)
        suffixes = self.config.source_suffix
        current_docname = self.env.docname
        glob = toctree['glob']

        # glob target documents
        all_docnames = self.env.found_docs.copy() | generated_docnames
        all_docnames.remove(current_docname)  # remove current document
        frozen_all_docnames = frozenset(all_docnames)

        ret: list[Node] = []
        excluded = Matcher(self.config.exclude_patterns)
        for entry in self.content:
            if not entry:
                continue

            # look for explicit titles ("Some Title <document>")
            explicit = explicit_title_re.match(entry)
            url_match = url_re.match(entry) is not None
            if glob and glob_re.match(entry) and not explicit and not url_match:
                pat_name = docname_join(current_docname, entry)
                doc_names = sorted(patfilter(all_docnames, pat_name))
                for docname in doc_names:
                    if docname in generated_docnames:
                        # don't include generated documents in globs
                        continue
                    all_docnames.remove(docname)  # don't include it again
                    toctree['entries'].append((None, docname))
                    toctree['includefiles'].append(docname)
                if not doc_names:
                    logger.warning(__("toctree glob pattern %r didn't match any documents"),
                                   entry, location=toctree)
                continue

            if explicit:
                ref = explicit.group(2)
                title = explicit.group(1)
                docname = ref
            else:
                ref = docname = entry
                title = None

            # remove suffixes (backwards compatibility)
            for suffix in suffixes:
                if docname.endswith(suffix):
                    docname = docname.removesuffix(suffix)
                    break

            # absolutise filenames
            docname = docname_join(current_docname, docname)
            if url_match or ref == 'self':
                toctree['entries'].append((title, ref))
                continue

            if docname not in frozen_all_docnames:
                if excluded(self.env.doc2path(docname, False)):
                    message = __('toctree contains reference to excluded document %r')
                    subtype = 'excluded'
                else:
                    message = __('toctree contains reference to nonexisting document %r')
                    subtype = 'not_readable'

                logger.warning(message, docname, type='toc', subtype=subtype,
                               location=toctree)
                self.env.note_reread()
                continue

            if docname in all_docnames:
                all_docnames.remove(docname)
            else:
                logger.warning(__('duplicated entry found in toctree: %s'), docname,
                               location=toctree)

            toctree['entries'].append((title, docname))
            toctree['includefiles'].append(docname)

        # entries contains all entries (self references, external links etc.)
        if 'reversed' in self.options:
            toctree['entries'] = list(reversed(toctree['entries']))
            toctree['includefiles'] = list(reversed(toctree['includefiles']))

        return ret


class Author(SphinxDirective):
    """
    Directive to give the name of the author of the current document
    or section. Shown in the output only if the show_authors option is on.
    """
    has_content = False
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec: OptionSpec = {}

    def run(self) -> list[Node]:
        if not self.config.show_authors:
            return []
        para: Element = nodes.paragraph(translatable=False)
        emph = nodes.emphasis()
        para += emph
        if self.name == 'sectionauthor':
            text = _('Section author: ')
        elif self.name == 'moduleauthor':
            text = _('Module author: ')
        elif self.name == 'codeauthor':
            text = _('Code author: ')
        else:
            text = _('Author: ')
        emph += nodes.Text(text)
        inodes, messages = self.state.inline_text(self.arguments[0], self.lineno)
        emph.extend(inodes)

        ret: list[Node] = [para]
        ret += messages
        return ret


class SeeAlso(BaseAdmonition):
    """
    An admonition mentioning things to look at as reference.
    """
    node_class = addnodes.seealso


class TabularColumns(SphinxDirective):
    """
    Directive to give an explicit tabulary column definition to LaTeX.
    """
    has_content = False
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec: OptionSpec = {}

    def run(self) -> list[Node]:
        node = addnodes.tabular_col_spec()
        node['spec'] = self.arguments[0]
        self.set_source_info(node)
        return [node]


class Centered(SphinxDirective):
    """
    Directive to create a centered line of bold text.
    """
    has_content = False
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec: OptionSpec = {}

    def run(self) -> list[Node]:
        if not self.arguments:
            return []
        subnode: Element = addnodes.centered()
        inodes, messages = self.state.inline_text(self.arguments[0], self.lineno)
        subnode.extend(inodes)

        ret: list[Node] = [subnode]
        ret += messages
        return ret


class Acks(SphinxDirective):
    """
    Directive for a list of names.
    """
    has_content = True
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec: OptionSpec = {}

    def run(self) -> list[Node]:
        node = addnodes.acks()
        node.document = self.state.document
        self.state.nested_parse(self.content, self.content_offset, node)
        if len(node.children) != 1 or not isinstance(node.children[0],
                                                     nodes.bullet_list):
            logger.warning(__('.. acks content is not a list'),
                           location=(self.env.docname, self.lineno))
            return []
        return [node]


class HList(SphinxDirective):
    """
    Directive for a list that gets compacted horizontally.
    """
    has_content = True
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec: OptionSpec = {
        'columns': int,
    }

    def run(self) -> list[Node]:
        ncolumns = self.options.get('columns', 2)
        node = nodes.paragraph()
        node.document = self.state.document
        self.state.nested_parse(self.content, self.content_offset, node)
        if len(node.children) != 1 or not isinstance(node.children[0],
                                                     nodes.bullet_list):
            logger.warning(__('.. hlist content is not a list'),
                           location=(self.env.docname, self.lineno))
            return []
        fulllist = node.children[0]
        # create a hlist node where the items are distributed
        npercol, nmore = divmod(len(fulllist), ncolumns)
        index = 0
        newnode = addnodes.hlist()
        newnode['ncolumns'] = str(ncolumns)
        for column in range(ncolumns):
            endindex = index + ((npercol + 1) if column < nmore else npercol)
            bullet_list = nodes.bullet_list()
            bullet_list += fulllist.children[index:endindex]
            newnode += addnodes.hlistcol('', bullet_list)
            index = endindex
        return [newnode]


class Only(SphinxDirective):
    """
    Directive to only include text if the given tag(s) are enabled.
    """
    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec: OptionSpec = {}

    def run(self) -> list[Node]:
        node = addnodes.only()
        node.document = self.state.document
        self.set_source_info(node)
        node['expr'] = self.arguments[0]

        # Same as util.nested_parse_with_titles but try to handle nested
        # sections which should be raised higher up the doctree.
        memo: Any = self.state.memo
        surrounding_title_styles = memo.title_styles
        surrounding_section_level = memo.section_level
        memo.title_styles = []
        memo.section_level = 0
        try:
            self.state.nested_parse(self.content, self.content_offset,
                                    node, match_titles=True)
            title_styles = memo.title_styles
            if (not surrounding_title_styles or
                    not title_styles or
                    title_styles[0] not in surrounding_title_styles or
                    not self.state.parent):
                # No nested sections so no special handling needed.
                return [node]
            # Calculate the depths of the current and nested sections.
            current_depth = 0
            parent = self.state.parent
            while parent:
                current_depth += 1
                parent = parent.parent
            current_depth -= 2
            title_style = title_styles[0]
            nested_depth = len(surrounding_title_styles)
            if title_style in surrounding_title_styles:
                nested_depth = surrounding_title_styles.index(title_style)
            # Use these depths to determine where the nested sections should
            # be placed in the doctree.
            n_sects_to_raise = current_depth - nested_depth + 1
            parent = cast(nodes.Element, self.state.parent)
            for _i in range(n_sects_to_raise):
                if parent.parent:
                    parent = parent.parent
            parent.append(node)
            return []
        finally:
            memo.title_styles = surrounding_title_styles
            memo.section_level = surrounding_section_level


class Include(BaseInclude, SphinxDirective):
    """
    Like the standard "Include" directive, but interprets absolute paths
    "correctly", i.e. relative to source directory.
    """

    def run(self) -> list[Node]:

        # To properly emit "include-read" events from included RST text,
        # we must patch the ``StateMachine.insert_input()`` method.
        # In the future, docutils will hopefully offer a way for Sphinx
        # to provide the RST parser to use
        # when parsing RST text that comes in via Include directive.
        def _insert_input(include_lines, source):
            # First, we need to combine the lines back into text so that
            # we can send it with the include-read event.
            # In docutils 0.18 and later, there are two lines at the end
            # that act as markers.
            # We must preserve them and leave them out of the include-read event:
            text = "\n".join(include_lines[:-2])

            path = Path(relpath(abspath(source), start=self.env.srcdir))
            docname = self.env.docname

            # Emit the "include-read" event
            arg = [text]
            self.env.app.events.emit('include-read', path, docname, arg)
            text = arg[0]

            # Split back into lines and reattach the two marker lines
            include_lines = text.splitlines() + include_lines[-2:]

            # Call the parent implementation.
            # Note that this snake does not eat its tail because we patch
            # the *Instance* method and this call is to the *Class* method.
            return StateMachine.insert_input(self.state_machine, include_lines, source)

        # Only enable this patch if there are listeners for 'include-read'.
        if self.env.app.events.listeners.get('include-read'):
            # See https://github.com/python/mypy/issues/2427 for details on the mypy issue
            self.state_machine.insert_input = _insert_input  # type: ignore[method-assign]

        if self.arguments[0].startswith('<') and \
           self.arguments[0].endswith('>'):
            # docutils "standard" includes, do not do path processing
            return super().run()
        rel_filename, filename = self.env.relfn2path(self.arguments[0])
        self.arguments[0] = filename
        self.env.note_included(filename)
        return super().run()


def setup(app: Sphinx) -> dict[str, Any]:
    directives.register_directive('toctree', TocTree)
    directives.register_directive('sectionauthor', Author)
    directives.register_directive('moduleauthor', Author)
    directives.register_directive('codeauthor', Author)
    directives.register_directive('seealso', SeeAlso)
    directives.register_directive('tabularcolumns', TabularColumns)
    directives.register_directive('centered', Centered)
    directives.register_directive('acks', Acks)
    directives.register_directive('hlist', HList)
    directives.register_directive('only', Only)
    directives.register_directive('include', Include)

    # register the standard rst class directive under a different name
    # only for backwards compatibility now
    directives.register_directive('cssclass', Class)
    # new standard name when default-domain with "class" is in effect
    directives.register_directive('rst-class', Class)

    return {
        'version': 'builtin',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/directives/other.py
NikolaBlagojevic/pyrecodes
"""The JavaScript domain."""

from __future__ import annotations

import contextlib
from typing import TYPE_CHECKING, Any, cast

from docutils import nodes
from docutils.parsers.rst import directives

from sphinx import addnodes
from sphinx.directives import ObjectDescription
from sphinx.domains import Domain, ObjType
from sphinx.domains.python import _pseudo_parse_arglist
from sphinx.locale import _, __
from sphinx.roles import XRefRole
from sphinx.util import logging
from sphinx.util.docfields import Field, GroupedField, TypedField
from sphinx.util.docutils import SphinxDirective
from sphinx.util.nodes import make_id, make_refnode, nested_parse_with_titles

if TYPE_CHECKING:
    from collections.abc import Iterator

    from docutils.nodes import Element, Node

    from sphinx.addnodes import desc_signature, pending_xref
    from sphinx.application import Sphinx
    from sphinx.builders import Builder
    from sphinx.environment import BuildEnvironment
    from sphinx.util.typing import OptionSpec

logger = logging.getLogger(__name__)


class JSObject(ObjectDescription[tuple[str, str]]):
    """
    Description of a JavaScript object.
    """
    #: If set to ``True`` this object is callable and a `desc_parameterlist` is
    #: added
    has_arguments = False

    #: If ``allow_nesting`` is ``True``, the object prefixes will be accumulated
    #: based on directive nesting
    allow_nesting = False

    option_spec: OptionSpec = {
        'no-index': directives.flag,
        'no-index-entry': directives.flag,
        'no-contents-entry': directives.flag,
        'no-typesetting': directives.flag,
        'noindex': directives.flag,
        'noindexentry': directives.flag,
        'nocontentsentry': directives.flag,
        'single-line-parameter-list': directives.flag,
    }

    def get_display_prefix(self) -> list[Node]:
        #: what is displayed right before the documentation entry
        return []

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        """Breaks down construct signatures

        Parses out prefix and argument list from construct definition. The
        namespace and class will be determined by the nesting of domain
        directives.
        """
        sig = sig.strip()
        if '(' in sig and sig[-1:] == ')':
            member, arglist = sig.split('(', 1)
            member = member.strip()
            arglist = arglist[:-1].strip()
        else:
            member = sig
            arglist = None
        # If construct is nested, prefix the current prefix
        prefix = self.env.ref_context.get('js:object', None)
        mod_name = self.env.ref_context.get('js:module')

        name = member
        try:
            member_prefix, member_name = member.rsplit('.', 1)
        except ValueError:
            member_name = name
            member_prefix = ''
        finally:
            name = member_name
            if prefix and member_prefix:
                prefix = '.'.join([prefix, member_prefix])
            elif prefix is None and member_prefix:
                prefix = member_prefix
        fullname = name
        if prefix:
            fullname = '.'.join([prefix, name])

        signode['module'] = mod_name
        signode['object'] = prefix
        signode['fullname'] = fullname

        max_len = (self.env.config.javascript_maximum_signature_line_length
                   or self.env.config.maximum_signature_line_length
                   or 0)
        multi_line_parameter_list = (
            'single-line-parameter-list' not in self.options
            and (len(sig) > max_len > 0)
        )

        display_prefix = self.get_display_prefix()
        if display_prefix:
            signode += addnodes.desc_annotation('', '', *display_prefix)

        actual_prefix = None
        if prefix:
            actual_prefix = prefix
        elif mod_name:
            actual_prefix = mod_name
        if actual_prefix:
            addName = addnodes.desc_addname('', '')
            for p in actual_prefix.split('.'):
                addName += addnodes.desc_sig_name(p, p)
                addName += addnodes.desc_sig_punctuation('.', '.')
            signode += addName
        signode += addnodes.desc_name('', '', addnodes.desc_sig_name(name, name))
        if self.has_arguments:
            if not arglist:
                signode += addnodes.desc_parameterlist()
            else:
                _pseudo_parse_arglist(signode, arglist, multi_line_parameter_list)
        return fullname, prefix

    def _object_hierarchy_parts(self, sig_node: desc_signature) -> tuple[str, ...]:
        if 'fullname' not in sig_node:
            return ()
        modname = sig_node.get('module')
        fullname = sig_node['fullname']

        if modname:
            return (modname, *fullname.split('.'))
        else:
            return tuple(fullname.split('.'))

    def add_target_and_index(self, name_obj: tuple[str, str], sig: str,
                             signode: desc_signature) -> None:
        mod_name = self.env.ref_context.get('js:module')
        fullname = (mod_name + '.' if mod_name else '') + name_obj[0]
        node_id = make_id(self.env, self.state.document, '', fullname)
        signode['ids'].append(node_id)
        self.state.document.note_explicit_target(signode)

        domain = cast(JavaScriptDomain, self.env.get_domain('js'))
        domain.note_object(fullname, self.objtype, node_id, location=signode)

        if 'no-index-entry' not in self.options:
            indextext = self.get_index_text(mod_name, name_obj)  # type: ignore[arg-type]
            if indextext:
                self.indexnode['entries'].append(('single', indextext, node_id, '', None))

    def get_index_text(self, objectname: str, name_obj: tuple[str, str]) -> str:
        name, obj = name_obj
        if self.objtype == 'function':
            if not obj:
                return _('%s() (built-in function)') % name
            return _('%s() (%s method)') % (name, obj)
        elif self.objtype == 'class':
            return _('%s() (class)') % name
        elif self.objtype == 'data':
            return _('%s (global variable or constant)') % name
        elif self.objtype == 'attribute':
            return _('%s (%s attribute)') % (name, obj)
        return ''

    def before_content(self) -> None:
        """Handle object nesting before content

        :py:class:`JSObject` represents JavaScript language constructs. For
        constructs that are nestable, this method will build up a stack of the
        nesting hierarchy so that it can be later de-nested correctly, in
        :py:meth:`after_content`.

        For constructs that aren't nestable, the stack is bypassed, and instead
        only the most recent object is tracked. This object prefix name will be
        removed with :py:meth:`after_content`.

        The following keys are used in ``self.env.ref_context``:

            js:objects
                Stores the object prefix history. With each nested element, we
                add the object prefix to this list. When we exit that object's
                nesting level, :py:meth:`after_content` is triggered and the
                prefix is removed from the end of the list.

            js:object
                Current object prefix. This should generally reflect the last
                element in the prefix history
        """
        prefix = None
        if self.names:
            (obj_name, obj_name_prefix) = self.names.pop()
            prefix = obj_name_prefix.strip('.') if obj_name_prefix else None
            if self.allow_nesting:
                prefix = obj_name
        if prefix:
            self.env.ref_context['js:object'] = prefix
            if self.allow_nesting:
                objects = self.env.ref_context.setdefault('js:objects', [])
                objects.append(prefix)

    def after_content(self) -> None:
        """Handle object de-nesting after content

        If this class is a nestable object, removing the last nested class prefix
        ends further nesting in the object.

        If this class is not a nestable object, the list of classes should not
        be altered as we didn't affect the nesting levels in
        :py:meth:`before_content`.
        """
        objects = self.env.ref_context.setdefault('js:objects', [])
        if self.allow_nesting:
            with contextlib.suppress(IndexError):
                objects.pop()

        self.env.ref_context['js:object'] = (objects[-1] if len(objects) > 0
                                             else None)

    def _toc_entry_name(self, sig_node: desc_signature) -> str:
        if not sig_node.get('_toc_parts'):
            return ''

        config = self.env.app.config
        objtype = sig_node.parent.get('objtype')
        if config.add_function_parentheses and objtype in {'function', 'method'}:
            parens = '()'
        else:
            parens = ''
        *parents, name = sig_node['_toc_parts']
        if config.toc_object_entries_show_parents == 'domain':
            return sig_node.get('fullname', name) + parens
        if config.toc_object_entries_show_parents == 'hide':
            return name + parens
        if config.toc_object_entries_show_parents == 'all':
            return '.'.join(parents + [name + parens])
        return ''


class JSCallable(JSObject):
    """Description of a JavaScript function, method or constructor."""
    has_arguments = True

    doc_field_types = [
        TypedField('arguments', label=_('Arguments'),
                   names=('argument', 'arg', 'parameter', 'param'),
                   typerolename='func', typenames=('paramtype', 'type')),
        GroupedField('errors', label=_('Throws'), rolename='func',
                     names=('throws', ),
                     can_collapse=True),
        Field('returnvalue', label=_('Returns'), has_arg=False,
              names=('returns', 'return')),
        Field('returntype', label=_('Return type'), has_arg=False,
              names=('rtype',)),
    ]


class JSConstructor(JSCallable):
    """Like a callable but with a different prefix."""

    allow_nesting = True

    def get_display_prefix(self) -> list[Node]:
        return [addnodes.desc_sig_keyword('class', 'class'),
                addnodes.desc_sig_space()]


class JSModule(SphinxDirective):
    """
    Directive to mark description of a new JavaScript module.

    This directive specifies the module name that will be used by objects that
    follow this directive.

    Options
    -------

    no-index
        If the ``:no-index:`` option is specified, no linkable elements will be
        created, and the module won't be added to the global module index. This
        is useful for splitting up the module definition across multiple
        sections or files.

    :param mod_name: Module name
    """

    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec: OptionSpec = {
        'no-index': directives.flag,
        'no-contents-entry': directives.flag,
        'no-typesetting': directives.flag,
        'noindex': directives.flag,
        'nocontentsentry': directives.flag,
    }

    def run(self) -> list[Node]:
        mod_name = self.arguments[0].strip()
        self.env.ref_context['js:module'] = mod_name
        no_index = 'no-index' in self.options or 'noindex' in self.options

        content_node: Element = nodes.section()
        # necessary so that the child nodes get the right source/line set
        content_node.document = self.state.document
        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)

        ret: list[Node] = []
        if not no_index:
            domain = cast(JavaScriptDomain, self.env.get_domain('js'))

            node_id = make_id(self.env, self.state.document, 'module', mod_name)
            domain.note_module(mod_name, node_id)
            # Make a duplicate entry in 'objects' to facilitate searching for
            # the module in JavaScriptDomain.find_obj()
            domain.note_object(mod_name, 'module', node_id,
                               location=(self.env.docname, self.lineno))

            # The node order is: index node first, then target node
            indextext = _('%s (module)') % mod_name
            inode = addnodes.index(entries=[('single', indextext, node_id, '', None)])
            ret.append(inode)
            target = nodes.target('', '', ids=[node_id], ismod=True)
            self.state.document.note_explicit_target(target)
            ret.append(target)
        ret.extend(content_node.children)
        return ret


class JSXRefRole(XRefRole):
    def process_link(self, env: BuildEnvironment, refnode: Element,
                     has_explicit_title: bool, title: str, target: str) -> tuple[str, str]:
        # basically what sphinx.domains.python.PyXRefRole does
        refnode['js:object'] = env.ref_context.get('js:object')
        refnode['js:module'] = env.ref_context.get('js:module')
        if not has_explicit_title:
            title = title.lstrip('.')
            target = target.lstrip('~')
            if title[0:1] == '~':
                title = title[1:]
                dot = title.rfind('.')
                if dot != -1:
                    title = title[dot + 1:]
        if target[0:1] == '.':
            target = target[1:]
            refnode['refspecific'] = True
        return title, target


class JavaScriptDomain(Domain):
    """JavaScript language domain."""
    name = 'js'
    label = 'JavaScript'
    # if you add a new object type make sure to edit JSObject.get_index_string
    object_types = {
        'function':  ObjType(_('function'),  'func'),
        'method':    ObjType(_('method'),    'meth'),
        'class':     ObjType(_('class'),     'class'),
        'data':      ObjType(_('data'),      'data'),
        'attribute': ObjType(_('attribute'), 'attr'),
        'module':    ObjType(_('module'),    'mod'),
    }
    directives = {
        'function':  JSCallable,
        'method':    JSCallable,
        'class':     JSConstructor,
        'data':      JSObject,
        'attribute': JSObject,
        'module':    JSModule,
    }
    roles = {
        'func':  JSXRefRole(fix_parens=True),
        'meth':  JSXRefRole(fix_parens=True),
        'class': JSXRefRole(fix_parens=True),
        'data':  JSXRefRole(),
        'attr':  JSXRefRole(),
        'mod':   JSXRefRole(),
    }
    initial_data: dict[str, dict[str, tuple[str, str]]] = {
        'objects': {},  # fullname -> docname, node_id, objtype
        'modules': {},  # modname  -> docname, node_id
    }

    @property
    def objects(self) -> dict[str, tuple[str, str, str]]:
        return self.data.setdefault('objects', {})  # fullname -> docname, node_id, objtype

    def note_object(self, fullname: str, objtype: str, node_id: str,
                    location: Any = None) -> None:
        if fullname in self.objects:
            docname = self.objects[fullname][0]
            logger.warning(__('duplicate %s description of %s, other %s in %s'),
                           objtype, fullname, objtype, docname, location=location)
        self.objects[fullname] = (self.env.docname, node_id, objtype)

    @property
    def modules(self) -> dict[str, tuple[str, str]]:
        return self.data.setdefault('modules', {})  # modname -> docname, node_id

    def note_module(self, modname: str, node_id: str) -> None:
        self.modules[modname] = (self.env.docname, node_id)

    def clear_doc(self, docname: str) -> None:
        for fullname, (pkg_docname, _node_id, _l) in list(self.objects.items()):
            if pkg_docname == docname:
                del self.objects[fullname]
        for modname, (pkg_docname, _node_id) in list(self.modules.items()):
            if pkg_docname == docname:
                del self.modules[modname]

    def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
        # XXX check duplicates
        for fullname, (fn, node_id, objtype) in otherdata['objects'].items():
            if fn in docnames:
                self.objects[fullname] = (fn, node_id, objtype)
        for mod_name, (pkg_docname, node_id) in otherdata['modules'].items():
            if pkg_docname in docnames:
                self.modules[mod_name] = (pkg_docname, node_id)

    def find_obj(
        self,
        env: BuildEnvironment,
        mod_name: str,
        prefix: str,
        name: str,
        typ: str | None,
        searchorder: int = 0,
    ) -> tuple[str | None, tuple[str, str, str] | None]:
        if name[-2:] == '()':
            name = name[:-2]

        searches = []
        if mod_name and prefix:
            searches.append('.'.join([mod_name, prefix, name]))
        if mod_name:
            searches.append('.'.join([mod_name, name]))
        if prefix:
            searches.append('.'.join([prefix, name]))
        searches.append(name)

        if searchorder == 0:
            searches.reverse()

        newname = None
        object_ = None
        for search_name in searches:
            if search_name in self.objects:
                newname = search_name
                object_ = self.objects[search_name]

        return newname, object_

    def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
                     typ: str, target: str, node: pending_xref, contnode: Element,
                     ) -> Element | None:
        mod_name = node.get('js:module')
        prefix = node.get('js:object')
        searchorder = 1 if node.hasattr('refspecific') else 0
        name, obj = self.find_obj(env, mod_name, prefix, target, typ, searchorder)
        if not obj:
            return None
        return make_refnode(builder, fromdocname, obj[0], obj[1], contnode, name)

    def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
                         target: str, node: pending_xref, contnode: Element,
                         ) -> list[tuple[str, Element]]:
        mod_name = node.get('js:module')
        prefix = node.get('js:object')
        name, obj = self.find_obj(env, mod_name, prefix, target, None, 1)
        if not obj:
            return []
        return [('js:' + self.role_for_objtype(obj[2]),  # type: ignore[operator]
                 make_refnode(builder, fromdocname, obj[0], obj[1], contnode, name))]

    def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]:
        for refname, (docname, node_id, typ) in list(self.objects.items()):
            yield refname, refname, typ, docname, node_id, 1

    def get_full_qualified_name(self, node: Element) -> str | None:
        modname = node.get('js:module')
        prefix = node.get('js:object')
        target = node.get('reftarget')
        if target is None:
            return None
        else:
            return '.'.join(filter(None, [modname, prefix, target]))


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_domain(JavaScriptDomain)
    app.add_config_value(
        'javascript_maximum_signature_line_length', None, 'env', types={int, None},
    )
    return {
        'version': 'builtin',
        'env_version': 3,
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/domains/javascript.py
NikolaBlagojevic/pyrecodes
"""The Python domain."""

from __future__ import annotations

import ast
import builtins
import contextlib
import inspect
import re
import token
import typing
from inspect import Parameter
from typing import TYPE_CHECKING, Any, NamedTuple, cast

from docutils import nodes
from docutils.parsers.rst import directives

from sphinx import addnodes
from sphinx.addnodes import desc_signature, pending_xref, pending_xref_condition
from sphinx.directives import ObjectDescription
from sphinx.domains import Domain, Index, IndexEntry, ObjType
from sphinx.locale import _, __
from sphinx.pycode.parser import Token, TokenProcessor
from sphinx.roles import XRefRole
from sphinx.util import logging
from sphinx.util.docfields import Field, GroupedField, TypedField
from sphinx.util.docutils import SphinxDirective
from sphinx.util.inspect import signature_from_str
from sphinx.util.nodes import (
    find_pending_xref_condition,
    make_id,
    make_refnode,
    nested_parse_with_titles,
)

if TYPE_CHECKING:
    from collections.abc import Iterable, Iterator

    from docutils.nodes import Element, Node
    from docutils.parsers.rst.states import Inliner

    from sphinx.application import Sphinx
    from sphinx.builders import Builder
    from sphinx.environment import BuildEnvironment
    from sphinx.util.typing import OptionSpec, TextlikeNode

logger = logging.getLogger(__name__)


# REs for Python signatures
py_sig_re = re.compile(
    r'''^ ([\w.]*\.)?            # class name(s)
          (\w+)  \s*             # thing name
          (?: \[\s*(.*)\s*])?    # optional: type parameters list
          (?: \(\s*(.*)\s*\)     # optional: arguments
           (?:\s* -> \s* (.*))?  #           return annotation
          )? $                   # and nothing more
          ''', re.VERBOSE)


pairindextypes = {
    'module': 'module',
    'keyword': 'keyword',
    'operator': 'operator',
    'object': 'object',
    'exception': 'exception',
    'statement': 'statement',
    'builtin': 'built-in function',
}


class ObjectEntry(NamedTuple):
    docname: str
    node_id: str
    objtype: str
    aliased: bool


class ModuleEntry(NamedTuple):
    docname: str
    node_id: str
    synopsis: str
    platform: str
    deprecated: bool


def parse_reftarget(reftarget: str, suppress_prefix: bool = False,
                    ) -> tuple[str, str, str, bool]:
    """Parse a type string and return (reftype, reftarget, title, refspecific flag)"""
    refspecific = False
    if reftarget.startswith('.'):
        reftarget = reftarget[1:]
        title = reftarget
        refspecific = True
    elif reftarget.startswith('~'):
        reftarget = reftarget[1:]
        title = reftarget.split('.')[-1]
    elif suppress_prefix:
        title = reftarget.split('.')[-1]
    elif reftarget.startswith('typing.'):
        title = reftarget[7:]
    else:
        title = reftarget

    if reftarget == 'None' or reftarget.startswith('typing.'):
        # typing module provides non-class types.  Obj reference is good to refer them.
        reftype = 'obj'
    else:
        reftype = 'class'

    return reftype, reftarget, title, refspecific


def type_to_xref(target: str, env: BuildEnvironment, *,
                 suppress_prefix: bool = False) -> addnodes.pending_xref:
    """Convert a type string to a cross reference node."""
    if env:
        kwargs = {'py:module': env.ref_context.get('py:module'),
                  'py:class': env.ref_context.get('py:class')}
    else:
        kwargs = {}

    reftype, target, title, refspecific = parse_reftarget(target, suppress_prefix)

    if env.config.python_use_unqualified_type_names:
        # Note: It would be better to use qualname to describe the object to support support
        # nested classes.  But python domain can't access the real python object because this
        # module should work not-dynamically.
        shortname = title.split('.')[-1]
        contnodes: list[Node] = [pending_xref_condition('', shortname, condition='resolved'),
                                 pending_xref_condition('', title, condition='*')]
    else:
        contnodes = [nodes.Text(title)]

    return pending_xref('', *contnodes,
                        refdomain='py', reftype=reftype, reftarget=target,
                        refspecific=refspecific, **kwargs)


def _parse_annotation(annotation: str, env: BuildEnvironment) -> list[Node]:
    """Parse type annotation."""
    short_literals = env.config.python_display_short_literal_types

    def unparse(node: ast.AST) -> list[Node]:
        if isinstance(node, ast.Attribute):
            return [nodes.Text(f"{unparse(node.value)[0]}.{node.attr}")]
        if isinstance(node, ast.BinOp):
            result: list[Node] = unparse(node.left)
            result.extend(unparse(node.op))
            result.extend(unparse(node.right))
            return result
        if isinstance(node, ast.BitOr):
            return [addnodes.desc_sig_space(),
                    addnodes.desc_sig_punctuation('', '|'),
                    addnodes.desc_sig_space()]
        if isinstance(node, ast.Constant):
            if node.value is Ellipsis:
                return [addnodes.desc_sig_punctuation('', "...")]
            if isinstance(node.value, bool):
                return [addnodes.desc_sig_keyword('', repr(node.value))]
            if isinstance(node.value, int):
                return [addnodes.desc_sig_literal_number('', repr(node.value))]
            if isinstance(node.value, str):
                return [addnodes.desc_sig_literal_string('', repr(node.value))]
            else:
                # handles None, which is further handled by type_to_xref later
                # and fallback for other types that should be converted
                return [nodes.Text(repr(node.value))]
        if isinstance(node, ast.Expr):
            return unparse(node.value)
        if isinstance(node, ast.Invert):
            return [addnodes.desc_sig_punctuation('', '~')]
        if isinstance(node, ast.List):
            result = [addnodes.desc_sig_punctuation('', '[')]
            if node.elts:
                # check if there are elements in node.elts to only pop the
                # last element of result if the for-loop was run at least
                # once
                for elem in node.elts:
                    result.extend(unparse(elem))
                    result.append(addnodes.desc_sig_punctuation('', ','))
                    result.append(addnodes.desc_sig_space())
                result.pop()
                result.pop()
            result.append(addnodes.desc_sig_punctuation('', ']'))
            return result
        if isinstance(node, ast.Module):
            return sum((unparse(e) for e in node.body), [])
        if isinstance(node, ast.Name):
            return [nodes.Text(node.id)]
        if isinstance(node, ast.Subscript):
            if getattr(node.value, 'id', '') in {'Optional', 'Union'}:
                return _unparse_pep_604_annotation(node)
            if short_literals and getattr(node.value, 'id', '') == 'Literal':
                return _unparse_pep_604_annotation(node)
            result = unparse(node.value)
            result.append(addnodes.desc_sig_punctuation('', '['))
            result.extend(unparse(node.slice))
            result.append(addnodes.desc_sig_punctuation('', ']'))

            # Wrap the Text nodes inside brackets by literal node if the subscript is a Literal
            if result[0] in ('Literal', 'typing.Literal'):
                for i, subnode in enumerate(result[1:], start=1):
                    if isinstance(subnode, nodes.Text):
                        result[i] = nodes.literal('', '', subnode)
            return result
        if isinstance(node, ast.UnaryOp):
            return unparse(node.op) + unparse(node.operand)
        if isinstance(node, ast.Tuple):
            if node.elts:
                result = []
                for elem in node.elts:
                    result.extend(unparse(elem))
                    result.append(addnodes.desc_sig_punctuation('', ','))
                    result.append(addnodes.desc_sig_space())
                result.pop()
                result.pop()
            else:
                result = [addnodes.desc_sig_punctuation('', '('),
                          addnodes.desc_sig_punctuation('', ')')]

            return result
        raise SyntaxError  # unsupported syntax

    def _unparse_pep_604_annotation(node: ast.Subscript) -> list[Node]:
        subscript = node.slice

        flattened: list[Node] = []
        if isinstance(subscript, ast.Tuple):
            flattened.extend(unparse(subscript.elts[0]))
            for elt in subscript.elts[1:]:
                flattened.extend(unparse(ast.BitOr()))
                flattened.extend(unparse(elt))
        else:
            # e.g. a Union[] inside an Optional[]
            flattened.extend(unparse(subscript))

        if getattr(node.value, 'id', '') == 'Optional':
            flattened.extend(unparse(ast.BitOr()))
            flattened.append(nodes.Text('None'))

        return flattened

    try:
        tree = ast.parse(annotation, type_comments=True)
        result: list[Node] = []
        for node in unparse(tree):
            if isinstance(node, nodes.literal):
                result.append(node[0])
            elif isinstance(node, nodes.Text) and node.strip():
                if (result and isinstance(result[-1], addnodes.desc_sig_punctuation) and
                        result[-1].astext() == '~'):
                    result.pop()
                    result.append(type_to_xref(str(node), env, suppress_prefix=True))
                else:
                    result.append(type_to_xref(str(node), env))
            else:
                result.append(node)
        return result
    except SyntaxError:
        return [type_to_xref(annotation, env)]


class _TypeParameterListParser(TokenProcessor):
    def __init__(self, sig: str) -> None:
        signature = sig.replace('\n', '').strip()
        super().__init__([signature])
        # Each item is a tuple (name, kind, default, annotation) mimicking
        # ``inspect.Parameter`` to allow default values on VAR_POSITIONAL
        # or VAR_KEYWORD parameters.
        self.type_params: list[tuple[str, int, Any, Any]] = []

    def fetch_type_param_spec(self) -> list[Token]:
        tokens = []
        while current := self.fetch_token():
            tokens.append(current)
            for ldelim, rdelim in ('(', ')'), ('{', '}'), ('[', ']'):
                if current == [token.OP, ldelim]:
                    tokens += self.fetch_until([token.OP, rdelim])
                    break
            else:
                if current == token.INDENT:
                    tokens += self.fetch_until(token.DEDENT)
                elif current.match(
                        [token.OP, ':'], [token.OP, '='], [token.OP, ',']):
                    tokens.pop()
                    break
        return tokens

    def parse(self) -> None:
        while current := self.fetch_token():
            if current == token.NAME:
                tp_name = current.value.strip()
                if self.previous and self.previous.match([token.OP, '*'], [token.OP, '**']):
                    if self.previous == [token.OP, '*']:
                        tp_kind = Parameter.VAR_POSITIONAL
                    else:
                        tp_kind = Parameter.VAR_KEYWORD  # type: ignore[assignment]
                else:
                    tp_kind = Parameter.POSITIONAL_OR_KEYWORD  # type: ignore[assignment]

                tp_ann: Any = Parameter.empty
                tp_default: Any = Parameter.empty

                current = self.fetch_token()
                if current and current.match([token.OP, ':'], [token.OP, '=']):
                    if current == [token.OP, ':']:
                        tokens = self.fetch_type_param_spec()
                        tp_ann = self._build_identifier(tokens)

                    if self.current and self.current == [token.OP, '=']:
                        tokens = self.fetch_type_param_spec()
                        tp_default = self._build_identifier(tokens)

                if tp_kind != Parameter.POSITIONAL_OR_KEYWORD and tp_ann != Parameter.empty:
                    msg = ('type parameter bound or constraint is not allowed '
                           f'for {tp_kind.description} parameters')
                    raise SyntaxError(msg)

                type_param = (tp_name, tp_kind, tp_default, tp_ann)
                self.type_params.append(type_param)

    def _build_identifier(self, tokens: list[Token]) -> str:
        from itertools import chain, tee

        def pairwise(iterable):
            a, b = tee(iterable)
            next(b, None)
            return zip(a, b)

        def triplewise(iterable):
            for (a, _z), (b, c) in pairwise(pairwise(iterable)):
                yield a, b, c

        idents: list[str] = []
        tokens: Iterable[Token] = iter(tokens)  # type: ignore[no-redef]
        # do not format opening brackets
        for tok in tokens:
            if not tok.match([token.OP, '('], [token.OP, '['], [token.OP, '{']):
                # check if the first non-delimiter character is an unpack operator
                is_unpack_operator = tok.match([token.OP, '*'], [token.OP, ['**']])
                idents.append(self._pformat_token(tok, native=is_unpack_operator))
                break
            idents.append(tok.value)

        # check the remaining tokens
        stop = Token(token.ENDMARKER, '', (-1, -1), (-1, -1), '<sentinel>')
        is_unpack_operator = False
        for tok, op, after in triplewise(chain(tokens, [stop, stop])):
            ident = self._pformat_token(tok, native=is_unpack_operator)
            idents.append(ident)
            # determine if the next token is an unpack operator depending
            # on the left and right hand side of the operator symbol
            is_unpack_operator = (
                op.match([token.OP, '*'], [token.OP, '**']) and not (
                    tok.match(token.NAME, token.NUMBER, token.STRING,
                              [token.OP, ')'], [token.OP, ']'], [token.OP, '}'])
                    and after.match(token.NAME, token.NUMBER, token.STRING,
                                    [token.OP, '('], [token.OP, '['], [token.OP, '{'])
                )
            )

        return ''.join(idents).strip()

    def _pformat_token(self, tok: Token, native: bool = False) -> str:
        if native:
            return tok.value

        if tok.match(token.NEWLINE, token.ENDMARKER):
            return ''

        if tok.match([token.OP, ':'], [token.OP, ','], [token.OP, '#']):
            return f'{tok.value} '

        # Arithmetic operators are allowed because PEP 695 specifies the
        # default type parameter to be *any* expression (so "T1 << T2" is
        # allowed if it makes sense). The caller is responsible to ensure
        # that a multiplication operator ("*") is not to be confused with
        # an unpack operator (which will not be surrounded by spaces).
        #
        # The operators are ordered according to how likely they are to
        # be used and for (possible) future implementations (e.g., "&" for
        # an intersection type).
        if tok.match(
            # Most likely operators to appear
            [token.OP, '='], [token.OP, '|'],
            # Type composition (future compatibility)
            [token.OP, '&'], [token.OP, '^'], [token.OP, '<'], [token.OP, '>'],
            # Unlikely type composition
            [token.OP, '+'], [token.OP, '-'], [token.OP, '*'], [token.OP, '**'],
            # Unlikely operators but included for completeness
            [token.OP, '@'], [token.OP, '/'], [token.OP, '//'], [token.OP, '%'],
            [token.OP, '<<'], [token.OP, '>>'], [token.OP, '>>>'],
            [token.OP, '<='], [token.OP, '>='], [token.OP, '=='], [token.OP, '!='],
        ):
            return f' {tok.value} '

        return tok.value


def _parse_type_list(
    tp_list: str, env: BuildEnvironment,
    multi_line_parameter_list: bool = False,
) -> addnodes.desc_type_parameter_list:
    """Parse a list of type parameters according to PEP 695."""
    type_params = addnodes.desc_type_parameter_list(tp_list)
    type_params['multi_line_parameter_list'] = multi_line_parameter_list
    # formal parameter names are interpreted as type parameter names and
    # type annotations are interpreted as type parameter bound or constraints
    parser = _TypeParameterListParser(tp_list)
    parser.parse()
    for (tp_name, tp_kind, tp_default, tp_ann) in parser.type_params:
        # no positional-only or keyword-only allowed in a type parameters list
        if tp_kind in {Parameter.POSITIONAL_ONLY, Parameter.KEYWORD_ONLY}:
            msg = ('positional-only or keyword-only parameters '
                   'are prohibited in type parameter lists')
            raise SyntaxError(msg)

        node = addnodes.desc_type_parameter()
        if tp_kind == Parameter.VAR_POSITIONAL:
            node += addnodes.desc_sig_operator('', '*')
        elif tp_kind == Parameter.VAR_KEYWORD:
            node += addnodes.desc_sig_operator('', '**')
        node += addnodes.desc_sig_name('', tp_name)

        if tp_ann is not Parameter.empty:
            annotation = _parse_annotation(tp_ann, env)
            if not annotation:
                continue

            node += addnodes.desc_sig_punctuation('', ':')
            node += addnodes.desc_sig_space()

            type_ann_expr = addnodes.desc_sig_name('', '',
                                                   *annotation)  # type: ignore[arg-type]
            # a type bound is ``T: U`` whereas type constraints
            # must be enclosed with parentheses. ``T: (U, V)``
            if tp_ann.startswith('(') and tp_ann.endswith(')'):
                type_ann_text = type_ann_expr.astext()
                if type_ann_text.startswith('(') and type_ann_text.endswith(')'):
                    node += type_ann_expr
                else:
                    # surrounding braces are lost when using _parse_annotation()
                    node += addnodes.desc_sig_punctuation('', '(')
                    node += type_ann_expr  # type constraint
                    node += addnodes.desc_sig_punctuation('', ')')
            else:
                node += type_ann_expr  # type bound

        if tp_default is not Parameter.empty:
            # Always surround '=' with spaces, even if there is no annotation
            node += addnodes.desc_sig_space()
            node += addnodes.desc_sig_operator('', '=')
            node += addnodes.desc_sig_space()
            node += nodes.inline('', tp_default,
                                 classes=['default_value'],
                                 support_smartquotes=False)

        type_params += node
    return type_params


def _parse_arglist(
    arglist: str, env: BuildEnvironment, multi_line_parameter_list: bool = False,
) -> addnodes.desc_parameterlist:
    """Parse a list of arguments using AST parser"""
    params = addnodes.desc_parameterlist(arglist)
    params['multi_line_parameter_list'] = multi_line_parameter_list
    sig = signature_from_str('(%s)' % arglist)
    last_kind = None
    for param in sig.parameters.values():
        if param.kind != param.POSITIONAL_ONLY and last_kind == param.POSITIONAL_ONLY:
            # PEP-570: Separator for Positional Only Parameter: /
            params += addnodes.desc_parameter('', '', addnodes.desc_sig_operator('', '/'))
        if param.kind == param.KEYWORD_ONLY and last_kind in (param.POSITIONAL_OR_KEYWORD,
                                                              param.POSITIONAL_ONLY,
                                                              None):
            # PEP-3102: Separator for Keyword Only Parameter: *
            params += addnodes.desc_parameter('', '', addnodes.desc_sig_operator('', '*'))

        node = addnodes.desc_parameter()
        if param.kind == param.VAR_POSITIONAL:
            node += addnodes.desc_sig_operator('', '*')
            node += addnodes.desc_sig_name('', param.name)
        elif param.kind == param.VAR_KEYWORD:
            node += addnodes.desc_sig_operator('', '**')
            node += addnodes.desc_sig_name('', param.name)
        else:
            node += addnodes.desc_sig_name('', param.name)

        if param.annotation is not param.empty:
            children = _parse_annotation(param.annotation, env)
            node += addnodes.desc_sig_punctuation('', ':')
            node += addnodes.desc_sig_space()
            node += addnodes.desc_sig_name('', '', *children)  # type: ignore[arg-type]
        if param.default is not param.empty:
            if param.annotation is not param.empty:
                node += addnodes.desc_sig_space()
                node += addnodes.desc_sig_operator('', '=')
                node += addnodes.desc_sig_space()
            else:
                node += addnodes.desc_sig_operator('', '=')
            node += nodes.inline('', param.default, classes=['default_value'],
                                 support_smartquotes=False)

        params += node
        last_kind = param.kind

    if last_kind == Parameter.POSITIONAL_ONLY:
        # PEP-570: Separator for Positional Only Parameter: /
        params += addnodes.desc_parameter('', '', addnodes.desc_sig_operator('', '/'))

    return params


def _pseudo_parse_arglist(
    signode: desc_signature, arglist: str, multi_line_parameter_list: bool = False,
) -> None:
    """"Parse" a list of arguments separated by commas.

    Arguments can have "optional" annotations given by enclosing them in
    brackets.  Currently, this will split at any comma, even if it's inside a
    string literal (e.g. default argument value).
    """
    paramlist = addnodes.desc_parameterlist()
    paramlist['multi_line_parameter_list'] = multi_line_parameter_list
    stack: list[Element] = [paramlist]
    try:
        for argument in arglist.split(','):
            argument = argument.strip()
            ends_open = ends_close = 0
            while argument.startswith('['):
                stack.append(addnodes.desc_optional())
                stack[-2] += stack[-1]
                argument = argument[1:].strip()
            while argument.startswith(']'):
                stack.pop()
                argument = argument[1:].strip()
            while argument.endswith(']') and not argument.endswith('[]'):
                ends_close += 1
                argument = argument[:-1].strip()
            while argument.endswith('['):
                ends_open += 1
                argument = argument[:-1].strip()
            if argument:
                stack[-1] += addnodes.desc_parameter(
                    '', '', addnodes.desc_sig_name(argument, argument))
            while ends_open:
                stack.append(addnodes.desc_optional())
                stack[-2] += stack[-1]
                ends_open -= 1
            while ends_close:
                stack.pop()
                ends_close -= 1
        if len(stack) != 1:
            raise IndexError
    except IndexError:
        # if there are too few or too many elements on the stack, just give up
        # and treat the whole argument list as one argument, discarding the
        # already partially populated paramlist node
        paramlist = addnodes.desc_parameterlist()
        paramlist += addnodes.desc_parameter(arglist, arglist)
        signode += paramlist
    else:
        signode += paramlist


# This override allows our inline type specifiers to behave like :class: link
# when it comes to handling "." and "~" prefixes.
class PyXrefMixin:
    def make_xref(
        self,
        rolename: str,
        domain: str,
        target: str,
        innernode: type[TextlikeNode] = nodes.emphasis,
        contnode: Node | None = None,
        env: BuildEnvironment | None = None,
        inliner: Inliner | None = None,
        location: Node | None = None,
    ) -> Node:
        # we use inliner=None to make sure we get the old behaviour with a single
        # pending_xref node
        result = super().make_xref(rolename, domain, target,  # type: ignore[misc]
                                   innernode, contnode,
                                   env, inliner=None, location=None)
        if isinstance(result, pending_xref):
            assert env is not None
            result['refspecific'] = True
            result['py:module'] = env.ref_context.get('py:module')
            result['py:class'] = env.ref_context.get('py:class')

            reftype, reftarget, reftitle, _ = parse_reftarget(target)
            if reftarget != reftitle:
                result['reftype'] = reftype
                result['reftarget'] = reftarget

                result.clear()
                result += innernode(reftitle, reftitle)
            elif env.config.python_use_unqualified_type_names:
                children = result.children
                result.clear()

                shortname = target.split('.')[-1]
                textnode = innernode('', shortname)
                contnodes = [pending_xref_condition('', '', textnode, condition='resolved'),
                             pending_xref_condition('', '', *children, condition='*')]
                result.extend(contnodes)

        return result

    def make_xrefs(
        self,
        rolename: str,
        domain: str,
        target: str,
        innernode: type[TextlikeNode] = nodes.emphasis,
        contnode: Node | None = None,
        env: BuildEnvironment | None = None,
        inliner: Inliner | None = None,
        location: Node | None = None,
    ) -> list[Node]:
        delims = r'(\s*[\[\]\(\),](?:\s*o[rf]\s)?\s*|\s+o[rf]\s+|\s*\|\s*|\.\.\.)'
        delims_re = re.compile(delims)
        sub_targets = re.split(delims, target)

        split_contnode = bool(contnode and contnode.astext() == target)

        in_literal = False
        results = []
        for sub_target in filter(None, sub_targets):
            if split_contnode:
                contnode = nodes.Text(sub_target)

            if in_literal or delims_re.match(sub_target):
                results.append(contnode or innernode(sub_target, sub_target))
            else:
                results.append(self.make_xref(rolename, domain, sub_target,
                                              innernode, contnode, env, inliner, location))

            if sub_target in ('Literal', 'typing.Literal', '~typing.Literal'):
                in_literal = True

        return results


class PyField(PyXrefMixin, Field):
    pass


class PyGroupedField(PyXrefMixin, GroupedField):
    pass


class PyTypedField(PyXrefMixin, TypedField):
    pass


class PyObject(ObjectDescription[tuple[str, str]]):
    """
    Description of a general Python object.

    :cvar allow_nesting: Class is an object that allows for nested namespaces
    :vartype allow_nesting: bool
    """
    option_spec: OptionSpec = {
        'no-index': directives.flag,
        'no-index-entry': directives.flag,
        'no-contents-entry': directives.flag,
        'no-typesetting': directives.flag,
        'noindex': directives.flag,
        'noindexentry': directives.flag,
        'nocontentsentry': directives.flag,
        'single-line-parameter-list': directives.flag,
        'single-line-type-parameter-list': directives.flag,
        'module': directives.unchanged,
        'canonical': directives.unchanged,
        'annotation': directives.unchanged,
    }

    doc_field_types = [
        PyTypedField('parameter', label=_('Parameters'),
                     names=('param', 'parameter', 'arg', 'argument',
                            'keyword', 'kwarg', 'kwparam'),
                     typerolename='class', typenames=('paramtype', 'type'),
                     can_collapse=True),
        PyTypedField('variable', label=_('Variables'),
                     names=('var', 'ivar', 'cvar'),
                     typerolename='class', typenames=('vartype',),
                     can_collapse=True),
        PyGroupedField('exceptions', label=_('Raises'), rolename='exc',
                       names=('raises', 'raise', 'exception', 'except'),
                       can_collapse=True),
        Field('returnvalue', label=_('Returns'), has_arg=False,
              names=('returns', 'return')),
        PyField('returntype', label=_('Return type'), has_arg=False,
                names=('rtype',), bodyrolename='class'),
    ]

    allow_nesting = False

    def get_signature_prefix(self, sig: str) -> list[nodes.Node]:
        """May return a prefix to put before the object name in the
        signature.
        """
        return []

    def needs_arglist(self) -> bool:
        """May return true if an empty argument list is to be generated even if
        the document contains none.
        """
        return False

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        """Transform a Python signature into RST nodes.

        Return (fully qualified name of the thing, classname if any).

        If inside a class, the current class name is handled intelligently:
        * it is stripped from the displayed name if present
        * it is added to the full name (return value) if not present
        """
        m = py_sig_re.match(sig)
        if m is None:
            raise ValueError
        prefix, name, tp_list, arglist, retann = m.groups()

        # determine module and class name (if applicable), as well as full name
        modname = self.options.get('module', self.env.ref_context.get('py:module'))
        classname = self.env.ref_context.get('py:class')
        if classname:
            add_module = False
            if prefix and (prefix == classname or
                           prefix.startswith(classname + ".")):
                fullname = prefix + name
                # class name is given again in the signature
                prefix = prefix[len(classname):].lstrip('.')
            elif prefix:
                # class name is given in the signature, but different
                # (shouldn't happen)
                fullname = classname + '.' + prefix + name
            else:
                # class name is not given in the signature
                fullname = classname + '.' + name
        else:
            add_module = True
            if prefix:
                classname = prefix.rstrip('.')
                fullname = prefix + name
            else:
                classname = ''
                fullname = name

        signode['module'] = modname
        signode['class'] = classname
        signode['fullname'] = fullname

        max_len = (self.env.config.python_maximum_signature_line_length
                   or self.env.config.maximum_signature_line_length
                   or 0)

        # determine if the function arguments (without its type parameters)
        # should be formatted on a multiline or not by removing the width of
        # the type parameters list (if any)
        sig_len = len(sig)
        tp_list_span = m.span(3)
        multi_line_parameter_list = (
            'single-line-parameter-list' not in self.options
            and (sig_len - (tp_list_span[1] - tp_list_span[0])) > max_len > 0
        )

        # determine whether the type parameter list must be wrapped or not
        arglist_span = m.span(4)
        multi_line_type_parameter_list = (
            'single-line-type-parameter-list' not in self.options
            and (sig_len - (arglist_span[1] - arglist_span[0])) > max_len > 0
        )

        sig_prefix = self.get_signature_prefix(sig)
        if sig_prefix:
            if type(sig_prefix) is str:
                msg = ("Python directive method get_signature_prefix()"
                       " must return a list of nodes."
                       f" Return value was '{sig_prefix}'.")
                raise TypeError(msg)
            signode += addnodes.desc_annotation(str(sig_prefix), '', *sig_prefix)

        if prefix:
            signode += addnodes.desc_addname(prefix, prefix)
        elif modname and add_module and self.env.config.add_module_names:
            nodetext = modname + '.'
            signode += addnodes.desc_addname(nodetext, nodetext)

        signode += addnodes.desc_name(name, name)

        if tp_list:
            try:
                signode += _parse_type_list(tp_list, self.env, multi_line_type_parameter_list)
            except Exception as exc:
                logger.warning("could not parse tp_list (%r): %s", tp_list, exc,
                               location=signode)

        if arglist:
            try:
                signode += _parse_arglist(arglist, self.env, multi_line_parameter_list)
            except SyntaxError:
                # fallback to parse arglist original parser
                # (this may happen if the argument list is incorrectly used
                # as a list of bases when documenting a class)
                # it supports to represent optional arguments (ex. "func(foo [, bar])")
                _pseudo_parse_arglist(signode, arglist, multi_line_parameter_list)
            except (NotImplementedError, ValueError) as exc:
                # duplicated parameter names raise ValueError and not a SyntaxError
                logger.warning("could not parse arglist (%r): %s", arglist, exc,
                               location=signode)
                _pseudo_parse_arglist(signode, arglist, multi_line_parameter_list)
        else:
            if self.needs_arglist():
                # for callables, add an empty parameter list
                signode += addnodes.desc_parameterlist()

        if retann:
            children = _parse_annotation(retann, self.env)
            signode += addnodes.desc_returns(retann, '', *children)

        anno = self.options.get('annotation')
        if anno:
            signode += addnodes.desc_annotation(' ' + anno, '',
                                                addnodes.desc_sig_space(),
                                                nodes.Text(anno))

        return fullname, prefix

    def _object_hierarchy_parts(self, sig_node: desc_signature) -> tuple[str, ...]:
        if 'fullname' not in sig_node:
            return ()
        modname = sig_node.get('module')
        fullname = sig_node['fullname']

        if modname:
            return (modname, *fullname.split('.'))
        else:
            return tuple(fullname.split('.'))

    def get_index_text(self, modname: str, name: tuple[str, str]) -> str:
        """Return the text for the index entry of the object."""
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

    def add_target_and_index(self, name_cls: tuple[str, str], sig: str,
                             signode: desc_signature) -> None:
        modname = self.options.get('module', self.env.ref_context.get('py:module'))
        fullname = (modname + '.' if modname else '') + name_cls[0]
        node_id = make_id(self.env, self.state.document, '', fullname)
        signode['ids'].append(node_id)
        self.state.document.note_explicit_target(signode)

        domain = cast(PythonDomain, self.env.get_domain('py'))
        domain.note_object(fullname, self.objtype, node_id, location=signode)

        canonical_name = self.options.get('canonical')
        if canonical_name:
            domain.note_object(canonical_name, self.objtype, node_id, aliased=True,
                               location=signode)

        if 'no-index-entry' not in self.options:
            indextext = self.get_index_text(modname, name_cls)
            if indextext:
                self.indexnode['entries'].append(('single', indextext, node_id, '', None))

    def before_content(self) -> None:
        """Handle object nesting before content

        :py:class:`PyObject` represents Python language constructs. For
        constructs that are nestable, such as a Python classes, this method will
        build up a stack of the nesting hierarchy so that it can be later
        de-nested correctly, in :py:meth:`after_content`.

        For constructs that aren't nestable, the stack is bypassed, and instead
        only the most recent object is tracked. This object prefix name will be
        removed with :py:meth:`after_content`.
        """
        prefix = None
        if self.names:
            # fullname and name_prefix come from the `handle_signature` method.
            # fullname represents the full object name that is constructed using
            # object nesting and explicit prefixes. `name_prefix` is the
            # explicit prefix given in a signature
            (fullname, name_prefix) = self.names[-1]
            if self.allow_nesting:
                prefix = fullname
            elif name_prefix:
                prefix = name_prefix.strip('.')
        if prefix:
            self.env.ref_context['py:class'] = prefix
            if self.allow_nesting:
                classes = self.env.ref_context.setdefault('py:classes', [])
                classes.append(prefix)
        if 'module' in self.options:
            modules = self.env.ref_context.setdefault('py:modules', [])
            modules.append(self.env.ref_context.get('py:module'))
            self.env.ref_context['py:module'] = self.options['module']

    def after_content(self) -> None:
        """Handle object de-nesting after content

        If this class is a nestable object, removing the last nested class prefix
        ends further nesting in the object.

        If this class is not a nestable object, the list of classes should not
        be altered as we didn't affect the nesting levels in
        :py:meth:`before_content`.
        """
        classes = self.env.ref_context.setdefault('py:classes', [])
        if self.allow_nesting:
            with contextlib.suppress(IndexError):
                classes.pop()

        self.env.ref_context['py:class'] = (classes[-1] if len(classes) > 0
                                            else None)
        if 'module' in self.options:
            modules = self.env.ref_context.setdefault('py:modules', [])
            if modules:
                self.env.ref_context['py:module'] = modules.pop()
            else:
                self.env.ref_context.pop('py:module')

    def _toc_entry_name(self, sig_node: desc_signature) -> str:
        if not sig_node.get('_toc_parts'):
            return ''

        config = self.env.app.config
        objtype = sig_node.parent.get('objtype')
        if config.add_function_parentheses and objtype in {'function', 'method'}:
            parens = '()'
        else:
            parens = ''
        *parents, name = sig_node['_toc_parts']
        if config.toc_object_entries_show_parents == 'domain':
            return sig_node.get('fullname', name) + parens
        if config.toc_object_entries_show_parents == 'hide':
            return name + parens
        if config.toc_object_entries_show_parents == 'all':
            return '.'.join(parents + [name + parens])
        return ''


class PyFunction(PyObject):
    """Description of a function."""

    option_spec: OptionSpec = PyObject.option_spec.copy()
    option_spec.update({
        'async': directives.flag,
    })

    def get_signature_prefix(self, sig: str) -> list[nodes.Node]:
        if 'async' in self.options:
            return [addnodes.desc_sig_keyword('', 'async'),
                    addnodes.desc_sig_space()]
        else:
            return []

    def needs_arglist(self) -> bool:
        return True

    def add_target_and_index(self, name_cls: tuple[str, str], sig: str,
                             signode: desc_signature) -> None:
        super().add_target_and_index(name_cls, sig, signode)
        if 'no-index-entry' not in self.options:
            modname = self.options.get('module', self.env.ref_context.get('py:module'))
            node_id = signode['ids'][0]

            name, cls = name_cls
            if modname:
                text = _('%s() (in module %s)') % (name, modname)
                self.indexnode['entries'].append(('single', text, node_id, '', None))
            else:
                text = f'built-in function; {name}()'
                self.indexnode['entries'].append(('pair', text, node_id, '', None))

    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
        # add index in own add_target_and_index() instead.
        return ''


class PyDecoratorFunction(PyFunction):
    """Description of a decorator."""

    def run(self) -> list[Node]:
        # a decorator function is a function after all
        self.name = 'py:function'
        return super().run()

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        ret = super().handle_signature(sig, signode)
        signode.insert(0, addnodes.desc_addname('@', '@'))
        return ret

    def needs_arglist(self) -> bool:
        return False


class PyVariable(PyObject):
    """Description of a variable."""

    option_spec: OptionSpec = PyObject.option_spec.copy()
    option_spec.update({
        'type': directives.unchanged,
        'value': directives.unchanged,
    })

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        fullname, prefix = super().handle_signature(sig, signode)

        typ = self.options.get('type')
        if typ:
            annotations = _parse_annotation(typ, self.env)
            signode += addnodes.desc_annotation(typ, '',
                                                addnodes.desc_sig_punctuation('', ':'),
                                                addnodes.desc_sig_space(), *annotations)

        value = self.options.get('value')
        if value:
            signode += addnodes.desc_annotation(value, '',
                                                addnodes.desc_sig_space(),
                                                addnodes.desc_sig_punctuation('', '='),
                                                addnodes.desc_sig_space(),
                                                nodes.Text(value))

        return fullname, prefix

    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
        name, cls = name_cls
        if modname:
            return _('%s (in module %s)') % (name, modname)
        else:
            return _('%s (built-in variable)') % name


class PyClasslike(PyObject):
    """
    Description of a class-like object (classes, interfaces, exceptions).
    """

    option_spec: OptionSpec = PyObject.option_spec.copy()
    option_spec.update({
        'final': directives.flag,
    })

    allow_nesting = True

    def get_signature_prefix(self, sig: str) -> list[nodes.Node]:
        if 'final' in self.options:
            return [nodes.Text('final'), addnodes.desc_sig_space(),
                    nodes.Text(self.objtype), addnodes.desc_sig_space()]
        else:
            return [nodes.Text(self.objtype), addnodes.desc_sig_space()]

    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
        if self.objtype == 'class':
            if not modname:
                return _('%s (built-in class)') % name_cls[0]
            return _('%s (class in %s)') % (name_cls[0], modname)
        elif self.objtype == 'exception':
            return name_cls[0]
        else:
            return ''


class PyMethod(PyObject):
    """Description of a method."""

    option_spec: OptionSpec = PyObject.option_spec.copy()
    option_spec.update({
        'abstractmethod': directives.flag,
        'async': directives.flag,
        'classmethod': directives.flag,
        'final': directives.flag,
        'staticmethod': directives.flag,
    })

    def needs_arglist(self) -> bool:
        return True

    def get_signature_prefix(self, sig: str) -> list[nodes.Node]:
        prefix: list[nodes.Node] = []
        if 'final' in self.options:
            prefix.append(nodes.Text('final'))
            prefix.append(addnodes.desc_sig_space())
        if 'abstractmethod' in self.options:
            prefix.append(nodes.Text('abstract'))
            prefix.append(addnodes.desc_sig_space())
        if 'async' in self.options:
            prefix.append(nodes.Text('async'))
            prefix.append(addnodes.desc_sig_space())
        if 'classmethod' in self.options:
            prefix.append(nodes.Text('classmethod'))
            prefix.append(addnodes.desc_sig_space())
        if 'staticmethod' in self.options:
            prefix.append(nodes.Text('static'))
            prefix.append(addnodes.desc_sig_space())
        return prefix

    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
        name, cls = name_cls
        try:
            clsname, methname = name.rsplit('.', 1)
            if modname and self.env.config.add_module_names:
                clsname = '.'.join([modname, clsname])
        except ValueError:
            if modname:
                return _('%s() (in module %s)') % (name, modname)
            else:
                return '%s()' % name

        if 'classmethod' in self.options:
            return _('%s() (%s class method)') % (methname, clsname)
        elif 'staticmethod' in self.options:
            return _('%s() (%s static method)') % (methname, clsname)
        else:
            return _('%s() (%s method)') % (methname, clsname)


class PyClassMethod(PyMethod):
    """Description of a classmethod."""

    option_spec: OptionSpec = PyObject.option_spec.copy()

    def run(self) -> list[Node]:
        self.name = 'py:method'
        self.options['classmethod'] = True

        return super().run()


class PyStaticMethod(PyMethod):
    """Description of a staticmethod."""

    option_spec: OptionSpec = PyObject.option_spec.copy()

    def run(self) -> list[Node]:
        self.name = 'py:method'
        self.options['staticmethod'] = True

        return super().run()


class PyDecoratorMethod(PyMethod):
    """Description of a decoratormethod."""

    def run(self) -> list[Node]:
        self.name = 'py:method'
        return super().run()

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        ret = super().handle_signature(sig, signode)
        signode.insert(0, addnodes.desc_addname('@', '@'))
        return ret

    def needs_arglist(self) -> bool:
        return False


class PyAttribute(PyObject):
    """Description of an attribute."""

    option_spec: OptionSpec = PyObject.option_spec.copy()
    option_spec.update({
        'type': directives.unchanged,
        'value': directives.unchanged,
    })

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        fullname, prefix = super().handle_signature(sig, signode)

        typ = self.options.get('type')
        if typ:
            annotations = _parse_annotation(typ, self.env)
            signode += addnodes.desc_annotation(typ, '',
                                                addnodes.desc_sig_punctuation('', ':'),
                                                addnodes.desc_sig_space(),
                                                *annotations)

        value = self.options.get('value')
        if value:
            signode += addnodes.desc_annotation(value, '',
                                                addnodes.desc_sig_space(),
                                                addnodes.desc_sig_punctuation('', '='),
                                                addnodes.desc_sig_space(),
                                                nodes.Text(value))

        return fullname, prefix

    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
        name, cls = name_cls
        try:
            clsname, attrname = name.rsplit('.', 1)
            if modname and self.env.config.add_module_names:
                clsname = '.'.join([modname, clsname])
        except ValueError:
            if modname:
                return _('%s (in module %s)') % (name, modname)
            else:
                return name

        return _('%s (%s attribute)') % (attrname, clsname)


class PyProperty(PyObject):
    """Description of an attribute."""

    option_spec = PyObject.option_spec.copy()
    option_spec.update({
        'abstractmethod': directives.flag,
        'classmethod': directives.flag,
        'type': directives.unchanged,
    })

    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
        fullname, prefix = super().handle_signature(sig, signode)

        typ = self.options.get('type')
        if typ:
            annotations = _parse_annotation(typ, self.env)
            signode += addnodes.desc_annotation(typ, '',
                                                addnodes.desc_sig_punctuation('', ':'),
                                                addnodes.desc_sig_space(),
                                                *annotations)

        return fullname, prefix

    def get_signature_prefix(self, sig: str) -> list[nodes.Node]:
        prefix: list[nodes.Node] = []
        if 'abstractmethod' in self.options:
            prefix.append(nodes.Text('abstract'))
            prefix.append(addnodes.desc_sig_space())
        if 'classmethod' in self.options:
            prefix.append(nodes.Text('class'))
            prefix.append(addnodes.desc_sig_space())

        prefix.append(nodes.Text('property'))
        prefix.append(addnodes.desc_sig_space())
        return prefix

    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
        name, cls = name_cls
        try:
            clsname, attrname = name.rsplit('.', 1)
            if modname and self.env.config.add_module_names:
                clsname = '.'.join([modname, clsname])
        except ValueError:
            if modname:
                return _('%s (in module %s)') % (name, modname)
            else:
                return name

        return _('%s (%s property)') % (attrname, clsname)


class PyModule(SphinxDirective):
    """
    Directive to mark description of a new module.
    """

    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec: OptionSpec = {
        'platform': lambda x: x,
        'synopsis': lambda x: x,
        'no-index': directives.flag,
        'no-contents-entry': directives.flag,
        'no-typesetting': directives.flag,
        'noindex': directives.flag,
        'nocontentsentry': directives.flag,
        'deprecated': directives.flag,
    }

    def run(self) -> list[Node]:
        domain = cast(PythonDomain, self.env.get_domain('py'))

        modname = self.arguments[0].strip()
        no_index = 'no-index' in self.options or 'noindex' in self.options
        self.env.ref_context['py:module'] = modname

        content_node: Element = nodes.section()
        # necessary so that the child nodes get the right source/line set
        content_node.document = self.state.document
        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)

        ret: list[Node] = []
        if not no_index:
            # note module to the domain
            node_id = make_id(self.env, self.state.document, 'module', modname)
            target = nodes.target('', '', ids=[node_id], ismod=True)
            self.set_source_info(target)
            self.state.document.note_explicit_target(target)

            domain.note_module(modname,
                               node_id,
                               self.options.get('synopsis', ''),
                               self.options.get('platform', ''),
                               'deprecated' in self.options)
            domain.note_object(modname, 'module', node_id, location=target)

            # the platform and synopsis aren't printed; in fact, they are only
            # used in the modindex currently
            indextext = f'module; {modname}'
            inode = addnodes.index(entries=[('pair', indextext, node_id, '', None)])
            # The node order is: index node first, then target node.
            ret.append(inode)
            ret.append(target)
        ret.extend(content_node.children)
        return ret


class PyCurrentModule(SphinxDirective):
    """
    This directive is just to tell Sphinx that we're documenting
    stuff in module foo, but links to module foo won't lead here.
    """

    has_content = False
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec: OptionSpec = {}

    def run(self) -> list[Node]:
        modname = self.arguments[0].strip()
        if modname == 'None':
            self.env.ref_context.pop('py:module', None)
        else:
            self.env.ref_context['py:module'] = modname
        return []


class PyXRefRole(XRefRole):
    def process_link(self, env: BuildEnvironment, refnode: Element,
                     has_explicit_title: bool, title: str, target: str) -> tuple[str, str]:
        refnode['py:module'] = env.ref_context.get('py:module')
        refnode['py:class'] = env.ref_context.get('py:class')
        if not has_explicit_title:
            title = title.lstrip('.')    # only has a meaning for the target
            target = target.lstrip('~')  # only has a meaning for the title
            # if the first character is a tilde, don't display the module/class
            # parts of the contents
            if title[0:1] == '~':
                title = title[1:]
                dot = title.rfind('.')
                if dot != -1:
                    title = title[dot + 1:]
        # if the first character is a dot, search more specific namespaces first
        # else search builtins first
        if target[0:1] == '.':
            target = target[1:]
            refnode['refspecific'] = True
        return title, target


def filter_meta_fields(app: Sphinx, domain: str, objtype: str, content: Element) -> None:
    """Filter ``:meta:`` field from its docstring."""
    if domain != 'py':
        return

    for node in content:
        if isinstance(node, nodes.field_list):
            fields = cast(list[nodes.field], node)
            # removing list items while iterating the list needs reversed()
            for field in reversed(fields):
                field_name = cast(nodes.field_body, field[0]).astext().strip()
                if field_name == 'meta' or field_name.startswith('meta '):
                    node.remove(field)


class PythonModuleIndex(Index):
    """
    Index subclass to provide the Python module index.
    """

    name = 'modindex'
    localname = _('Python Module Index')
    shortname = _('modules')

    def generate(self, docnames: Iterable[str] | None = None,
                 ) -> tuple[list[tuple[str, list[IndexEntry]]], bool]:
        content: dict[str, list[IndexEntry]] = {}
        # list of prefixes to ignore
        ignores: list[str] = self.domain.env.config['modindex_common_prefix']
        ignores = sorted(ignores, key=len, reverse=True)
        # list of all modules, sorted by module name
        modules = sorted(self.domain.data['modules'].items(),
                         key=lambda x: x[0].lower())
        # sort out collapsible modules
        prev_modname = ''
        num_toplevels = 0
        for modname, (docname, node_id, synopsis, platforms, deprecated) in modules:
            if docnames and docname not in docnames:
                continue

            for ignore in ignores:
                if modname.startswith(ignore):
                    modname = modname[len(ignore):]
                    stripped = ignore
                    break
            else:
                stripped = ''

            # we stripped the whole module name?
            if not modname:
                modname, stripped = stripped, ''

            entries = content.setdefault(modname[0].lower(), [])

            package = modname.split('.')[0]
            if package != modname:
                # it's a submodule
                if prev_modname == package:
                    # first submodule - make parent a group head
                    if entries:
                        last = entries[-1]
                        entries[-1] = IndexEntry(last[0], 1, last[2], last[3],
                                                 last[4], last[5], last[6])
                elif not prev_modname.startswith(package):
                    # submodule without parent in list, add dummy entry
                    entries.append(IndexEntry(stripped + package, 1, '', '', '', '', ''))
                subtype = 2
            else:
                num_toplevels += 1
                subtype = 0

            qualifier = _('Deprecated') if deprecated else ''
            entries.append(IndexEntry(stripped + modname, subtype, docname,
                                      node_id, platforms, qualifier, synopsis))
            prev_modname = modname

        # apply heuristics when to collapse modindex at page load:
        # only collapse if number of toplevel modules is larger than
        # number of submodules
        collapse = len(modules) - num_toplevels < num_toplevels

        # sort by first letter
        sorted_content = sorted(content.items())

        return sorted_content, collapse


class PythonDomain(Domain):
    """Python language domain."""
    name = 'py'
    label = 'Python'
    object_types: dict[str, ObjType] = {
        'function':     ObjType(_('function'),      'func', 'obj'),
        'data':         ObjType(_('data'),          'data', 'obj'),
        'class':        ObjType(_('class'),         'class', 'exc', 'obj'),
        'exception':    ObjType(_('exception'),     'exc', 'class', 'obj'),
        'method':       ObjType(_('method'),        'meth', 'obj'),
        'classmethod':  ObjType(_('class method'),  'meth', 'obj'),
        'staticmethod': ObjType(_('static method'), 'meth', 'obj'),
        'attribute':    ObjType(_('attribute'),     'attr', 'obj'),
        'property':     ObjType(_('property'),      'attr', '_prop', 'obj'),
        'module':       ObjType(_('module'),        'mod', 'obj'),
    }

    directives = {
        'function':        PyFunction,
        'data':            PyVariable,
        'class':           PyClasslike,
        'exception':       PyClasslike,
        'method':          PyMethod,
        'classmethod':     PyClassMethod,
        'staticmethod':    PyStaticMethod,
        'attribute':       PyAttribute,
        'property':        PyProperty,
        'module':          PyModule,
        'currentmodule':   PyCurrentModule,
        'decorator':       PyDecoratorFunction,
        'decoratormethod': PyDecoratorMethod,
    }
    roles = {
        'data':  PyXRefRole(),
        'exc':   PyXRefRole(),
        'func':  PyXRefRole(fix_parens=True),
        'class': PyXRefRole(),
        'const': PyXRefRole(),
        'attr':  PyXRefRole(),
        'meth':  PyXRefRole(fix_parens=True),
        'mod':   PyXRefRole(),
        'obj':   PyXRefRole(),
    }
    initial_data: dict[str, dict[str, tuple[Any]]] = {
        'objects': {},  # fullname -> docname, objtype
        'modules': {},  # modname -> docname, synopsis, platform, deprecated
    }
    indices = [
        PythonModuleIndex,
    ]

    @property
    def objects(self) -> dict[str, ObjectEntry]:
        return self.data.setdefault('objects', {})  # fullname -> ObjectEntry

    def note_object(self, name: str, objtype: str, node_id: str,
                    aliased: bool = False, location: Any = None) -> None:
        """Note a python object for cross reference.

        .. versionadded:: 2.1
        """
        if name in self.objects:
            other = self.objects[name]
            if other.aliased and aliased is False:
                # The original definition found. Override it!
                pass
            elif other.aliased is False and aliased:
                # The original definition is already registered.
                return
            else:
                # duplicated
                logger.warning(__('duplicate object description of %s, '
                                  'other instance in %s, use :no-index: for one of them'),
                               name, other.docname, location=location)
        self.objects[name] = ObjectEntry(self.env.docname, node_id, objtype, aliased)

    @property
    def modules(self) -> dict[str, ModuleEntry]:
        return self.data.setdefault('modules', {})  # modname -> ModuleEntry

    def note_module(self, name: str, node_id: str, synopsis: str,
                    platform: str, deprecated: bool) -> None:
        """Note a python module for cross reference.

        .. versionadded:: 2.1
        """
        self.modules[name] = ModuleEntry(self.env.docname, node_id,
                                         synopsis, platform, deprecated)

    def clear_doc(self, docname: str) -> None:
        for fullname, obj in list(self.objects.items()):
            if obj.docname == docname:
                del self.objects[fullname]
        for modname, mod in list(self.modules.items()):
            if mod.docname == docname:
                del self.modules[modname]

    def merge_domaindata(self, docnames: list[str], otherdata: dict[str, Any]) -> None:
        # XXX check duplicates?
        for fullname, obj in otherdata['objects'].items():
            if obj.docname in docnames:
                self.objects[fullname] = obj
        for modname, mod in otherdata['modules'].items():
            if mod.docname in docnames:
                self.modules[modname] = mod

    def find_obj(self, env: BuildEnvironment, modname: str, classname: str,
                 name: str, type: str | None, searchmode: int = 0,
                 ) -> list[tuple[str, ObjectEntry]]:
        """Find a Python object for "name", perhaps using the given module
        and/or classname.  Returns a list of (name, object entry) tuples.
        """
        # skip parens
        if name[-2:] == '()':
            name = name[:-2]

        if not name:
            return []

        matches: list[tuple[str, ObjectEntry]] = []

        newname = None
        if searchmode == 1:
            if type is None:
                objtypes: list[str] | None = list(self.object_types)
            else:
                objtypes = self.objtypes_for_role(type)
            if objtypes is not None:
                if modname and classname:
                    fullname = modname + '.' + classname + '.' + name
                    if fullname in self.objects and self.objects[fullname].objtype in objtypes:
                        newname = fullname
                if not newname:
                    if modname and modname + '.' + name in self.objects and \
                       self.objects[modname + '.' + name].objtype in objtypes:
                        newname = modname + '.' + name
                    elif name in self.objects and self.objects[name].objtype in objtypes:
                        newname = name
                    else:
                        # "fuzzy" searching mode
                        searchname = '.' + name
                        matches = [(oname, self.objects[oname]) for oname in self.objects
                                   if oname.endswith(searchname) and
                                   self.objects[oname].objtype in objtypes]
        else:
            # NOTE: searching for exact match, object type is not considered
            if name in self.objects:
                newname = name
            elif type == 'mod':
                # only exact matches allowed for modules
                return []
            elif classname and classname + '.' + name in self.objects:
                newname = classname + '.' + name
            elif modname and modname + '.' + name in self.objects:
                newname = modname + '.' + name
            elif modname and classname and \
                    modname + '.' + classname + '.' + name in self.objects:
                newname = modname + '.' + classname + '.' + name
        if newname is not None:
            matches.append((newname, self.objects[newname]))
        return matches

    def resolve_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
                     type: str, target: str, node: pending_xref, contnode: Element,
                     ) -> Element | None:
        modname = node.get('py:module')
        clsname = node.get('py:class')
        searchmode = 1 if node.hasattr('refspecific') else 0
        matches = self.find_obj(env, modname, clsname, target,
                                type, searchmode)

        if not matches and type == 'attr':
            # fallback to meth (for property; Sphinx 2.4.x)
            # this ensures that `:attr:` role continues to refer to the old property entry
            # that defined by ``method`` directive in old reST files.
            matches = self.find_obj(env, modname, clsname, target, 'meth', searchmode)
        if not matches and type == 'meth':
            # fallback to attr (for property)
            # this ensures that `:meth:` in the old reST files can refer to the property
            # entry that defined by ``property`` directive.
            #
            # Note: _prop is a secret role only for internal look-up.
            matches = self.find_obj(env, modname, clsname, target, '_prop', searchmode)

        if not matches:
            return None
        elif len(matches) > 1:
            canonicals = [m for m in matches if not m[1].aliased]
            if len(canonicals) == 1:
                matches = canonicals
            else:
                logger.warning(__('more than one target found for cross-reference %r: %s'),
                               target, ', '.join(match[0] for match in matches),
                               type='ref', subtype='python', location=node)
        name, obj = matches[0]

        if obj[2] == 'module':
            return self._make_module_refnode(builder, fromdocname, name, contnode)
        else:
            # determine the content of the reference by conditions
            content = find_pending_xref_condition(node, 'resolved')
            if content:
                children = content.children
            else:
                # if not found, use contnode
                children = [contnode]

            return make_refnode(builder, fromdocname, obj[0], obj[1], children, name)

    def resolve_any_xref(self, env: BuildEnvironment, fromdocname: str, builder: Builder,
                         target: str, node: pending_xref, contnode: Element,
                         ) -> list[tuple[str, Element]]:
        modname = node.get('py:module')
        clsname = node.get('py:class')
        results: list[tuple[str, Element]] = []

        # always search in "refspecific" mode with the :any: role
        matches = self.find_obj(env, modname, clsname, target, None, 1)
        multiple_matches = len(matches) > 1

        for name, obj in matches:

            if multiple_matches and obj.aliased:
                # Skip duplicated matches
                continue

            if obj[2] == 'module':
                results.append(('py:mod',
                                self._make_module_refnode(builder, fromdocname,
                                                          name, contnode)))
            else:
                # determine the content of the reference by conditions
                content = find_pending_xref_condition(node, 'resolved')
                if content:
                    children = content.children
                else:
                    # if not found, use contnode
                    children = [contnode]

                role = 'py:' + self.role_for_objtype(obj[2])  # type: ignore[operator]
                results.append((role, make_refnode(builder, fromdocname, obj[0], obj[1],
                                                   children, name)))
        return results

    def _make_module_refnode(self, builder: Builder, fromdocname: str, name: str,
                             contnode: Node) -> Element:
        # get additional info for modules
        module = self.modules[name]
        title = name
        if module.synopsis:
            title += ': ' + module.synopsis
        if module.deprecated:
            title += _(' (deprecated)')
        if module.platform:
            title += ' (' + module.platform + ')'
        return make_refnode(builder, fromdocname, module.docname, module.node_id,
                            contnode, title)

    def get_objects(self) -> Iterator[tuple[str, str, str, str, str, int]]:
        for modname, mod in self.modules.items():
            yield (modname, modname, 'module', mod.docname, mod.node_id, 0)
        for refname, obj in self.objects.items():
            if obj.objtype != 'module':  # modules are already handled
                if obj.aliased:
                    # aliased names are not full-text searchable.
                    yield (refname, refname, obj.objtype, obj.docname, obj.node_id, -1)
                else:
                    yield (refname, refname, obj.objtype, obj.docname, obj.node_id, 1)

    def get_full_qualified_name(self, node: Element) -> str | None:
        modname = node.get('py:module')
        clsname = node.get('py:class')
        target = node.get('reftarget')
        if target is None:
            return None
        else:
            return '.'.join(filter(None, [modname, clsname, target]))


def builtin_resolver(app: Sphinx, env: BuildEnvironment,
                     node: pending_xref, contnode: Element) -> Element | None:
    """Do not emit nitpicky warnings for built-in types."""
    def istyping(s: str) -> bool:
        if s.startswith('typing.'):
            s = s.split('.', 1)[1]

        return s in typing.__all__

    if node.get('refdomain') != 'py':
        return None
    elif node.get('reftype') in ('class', 'obj') and node.get('reftarget') == 'None':
        return contnode
    elif node.get('reftype') in ('class', 'obj', 'exc'):
        reftarget = node.get('reftarget')
        if inspect.isclass(getattr(builtins, reftarget, None)):
            # built-in class
            return contnode
        if istyping(reftarget):
            # typing class
            return contnode

    return None


def setup(app: Sphinx) -> dict[str, Any]:
    app.setup_extension('sphinx.directives')

    app.add_domain(PythonDomain)
    app.add_config_value('python_use_unqualified_type_names', False, 'env')
    app.add_config_value('python_maximum_signature_line_length', None, 'env',
                         types={int, None})
    app.add_config_value('python_display_short_literal_types', False, 'env')
    app.connect('object-description-transform', filter_meta_fields)
    app.connect('missing-reference', builtin_resolver, priority=900)

    return {
        'version': 'builtin',
        'env_version': 4,
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/domains/python.py
NikolaBlagojevic/pyrecodes
"""Global creation environment."""

from __future__ import annotations

import functools
import os
import pickle
import time
from collections import defaultdict
from copy import copy
from os import path
from typing import TYPE_CHECKING, Any, Callable

from sphinx import addnodes
from sphinx.environment.adapters import toctree as toctree_adapters
from sphinx.errors import BuildEnvironmentError, DocumentError, ExtensionError, SphinxError
from sphinx.locale import __
from sphinx.transforms import SphinxTransformer
from sphinx.util import DownloadFiles, FilenameUniqDict, logging
from sphinx.util.docutils import LoggingReporter
from sphinx.util.i18n import CatalogRepository, docname_to_domain
from sphinx.util.nodes import is_translatable
from sphinx.util.osutil import canon_path, os_path

if TYPE_CHECKING:
    from collections.abc import Generator, Iterator
    from pathlib import Path

    from docutils import nodes
    from docutils.nodes import Node

    from sphinx.application import Sphinx
    from sphinx.builders import Builder
    from sphinx.config import Config
    from sphinx.domains import Domain
    from sphinx.events import EventManager
    from sphinx.project import Project

logger = logging.getLogger(__name__)

default_settings: dict[str, Any] = {
    'auto_id_prefix': 'id',
    'image_loading': 'link',
    'embed_stylesheet': False,
    'cloak_email_addresses': True,
    'pep_base_url': 'https://peps.python.org/',
    'pep_references': None,
    'rfc_base_url': 'https://datatracker.ietf.org/doc/html/',
    'rfc_references': None,
    'input_encoding': 'utf-8-sig',
    'doctitle_xform': False,
    'sectsubtitle_xform': False,
    'section_self_link': False,
    'halt_level': 5,
    'file_insertion_enabled': True,
    'smartquotes_locales': [],
}

# This is increased every time an environment attribute is added
# or changed to properly invalidate pickle files.
ENV_VERSION = 60

# config status
CONFIG_UNSET = -1
CONFIG_OK = 1
CONFIG_NEW = 2
CONFIG_CHANGED = 3
CONFIG_EXTENSIONS_CHANGED = 4

CONFIG_CHANGED_REASON = {
    CONFIG_NEW: __('new config'),
    CONFIG_CHANGED: __('config changed'),
    CONFIG_EXTENSIONS_CHANGED: __('extensions changed'),
}


versioning_conditions: dict[str, bool | Callable] = {
    'none': False,
    'text': is_translatable,
}

if TYPE_CHECKING:
    from collections.abc import MutableMapping
    from typing import Literal

    from typing_extensions import overload

    from sphinx.domains.c import CDomain
    from sphinx.domains.changeset import ChangeSetDomain
    from sphinx.domains.citation import CitationDomain
    from sphinx.domains.cpp import CPPDomain
    from sphinx.domains.index import IndexDomain
    from sphinx.domains.javascript import JavaScriptDomain
    from sphinx.domains.math import MathDomain
    from sphinx.domains.python import PythonDomain
    from sphinx.domains.rst import ReSTDomain
    from sphinx.domains.std import StandardDomain
    from sphinx.ext.duration import DurationDomain
    from sphinx.ext.todo import TodoDomain

    class _DomainsType(MutableMapping[str, Domain]):
        @overload
        def __getitem__(self, key: Literal["c"]) -> CDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["cpp"]) -> CPPDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["changeset"]) -> ChangeSetDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["citation"]) -> CitationDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["index"]) -> IndexDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["js"]) -> JavaScriptDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["math"]) -> MathDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["py"]) -> PythonDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["rst"]) -> ReSTDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["std"]) -> StandardDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["duration"]) -> DurationDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: Literal["todo"]) -> TodoDomain: ...  # NoQA: E704
        @overload
        def __getitem__(self, key: str) -> Domain: ...  # NoQA: E704
        def __getitem__(self, key): raise NotImplementedError  # NoQA: E704
        def __setitem__(self, key, value): raise NotImplementedError  # NoQA: E704
        def __delitem__(self, key): raise NotImplementedError  # NoQA: E704
        def __iter__(self): raise NotImplementedError  # NoQA: E704
        def __len__(self): raise NotImplementedError  # NoQA: E704

else:
    _DomainsType = dict


class BuildEnvironment:
    """
    The environment in which the ReST files are translated.
    Stores an inventory of cross-file targets and provides doctree
    transformations to resolve links to them.
    """

    domains: _DomainsType

    # --------- ENVIRONMENT INITIALIZATION -------------------------------------

    def __init__(self, app: Sphinx):
        self.app: Sphinx = app
        self.doctreedir: Path = app.doctreedir
        self.srcdir: Path = app.srcdir
        self.config: Config = None  # type: ignore[assignment]
        self.config_status: int = CONFIG_UNSET
        self.config_status_extra: str = ''
        self.events: EventManager = app.events
        self.project: Project = app.project
        self.version: dict[str, str] = app.registry.get_envversion(app)

        # the method of doctree versioning; see set_versioning_method
        self.versioning_condition: bool | Callable | None = None
        self.versioning_compare: bool | None = None

        # all the registered domains, set by the application
        self.domains = _DomainsType()

        # the docutils settings for building
        self.settings: dict[str, Any] = default_settings.copy()
        self.settings['env'] = self

        # All "docnames" here are /-separated and relative and exclude
        # the source suffix.

        # docname -> time of reading (in integer microseconds)
        # contains all read docnames
        self.all_docs: dict[str, int] = {}
        # docname -> set of dependent file
        # names, relative to documentation root
        self.dependencies: dict[str, set[str]] = defaultdict(set)
        # docname -> set of included file
        # docnames included from other documents
        self.included: dict[str, set[str]] = defaultdict(set)
        # docnames to re-read unconditionally on next build
        self.reread_always: set[str] = set()

        # docname -> pickled doctree
        self._pickled_doctree_cache: dict[str, bytes] = {}

        # docname -> doctree
        self._write_doc_doctree_cache: dict[str, nodes.document] = {}

        # File metadata
        # docname -> dict of metadata items
        self.metadata: dict[str, dict[str, Any]] = defaultdict(dict)

        # TOC inventory
        # docname -> title node
        self.titles: dict[str, nodes.title] = {}
        # docname -> title node; only different if
        # set differently with title directive
        self.longtitles: dict[str, nodes.title] = {}
        # docname -> table of contents nodetree
        self.tocs: dict[str, nodes.bullet_list] = {}
        # docname -> number of real entries
        self.toc_num_entries: dict[str, int] = {}

        # used to determine when to show the TOC
        # in a sidebar (don't show if it's only one item)
        # docname -> dict of sectionid -> number
        self.toc_secnumbers: dict[str, dict[str, tuple[int, ...]]] = {}
        # docname -> dict of figtype -> dict of figureid -> number
        self.toc_fignumbers: dict[str, dict[str, dict[str, tuple[int, ...]]]] = {}

        # docname -> list of toctree includefiles
        self.toctree_includes: dict[str, list[str]] = {}
        # docname -> set of files (containing its TOCs) to rebuild too
        self.files_to_rebuild: dict[str, set[str]] = {}
        # docnames that have :glob: toctrees
        self.glob_toctrees: set[str] = set()
        # docnames that have :numbered: toctrees
        self.numbered_toctrees: set[str] = set()

        # domain-specific inventories, here to be pickled
        # domainname -> domain-specific dict
        self.domaindata: dict[str, dict] = {}

        # these map absolute path -> (docnames, unique filename)
        self.images: FilenameUniqDict = FilenameUniqDict()
        # filename -> (set of docnames, destination)
        self.dlfiles: DownloadFiles = DownloadFiles()

        # the original URI for images
        self.original_image_uri: dict[str, str] = {}

        # temporary data storage while reading a document
        self.temp_data: dict[str, Any] = {}
        # context for cross-references (e.g. current module or class)
        # this is similar to temp_data, but will for example be copied to
        # attributes of "any" cross references
        self.ref_context: dict[str, Any] = {}

        # search index data

        # docname -> title
        self._search_index_titles: dict[str, str] = {}
        # docname -> filename
        self._search_index_filenames: dict[str, str] = {}
        # stemmed words -> set(docname)
        self._search_index_mapping: dict[str, set[str]] = {}
        # stemmed words in titles -> set(docname)
        self._search_index_title_mapping: dict[str, set[str]] = {}
        # docname -> all titles in document
        self._search_index_all_titles: dict[str, list[tuple[str, str]]] = {}
        # docname -> list(index entry)
        self._search_index_index_entries: dict[str, list[tuple[str, str, str]]] = {}
        # objtype -> index
        self._search_index_objtypes: dict[tuple[str, str], int] = {}
        # objtype index -> (domain, type, objname (localized))
        self._search_index_objnames: dict[int, tuple[str, str, str]] = {}

        # set up environment
        self.setup(app)

    def __getstate__(self) -> dict:
        """Obtains serializable data for pickling."""
        __dict__ = self.__dict__.copy()
        __dict__.update(app=None, domains={}, events=None)  # clear unpickable attributes
        return __dict__

    def __setstate__(self, state: dict) -> None:
        self.__dict__.update(state)

    def setup(self, app: Sphinx) -> None:
        """Set up BuildEnvironment object."""
        if self.version and self.version != app.registry.get_envversion(app):
            raise BuildEnvironmentError(__('build environment version not current'))
        if self.srcdir and self.srcdir != app.srcdir:
            raise BuildEnvironmentError(__('source directory has changed'))

        if self.project:
            app.project.restore(self.project)

        self.app = app
        self.doctreedir = app.doctreedir
        self.events = app.events
        self.srcdir = app.srcdir
        self.project = app.project
        self.version = app.registry.get_envversion(app)

        # initialize domains
        self.domains = _DomainsType()
        for domain in app.registry.create_domains(self):
            self.domains[domain.name] = domain

        # setup domains (must do after all initialization)
        for domain in self.domains.values():
            domain.setup()

        # initialize config
        self._update_config(app.config)

        # initialie settings
        self._update_settings(app.config)

    def _update_config(self, config: Config) -> None:
        """Update configurations by new one."""
        self.config_status = CONFIG_OK
        self.config_status_extra = ''
        if self.config is None:
            self.config_status = CONFIG_NEW
        elif self.config.extensions != config.extensions:
            self.config_status = CONFIG_EXTENSIONS_CHANGED
            extensions = sorted(
                set(self.config.extensions) ^ set(config.extensions))
            if len(extensions) == 1:
                extension = extensions[0]
            else:
                extension = '%d' % (len(extensions),)
            self.config_status_extra = f' ({extension!r})'
        else:
            # check if a config value was changed that affects how
            # doctrees are read
            for item in config.filter('env'):
                if self.config[item.name] != item.value:
                    self.config_status = CONFIG_CHANGED
                    self.config_status_extra = f' ({item.name!r})'
                    break

        self.config = config

    def _update_settings(self, config: Config) -> None:
        """Update settings by new config."""
        self.settings['input_encoding'] = config.source_encoding
        self.settings['trim_footnote_reference_space'] = config.trim_footnote_reference_space
        self.settings['language_code'] = config.language

        # Allow to disable by 3rd party extension (workaround)
        self.settings.setdefault('smart_quotes', True)

    def set_versioning_method(self, method: str | Callable, compare: bool) -> None:
        """This sets the doctree versioning method for this environment.

        Versioning methods are a builder property; only builders with the same
        versioning method can share the same doctree directory.  Therefore, we
        raise an exception if the user tries to use an environment with an
        incompatible versioning method.
        """
        condition: bool | Callable
        if callable(method):
            condition = method
        else:
            if method not in versioning_conditions:
                raise ValueError('invalid versioning method: %r' % method)
            condition = versioning_conditions[method]

        if self.versioning_condition not in (None, condition):
            raise SphinxError(__('This environment is incompatible with the '
                                 'selected builder, please choose another '
                                 'doctree directory.'))
        self.versioning_condition = condition
        self.versioning_compare = compare

    def clear_doc(self, docname: str) -> None:
        """Remove all traces of a source file in the inventory."""
        if docname in self.all_docs:
            self.all_docs.pop(docname, None)
            self.included.pop(docname, None)
            self.reread_always.discard(docname)

        for domain in self.domains.values():
            domain.clear_doc(docname)

    def merge_info_from(self, docnames: list[str], other: BuildEnvironment,
                        app: Sphinx) -> None:
        """Merge global information gathered about *docnames* while reading them
        from the *other* environment.

        This possibly comes from a parallel build process.
        """
        docnames = set(docnames)  # type: ignore[assignment]
        for docname in docnames:
            self.all_docs[docname] = other.all_docs[docname]
            self.included[docname] = other.included[docname]
            if docname in other.reread_always:
                self.reread_always.add(docname)

        for domainname, domain in self.domains.items():
            domain.merge_domaindata(docnames, other.domaindata[domainname])
        self.events.emit('env-merge-info', self, docnames, other)

    def path2doc(self, filename: str | os.PathLike[str]) -> str | None:
        """Return the docname for the filename if the file is document.

        *filename* should be absolute or relative to the source directory.
        """
        return self.project.path2doc(filename)

    def doc2path(self, docname: str, base: bool = True) -> str:
        """Return the filename for the document name.

        If *base* is True, return absolute path under self.srcdir.
        If *base* is False, return relative path to self.srcdir.
        """
        return self.project.doc2path(docname, base)

    def relfn2path(self, filename: str, docname: str | None = None) -> tuple[str, str]:
        """Return paths to a file referenced from a document, relative to
        documentation root and absolute.

        In the input "filename", absolute filenames are taken as relative to the
        source dir, while relative filenames are relative to the dir of the
        containing document.
        """
        filename = os_path(filename)
        if filename.startswith(('/', os.sep)):
            rel_fn = filename[1:]
        else:
            docdir = path.dirname(self.doc2path(docname or self.docname,
                                                base=False))
            rel_fn = path.join(docdir, filename)

        return (canon_path(path.normpath(rel_fn)),
                path.normpath(path.join(self.srcdir, rel_fn)))

    @property
    def found_docs(self) -> set[str]:
        """contains all existing docnames."""
        return self.project.docnames

    def find_files(self, config: Config, builder: Builder) -> None:
        """Find all source files in the source dir and put them in
        self.found_docs.
        """
        try:
            exclude_paths = (self.config.exclude_patterns +
                             self.config.templates_path +
                             builder.get_asset_paths())
            self.project.discover(exclude_paths, self.config.include_patterns)

            # Current implementation is applying translated messages in the reading
            # phase.Therefore, in order to apply the updated message catalog, it is
            # necessary to re-process from the reading phase. Here, if dependency
            # is set for the doc source and the mo file, it is processed again from
            # the reading phase when mo is updated. In the future, we would like to
            # move i18n process into the writing phase, and remove these lines.
            if builder.use_message_catalog:
                # add catalog mo file dependency
                repo = CatalogRepository(self.srcdir, self.config.locale_dirs,
                                         self.config.language, self.config.source_encoding)
                mo_paths = {c.domain: c.mo_path for c in repo.catalogs}
                for docname in self.found_docs:
                    domain = docname_to_domain(docname, self.config.gettext_compact)
                    if domain in mo_paths:
                        self.dependencies[docname].add(mo_paths[domain])
        except OSError as exc:
            raise DocumentError(__('Failed to scan documents in %s: %r') %
                                (self.srcdir, exc)) from exc

    def get_outdated_files(self, config_changed: bool) -> tuple[set[str], set[str], set[str]]:
        """Return (added, changed, removed) sets."""
        # clear all files no longer present
        removed = set(self.all_docs) - self.found_docs

        added: set[str] = set()
        changed: set[str] = set()

        if config_changed:
            # config values affect e.g. substitutions
            added = self.found_docs
        else:
            for docname in self.found_docs:
                if docname not in self.all_docs:
                    logger.debug('[build target] added %r', docname)
                    added.add(docname)
                    continue
                # if the doctree file is not there, rebuild
                filename = path.join(self.doctreedir, docname + '.doctree')
                if not path.isfile(filename):
                    logger.debug('[build target] changed %r', docname)
                    changed.add(docname)
                    continue
                # check the "reread always" list
                if docname in self.reread_always:
                    logger.debug('[build target] changed %r', docname)
                    changed.add(docname)
                    continue
                # check the mtime of the document
                mtime = self.all_docs[docname]
                newmtime = _last_modified_time(self.doc2path(docname))
                if newmtime > mtime:
                    logger.debug('[build target] outdated %r: %s -> %s',
                                 docname,
                                 _format_modified_time(mtime), _format_modified_time(newmtime))
                    changed.add(docname)
                    continue
                # finally, check the mtime of dependencies
                for dep in self.dependencies[docname]:
                    try:
                        # this will do the right thing when dep is absolute too
                        deppath = path.join(self.srcdir, dep)
                        if not path.isfile(deppath):
                            logger.debug(
                                '[build target] changed %r missing dependency %r',
                                docname, deppath,
                            )
                            changed.add(docname)
                            break
                        depmtime = _last_modified_time(deppath)
                        if depmtime > mtime:
                            logger.debug(
                                '[build target] outdated %r from dependency %r: %s -> %s',
                                docname, deppath,
                                _format_modified_time(mtime), _format_modified_time(depmtime),
                            )
                            changed.add(docname)
                            break
                    except OSError:
                        # give it another chance
                        changed.add(docname)
                        break

        return added, changed, removed

    def check_dependents(self, app: Sphinx, already: set[str]) -> Generator[str, None, None]:
        to_rewrite: list[str] = []
        for docnames in self.events.emit('env-get-updated', self):
            to_rewrite.extend(docnames)
        for docname in set(to_rewrite):
            if docname not in already:
                yield docname

    # --------- SINGLE FILE READING --------------------------------------------

    def prepare_settings(self, docname: str) -> None:
        """Prepare to set up environment for reading."""
        self.temp_data['docname'] = docname
        # defaults to the global default, but can be re-set in a document
        self.temp_data['default_role'] = self.config.default_role
        self.temp_data['default_domain'] = \
            self.domains.get(self.config.primary_domain)

    # utilities to use while reading a document

    @property
    def docname(self) -> str:
        """Returns the docname of the document currently being parsed."""
        return self.temp_data['docname']

    def new_serialno(self, category: str = '') -> int:
        """Return a serial number, e.g. for index entry targets.

        The number is guaranteed to be unique in the current document.
        """
        key = category + 'serialno'
        cur = self.temp_data.get(key, 0)
        self.temp_data[key] = cur + 1
        return cur

    def note_dependency(self, filename: str) -> None:
        """Add *filename* as a dependency of the current document.

        This means that the document will be rebuilt if this file changes.

        *filename* should be absolute or relative to the source directory.
        """
        self.dependencies[self.docname].add(filename)

    def note_included(self, filename: str) -> None:
        """Add *filename* as a included from other document.

        This means the document is not orphaned.

        *filename* should be absolute or relative to the source directory.
        """
        doc = self.path2doc(filename)
        if doc:
            self.included[self.docname].add(doc)

    def note_reread(self) -> None:
        """Add the current document to the list of documents that will
        automatically be re-read at the next build.
        """
        self.reread_always.add(self.docname)

    def get_domain(self, domainname: str) -> Domain:
        """Return the domain instance with the specified name.

        Raises an ExtensionError if the domain is not registered.
        """
        try:
            return self.domains[domainname]
        except KeyError as exc:
            raise ExtensionError(__('Domain %r is not registered') % domainname) from exc

    # --------- RESOLVING REFERENCES AND TOCTREES ------------------------------

    def get_doctree(self, docname: str) -> nodes.document:
        """Read the doctree for a file from the pickle and return it."""
        try:
            serialised = self._pickled_doctree_cache[docname]
        except KeyError:
            filename = path.join(self.doctreedir, docname + '.doctree')
            with open(filename, 'rb') as f:
                serialised = self._pickled_doctree_cache[docname] = f.read()

        doctree = pickle.loads(serialised)
        doctree.settings.env = self
        doctree.reporter = LoggingReporter(self.doc2path(docname))
        return doctree

    @functools.cached_property
    def master_doctree(self) -> nodes.document:
        return self.get_doctree(self.config.root_doc)

    def get_and_resolve_doctree(
        self,
        docname: str,
        builder: Builder,
        doctree: nodes.document | None = None,
        prune_toctrees: bool = True,
        includehidden: bool = False,
    ) -> nodes.document:
        """Read the doctree from the pickle, resolve cross-references and
        toctrees and return it.
        """
        if doctree is None:
            try:
                doctree = self._write_doc_doctree_cache.pop(docname)
                doctree.settings.env = self
                doctree.reporter = LoggingReporter(self.doc2path(docname))
            except KeyError:
                doctree = self.get_doctree(docname)

        # resolve all pending cross-references
        self.apply_post_transforms(doctree, docname)

        # now, resolve all toctree nodes
        for toctreenode in doctree.findall(addnodes.toctree):
            result = toctree_adapters._resolve_toctree(
                self, docname, builder, toctreenode,
                prune=prune_toctrees,
                includehidden=includehidden,
            )
            if result is None:
                toctreenode.parent.replace(toctreenode, [])
            else:
                toctreenode.replace_self(result)

        return doctree

    def resolve_toctree(self, docname: str, builder: Builder, toctree: addnodes.toctree,
                        prune: bool = True, maxdepth: int = 0, titles_only: bool = False,
                        collapse: bool = False, includehidden: bool = False) -> Node | None:
        """Resolve a *toctree* node into individual bullet lists with titles
        as items, returning None (if no containing titles are found) or
        a new node.

        If *prune* is True, the tree is pruned to *maxdepth*, or if that is 0,
        to the value of the *maxdepth* option on the *toctree* node.
        If *titles_only* is True, only toplevel document titles will be in the
        resulting tree.
        If *collapse* is True, all branches not containing docname will
        be collapsed.
        """
        return toctree_adapters._resolve_toctree(
            self, docname, builder, toctree,
            prune=prune,
            maxdepth=maxdepth,
            titles_only=titles_only,
            collapse=collapse,
            includehidden=includehidden,
        )

    def resolve_references(self, doctree: nodes.document, fromdocname: str,
                           builder: Builder) -> None:
        self.apply_post_transforms(doctree, fromdocname)

    def apply_post_transforms(self, doctree: nodes.document, docname: str) -> None:
        """Apply all post-transforms."""
        try:
            # set env.docname during applying post-transforms
            backup = copy(self.temp_data)
            self.temp_data['docname'] = docname

            transformer = SphinxTransformer(doctree)
            transformer.set_environment(self)
            transformer.add_transforms(self.app.registry.get_post_transforms())
            transformer.apply_transforms()
        finally:
            self.temp_data = backup

        # allow custom references to be resolved
        self.events.emit('doctree-resolved', doctree, docname)

    def collect_relations(self) -> dict[str, list[str | None]]:
        traversed: set[str] = set()

        relations = {}
        docnames = _traverse_toctree(
            traversed, None, self.config.root_doc, self.toctree_includes,
        )
        prev_doc = None
        parent, docname = next(docnames)
        for next_parent, next_doc in docnames:
            relations[docname] = [parent, prev_doc, next_doc]
            prev_doc = docname
            docname = next_doc
            parent = next_parent

        relations[docname] = [parent, prev_doc, None]

        return relations

    def check_consistency(self) -> None:
        """Do consistency checks."""
        included = set().union(*self.included.values())
        for docname in sorted(self.all_docs):
            if docname not in self.files_to_rebuild:
                if docname == self.config.root_doc:
                    # the master file is not included anywhere ;)
                    continue
                if docname in included:
                    # the document is included from other documents
                    continue
                if 'orphan' in self.metadata[docname]:
                    continue
                logger.warning(__("document isn't included in any toctree"),
                               location=docname)

        # call check-consistency for all extensions
        for domain in self.domains.values():
            domain.check_consistency()
        self.events.emit('env-check-consistency', self)


def _last_modified_time(filename: str | os.PathLike[str]) -> int:
    """Return the last modified time of ``filename``.

    The time is returned as integer microseconds.
    The lowest common denominator of modern file-systems seems to be
    microsecond-level precision.

    We prefer to err on the side of re-rendering a file,
    so we round up to the nearest microsecond.
    """

    # upside-down floor division to get the ceiling
    return -(os.stat(filename).st_mtime_ns // -1_000)


def _format_modified_time(timestamp: int) -> str:
    """Return an RFC 3339 formatted string representing the given timestamp."""
    seconds, fraction = divmod(timestamp, 10**6)
    return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(seconds)) + f'.{fraction//1_000}'


def _traverse_toctree(
    traversed: set[str],
    parent: str | None,
    docname: str,
    toctree_includes: dict[str, list[str]],
) -> Iterator[tuple[str | None, str]]:
    if parent == docname:
        logger.warning(__('self referenced toctree found. Ignored.'),
                       location=docname, type='toc',
                       subtype='circular')
        return

    # traverse toctree by pre-order
    yield parent, docname
    traversed.add(docname)

    for child in toctree_includes.get(docname, ()):
        for sub_parent, sub_docname in _traverse_toctree(
            traversed, docname, child, toctree_includes,
        ):
            if sub_docname not in traversed:
                yield sub_parent, sub_docname
                traversed.add(sub_docname)

env_pyrecodes/lib/python3.9/site-packages/sphinx/environment/__init__.py
NikolaBlagojevic/pyrecodes
"""Toctree adapter for sphinx.environment."""

from __future__ import annotations

from typing import TYPE_CHECKING, Any, TypeVar

from docutils import nodes
from docutils.nodes import Element, Node

from sphinx import addnodes
from sphinx.locale import __
from sphinx.util import logging, url_re
from sphinx.util.matching import Matcher
from sphinx.util.nodes import _only_node_keep_children, clean_astext

if TYPE_CHECKING:
    from collections.abc import Iterable, Set

    from sphinx.builders import Builder
    from sphinx.environment import BuildEnvironment
    from sphinx.util.tags import Tags


logger = logging.getLogger(__name__)


def note_toctree(env: BuildEnvironment, docname: str, toctreenode: addnodes.toctree) -> None:
    """Note a TOC tree directive in a document and gather information about
    file relations from it.
    """
    if toctreenode['glob']:
        env.glob_toctrees.add(docname)
    if toctreenode.get('numbered'):
        env.numbered_toctrees.add(docname)
    include_files = toctreenode['includefiles']
    for include_file in include_files:
        # note that if the included file is rebuilt, this one must be
        # too (since the TOC of the included file could have changed)
        env.files_to_rebuild.setdefault(include_file, set()).add(docname)
    env.toctree_includes.setdefault(docname, []).extend(include_files)


def document_toc(env: BuildEnvironment, docname: str, tags: Tags) -> Node:
    """Get the (local) table of contents for a document.

    Note that this is only the sections within the document.
    For a ToC tree that shows the document's place in the
    ToC structure, use `get_toctree_for`.
    """

    tocdepth = env.metadata[docname].get('tocdepth', 0)
    try:
        toc = _toctree_copy(env.tocs[docname], 2, tocdepth, False, tags)
    except KeyError:
        # the document does not exist any more:
        # return a dummy node that renders to nothing
        return nodes.paragraph()

    for node in toc.findall(nodes.reference):
        node['refuri'] = node['anchorname'] or '#'
    return toc


def global_toctree_for_doc(
    env: BuildEnvironment,
    docname: str,
    builder: Builder,
    collapse: bool = False,
    includehidden: bool = True,
    maxdepth: int = 0,
    titles_only: bool = False,
) -> Element | None:
    """Get the global ToC tree at a given document.

    This gives the global ToC, with all ancestors and their siblings.
    """

    toctrees: list[Element] = []
    for toctree_node in env.master_doctree.findall(addnodes.toctree):
        if toctree := _resolve_toctree(
            env,
            docname,
            builder,
            toctree_node,
            prune=True,
            maxdepth=int(maxdepth),
            titles_only=titles_only,
            collapse=collapse,
            includehidden=includehidden,
        ):
            toctrees.append(toctree)
    if not toctrees:
        return None
    result = toctrees[0]
    for toctree in toctrees[1:]:
        result.extend(toctree.children)
    return result


def _resolve_toctree(
    env: BuildEnvironment, docname: str, builder: Builder, toctree: addnodes.toctree, *,
    prune: bool = True, maxdepth: int = 0, titles_only: bool = False,
    collapse: bool = False, includehidden: bool = False,
) -> Element | None:
    """Resolve a *toctree* node into individual bullet lists with titles
    as items, returning None (if no containing titles are found) or
    a new node.

    If *prune* is True, the tree is pruned to *maxdepth*, or if that is 0,
    to the value of the *maxdepth* option on the *toctree* node.
    If *titles_only* is True, only toplevel document titles will be in the
    resulting tree.
    If *collapse* is True, all branches not containing docname will
    be collapsed.
    """

    if toctree.get('hidden', False) and not includehidden:
        return None

    # For reading the following two helper function, it is useful to keep
    # in mind the node structure of a toctree (using HTML-like node names
    # for brevity):
    #
    # <ul>
    #   <li>
    #     <p><a></p>
    #     <p><a></p>
    #     ...
    #     <ul>
    #       ...
    #     </ul>
    #   </li>
    # </ul>
    #
    # The transformation is made in two passes in order to avoid
    # interactions between marking and pruning the tree (see bug #1046).

    toctree_ancestors = _get_toctree_ancestors(env.toctree_includes, docname)
    included = Matcher(env.config.include_patterns)
    excluded = Matcher(env.config.exclude_patterns)

    maxdepth = maxdepth or toctree.get('maxdepth', -1)
    if not titles_only and toctree.get('titlesonly', False):
        titles_only = True
    if not includehidden and toctree.get('includehidden', False):
        includehidden = True

    tocentries = _entries_from_toctree(
        env,
        prune,
        titles_only,
        collapse,
        includehidden,
        builder.tags,
        toctree_ancestors,
        included,
        excluded,
        toctree,
        [],
    )
    if not tocentries:
        return None

    newnode = addnodes.compact_paragraph('', '')
    if caption := toctree.attributes.get('caption'):
        caption_node = nodes.title(caption, '', *[nodes.Text(caption)])
        caption_node.line = toctree.line
        caption_node.source = toctree.source
        caption_node.rawsource = toctree['rawcaption']
        if hasattr(toctree, 'uid'):
            # move uid to caption_node to translate it
            caption_node.uid = toctree.uid  # type: ignore[attr-defined]
            del toctree.uid
        newnode.append(caption_node)
    newnode.extend(tocentries)
    newnode['toctree'] = True

    # prune the tree to maxdepth, also set toc depth and current classes
    _toctree_add_classes(newnode, 1, docname)
    newnode = _toctree_copy(newnode, 1, maxdepth if prune else 0, collapse, builder.tags)

    if isinstance(newnode[-1], nodes.Element) and len(newnode[-1]) == 0:  # No titles found
        return None

    # set the target paths in the toctrees (they are not known at TOC
    # generation time)
    for refnode in newnode.findall(nodes.reference):
        if url_re.match(refnode['refuri']) is None:
            rel_uri = builder.get_relative_uri(docname, refnode['refuri'])
            refnode['refuri'] = rel_uri + refnode['anchorname']
    return newnode


def _entries_from_toctree(
    env: BuildEnvironment,
    prune: bool,
    titles_only: bool,
    collapse: bool,
    includehidden: bool,
    tags: Tags,
    toctree_ancestors: Set[str],
    included: Matcher,
    excluded: Matcher,
    toctreenode: addnodes.toctree,
    parents: list[str],
    subtree: bool = False,
) -> list[Element]:
    """Return TOC entries for a toctree node."""
    entries: list[Element] = []
    for (title, ref) in toctreenode['entries']:
        try:
            toc, refdoc = _toctree_entry(
                title, ref, env, prune, collapse, tags, toctree_ancestors,
                included, excluded, toctreenode, parents,
            )
        except LookupError:
            continue

        # children of toc are:
        # - list_item + compact_paragraph + (reference and subtoc)
        # - only + subtoc
        # - toctree
        children: Iterable[nodes.Element] = toc.children  # type: ignore[assignment]

        # if titles_only is given, only keep the main title and
        # sub-toctrees
        if titles_only:
            # delete everything but the toplevel title(s)
            # and toctrees
            for top_level in children:
                # nodes with length 1 don't have any children anyway
                if len(top_level) > 1:
                    if subtrees := list(top_level.findall(addnodes.toctree)):
                        top_level[1][:] = subtrees  # type: ignore[index]
                    else:
                        top_level.pop(1)
        # resolve all sub-toctrees
        for sub_toc_node in list(toc.findall(addnodes.toctree)):
            if sub_toc_node.get('hidden', False) and not includehidden:
                continue
            for i, entry in enumerate(
                _entries_from_toctree(
                    env,
                    prune,
                    titles_only,
                    collapse,
                    includehidden,
                    tags,
                    toctree_ancestors,
                    included,
                    excluded,
                    sub_toc_node,
                    [refdoc] + parents,
                    subtree=True,
                ),
                start=sub_toc_node.parent.index(sub_toc_node) + 1,
            ):
                sub_toc_node.parent.insert(i, entry)
            sub_toc_node.parent.remove(sub_toc_node)

        entries.extend(children)

    if not subtree:
        ret = nodes.bullet_list()
        ret += entries
        return [ret]

    return entries


def _toctree_entry(
    title: str,
    ref: str,
    env: BuildEnvironment,
    prune: bool,
    collapse: bool,
    tags: Tags,
    toctree_ancestors: Set[str],
    included: Matcher,
    excluded: Matcher,
    toctreenode: addnodes.toctree,
    parents: list[str],
) -> tuple[Element, str]:
    from sphinx.domains.std import StandardDomain

    try:
        refdoc = ''
        if url_re.match(ref):
            toc = _toctree_url_entry(title, ref)
        elif ref == 'self':
            toc = _toctree_self_entry(title, toctreenode['parent'], env.titles)
        elif ref in StandardDomain._virtual_doc_names:
            toc = _toctree_generated_entry(title, ref)
        else:
            if ref in parents:
                logger.warning(__('circular toctree references '
                                  'detected, ignoring: %s <- %s'),
                               ref, ' <- '.join(parents),
                               location=ref, type='toc', subtype='circular')
                msg = 'circular reference'
                raise LookupError(msg)

            toc, refdoc = _toctree_standard_entry(
                title,
                ref,
                env.metadata[ref].get('tocdepth', 0),
                env.tocs[ref],
                toctree_ancestors,
                prune,
                collapse,
                tags,
            )

        if not toc.children:
            # empty toc means: no titles will show up in the toctree
            logger.warning(__('toctree contains reference to document %r that '
                              "doesn't have a title: no link will be generated"),
                           ref, location=toctreenode)
    except KeyError:
        # this is raised if the included file does not exist
        ref_path = env.doc2path(ref, False)
        if excluded(ref_path):
            message = __('toctree contains reference to excluded document %r')
        elif not included(ref_path):
            message = __('toctree contains reference to non-included document %r')
        else:
            message = __('toctree contains reference to nonexisting document %r')

        logger.warning(message, ref, location=toctreenode)
        raise
    return toc, refdoc


def _toctree_url_entry(title: str, ref: str) -> nodes.bullet_list:
    if title is None:
        title = ref
    reference = nodes.reference('', '', internal=False,
                                refuri=ref, anchorname='',
                                *[nodes.Text(title)])
    para = addnodes.compact_paragraph('', '', reference)
    item = nodes.list_item('', para)
    toc = nodes.bullet_list('', item)
    return toc


def _toctree_self_entry(
    title: str, ref: str, titles: dict[str, nodes.title],
) -> nodes.bullet_list:
    # 'self' refers to the document from which this
    # toctree originates
    if not title:
        title = clean_astext(titles[ref])
    reference = nodes.reference('', '', internal=True,
                                refuri=ref,
                                anchorname='',
                                *[nodes.Text(title)])
    para = addnodes.compact_paragraph('', '', reference)
    item = nodes.list_item('', para)
    # don't show subitems
    toc = nodes.bullet_list('', item)
    return toc


def _toctree_generated_entry(title: str, ref: str) -> nodes.bullet_list:
    from sphinx.domains.std import StandardDomain

    docname, sectionname = StandardDomain._virtual_doc_names[ref]
    if not title:
        title = sectionname
    reference = nodes.reference('', title, internal=True,
                                refuri=docname, anchorname='')
    para = addnodes.compact_paragraph('', '', reference)
    item = nodes.list_item('', para)
    # don't show subitems
    toc = nodes.bullet_list('', item)
    return toc


def _toctree_standard_entry(
    title: str,
    ref: str,
    maxdepth: int,
    toc: nodes.bullet_list,
    toctree_ancestors: Set[str],
    prune: bool,
    collapse: bool,
    tags: Tags,
) -> tuple[nodes.bullet_list, str]:
    refdoc = ref
    if ref in toctree_ancestors and (not prune or maxdepth <= 0):
        toc = toc.deepcopy()
    else:
        toc = _toctree_copy(toc, 2, maxdepth, collapse, tags)

    if title and toc.children and len(toc.children) == 1:
        child = toc.children[0]
        for refnode in child.findall(nodes.reference):
            if refnode['refuri'] == ref and not refnode['anchorname']:
                refnode.children[:] = [nodes.Text(title)]
    return toc, refdoc


def _toctree_add_classes(node: Element, depth: int, docname: str) -> None:
    """Add 'toctree-l%d' and 'current' classes to the toctree."""
    for subnode in node.children:
        if isinstance(subnode, (addnodes.compact_paragraph, nodes.list_item)):
            # for <p> and <li>, indicate the depth level and recurse
            subnode['classes'].append(f'toctree-l{depth - 1}')
            _toctree_add_classes(subnode, depth, docname)
        elif isinstance(subnode, nodes.bullet_list):
            # for <ul>, just recurse
            _toctree_add_classes(subnode, depth + 1, docname)
        elif isinstance(subnode, nodes.reference):
            # for <a>, identify which entries point to the current
            # document and therefore may not be collapsed
            if subnode['refuri'] == docname:
                if not subnode['anchorname']:
                    # give the whole branch a 'current' class
                    # (useful for styling it differently)
                    branchnode: Element = subnode
                    while branchnode:
                        branchnode['classes'].append('current')
                        branchnode = branchnode.parent
                # mark the list_item as "on current page"
                if subnode.parent.parent.get('iscurrent'):
                    # but only if it's not already done
                    return
                while subnode:
                    subnode['iscurrent'] = True
                    subnode = subnode.parent


ET = TypeVar('ET', bound=Element)


def _toctree_copy(node: ET, depth: int, maxdepth: int, collapse: bool, tags: Tags) -> ET:
    """Utility: Cut and deep-copy a TOC at a specified depth."""
    keep_bullet_list_sub_nodes = (depth <= 1
                                  or ((depth <= maxdepth or maxdepth <= 0)
                                      and (not collapse or 'iscurrent' in node)))

    copy = node.copy()
    for subnode in node.children:
        if isinstance(subnode, (addnodes.compact_paragraph, nodes.list_item)):
            # for <p> and <li>, just recurse
            copy.append(_toctree_copy(subnode, depth, maxdepth, collapse, tags))
        elif isinstance(subnode, nodes.bullet_list):
            # for <ul>, copy if the entry is top-level
            # or, copy if the depth is within bounds and;
            # collapsing is disabled or the sub-entry's parent is 'current'.
            # The boolean is constant so is calculated outwith the loop.
            if keep_bullet_list_sub_nodes:
                copy.append(_toctree_copy(subnode, depth + 1, maxdepth, collapse, tags))
        elif isinstance(subnode, addnodes.toctree):
            # copy sub toctree nodes for later processing
            copy.append(subnode.copy())
        elif isinstance(subnode, addnodes.only):
            # only keep children if the only node matches the tags
            if _only_node_keep_children(subnode, tags):
                for child in subnode.children:
                    copy.append(_toctree_copy(
                        child, depth, maxdepth, collapse, tags,  # type: ignore[type-var]
                    ))
        elif isinstance(subnode, (nodes.reference, nodes.title)):
            # deep copy references and captions
            sub_node_copy = subnode.copy()
            sub_node_copy.children = [child.deepcopy() for child in subnode.children]
            for child in sub_node_copy.children:
                child.parent = sub_node_copy
            copy.append(sub_node_copy)
        else:
            msg = f'Unexpected node type {subnode.__class__.__name__!r}!'
            raise ValueError(msg)
    return copy


def _get_toctree_ancestors(
    toctree_includes: dict[str, list[str]], docname: str,
) -> Set[str]:
    parent: dict[str, str] = {}
    for p, children in toctree_includes.items():
        parent |= dict.fromkeys(children, p)
    ancestors: list[str] = []
    d = docname
    while d in parent and d not in ancestors:
        ancestors.append(d)
        d = parent[d]
    # use dict keys for ordered set operations
    return dict.fromkeys(ancestors).keys()


class TocTree:
    def __init__(self, env: BuildEnvironment) -> None:
        self.env = env

    def note(self, docname: str, toctreenode: addnodes.toctree) -> None:
        note_toctree(self.env, docname, toctreenode)

    def resolve(self, docname: str, builder: Builder, toctree: addnodes.toctree,
                prune: bool = True, maxdepth: int = 0, titles_only: bool = False,
                collapse: bool = False, includehidden: bool = False) -> Element | None:
        return _resolve_toctree(
            self.env, docname, builder, toctree,
            prune=prune,
            maxdepth=maxdepth,
            titles_only=titles_only,
            collapse=collapse,
            includehidden=includehidden,
        )

    def get_toctree_ancestors(self, docname: str) -> list[str]:
        return [*_get_toctree_ancestors(self.env.toctree_includes, docname)]

    def get_toc_for(self, docname: str, builder: Builder) -> Node:
        return document_toc(self.env, docname, self.env.app.builder.tags)

    def get_toctree_for(
        self, docname: str, builder: Builder, collapse: bool, **kwargs: Any,
    ) -> Element | None:
        return global_toctree_for_doc(self.env, docname, builder, collapse=collapse, **kwargs)

env_pyrecodes/lib/python3.9/site-packages/sphinx/environment/adapters/toctree.py
NikolaBlagojevic/pyrecodes
"""Sphinx core events.

Gracefully adapted from the TextPress system by Armin.
"""

from __future__ import annotations

import contextlib
from collections import defaultdict
from operator import attrgetter
from typing import TYPE_CHECKING, Any, Callable, NamedTuple

from sphinx.errors import ExtensionError, SphinxError
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.inspect import safe_getattr

if TYPE_CHECKING:
    from sphinx.application import Sphinx


logger = logging.getLogger(__name__)


class EventListener(NamedTuple):
    id: int
    handler: Callable
    priority: int


# List of all known core events. Maps name to arguments description.
core_events = {
    'builder-inited': '',
    'config-inited': 'config',
    'env-get-outdated': 'env, added, changed, removed',
    'env-get-updated': 'env',
    'env-purge-doc': 'env, docname',
    'env-before-read-docs': 'env, docnames',
    'env-check-consistency': 'env',
    'source-read': 'docname, source text',
    'include-read': 'relative path, parent docname, source text',
    'doctree-read': 'the doctree before being pickled',
    'env-merge-info': 'env, read docnames, other env instance',
    'missing-reference': 'env, node, contnode',
    'warn-missing-reference': 'domain, node',
    'doctree-resolved': 'doctree, docname',
    'env-updated': 'env',
    'build-finished': 'exception',
}


class EventManager:
    """Event manager for Sphinx."""

    def __init__(self, app: Sphinx) -> None:
        self.app = app
        self.events = core_events.copy()
        self.listeners: dict[str, list[EventListener]] = defaultdict(list)
        self.next_listener_id = 0

    def add(self, name: str) -> None:
        """Register a custom Sphinx event."""
        if name in self.events:
            raise ExtensionError(__('Event %r already present') % name)
        self.events[name] = ''

    def connect(self, name: str, callback: Callable, priority: int) -> int:
        """Connect a handler to specific event."""
        if name not in self.events:
            raise ExtensionError(__('Unknown event name: %s') % name)

        listener_id = self.next_listener_id
        self.next_listener_id += 1
        self.listeners[name].append(EventListener(listener_id, callback, priority))
        return listener_id

    def disconnect(self, listener_id: int) -> None:
        """Disconnect a handler."""
        for listeners in self.listeners.values():
            for listener in listeners[:]:
                if listener.id == listener_id:
                    listeners.remove(listener)

    def emit(self, name: str, *args: Any,
             allowed_exceptions: tuple[type[Exception], ...] = ()) -> list:
        """Emit a Sphinx event."""

        # not every object likes to be repr()'d (think
        # random stuff coming via autodoc)
        with contextlib.suppress(Exception):
            logger.debug('[app] emitting event: %r%s', name, repr(args)[:100])

        results = []
        listeners = sorted(self.listeners[name], key=attrgetter("priority"))
        for listener in listeners:
            try:
                results.append(listener.handler(self.app, *args))
            except allowed_exceptions:
                # pass through the errors specified as *allowed_exceptions*
                raise
            except SphinxError:
                raise
            except Exception as exc:
                if self.app.pdb:
                    # Just pass through the error, so that it can be debugged.
                    raise
                modname = safe_getattr(listener.handler, '__module__', None)
                raise ExtensionError(__("Handler %r for event %r threw an exception") %
                                     (listener.handler, name), exc, modname=modname) from exc
        return results

    def emit_firstresult(self, name: str, *args: Any,
                         allowed_exceptions: tuple[type[Exception], ...] = ()) -> Any:
        """Emit a Sphinx event and returns first result.

        This returns the result of the first handler that doesn't return ``None``.
        """
        for result in self.emit(name, *args, allowed_exceptions=allowed_exceptions):
            if result is not None:
                return result
        return None

env_pyrecodes/lib/python3.9/site-packages/sphinx/events.py
NikolaBlagojevic/pyrecodes
"""Creates reST files corresponding to Python modules for code documentation.

Parses a directory tree looking for Python modules and packages and creates
ReST files appropriately to create code documentation with Sphinx.  It also
creates a modules index (named modules.<suffix>).

This is derived from the "sphinx-autopackage" script, which is:
Copyright 2008 Société des arts technologiques (SAT),
https://sat.qc.ca/
"""

from __future__ import annotations

import argparse
import fnmatch
import glob
import locale
import os
import re
import sys
from copy import copy
from importlib.machinery import EXTENSION_SUFFIXES
from os import path
from typing import TYPE_CHECKING, Any

import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.cmd.quickstart import EXTENSIONS
from sphinx.locale import __
from sphinx.util import logging
from sphinx.util.osutil import FileAvoidWrite, ensuredir
from sphinx.util.template import ReSTRenderer

if TYPE_CHECKING:
    from collections.abc import Generator, Sequence

logger = logging.getLogger(__name__)

# automodule options
if 'SPHINX_APIDOC_OPTIONS' in os.environ:
    OPTIONS = os.environ['SPHINX_APIDOC_OPTIONS'].split(',')
else:
    OPTIONS = [
        'members',
        'undoc-members',
        # 'inherited-members', # disabled because there's a bug in sphinx
        'show-inheritance',
    ]

PY_SUFFIXES = ('.py', '.pyx') + tuple(EXTENSION_SUFFIXES)

template_dir = path.join(package_dir, 'templates', 'apidoc')


def is_initpy(filename: str) -> bool:
    """Check *filename* is __init__ file or not."""
    basename = path.basename(filename)
    return any(
        basename == '__init__' + suffix
        for suffix in sorted(PY_SUFFIXES, key=len, reverse=True)
    )


def module_join(*modnames: str | None) -> str:
    """Join module names with dots."""
    return '.'.join(filter(None, modnames))


def is_packagedir(dirname: str | None = None, files: list[str] | None = None) -> bool:
    """Check given *files* contains __init__ file."""
    if files is None and dirname is None:
        return False

    if files is None:
        files = os.listdir(dirname)
    return any(f for f in files if is_initpy(f))


def write_file(name: str, text: str, opts: Any) -> None:
    """Write the output file for module/package <name>."""
    quiet = getattr(opts, 'quiet', None)

    fname = path.join(opts.destdir, f'{name}.{opts.suffix}')
    if opts.dryrun:
        if not quiet:
            logger.info(__('Would create file %s.'), fname)
        return
    if not opts.force and path.isfile(fname):
        if not quiet:
            logger.info(__('File %s already exists, skipping.'), fname)
    else:
        if not quiet:
            logger.info(__('Creating file %s.'), fname)
        with FileAvoidWrite(fname) as f:
            f.write(text)


def create_module_file(package: str | None, basename: str, opts: Any,
                       user_template_dir: str | None = None) -> None:
    """Build the text of the file and write the file."""
    options = copy(OPTIONS)
    if opts.includeprivate and 'private-members' not in options:
        options.append('private-members')

    qualname = module_join(package, basename)
    context = {
        'show_headings': not opts.noheadings,
        'basename': basename,
        'qualname': qualname,
        'automodule_options': options,
    }
    if user_template_dir is not None:
        template_path = [user_template_dir, template_dir]
    else:
        template_path = [template_dir]
    text = ReSTRenderer(template_path).render('module.rst_t', context)
    write_file(qualname, text, opts)


def create_package_file(root: str, master_package: str | None, subroot: str,
                        py_files: list[str],
                        opts: Any, subs: list[str], is_namespace: bool,
                        excludes: Sequence[re.Pattern[str]] = (),
                        user_template_dir: str | None = None,
                        ) -> None:
    """Build the text of the file and write the file."""
    # build a list of sub packages (directories containing an __init__ file)
    subpackages = [module_join(master_package, subroot, pkgname)
                   for pkgname in subs
                   if not is_skipped_package(path.join(root, pkgname), opts, excludes)]
    # build a list of sub modules
    submodules = [sub.split('.')[0] for sub in py_files
                  if not is_skipped_module(path.join(root, sub), opts, excludes) and
                  not is_initpy(sub)]
    submodules = sorted(set(submodules))
    submodules = [module_join(master_package, subroot, modname)
                  for modname in submodules]
    options = copy(OPTIONS)
    if opts.includeprivate and 'private-members' not in options:
        options.append('private-members')

    pkgname = module_join(master_package, subroot)
    context = {
        'pkgname': pkgname,
        'subpackages': subpackages,
        'submodules': submodules,
        'is_namespace': is_namespace,
        'modulefirst': opts.modulefirst,
        'separatemodules': opts.separatemodules,
        'automodule_options': options,
        'show_headings': not opts.noheadings,
        'maxdepth': opts.maxdepth,
    }
    if user_template_dir is not None:
        template_path = [user_template_dir, template_dir]
    else:
        template_path = [template_dir]
    text = ReSTRenderer(template_path).render('package.rst_t', context)
    write_file(pkgname, text, opts)

    if submodules and opts.separatemodules:
        for submodule in submodules:
            create_module_file(None, submodule, opts, user_template_dir)


def create_modules_toc_file(modules: list[str], opts: Any, name: str = 'modules',
                            user_template_dir: str | None = None) -> None:
    """Create the module's index."""
    modules.sort()
    prev_module = ''
    for module in modules[:]:
        # look if the module is a subpackage and, if yes, ignore it
        if module.startswith(prev_module + '.'):
            modules.remove(module)
        else:
            prev_module = module

    context = {
        'header': opts.header,
        'maxdepth': opts.maxdepth,
        'docnames': modules,
    }
    if user_template_dir is not None:
        template_path = [user_template_dir, template_dir]
    else:
        template_path = [template_dir]
    text = ReSTRenderer(template_path).render('toc.rst_t', context)
    write_file(name, text, opts)


def is_skipped_package(dirname: str, opts: Any,
                       excludes: Sequence[re.Pattern[str]] = ()) -> bool:
    """Check if we want to skip this module."""
    if not path.isdir(dirname):
        return False

    files = glob.glob(path.join(dirname, '*.py'))
    regular_package = any(f for f in files if is_initpy(f))
    if not regular_package and not opts.implicit_namespaces:
        # *dirname* is not both a regular package and an implicit namespace package
        return True

    # Check there is some showable module inside package
    return all(is_excluded(path.join(dirname, f), excludes) for f in files)


def is_skipped_module(filename: str, opts: Any, _excludes: Sequence[re.Pattern[str]]) -> bool:
    """Check if we want to skip this module."""
    if not path.exists(filename):
        # skip if the file doesn't exist
        return True
    if path.basename(filename).startswith('_') and not opts.includeprivate:
        # skip if the module has a "private" name
        return True
    return False


def walk(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: Any,
         ) -> Generator[tuple[str, list[str], list[str]], None, None]:
    """Walk through the directory and list files and subdirectories up."""
    followlinks = getattr(opts, 'followlinks', False)
    includeprivate = getattr(opts, 'includeprivate', False)

    for root, subs, files in os.walk(rootpath, followlinks=followlinks):
        # document only Python module files (that aren't excluded)
        files = sorted(f for f in files
                       if f.endswith(PY_SUFFIXES) and
                       not is_excluded(path.join(root, f), excludes))

        # remove hidden ('.') and private ('_') directories, as well as
        # excluded dirs
        if includeprivate:
            exclude_prefixes: tuple[str, ...] = ('.',)
        else:
            exclude_prefixes = ('.', '_')

        subs[:] = sorted(sub for sub in subs if not sub.startswith(exclude_prefixes) and
                         not is_excluded(path.join(root, sub), excludes))

        yield root, subs, files


def has_child_module(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: Any) -> bool:
    """Check the given directory contains child module/s (at least one)."""
    return any(
        files
        for _root, _subs, files in walk(rootpath, excludes, opts)
    )


def recurse_tree(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: Any,
                 user_template_dir: str | None = None) -> list[str]:
    """
    Look for every file in the directory tree and create the corresponding
    ReST files.
    """
    implicit_namespaces = getattr(opts, 'implicit_namespaces', False)

    # check if the base directory is a package and get its name
    if is_packagedir(rootpath) or implicit_namespaces:
        root_package = rootpath.split(path.sep)[-1]
    else:
        # otherwise, the base is a directory with packages
        root_package = None

    toplevels = []
    for root, subs, files in walk(rootpath, excludes, opts):
        is_pkg = is_packagedir(None, files)
        is_namespace = not is_pkg and implicit_namespaces
        if is_pkg:
            for f in files[:]:
                if is_initpy(f):
                    files.remove(f)
                    files.insert(0, f)
        elif root != rootpath:
            # only accept non-package at toplevel unless using implicit namespaces
            if not implicit_namespaces:
                del subs[:]
                continue

        if is_pkg or is_namespace:
            # we are in a package with something to document
            if subs or len(files) > 1 or not is_skipped_package(root, opts):
                subpackage = root[len(rootpath):].lstrip(path.sep).\
                    replace(path.sep, '.')
                # if this is not a namespace or
                # a namespace and there is something there to document
                if not is_namespace or has_child_module(root, excludes, opts):
                    create_package_file(root, root_package, subpackage,
                                        files, opts, subs, is_namespace, excludes,
                                        user_template_dir)
                    toplevels.append(module_join(root_package, subpackage))
        else:
            # if we are at the root level, we don't require it to be a package
            assert root == rootpath
            assert root_package is None
            for py_file in files:
                if not is_skipped_module(path.join(rootpath, py_file), opts, excludes):
                    module = py_file.split('.')[0]
                    create_module_file(root_package, module, opts, user_template_dir)
                    toplevels.append(module)

    return toplevels


def is_excluded(root: str, excludes: Sequence[re.Pattern[str]]) -> bool:
    """Check if the directory is in the exclude list.

    Note: by having trailing slashes, we avoid common prefix issues, like
          e.g. an exclude "foo" also accidentally excluding "foobar".
    """
    return any(exclude.match(root) for exclude in excludes)


def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        usage='%(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> '
              '[EXCLUDE_PATTERN, ...]',
        epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'),
        description=__("""
Look recursively in <MODULE_PATH> for Python modules and packages and create
one reST file with automodule directives per package in the <OUTPUT_PATH>.

The <EXCLUDE_PATTERN>s can be file and/or directory patterns that will be
excluded from generation.

Note: By default this script will not overwrite already created files."""))

    parser.add_argument('--version', action='version', dest='show_version',
                        version='%%(prog)s %s' % __display_version__)

    parser.add_argument('module_path',
                        help=__('path to module to document'))
    parser.add_argument('exclude_pattern', nargs='*',
                        help=__('fnmatch-style file and/or directory patterns '
                                'to exclude from generation'))

    parser.add_argument('-o', '--output-dir', action='store', dest='destdir',
                        required=True,
                        help=__('directory to place all output'))
    parser.add_argument('-q', action='store_true', dest='quiet',
                        help=__('no output on stdout, just warnings on stderr'))
    parser.add_argument('-d', '--maxdepth', action='store', dest='maxdepth',
                        type=int, default=4,
                        help=__('maximum depth of submodules to show in the TOC '
                                '(default: 4)'))
    parser.add_argument('-f', '--force', action='store_true', dest='force',
                        help=__('overwrite existing files'))
    parser.add_argument('-l', '--follow-links', action='store_true',
                        dest='followlinks', default=False,
                        help=__('follow symbolic links. Powerful when combined '
                                'with collective.recipe.omelette.'))
    parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun',
                        help=__('run the script without creating files'))
    parser.add_argument('-e', '--separate', action='store_true',
                        dest='separatemodules',
                        help=__('put documentation for each module on its own page'))
    parser.add_argument('-P', '--private', action='store_true',
                        dest='includeprivate',
                        help=__('include "_private" modules'))
    parser.add_argument('--tocfile', action='store', dest='tocfile', default='modules',
                        help=__("filename of table of contents (default: modules)"))
    parser.add_argument('-T', '--no-toc', action='store_false', dest='tocfile',
                        help=__("don't create a table of contents file"))
    parser.add_argument('-E', '--no-headings', action='store_true',
                        dest='noheadings',
                        help=__("don't create headings for the module/package "
                                "packages (e.g. when the docstrings already "
                                "contain them)"))
    parser.add_argument('-M', '--module-first', action='store_true',
                        dest='modulefirst',
                        help=__('put module documentation before submodule '
                                'documentation'))
    parser.add_argument('--implicit-namespaces', action='store_true',
                        dest='implicit_namespaces',
                        help=__('interpret module paths according to PEP-0420 '
                                'implicit namespaces specification'))
    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
                        default='rst',
                        help=__('file suffix (default: rst)'))
    parser.add_argument('-F', '--full', action='store_true', dest='full',
                        help=__('generate a full project with sphinx-quickstart'))
    parser.add_argument('-a', '--append-syspath', action='store_true',
                        dest='append_syspath',
                        help=__('append module_path to sys.path, used when --full is given'))
    parser.add_argument('-H', '--doc-project', action='store', dest='header',
                        help=__('project name (default: root module name)'))
    parser.add_argument('-A', '--doc-author', action='store', dest='author',
                        help=__('project author(s), used when --full is given'))
    parser.add_argument('-V', '--doc-version', action='store', dest='version',
                        help=__('project version, used when --full is given'))
    parser.add_argument('-R', '--doc-release', action='store', dest='release',
                        help=__('project release, used when --full is given, '
                                'defaults to --doc-version'))

    group = parser.add_argument_group(__('extension options'))
    group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions',
                       action='append', help=__('enable arbitrary extensions'))
    for ext in EXTENSIONS:
        group.add_argument('--ext-%s' % ext, action='append_const',
                           const='sphinx.ext.%s' % ext, dest='extensions',
                           help=__('enable %s extension') % ext)

    group = parser.add_argument_group(__('Project templating'))
    group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR',
                       dest='templatedir',
                       help=__('template directory for template files'))

    return parser


def main(argv: Sequence[str] = (), /) -> int:
    """Parse and check the command line arguments."""
    locale.setlocale(locale.LC_ALL, '')
    sphinx.locale.init_console()

    parser = get_parser()
    args = parser.parse_args(argv or sys.argv[1:])

    rootpath = path.abspath(args.module_path)

    # normalize opts

    if args.header is None:
        args.header = rootpath.split(path.sep)[-1]
    if args.suffix.startswith('.'):
        args.suffix = args.suffix[1:]
    if not path.isdir(rootpath):
        logger.error(__('%s is not a directory.'), rootpath)
        raise SystemExit(1)
    if not args.dryrun:
        ensuredir(args.destdir)
    excludes = tuple(
        re.compile(fnmatch.translate(path.abspath(exclude)))
        for exclude in dict.fromkeys(args.exclude_pattern)
    )
    modules = recurse_tree(rootpath, excludes, args, args.templatedir)

    if args.full:
        from sphinx.cmd import quickstart as qs
        modules.sort()
        prev_module = ''
        text = ''
        for module in modules:
            if module.startswith(prev_module + '.'):
                continue
            prev_module = module
            text += '   %s\n' % module
        d = {
            'path': args.destdir,
            'sep': False,
            'dot': '_',
            'project': args.header,
            'author': args.author or 'Author',
            'version': args.version or '',
            'release': args.release or args.version or '',
            'suffix': '.' + args.suffix,
            'master': 'index',
            'epub': True,
            'extensions': ['sphinx.ext.autodoc', 'sphinx.ext.viewcode',
                           'sphinx.ext.todo'],
            'makefile': True,
            'batchfile': True,
            'make_mode': True,
            'mastertocmaxdepth': args.maxdepth,
            'mastertoctree': text,
            'language': 'en',
            'module_path': rootpath,
            'append_syspath': args.append_syspath,
        }
        if args.extensions:
            d['extensions'].extend(args.extensions)
        if args.quiet:
            d['quiet'] = True

        for ext in d['extensions'][:]:
            if ',' in ext:
                d['extensions'].remove(ext)
                d['extensions'].extend(ext.split(','))

        if not args.dryrun:
            qs.generate(d, silent=True, overwrite=args.force,
                        templatedir=args.templatedir)
    elif args.tocfile:
        create_modules_toc_file(modules, args, args.tocfile, args.templatedir)

    return 0


# So program can be started with "python -m sphinx.apidoc ..."
if __name__ == "__main__":
    raise SystemExit(main(sys.argv[1:]))

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/apidoc.py
NikolaBlagojevic/pyrecodes
"""Extension to create automatic documentation from code docstrings.

Automatically insert docstrings for functions, classes or whole modules into
the doctree, thus avoiding duplication between docstrings and documentation
for those who like elaborate docstrings.
"""

from __future__ import annotations

import re
import sys
import warnings
from inspect import Parameter, Signature
from typing import TYPE_CHECKING, Any, Callable, TypeVar

from docutils.statemachine import StringList

import sphinx
from sphinx.config import ENUM, Config
from sphinx.deprecation import RemovedInSphinx80Warning
from sphinx.ext.autodoc.importer import get_class_members, import_module, import_object
from sphinx.ext.autodoc.mock import ismock, mock, undecorate
from sphinx.locale import _, __
from sphinx.pycode import ModuleAnalyzer, PycodeError
from sphinx.util import inspect, logging
from sphinx.util.docstrings import prepare_docstring, separate_metadata
from sphinx.util.inspect import (
    evaluate_signature,
    getdoc,
    object_description,
    safe_getattr,
    stringify_signature,
)
from sphinx.util.typing import OptionSpec, get_type_hints, restify, stringify_annotation

if TYPE_CHECKING:
    from collections.abc import Iterator, Sequence
    from types import ModuleType

    from sphinx.application import Sphinx
    from sphinx.environment import BuildEnvironment
    from sphinx.ext.autodoc.directive import DocumenterBridge

logger = logging.getLogger(__name__)


# This type isn't exposed directly in any modules, but can be found
# here in most Python versions
MethodDescriptorType = type(type.__subclasses__)


#: extended signature RE: with explicit module name separated by ::
py_ext_sig_re = re.compile(
    r'''^ ([\w.]+::)?            # explicit module name
          ([\w.]+\.)?            # module and/or class name(s)
          (\w+)  \s*             # thing name
          (?: \[\s*(.*)\s*])?    # optional: type parameters list
          (?: \((.*)\)           # optional: arguments
           (?:\s* -> \s* (.*))?  #           return annotation
          )? $                   # and nothing more
          ''', re.VERBOSE)
special_member_re = re.compile(r'^__\S+__$')


def identity(x: Any) -> Any:
    return x


class _All:
    """A special value for :*-members: that matches to any member."""

    def __contains__(self, item: Any) -> bool:
        return True

    def append(self, item: Any) -> None:
        pass  # nothing


class _Empty:
    """A special value for :exclude-members: that never matches to any member."""

    def __contains__(self, item: Any) -> bool:
        return False


ALL = _All()
EMPTY = _Empty()
UNINITIALIZED_ATTR = object()
INSTANCEATTR = object()
SLOTSATTR = object()


def members_option(arg: Any) -> object | list[str]:
    """Used to convert the :members: option to auto directives."""
    if arg in (None, True):
        return ALL
    elif arg is False:
        return None
    else:
        return [x.strip() for x in arg.split(',') if x.strip()]


def exclude_members_option(arg: Any) -> object | set[str]:
    """Used to convert the :exclude-members: option."""
    if arg in (None, True):
        return EMPTY
    return {x.strip() for x in arg.split(',') if x.strip()}


def inherited_members_option(arg: Any) -> set[str]:
    """Used to convert the :inherited-members: option to auto directives."""
    if arg in (None, True):
        return {'object'}
    elif arg:
        return {x.strip() for x in arg.split(',')}
    else:
        return set()


def member_order_option(arg: Any) -> str | None:
    """Used to convert the :member-order: option to auto directives."""
    if arg in (None, True):
        return None
    elif arg in ('alphabetical', 'bysource', 'groupwise'):
        return arg
    else:
        raise ValueError(__('invalid value for member-order option: %s') % arg)


def class_doc_from_option(arg: Any) -> str | None:
    """Used to convert the :class-doc-from: option to autoclass directives."""
    if arg in ('both', 'class', 'init'):
        return arg
    else:
        raise ValueError(__('invalid value for class-doc-from option: %s') % arg)


SUPPRESS = object()


def annotation_option(arg: Any) -> Any:
    if arg in (None, True):
        # suppress showing the representation of the object
        return SUPPRESS
    else:
        return arg


def bool_option(arg: Any) -> bool:
    """Used to convert flag options to auto directives.  (Instead of
    directives.flag(), which returns None).
    """
    return True


def merge_members_option(options: dict) -> None:
    """Merge :private-members: and :special-members: options to the
    :members: option.
    """
    if options.get('members') is ALL:
        # merging is not needed when members: ALL
        return

    members = options.setdefault('members', [])
    for key in {'private-members', 'special-members'}:
        if key in options and options[key] not in (ALL, None):
            for member in options[key]:
                if member not in members:
                    members.append(member)


# Some useful event listener factories for autodoc-process-docstring.

def cut_lines(pre: int, post: int = 0, what: str | None = None) -> Callable:
    """Return a listener that removes the first *pre* and last *post*
    lines of every docstring.  If *what* is a sequence of strings,
    only docstrings of a type in *what* will be processed.

    Use like this (e.g. in the ``setup()`` function of :file:`conf.py`)::

       from sphinx.ext.autodoc import cut_lines
       app.connect('autodoc-process-docstring', cut_lines(4, what=['module']))

    This can (and should) be used in place of :confval:`automodule_skip_lines`.
    """
    def process(app: Sphinx, what_: str, name: str, obj: Any, options: Any, lines: list[str],
                ) -> None:
        if what and what_ not in what:
            return
        del lines[:pre]
        if post:
            # remove one trailing blank line.
            if lines and not lines[-1]:
                lines.pop(-1)
            del lines[-post:]
        # make sure there is a blank line at the end
        if lines and lines[-1]:
            lines.append('')
    return process


def between(
    marker: str,
    what: Sequence[str] | None = None,
    keepempty: bool = False,
    exclude: bool = False,
) -> Callable:
    """Return a listener that either keeps, or if *exclude* is True excludes,
    lines between lines that match the *marker* regular expression.  If no line
    matches, the resulting docstring would be empty, so no change will be made
    unless *keepempty* is true.

    If *what* is a sequence of strings, only docstrings of a type in *what* will
    be processed.
    """
    marker_re = re.compile(marker)

    def process(app: Sphinx, what_: str, name: str, obj: Any, options: Any, lines: list[str],
                ) -> None:
        if what and what_ not in what:
            return
        deleted = 0
        delete = not exclude
        orig_lines = lines[:]
        for i, line in enumerate(orig_lines):
            if delete:
                lines.pop(i - deleted)
                deleted += 1
            if marker_re.match(line):
                delete = not delete
                if delete:
                    lines.pop(i - deleted)
                    deleted += 1
        if not lines and not keepempty:
            lines[:] = orig_lines
        # make sure there is a blank line at the end
        if lines and lines[-1]:
            lines.append('')
    return process


# This class is used only in ``sphinx.ext.autodoc.directive``,
# But we define this class here to keep compatibility (see #4538)
class Options(dict):
    """A dict/attribute hybrid that returns None on nonexisting keys."""
    def copy(self) -> Options:
        return Options(super().copy())

    def __getattr__(self, name: str) -> Any:
        try:
            return self[name.replace('_', '-')]
        except KeyError:
            return None


class ObjectMember:
    """A member of object.

    This is used for the result of `Documenter.get_module_members()` to
    represent each member of the object.

    .. Note::

       An instance of this class behaves as a tuple of (name, object)
       for compatibility to old Sphinx.  The behavior will be dropped
       in the future.  Therefore extensions should not use the tuple
       interface.
    """

    def __init__(self, name: str, obj: Any, *, docstring: str | None = None,
                 class_: Any = None, skipped: bool = False) -> None:
        self.__name__ = name
        self.object = obj
        self.docstring = docstring
        self.skipped = skipped
        self.class_ = class_

    def __getitem__(self, index):
        warnings.warn('The tuple interface of ObjectMember is deprecated. '
                      'Use (obj.__name__, obj.object) instead.',
                      RemovedInSphinx80Warning, stacklevel=2)
        return (self.__name__, self.object)[index]


class Documenter:
    """
    A Documenter knows how to autodocument a single object type.  When
    registered with the AutoDirective, it will be used to document objects
    of that type when needed by autodoc.

    Its *objtype* attribute selects what auto directive it is assigned to
    (the directive name is 'auto' + objtype), and what directive it generates
    by default, though that can be overridden by an attribute called
    *directivetype*.

    A Documenter has an *option_spec* that works like a docutils directive's;
    in fact, it will be used to parse an auto directive's options that matches
    the Documenter.
    """
    #: name by which the directive is called (auto...) and the default
    #: generated directive name
    objtype = 'object'
    #: indentation by which to indent the directive content
    content_indent = '   '
    #: priority if multiple documenters return True from can_document_member
    priority = 0
    #: order if autodoc_member_order is set to 'groupwise'
    member_order = 0
    #: true if the generated content may contain titles
    titles_allowed = True

    option_spec: OptionSpec = {
        'no-index': bool_option,
        'noindex': bool_option,
    }

    def get_attr(self, obj: Any, name: str, *defargs: Any) -> Any:
        """getattr() override for types such as Zope interfaces."""
        return autodoc_attrgetter(self.env.app, obj, name, *defargs)

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        """Called to see if a member can be documented by this Documenter."""
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

    def __init__(self, directive: DocumenterBridge, name: str, indent: str = '') -> None:
        self.directive = directive
        self.config: Config = directive.env.config
        self.env: BuildEnvironment = directive.env
        self.options = directive.genopt
        self.name = name
        self.indent = indent
        # the module and object path within the module, and the fully
        # qualified name (all set after resolve_name succeeds)
        self.modname: str = ''
        self.module: ModuleType | None = None
        self.objpath: list[str] = []
        self.fullname = ''
        # extra signature items (arguments and return annotation,
        # also set after resolve_name succeeds)
        self.args: str | None = None
        self.retann: str = ''
        # the object to document (set after import_object succeeds)
        self.object: Any = None
        self.object_name = ''
        # the parent/owner of the object to document
        self.parent: Any = None
        # the module analyzer to get at attribute docs, or None
        self.analyzer: ModuleAnalyzer | None = None

    @property
    def documenters(self) -> dict[str, type[Documenter]]:
        """Returns registered Documenter classes"""
        return self.env.app.registry.documenters

    def add_line(self, line: str, source: str, *lineno: int) -> None:
        """Append one line of generated reST to the output."""
        if line.strip():  # not a blank line
            self.directive.result.append(self.indent + line, source, *lineno)
        else:
            self.directive.result.append('', source, *lineno)

    def resolve_name(self, modname: str | None, parents: Any, path: str, base: str,
                     ) -> tuple[str | None, list[str]]:
        """Resolve the module and name of the object to document given by the
        arguments and the current module/class.

        Must return a pair of the module name and a chain of attributes; for
        example, it would return ``('zipfile', ['ZipFile', 'open'])`` for the
        ``zipfile.ZipFile.open`` method.
        """
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

    def parse_name(self) -> bool:
        """Determine what module to import and what attribute to document.

        Returns True and sets *self.modname*, *self.objpath*, *self.fullname*,
        *self.args* and *self.retann* if parsing and resolving was successful.
        """
        # first, parse the definition -- auto directives for classes and
        # functions can contain a signature which is then used instead of
        # an autogenerated one
        matched = py_ext_sig_re.match(self.name)
        if matched is None:
            logger.warning(__('invalid signature for auto%s (%r)') % (self.objtype, self.name),
                           type='autodoc')
            return False
        explicit_modname, path, base, tp_list, args, retann = matched.groups()

        # support explicit module and class name separation via ::
        if explicit_modname is not None:
            modname = explicit_modname[:-2]
            parents = path.rstrip('.').split('.') if path else []
        else:
            modname = None
            parents = []

        with mock(self.config.autodoc_mock_imports):
            modname, self.objpath = self.resolve_name(modname, parents, path, base)

        if not modname:
            return False

        self.modname = modname
        self.args = args
        self.retann = retann
        self.fullname = ((self.modname or '') +
                         ('.' + '.'.join(self.objpath) if self.objpath else ''))
        return True

    def import_object(self, raiseerror: bool = False) -> bool:
        """Import the object given by *self.modname* and *self.objpath* and set
        it as *self.object*.

        Returns True if successful, False if an error occurred.
        """
        with mock(self.config.autodoc_mock_imports):
            try:
                ret = import_object(self.modname, self.objpath, self.objtype,
                                    attrgetter=self.get_attr,
                                    warningiserror=self.config.autodoc_warningiserror)
                self.module, self.parent, self.object_name, self.object = ret
                if ismock(self.object):
                    self.object = undecorate(self.object)
                return True
            except ImportError as exc:
                if raiseerror:
                    raise
                logger.warning(exc.args[0], type='autodoc', subtype='import_object')
                self.env.note_reread()
                return False

    def get_real_modname(self) -> str:
        """Get the real module name of an object to document.

        It can differ from the name of the module through which the object was
        imported.
        """
        return self.get_attr(self.object, '__module__', None) or self.modname

    def check_module(self) -> bool:
        """Check if *self.object* is really defined in the module given by
        *self.modname*.
        """
        if self.options.imported_members:
            return True

        subject = inspect.unpartial(self.object)
        modname = self.get_attr(subject, '__module__', None)
        if modname and modname != self.modname:
            return False
        return True

    def format_args(self, **kwargs: Any) -> str:
        """Format the argument signature of *self.object*.

        Should return None if the object does not have a signature.
        """
        return ''

    def format_name(self) -> str:
        """Format the name of *self.object*.

        This normally should be something that can be parsed by the generated
        directive, but doesn't need to be (Sphinx will display it unparsed
        then).
        """
        # normally the name doesn't contain the module (except for module
        # directives of course)
        return '.'.join(self.objpath) or self.modname

    def _call_format_args(self, **kwargs: Any) -> str:
        if kwargs:
            try:
                return self.format_args(**kwargs)
            except TypeError:
                # avoid chaining exceptions, by putting nothing here
                pass

        # retry without arguments for old documenters
        return self.format_args()

    def format_signature(self, **kwargs: Any) -> str:
        """Format the signature (arguments and return annotation) of the object.

        Let the user process it via the ``autodoc-process-signature`` event.
        """
        if self.args is not None:
            # signature given explicitly
            args = "(%s)" % self.args
            retann = self.retann
        else:
            # try to introspect the signature
            try:
                retann = None
                args = self._call_format_args(**kwargs)
                if args:
                    matched = re.match(r'^(\(.*\))\s+->\s+(.*)$', args)
                    if matched:
                        args = matched.group(1)
                        retann = matched.group(2)
            except Exception as exc:
                logger.warning(__('error while formatting arguments for %s: %s'),
                               self.fullname, exc, type='autodoc')
                args = None

        result = self.env.events.emit_firstresult('autodoc-process-signature',
                                                  self.objtype, self.fullname,
                                                  self.object, self.options, args, retann)
        if result:
            args, retann = result

        if args is not None:
            return args + ((' -> %s' % retann) if retann else '')
        else:
            return ''

    def add_directive_header(self, sig: str) -> None:
        """Add the directive header and options to the generated content."""
        domain = getattr(self, 'domain', 'py')
        directive = getattr(self, 'directivetype', self.objtype)
        name = self.format_name()
        sourcename = self.get_sourcename()

        # one signature per line, indented by column
        prefix = f'.. {domain}:{directive}:: '
        for i, sig_line in enumerate(sig.split("\n")):
            self.add_line(f'{prefix}{name}{sig_line}',
                          sourcename)
            if i == 0:
                prefix = " " * len(prefix)

        if self.options.no_index or self.options.noindex:
            self.add_line('   :no-index:', sourcename)
        if self.objpath:
            # Be explicit about the module, this is necessary since .. class::
            # etc. don't support a prepended module name
            self.add_line('   :module: %s' % self.modname, sourcename)

    def get_doc(self) -> list[list[str]] | None:
        """Decode and return lines of the docstring(s) for the object.

        When it returns None, autodoc-process-docstring will not be called for this
        object.
        """
        docstring = getdoc(self.object, self.get_attr, self.config.autodoc_inherit_docstrings,
                           self.parent, self.object_name)
        if docstring:
            tab_width = self.directive.state.document.settings.tab_width
            return [prepare_docstring(docstring, tab_width)]
        return []

    def process_doc(self, docstrings: list[list[str]]) -> Iterator[str]:
        """Let the user process the docstrings before adding them."""
        for docstringlines in docstrings:
            if self.env.app:
                # let extensions preprocess docstrings
                self.env.app.emit('autodoc-process-docstring',
                                  self.objtype, self.fullname, self.object,
                                  self.options, docstringlines)

                if docstringlines and docstringlines[-1] != '':
                    # append a blank line to the end of the docstring
                    docstringlines.append('')

            yield from docstringlines

    def get_sourcename(self) -> str:
        if (inspect.safe_getattr(self.object, '__module__', None) and
                inspect.safe_getattr(self.object, '__qualname__', None)):
            # Get the correct location of docstring from self.object
            # to support inherited methods
            fullname = f'{self.object.__module__}.{self.object.__qualname__}'
        else:
            fullname = self.fullname

        if self.analyzer:
            return f'{self.analyzer.srcname}:docstring of {fullname}'
        else:
            return 'docstring of %s' % fullname

    def add_content(self, more_content: StringList | None) -> None:
        """Add content from docstrings, attribute documentation and user."""
        docstring = True

        # set sourcename and add content from attribute documentation
        sourcename = self.get_sourcename()
        if self.analyzer:
            attr_docs = self.analyzer.find_attr_docs()
            if self.objpath:
                key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
                if key in attr_docs:
                    docstring = False
                    # make a copy of docstring for attributes to avoid cache
                    # the change of autodoc-process-docstring event.
                    attribute_docstrings = [list(attr_docs[key])]

                    for i, line in enumerate(self.process_doc(attribute_docstrings)):
                        self.add_line(line, sourcename, i)

        # add content from docstrings
        if docstring:
            docstrings = self.get_doc()
            if docstrings is None:
                # Do not call autodoc-process-docstring on get_doc() returns None.
                pass
            else:
                if not docstrings:
                    # append at least a dummy docstring, so that the event
                    # autodoc-process-docstring is fired and can add some
                    # content if desired
                    docstrings.append([])
                for i, line in enumerate(self.process_doc(docstrings)):
                    self.add_line(line, sourcename, i)

        # add additional content (e.g. from document), if present
        if more_content:
            for line, src in zip(more_content.data, more_content.items):
                self.add_line(line, src[0], src[1])

    def get_object_members(self, want_all: bool) -> tuple[bool, list[ObjectMember]]:
        """Return `(members_check_module, members)` where `members` is a
        list of `(membername, member)` pairs of the members of *self.object*.

        If *want_all* is True, return all members.  Else, only return those
        members given by *self.options.members* (which may also be None).
        """
        msg = 'must be implemented in subclasses'
        raise NotImplementedError(msg)

    def filter_members(self, members: list[ObjectMember], want_all: bool,
                       ) -> list[tuple[str, Any, bool]]:
        """Filter the given member list.

        Members are skipped if

        - they are private (except if given explicitly or the private-members
          option is set)
        - they are special methods (except if given explicitly or the
          special-members option is set)
        - they are undocumented (except if the undoc-members option is set)

        The user can override the skipping decision by connecting to the
        ``autodoc-skip-member`` event.
        """
        def is_filtered_inherited_member(name: str, obj: Any) -> bool:
            inherited_members = self.options.inherited_members or set()

            if inspect.isclass(self.object):
                for cls in self.object.__mro__:
                    if cls.__name__ in inherited_members and cls != self.object:
                        # given member is a member of specified *super class*
                        return True
                    if name in cls.__dict__:
                        return False
                    if name in self.get_attr(cls, '__annotations__', {}):
                        return False
                    if isinstance(obj, ObjectMember) and obj.class_ is cls:
                        return False

            return False

        ret = []

        # search for members in source code too
        namespace = '.'.join(self.objpath)  # will be empty for modules

        if self.analyzer:
            attr_docs = self.analyzer.find_attr_docs()
        else:
            attr_docs = {}

        # process members and determine which to skip
        for obj in members:
            try:
                membername = obj.__name__
                member = obj.object
            except AttributeError:
                if isinstance(obj, ObjectMember):
                    raise
                # To be removed, retained for compatibility.
                # See https://github.com/sphinx-doc/sphinx/issues/11631
                membername, member = obj
                warnings.warn(
                    'Returning tuples of (name, object) as '
                    'the second return value from get_object_members() is deprecated. '
                    'Return ObjectMember(name, object) instances instead.',
                    RemovedInSphinx80Warning, stacklevel=2,
                )

            # if isattr is True, the member is documented as an attribute
            isattr = member is INSTANCEATTR or (namespace, membername) in attr_docs

            try:
                doc = getdoc(member, self.get_attr, self.config.autodoc_inherit_docstrings,
                             self.object, membername)
                if not isinstance(doc, str):
                    # Ignore non-string __doc__
                    doc = None

                # if the member __doc__ is the same as self's __doc__, it's just
                # inherited and therefore not the member's doc
                cls = self.get_attr(member, '__class__', None)
                if cls:
                    cls_doc = self.get_attr(cls, '__doc__', None)
                    if cls_doc == doc:
                        doc = None

                if isinstance(obj, ObjectMember) and obj.docstring:
                    # hack for ClassDocumenter to inject docstring via ObjectMember
                    doc = obj.docstring

                doc, metadata = separate_metadata(doc)
                has_doc = bool(doc)

                if 'private' in metadata:
                    # consider a member private if docstring has "private" metadata
                    isprivate = True
                elif 'public' in metadata:
                    # consider a member public if docstring has "public" metadata
                    isprivate = False
                else:
                    isprivate = membername.startswith('_')

                keep = False
                if ismock(member) and (namespace, membername) not in attr_docs:
                    # mocked module or object
                    pass
                elif (self.options.exclude_members and
                      membername in self.options.exclude_members):
                    # remove members given by exclude-members
                    keep = False
                elif want_all and special_member_re.match(membername):
                    # special __methods__
                    if (self.options.special_members and
                            membername in self.options.special_members):
                        if membername == '__doc__':  # NoQA: SIM114
                            keep = False
                        elif is_filtered_inherited_member(membername, obj):
                            keep = False
                        else:
                            keep = has_doc or self.options.undoc_members
                    else:
                        keep = False
                elif (namespace, membername) in attr_docs:
                    if want_all and isprivate:
                        if self.options.private_members is None:
                            keep = False
                        else:
                            keep = membername in self.options.private_members
                    else:
                        # keep documented attributes
                        keep = True
                elif want_all and isprivate:
                    if has_doc or self.options.undoc_members:
                        if self.options.private_members is None:  # NoQA: SIM114
                            keep = False
                        elif is_filtered_inherited_member(membername, obj):
                            keep = False
                        else:
                            keep = membername in self.options.private_members
                    else:
                        keep = False
                else:
                    if (self.options.members is ALL and
                            is_filtered_inherited_member(membername, obj)):
                        keep = False
                    else:
                        # ignore undocumented members if :undoc-members: is not given
                        keep = has_doc or self.options.undoc_members

                if isinstance(obj, ObjectMember) and obj.skipped:
                    # forcedly skipped member (ex. a module attribute not defined in __all__)
                    keep = False

                # give the user a chance to decide whether this member
                # should be skipped
                if self.env.app:
                    # let extensions preprocess docstrings
                    skip_user = self.env.app.emit_firstresult(
                        'autodoc-skip-member', self.objtype, membername, member,
                        not keep, self.options)
                    if skip_user is not None:
                        keep = not skip_user
            except Exception as exc:
                logger.warning(__('autodoc: failed to determine %s.%s (%r) to be documented, '
                                  'the following exception was raised:\n%s'),
                               self.name, membername, member, exc, type='autodoc')
                keep = False

            if keep:
                ret.append((membername, member, isattr))

        return ret

    def document_members(self, all_members: bool = False) -> None:
        """Generate reST for member documentation.

        If *all_members* is True, document all members, else those given by
        *self.options.members*.
        """
        # set current namespace for finding members
        self.env.temp_data['autodoc:module'] = self.modname
        if self.objpath:
            self.env.temp_data['autodoc:class'] = self.objpath[0]

        want_all = (all_members or
                    self.options.inherited_members or
                    self.options.members is ALL)
        # find out which members are documentable
        members_check_module, members = self.get_object_members(want_all)

        # document non-skipped members
        memberdocumenters: list[tuple[Documenter, bool]] = []
        for (mname, member, isattr) in self.filter_members(members, want_all):
            classes = [cls for cls in self.documenters.values()
                       if cls.can_document_member(member, mname, isattr, self)]
            if not classes:
                # don't know how to document this member
                continue
            # prefer the documenter with the highest priority
            classes.sort(key=lambda cls: cls.priority)
            # give explicitly separated module name, so that members
            # of inner classes can be documented
            full_mname = f'{self.modname}::' + '.'.join((*self.objpath, mname))
            documenter = classes[-1](self.directive, full_mname, self.indent)
            memberdocumenters.append((documenter, isattr))

        member_order = self.options.member_order or self.config.autodoc_member_order
        memberdocumenters = self.sort_members(memberdocumenters, member_order)

        for documenter, isattr in memberdocumenters:
            documenter.generate(
                all_members=True, real_modname=self.real_modname,
                check_module=members_check_module and not isattr)

        # reset current objects
        self.env.temp_data['autodoc:module'] = None
        self.env.temp_data['autodoc:class'] = None

    def sort_members(self, documenters: list[tuple[Documenter, bool]],
                     order: str) -> list[tuple[Documenter, bool]]:
        """Sort the given member list."""
        if order == 'groupwise':
            # sort by group; alphabetically within groups
            documenters.sort(key=lambda e: (e[0].member_order, e[0].name))
        elif order == 'bysource':
            # By default, member discovery order matches source order,
            # as dicts are insertion-ordered from Python 3.7.
            if self.analyzer:
                # sort by source order, by virtue of the module analyzer
                tagorder = self.analyzer.tagorder

                def keyfunc(entry: tuple[Documenter, bool]) -> int:
                    fullname = entry[0].name.split('::')[1]
                    return tagorder.get(fullname, len(tagorder))
                documenters.sort(key=keyfunc)
        else:  # alphabetical
            documenters.sort(key=lambda e: e[0].name)

        return documenters

    def generate(
        self,
        more_content: StringList | None = None,
        real_modname: str | None = None,
        check_module: bool = False,
        all_members: bool = False,
    ) -> None:
        """Generate reST for the object given by *self.name*, and possibly for
        its members.

        If *more_content* is given, include that content. If *real_modname* is
        given, use that module name to find attribute docs. If *check_module* is
        True, only generate if the object is defined in the module name it is
        imported from. If *all_members* is True, document all members.
        """
        if not self.parse_name():
            # need a module to import
            logger.warning(
                __("don't know which module to import for autodocumenting "
                   '%r (try placing a "module" or "currentmodule" directive '
                   'in the document, or giving an explicit module name)') %
                self.name, type='autodoc')
            return

        # now, import the module and get object to document
        if not self.import_object():
            return

        # If there is no real module defined, figure out which to use.
        # The real module is used in the module analyzer to look up the module
        # where the attribute documentation would actually be found in.
        # This is used for situations where you have a module that collects the
        # functions and classes of internal submodules.
        guess_modname = self.get_real_modname()
        self.real_modname: str = real_modname or guess_modname

        # try to also get a source code analyzer for attribute docs
        try:
            self.analyzer = ModuleAnalyzer.for_module(self.real_modname)
            # parse right now, to get PycodeErrors on parsing (results will
            # be cached anyway)
            self.analyzer.find_attr_docs()
        except PycodeError as exc:
            logger.debug('[autodoc] module analyzer failed: %s', exc)
            # no source file -- e.g. for builtin and C modules
            self.analyzer = None
            # at least add the module.__file__ as a dependency
            if module___file__ := getattr(self.module, '__file__', ''):
                self.directive.record_dependencies.add(module___file__)
        else:
            self.directive.record_dependencies.add(self.analyzer.srcname)

        if self.real_modname != guess_modname:
            # Add module to dependency list if target object is defined in other module.
            try:
                analyzer = ModuleAnalyzer.for_module(guess_modname)
                self.directive.record_dependencies.add(analyzer.srcname)
            except PycodeError:
                pass

        docstrings: list[str] = sum(self.get_doc() or [], [])
        if ismock(self.object) and not docstrings:
            logger.warning(__('A mocked object is detected: %r'),
                           self.name, type='autodoc')

        # check __module__ of object (for members not given explicitly)
        if check_module:
            if not self.check_module():
                return

        sourcename = self.get_sourcename()

        # make sure that the result starts with an empty line.  This is
        # necessary for some situations where another directive preprocesses
        # reST and no starting newline is present
        self.add_line('', sourcename)

        # format the object's signature, if any
        try:
            sig = self.format_signature()
        except Exception as exc:
            logger.warning(__('error while formatting signature for %s: %s'),
                           self.fullname, exc, type='autodoc')
            return

        # generate the directive header and options, if applicable
        self.add_directive_header(sig)
        self.add_line('', sourcename)

        # e.g. the module directive doesn't have content
        self.indent += self.content_indent

        # add all content (from docstrings, attribute docs etc.)
        self.add_content(more_content)

        # document members, if possible
        self.document_members(all_members)


class ModuleDocumenter(Documenter):
    """
    Specialized Documenter subclass for modules.
    """
    objtype = 'module'
    content_indent = ''
    _extra_indent = '   '

    option_spec: OptionSpec = {
        'members': members_option, 'undoc-members': bool_option,
        'no-index': bool_option, 'inherited-members': inherited_members_option,
        'show-inheritance': bool_option, 'synopsis': identity,
        'platform': identity, 'deprecated': bool_option,
        'member-order': member_order_option, 'exclude-members': exclude_members_option,
        'private-members': members_option, 'special-members': members_option,
        'imported-members': bool_option, 'ignore-module-all': bool_option,
        'no-value': bool_option,
        'noindex': bool_option,
    }

    def __init__(self, *args: Any) -> None:
        super().__init__(*args)
        merge_members_option(self.options)
        self.__all__: Sequence[str] | None = None

    def add_content(self, more_content: StringList | None) -> None:
        old_indent = self.indent
        self.indent += self._extra_indent
        super().add_content(None)
        self.indent = old_indent
        if more_content:
            for line, src in zip(more_content.data, more_content.items):
                self.add_line(line, src[0], src[1])

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        # don't document submodules automatically
        return False

    def resolve_name(self, modname: str | None, parents: Any, path: str, base: str,
                     ) -> tuple[str | None, list[str]]:
        if modname is not None:
            logger.warning(__('"::" in automodule name doesn\'t make sense'),
                           type='autodoc')
        return (path or '') + base, []

    def parse_name(self) -> bool:
        ret = super().parse_name()
        if self.args or self.retann:
            logger.warning(__('signature arguments or return annotation '
                              'given for automodule %s') % self.fullname,
                           type='autodoc')
        return ret

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)

        try:
            if not self.options.ignore_module_all:
                self.__all__ = inspect.getall(self.object)
        except ValueError as exc:
            # invalid __all__ found.
            logger.warning(__('__all__ should be a list of strings, not %r '
                              '(in module %s) -- ignoring __all__') %
                           (exc.args[0], self.fullname), type='autodoc')

        return ret

    def add_directive_header(self, sig: str) -> None:
        Documenter.add_directive_header(self, sig)

        sourcename = self.get_sourcename()

        # add some module-specific options
        if self.options.synopsis:
            self.add_line('   :synopsis: ' + self.options.synopsis, sourcename)
        if self.options.platform:
            self.add_line('   :platform: ' + self.options.platform, sourcename)
        if self.options.deprecated:
            self.add_line('   :deprecated:', sourcename)

    def get_module_members(self) -> dict[str, ObjectMember]:
        """Get members of target module."""
        if self.analyzer:
            attr_docs = self.analyzer.attr_docs
        else:
            attr_docs = {}

        members: dict[str, ObjectMember] = {}
        for name in dir(self.object):
            try:
                value = safe_getattr(self.object, name, None)
                if ismock(value):
                    value = undecorate(value)
                docstring = attr_docs.get(('', name), [])
                members[name] = ObjectMember(name, value, docstring="\n".join(docstring))
            except AttributeError:
                continue

        # annotation only member (ex. attr: int)
        for name in inspect.getannotations(self.object):
            if name not in members:
                docstring = attr_docs.get(('', name), [])
                members[name] = ObjectMember(name, INSTANCEATTR,
                                             docstring="\n".join(docstring))

        return members

    def get_object_members(self, want_all: bool) -> tuple[bool, list[ObjectMember]]:
        members = self.get_module_members()
        if want_all:
            if self.__all__ is None:
                # for implicit module members, check __module__ to avoid
                # documenting imported objects
                return True, list(members.values())
            else:
                for member in members.values():
                    if member.__name__ not in self.__all__:
                        member.skipped = True

                return False, list(members.values())
        else:
            memberlist = self.options.members or []
            ret = []
            for name in memberlist:
                if name in members:
                    ret.append(members[name])
                else:
                    logger.warning(__('missing attribute mentioned in :members: option: '
                                      'module %s, attribute %s') %
                                   (safe_getattr(self.object, '__name__', '???'), name),
                                   type='autodoc')
            return False, ret

    def sort_members(self, documenters: list[tuple[Documenter, bool]],
                     order: str) -> list[tuple[Documenter, bool]]:
        if order == 'bysource' and self.__all__:
            assert self.__all__ is not None
            module_all = self.__all__
            module_all_set = set(module_all)
            module_all_len = len(module_all)

            # Sort alphabetically first (for members not listed on the __all__)
            documenters.sort(key=lambda e: e[0].name)

            # Sort by __all__
            def keyfunc(entry: tuple[Documenter, bool]) -> int:
                name = entry[0].name.split('::')[1]
                if name in module_all_set:
                    return module_all.index(name)
                else:
                    return module_all_len
            documenters.sort(key=keyfunc)

            return documenters
        else:
            return super().sort_members(documenters, order)


class ModuleLevelDocumenter(Documenter):
    """
    Specialized Documenter subclass for objects on module level (functions,
    classes, data/constants).
    """
    def resolve_name(self, modname: str | None, parents: Any, path: str, base: str,
                     ) -> tuple[str | None, list[str]]:
        if modname is not None:
            return modname, parents + [base]
        if path:
            modname = path.rstrip('.')
            return modname, parents + [base]

        # if documenting a toplevel object without explicit module,
        # it can be contained in another auto directive ...
        modname = self.env.temp_data.get('autodoc:module')
        # ... or in the scope of a module directive
        if not modname:
            modname = self.env.ref_context.get('py:module')
        # ... else, it stays None, which means invalid
        return modname, parents + [base]


class ClassLevelDocumenter(Documenter):
    """
    Specialized Documenter subclass for objects on class level (methods,
    attributes).
    """
    def resolve_name(self, modname: str | None, parents: Any, path: str, base: str,
                     ) -> tuple[str | None, list[str]]:
        if modname is not None:
            return modname, parents + [base]

        if path:
            mod_cls = path.rstrip('.')
        else:
            # if documenting a class-level object without path,
            # there must be a current class, either from a parent
            # auto directive ...
            mod_cls_ = self.env.temp_data.get('autodoc:class')
            # ... or from a class directive
            if mod_cls_ is None:
                mod_cls_ = self.env.ref_context.get('py:class')
                # ... if still None, there's no way to know
                if mod_cls_ is None:
                    return None, []
            mod_cls = mod_cls_
        modname, sep, cls = mod_cls.rpartition('.')
        parents = [cls]
        # if the module name is still missing, get it like above
        if not modname:
            modname = self.env.temp_data.get('autodoc:module')
        if not modname:
            modname = self.env.ref_context.get('py:module')
        # ... else, it stays None, which means invalid
        return modname, parents + [base]


class DocstringSignatureMixin:
    """
    Mixin for FunctionDocumenter and MethodDocumenter to provide the
    feature of reading the signature from the docstring.
    """
    _new_docstrings: list[list[str]] | None = None
    _signatures: list[str] = []

    def _find_signature(self) -> tuple[str | None, str | None] | None:
        # candidates of the object name
        valid_names = [self.objpath[-1]]  # type: ignore[attr-defined]
        if isinstance(self, ClassDocumenter):
            valid_names.append('__init__')
            if hasattr(self.object, '__mro__'):
                valid_names.extend(cls.__name__ for cls in self.object.__mro__)

        docstrings = self.get_doc()
        if docstrings is None:
            return None, None
        self._new_docstrings = docstrings[:]
        self._signatures = []
        result = None
        for i, doclines in enumerate(docstrings):
            for j, line in enumerate(doclines):
                if not line:
                    # no lines in docstring, no match
                    break

                if line.endswith('\\'):
                    line = line.rstrip('\\').rstrip()

                # match first line of docstring against signature RE
                match = py_ext_sig_re.match(line)
                if not match:
                    break
                exmod, path, base, tp_list, args, retann = match.groups()

                # the base name must match ours
                if base not in valid_names:
                    break

                # re-prepare docstring to ignore more leading indentation
                directive = self.directive  # type: ignore[attr-defined]
                tab_width = directive.state.document.settings.tab_width
                self._new_docstrings[i] = prepare_docstring('\n'.join(doclines[j + 1:]),
                                                            tab_width)

                if result is None:
                    # first signature
                    result = args, retann
                else:
                    # subsequent signatures
                    self._signatures.append(f"({args}) -> {retann}")

            if result is not None:
                # finish the loop when signature found
                break

        return result

    def get_doc(self) -> list[list[str]] | None:
        if self._new_docstrings is not None:
            return self._new_docstrings
        return super().get_doc()  # type: ignore[misc]

    def format_signature(self, **kwargs: Any) -> str:
        self.args: str | None
        if (self.args is None
                and self.config.autodoc_docstring_signature):  # type: ignore[attr-defined]
            # only act if a signature is not explicitly given already, and if
            # the feature is enabled
            result = self._find_signature()
            if result is not None:
                self.args, self.retann = result
        sig = super().format_signature(**kwargs)  # type: ignore[misc]
        if self._signatures:
            return "\n".join([sig] + self._signatures)
        else:
            return sig


class DocstringStripSignatureMixin(DocstringSignatureMixin):
    """
    Mixin for AttributeDocumenter to provide the
    feature of stripping any function signature from the docstring.
    """
    def format_signature(self, **kwargs: Any) -> str:
        if (
            self.args is None
            and self.config.autodoc_docstring_signature  # type: ignore[attr-defined]
        ):
            # only act if a signature is not explicitly given already, and if
            # the feature is enabled
            result = self._find_signature()
            if result is not None:
                # Discarding _args is a only difference with
                # DocstringSignatureMixin.format_signature.
                # Documenter.format_signature use self.args value to format.
                _args, self.retann = result
        return super().format_signature(**kwargs)


class FunctionDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type: ignore[misc]
    """
    Specialized Documenter subclass for functions.
    """
    objtype = 'function'
    member_order = 30

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        # supports functions, builtins and bound methods exported at the module level
        return (inspect.isfunction(member) or inspect.isbuiltin(member) or
                (inspect.isroutine(member) and isinstance(parent, ModuleDocumenter)))

    def format_args(self, **kwargs: Any) -> str:
        if self.config.autodoc_typehints in ('none', 'description'):
            kwargs.setdefault('show_annotation', False)
        if self.config.autodoc_typehints_format == "short":
            kwargs.setdefault('unqualified_typehints', True)

        try:
            self.env.app.emit('autodoc-before-process-signature', self.object, False)
            sig = inspect.signature(self.object, type_aliases=self.config.autodoc_type_aliases)
            args = stringify_signature(sig, **kwargs)
        except TypeError as exc:
            logger.warning(__("Failed to get a function signature for %s: %s"),
                           self.fullname, exc)
            return ''
        except ValueError:
            args = ''

        if self.config.strip_signature_backslash:
            # escape backslashes for reST
            args = args.replace('\\', '\\\\')
        return args

    def document_members(self, all_members: bool = False) -> None:
        pass

    def add_directive_header(self, sig: str) -> None:
        sourcename = self.get_sourcename()
        super().add_directive_header(sig)

        if inspect.iscoroutinefunction(self.object) or inspect.isasyncgenfunction(self.object):
            self.add_line('   :async:', sourcename)

    def format_signature(self, **kwargs: Any) -> str:
        if self.config.autodoc_typehints_format == "short":
            kwargs.setdefault('unqualified_typehints', True)

        sigs = []
        if (self.analyzer and
                '.'.join(self.objpath) in self.analyzer.overloads and
                self.config.autodoc_typehints != 'none'):
            # Use signatures for overloaded functions instead of the implementation function.
            overloaded = True
        else:
            overloaded = False
            sig = super().format_signature(**kwargs)
            sigs.append(sig)

        if inspect.is_singledispatch_function(self.object):
            # append signature of singledispatch'ed functions
            for typ, func in self.object.registry.items():
                if typ is object:
                    pass  # default implementation. skipped.
                else:
                    dispatchfunc = self.annotate_to_first_argument(func, typ)
                    if dispatchfunc:
                        documenter = FunctionDocumenter(self.directive, '')
                        documenter.object = dispatchfunc
                        documenter.objpath = ['']
                        sigs.append(documenter.format_signature())
        if overloaded and self.analyzer is not None:
            actual = inspect.signature(self.object,
                                       type_aliases=self.config.autodoc_type_aliases)
            __globals__ = safe_getattr(self.object, '__globals__', {})
            for overload in self.analyzer.overloads['.'.join(self.objpath)]:
                overload = self.merge_default_value(actual, overload)
                overload = evaluate_signature(overload, __globals__,
                                              self.config.autodoc_type_aliases)

                sig = stringify_signature(overload, **kwargs)
                sigs.append(sig)

        return "\n".join(sigs)

    def merge_default_value(self, actual: Signature, overload: Signature) -> Signature:
        """Merge default values of actual implementation to the overload variants."""
        parameters = list(overload.parameters.values())
        for i, param in enumerate(parameters):
            actual_param = actual.parameters.get(param.name)
            if actual_param and param.default == '...':
                parameters[i] = param.replace(default=actual_param.default)

        return overload.replace(parameters=parameters)

    def annotate_to_first_argument(self, func: Callable, typ: type) -> Callable | None:
        """Annotate type hint to the first argument of function if needed."""
        try:
            sig = inspect.signature(func, type_aliases=self.config.autodoc_type_aliases)
        except TypeError as exc:
            logger.warning(__("Failed to get a function signature for %s: %s"),
                           self.fullname, exc)
            return None
        except ValueError:
            return None

        if len(sig.parameters) == 0:
            return None

        def dummy():
            pass

        params = list(sig.parameters.values())
        if params[0].annotation is Parameter.empty:
            params[0] = params[0].replace(annotation=typ)
            try:
                dummy.__signature__ = sig.replace(  # type: ignore[attr-defined]
                    parameters=params)
                return dummy
            except (AttributeError, TypeError):
                # failed to update signature (ex. built-in or extension types)
                return None

        return func


class DecoratorDocumenter(FunctionDocumenter):
    """
    Specialized Documenter subclass for decorator functions.
    """
    objtype = 'decorator'

    # must be lower than FunctionDocumenter
    priority = -1

    def format_args(self, **kwargs: Any) -> str:
        args = super().format_args(**kwargs)
        if ',' in args:
            return args
        else:
            return ''


# Types which have confusing metaclass signatures it would be best not to show.
# These are listed by name, rather than storing the objects themselves, to avoid
# needing to import the modules.
_METACLASS_CALL_BLACKLIST = [
    'enum.EnumMeta.__call__',
]


# Types whose __new__ signature is a pass-through.
_CLASS_NEW_BLACKLIST = [
    'typing.Generic.__new__',
]


class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type: ignore[misc]
    """
    Specialized Documenter subclass for classes.
    """
    objtype = 'class'
    member_order = 20
    option_spec: OptionSpec = {
        'members': members_option, 'undoc-members': bool_option,
        'no-index': bool_option, 'inherited-members': inherited_members_option,
        'show-inheritance': bool_option, 'member-order': member_order_option,
        'exclude-members': exclude_members_option,
        'private-members': members_option, 'special-members': members_option,
        'class-doc-from': class_doc_from_option,
        'noindex': bool_option,
    }

    # Must be higher than FunctionDocumenter, ClassDocumenter, and
    # AttributeDocumenter as NewType can be an attribute and is a class
    # after Python 3.10. Before 3.10 it is a kind of function object
    priority = 15

    _signature_class: Any = None
    _signature_method_name: str = ''

    def __init__(self, *args: Any) -> None:
        super().__init__(*args)

        if self.config.autodoc_class_signature == 'separated':
            self.options = self.options.copy()

            # show __init__() method
            if self.options.special_members is None:
                self.options['special-members'] = ['__new__', '__init__']
            else:
                self.options.special_members.append('__new__')
                self.options.special_members.append('__init__')

        merge_members_option(self.options)

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        return isinstance(member, type) or (
            isattr and (inspect.isNewType(member) or isinstance(member, TypeVar)))

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)
        # if the class is documented under another name, document it
        # as data/attribute
        if ret:
            if hasattr(self.object, '__name__'):
                self.doc_as_attr = (self.objpath[-1] != self.object.__name__)
            else:
                self.doc_as_attr = True
            if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
                modname = getattr(self.object, '__module__', self.modname)
                if modname != self.modname and self.modname.startswith(modname):
                    bases = self.modname[len(modname):].strip('.').split('.')
                    self.objpath = bases + self.objpath
                    self.modname = modname
        return ret

    def _get_signature(self) -> tuple[Any | None, str | None, Signature | None]:
        if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
            # Suppress signature
            return None, None, None

        def get_user_defined_function_or_method(obj: Any, attr: str) -> Any:
            """ Get the `attr` function or method from `obj`, if it is user-defined. """
            if inspect.is_builtin_class_method(obj, attr):
                return None
            attr = self.get_attr(obj, attr, None)
            if not (inspect.ismethod(attr) or inspect.isfunction(attr)):
                return None
            return attr

        # This sequence is copied from inspect._signature_from_callable.
        # ValueError means that no signature could be found, so we keep going.

        # First, we check the obj has a __signature__ attribute
        if (hasattr(self.object, '__signature__') and
                isinstance(self.object.__signature__, Signature)):
            return None, None, self.object.__signature__

        # Next, let's see if it has an overloaded __call__ defined
        # in its metaclass
        call = get_user_defined_function_or_method(type(self.object), '__call__')

        if call is not None:
            if f"{call.__module__}.{call.__qualname__}" in _METACLASS_CALL_BLACKLIST:
                call = None

        if call is not None:
            self.env.app.emit('autodoc-before-process-signature', call, True)
            try:
                sig = inspect.signature(call, bound_method=True,
                                        type_aliases=self.config.autodoc_type_aliases)
                return type(self.object), '__call__', sig
            except ValueError:
                pass

        # Now we check if the 'obj' class has a '__new__' method
        new = get_user_defined_function_or_method(self.object, '__new__')

        if new is not None:
            if f"{new.__module__}.{new.__qualname__}" in _CLASS_NEW_BLACKLIST:
                new = None

        if new is not None:
            self.env.app.emit('autodoc-before-process-signature', new, True)
            try:
                sig = inspect.signature(new, bound_method=True,
                                        type_aliases=self.config.autodoc_type_aliases)
                return self.object, '__new__', sig
            except ValueError:
                pass

        # Finally, we should have at least __init__ implemented
        init = get_user_defined_function_or_method(self.object, '__init__')
        if init is not None:
            self.env.app.emit('autodoc-before-process-signature', init, True)
            try:
                sig = inspect.signature(init, bound_method=True,
                                        type_aliases=self.config.autodoc_type_aliases)
                return self.object, '__init__', sig
            except ValueError:
                pass

        # None of the attributes are user-defined, so fall back to let inspect
        # handle it.
        # We don't know the exact method that inspect.signature will read
        # the signature from, so just pass the object itself to our hook.
        self.env.app.emit('autodoc-before-process-signature', self.object, False)
        try:
            sig = inspect.signature(self.object, bound_method=False,
                                    type_aliases=self.config.autodoc_type_aliases)
            return None, None, sig
        except ValueError:
            pass

        # Still no signature: happens e.g. for old-style classes
        # with __init__ in C and no `__text_signature__`.
        return None, None, None

    def format_args(self, **kwargs: Any) -> str:
        if self.config.autodoc_typehints in ('none', 'description'):
            kwargs.setdefault('show_annotation', False)
        if self.config.autodoc_typehints_format == "short":
            kwargs.setdefault('unqualified_typehints', True)

        try:
            self._signature_class, _signature_method_name, sig = self._get_signature()
        except TypeError as exc:
            # __signature__ attribute contained junk
            logger.warning(__("Failed to get a constructor signature for %s: %s"),
                           self.fullname, exc)
            return ''
        self._signature_method_name = _signature_method_name or ''

        if sig is None:
            return ''

        return stringify_signature(sig, show_return_annotation=False, **kwargs)

    def _find_signature(self) -> tuple[str | None, str | None] | None:
        result = super()._find_signature()
        if result is not None:
            # Strip a return value from signature of constructor in docstring (first entry)
            result = (result[0], None)

        for i, sig in enumerate(self._signatures):
            if sig.endswith(' -> None'):
                # Strip a return value from signatures of constructor in docstring (subsequent
                # entries)
                self._signatures[i] = sig[:-8]

        return result

    def format_signature(self, **kwargs: Any) -> str:
        if self.doc_as_attr:
            return ''
        if self.config.autodoc_class_signature == 'separated':
            # do not show signatures
            return ''

        if self.config.autodoc_typehints_format == "short":
            kwargs.setdefault('unqualified_typehints', True)

        sig = super().format_signature()
        sigs = []

        overloads = self.get_overloaded_signatures()
        if overloads and self.config.autodoc_typehints != 'none':
            # Use signatures for overloaded methods instead of the implementation method.
            method = safe_getattr(self._signature_class, self._signature_method_name, None)
            __globals__ = safe_getattr(method, '__globals__', {})
            for overload in overloads:
                overload = evaluate_signature(overload, __globals__,
                                              self.config.autodoc_type_aliases)

                parameters = list(overload.parameters.values())
                overload = overload.replace(parameters=parameters[1:],
                                            return_annotation=Parameter.empty)
                sig = stringify_signature(overload, **kwargs)
                sigs.append(sig)
        else:
            sigs.append(sig)

        return "\n".join(sigs)

    def get_overloaded_signatures(self) -> list[Signature]:
        if self._signature_class and self._signature_method_name:
            for cls in self._signature_class.__mro__:
                try:
                    analyzer = ModuleAnalyzer.for_module(cls.__module__)
                    analyzer.analyze()
                    qualname = '.'.join([cls.__qualname__, self._signature_method_name])
                    if qualname in analyzer.overloads:
                        return analyzer.overloads.get(qualname, [])
                    elif qualname in analyzer.tagorder:
                        # the constructor is defined in the class, but not overridden.
                        return []
                except PycodeError:
                    pass

        return []

    def get_canonical_fullname(self) -> str | None:
        __modname__ = safe_getattr(self.object, '__module__', self.modname)
        __qualname__ = safe_getattr(self.object, '__qualname__', None)
        if __qualname__ is None:
            __qualname__ = safe_getattr(self.object, '__name__', None)
        if __qualname__ and '<locals>' in __qualname__:
            # No valid qualname found if the object is defined as locals
            __qualname__ = None

        if __modname__ and __qualname__:
            return '.'.join([__modname__, __qualname__])
        else:
            return None

    def add_directive_header(self, sig: str) -> None:
        sourcename = self.get_sourcename()

        if self.doc_as_attr:
            self.directivetype = 'attribute'
        super().add_directive_header(sig)

        if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
            return

        if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
            self.add_line('   :final:', sourcename)

        canonical_fullname = self.get_canonical_fullname()
        if (not self.doc_as_attr and not inspect.isNewType(self.object)
                and canonical_fullname and self.fullname != canonical_fullname):
            self.add_line('   :canonical: %s' % canonical_fullname, sourcename)

        # add inheritance info, if wanted
        if not self.doc_as_attr and self.options.show_inheritance:
            if inspect.getorigbases(self.object):
                # A subclass of generic types
                # refs: PEP-560 <https://peps.python.org/pep-0560/>
                bases = list(self.object.__orig_bases__)
            elif hasattr(self.object, '__bases__') and len(self.object.__bases__):
                # A normal class
                bases = list(self.object.__bases__)
            else:
                bases = []

            self.env.events.emit('autodoc-process-bases',
                                 self.fullname, self.object, self.options, bases)

            if self.config.autodoc_typehints_format == "short":
                base_classes = [restify(cls, "smart") for cls in bases]
            else:
                base_classes = [restify(cls) for cls in bases]

            sourcename = self.get_sourcename()
            self.add_line('', sourcename)
            self.add_line('   ' + _('Bases: %s') % ', '.join(base_classes), sourcename)

    def get_object_members(self, want_all: bool) -> tuple[bool, list[ObjectMember]]:
        members = get_class_members(self.object, self.objpath, self.get_attr,
                                    self.config.autodoc_inherit_docstrings)
        if not want_all:
            if not self.options.members:
                return False, []
            # specific members given
            selected = []
            for name in self.options.members:
                if name in members:
                    selected.append(members[name])
                else:
                    logger.warning(__('missing attribute %s in object %s') %
                                   (name, self.fullname), type='autodoc')
            return False, selected
        elif self.options.inherited_members:
            return False, list(members.values())
        else:
            return False, [m for m in members.values() if m.class_ == self.object]

    def get_doc(self) -> list[list[str]] | None:
        if isinstance(self.object, TypeVar):
            if self.object.__doc__ == TypeVar.__doc__:
                return []
        if sys.version_info[:2] < (3, 10):
            if inspect.isNewType(self.object) or isinstance(self.object, TypeVar):
                parts = self.modname.strip('.').split('.')
                orig_objpath = self.objpath
                for i in range(len(parts)):
                    new_modname = '.'.join(parts[:len(parts) - i])
                    new_objpath = parts[len(parts) - i:] + orig_objpath
                    try:
                        analyzer = ModuleAnalyzer.for_module(new_modname)
                        analyzer.analyze()
                        key = ('', new_objpath[-1])
                        comment = list(analyzer.attr_docs.get(key, []))
                        if comment:
                            self.objpath = new_objpath
                            self.modname = new_modname
                            return [comment]
                    except PycodeError:
                        pass
        if self.doc_as_attr:
            # Don't show the docstring of the class when it is an alias.
            if self.get_variable_comment():
                return []
            else:
                return None

        lines = getattr(self, '_new_docstrings', None)
        if lines is not None:
            return lines

        classdoc_from = self.options.get('class-doc-from', self.config.autoclass_content)

        docstrings = []
        attrdocstring = getdoc(self.object, self.get_attr)
        if attrdocstring:
            docstrings.append(attrdocstring)

        # for classes, what the "docstring" is can be controlled via a
        # config value; the default is only the class docstring
        if classdoc_from in ('both', 'init'):
            __init__ = self.get_attr(self.object, '__init__', None)
            initdocstring = getdoc(__init__, self.get_attr,
                                   self.config.autodoc_inherit_docstrings,
                                   self.object, '__init__')
            # for new-style classes, no __init__ means default __init__
            if (initdocstring is not None and
                (initdocstring == object.__init__.__doc__ or  # for pypy
                 initdocstring.strip() == object.__init__.__doc__)):  # for !pypy
                initdocstring = None
            if not initdocstring:
                # try __new__
                __new__ = self.get_attr(self.object, '__new__', None)
                initdocstring = getdoc(__new__, self.get_attr,
                                       self.config.autodoc_inherit_docstrings,
                                       self.object, '__new__')
                # for new-style classes, no __new__ means default __new__
                if (initdocstring is not None and
                    (initdocstring == object.__new__.__doc__ or  # for pypy
                     initdocstring.strip() == object.__new__.__doc__)):  # for !pypy
                    initdocstring = None
            if initdocstring:
                if classdoc_from == 'init':
                    docstrings = [initdocstring]
                else:
                    docstrings.append(initdocstring)

        tab_width = self.directive.state.document.settings.tab_width
        return [prepare_docstring(docstring, tab_width) for docstring in docstrings]

    def get_variable_comment(self) -> list[str] | None:
        try:
            key = ('', '.'.join(self.objpath))
            if self.doc_as_attr:
                analyzer = ModuleAnalyzer.for_module(self.modname)
            else:
                analyzer = ModuleAnalyzer.for_module(self.get_real_modname())
            analyzer.analyze()
            return list(analyzer.attr_docs.get(key, []))
        except PycodeError:
            return None

    def add_content(self, more_content: StringList | None) -> None:
        if inspect.isNewType(self.object):
            if self.config.autodoc_typehints_format == "short":
                supertype = restify(self.object.__supertype__, "smart")
            else:
                supertype = restify(self.object.__supertype__)

            more_content = StringList([_('alias of %s') % supertype, ''], source='')
        if isinstance(self.object, TypeVar):
            attrs = [repr(self.object.__name__)]
            for constraint in self.object.__constraints__:
                if self.config.autodoc_typehints_format == "short":
                    attrs.append(stringify_annotation(constraint, "smart"))
                else:
                    attrs.append(stringify_annotation(constraint))
            if self.object.__bound__:
                if self.config.autodoc_typehints_format == "short":
                    bound = restify(self.object.__bound__, "smart")
                else:
                    bound = restify(self.object.__bound__)
                attrs.append(r"bound=\ " + bound)
            if self.object.__covariant__:
                attrs.append("covariant=True")
            if self.object.__contravariant__:
                attrs.append("contravariant=True")

            more_content = StringList(
                [_('alias of TypeVar(%s)') % ", ".join(attrs), ''],
                source='',
            )
        if self.doc_as_attr and self.modname != self.get_real_modname():
            try:
                # override analyzer to obtain doccomment around its definition.
                self.analyzer = ModuleAnalyzer.for_module(self.modname)
                self.analyzer.analyze()
            except PycodeError:
                pass

        if self.doc_as_attr and not self.get_variable_comment():
            try:
                if self.config.autodoc_typehints_format == "short":
                    alias = restify(self.object, "smart")
                else:
                    alias = restify(self.object)
                more_content = StringList([_('alias of %s') % alias], source='')
            except AttributeError:
                pass  # Invalid class object is passed.

        super().add_content(more_content)

    def document_members(self, all_members: bool = False) -> None:
        if self.doc_as_attr:
            return
        super().document_members(all_members)

    def generate(
        self,
        more_content: StringList | None = None,
        real_modname: str | None = None,
        check_module: bool = False,
        all_members: bool = False,
    ) -> None:
        # Do not pass real_modname and use the name from the __module__
        # attribute of the class.
        # If a class gets imported into the module real_modname
        # the analyzer won't find the source of the class, if
        # it looks in real_modname.
        return super().generate(more_content=more_content,
                                check_module=check_module,
                                all_members=all_members)


class ExceptionDocumenter(ClassDocumenter):
    """
    Specialized ClassDocumenter subclass for exceptions.
    """
    objtype = 'exception'
    member_order = 10

    # needs a higher priority than ClassDocumenter
    priority = ClassDocumenter.priority + 5

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        try:
            return isinstance(member, type) and issubclass(member, BaseException)
        except TypeError as exc:
            # It's possible for a member to be considered a type, but fail
            # issubclass checks due to not being a class. For example:
            # https://github.com/sphinx-doc/sphinx/issues/11654#issuecomment-1696790436
            msg = (
                f'{cls.__name__} failed to discern if member {member} with'
                f' membername {membername} is a BaseException subclass.'
            )
            raise ValueError(msg) from exc


class DataDocumenterMixinBase:
    # define types of instance variables
    config: Config
    env: BuildEnvironment
    modname: str
    parent: Any
    object: Any
    objpath: list[str]

    def should_suppress_directive_header(self) -> bool:
        """Check directive header should be suppressed."""
        return False

    def should_suppress_value_header(self) -> bool:
        """Check :value: header should be suppressed."""
        return False

    def update_content(self, more_content: StringList) -> None:
        """Update docstring, for example with TypeVar variance."""
        pass


class GenericAliasMixin(DataDocumenterMixinBase):
    """
    Mixin for DataDocumenter and AttributeDocumenter to provide the feature for
    supporting GenericAliases.
    """

    def should_suppress_directive_header(self) -> bool:
        return (inspect.isgenericalias(self.object) or
                super().should_suppress_directive_header())

    def update_content(self, more_content: StringList) -> None:
        if inspect.isgenericalias(self.object):
            if self.config.autodoc_typehints_format == "short":
                alias = restify(self.object, "smart")
            else:
                alias = restify(self.object)

            more_content.append(_('alias of %s') % alias, '')
            more_content.append('', '')

        super().update_content(more_content)


class UninitializedGlobalVariableMixin(DataDocumenterMixinBase):
    """
    Mixin for DataDocumenter to provide the feature for supporting uninitialized
    (type annotation only) global variables.
    """

    def import_object(self, raiseerror: bool = False) -> bool:
        try:
            return super().import_object(raiseerror=True)  # type: ignore[misc]
        except ImportError as exc:
            # annotation only instance variable (PEP-526)
            try:
                with mock(self.config.autodoc_mock_imports):
                    parent = import_module(self.modname, self.config.autodoc_warningiserror)
                    annotations = get_type_hints(parent, None,
                                                 self.config.autodoc_type_aliases)
                    if self.objpath[-1] in annotations:
                        self.object = UNINITIALIZED_ATTR
                        self.parent = parent
                        return True
            except ImportError:
                pass

            if raiseerror:
                raise
            logger.warning(exc.args[0], type='autodoc', subtype='import_object')
            self.env.note_reread()
            return False

    def should_suppress_value_header(self) -> bool:
        return (self.object is UNINITIALIZED_ATTR or
                super().should_suppress_value_header())

    def get_doc(self) -> list[list[str]] | None:
        if self.object is UNINITIALIZED_ATTR:
            return []
        else:
            return super().get_doc()  # type: ignore[misc]


class DataDocumenter(GenericAliasMixin,
                     UninitializedGlobalVariableMixin, ModuleLevelDocumenter):
    """
    Specialized Documenter subclass for data items.
    """
    objtype = 'data'
    member_order = 40
    priority = -10
    option_spec: OptionSpec = dict(ModuleLevelDocumenter.option_spec)
    option_spec["annotation"] = annotation_option
    option_spec["no-value"] = bool_option

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        return isinstance(parent, ModuleDocumenter) and isattr

    def update_annotations(self, parent: Any) -> None:
        """Update __annotations__ to support type_comment and so on."""
        annotations = dict(inspect.getannotations(parent))
        parent.__annotations__ = annotations

        try:
            analyzer = ModuleAnalyzer.for_module(self.modname)
            analyzer.analyze()
            for (classname, attrname), annotation in analyzer.annotations.items():
                if classname == '' and attrname not in annotations:
                    annotations[attrname] = annotation
        except PycodeError:
            pass

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)
        if self.parent:
            self.update_annotations(self.parent)

        return ret

    def should_suppress_value_header(self) -> bool:
        if super().should_suppress_value_header():
            return True
        else:
            doc = self.get_doc() or []
            docstring, metadata = separate_metadata('\n'.join(sum(doc, [])))
            if 'hide-value' in metadata:
                return True

        return False

    def add_directive_header(self, sig: str) -> None:
        super().add_directive_header(sig)
        sourcename = self.get_sourcename()
        if self.options.annotation is SUPPRESS or self.should_suppress_directive_header():
            pass
        elif self.options.annotation:
            self.add_line('   :annotation: %s' % self.options.annotation,
                          sourcename)
        else:
            if self.config.autodoc_typehints != 'none':
                # obtain annotation for this data
                annotations = get_type_hints(self.parent, None,
                                             self.config.autodoc_type_aliases)
                if self.objpath[-1] in annotations:
                    if self.config.autodoc_typehints_format == "short":
                        objrepr = stringify_annotation(annotations.get(self.objpath[-1]),
                                                       "smart")
                    else:
                        objrepr = stringify_annotation(annotations.get(self.objpath[-1]),
                                                       "fully-qualified-except-typing")
                    self.add_line('   :type: ' + objrepr, sourcename)

            try:
                if (self.options.no_value or self.should_suppress_value_header() or
                        ismock(self.object)):
                    pass
                else:
                    objrepr = object_description(self.object)
                    self.add_line('   :value: ' + objrepr, sourcename)
            except ValueError:
                pass

    def document_members(self, all_members: bool = False) -> None:
        pass

    def get_real_modname(self) -> str:
        real_modname = self.get_attr(self.parent or self.object, '__module__', None)
        return real_modname or self.modname

    def get_module_comment(self, attrname: str) -> list[str] | None:
        try:
            analyzer = ModuleAnalyzer.for_module(self.modname)
            analyzer.analyze()
            key = ('', attrname)
            if key in analyzer.attr_docs:
                return list(analyzer.attr_docs[key])
        except PycodeError:
            pass

        return None

    def get_doc(self) -> list[list[str]] | None:
        # Check the variable has a docstring-comment
        comment = self.get_module_comment(self.objpath[-1])
        if comment:
            return [comment]
        else:
            return super().get_doc()

    def add_content(self, more_content: StringList | None) -> None:
        # Disable analyzing variable comment on Documenter.add_content() to control it on
        # DataDocumenter.add_content()
        self.analyzer = None

        if not more_content:
            more_content = StringList()

        self.update_content(more_content)
        super().add_content(more_content)


class MethodDocumenter(DocstringSignatureMixin, ClassLevelDocumenter):  # type: ignore[misc]
    """
    Specialized Documenter subclass for methods (normal, static and class).
    """
    objtype = 'method'
    directivetype = 'method'
    member_order = 50
    priority = 1  # must be more than FunctionDocumenter

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        return inspect.isroutine(member) and not isinstance(parent, ModuleDocumenter)

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)
        if not ret:
            return ret

        # to distinguish classmethod/staticmethod
        obj = self.parent.__dict__.get(self.object_name)
        if obj is None:
            obj = self.object

        if (inspect.isclassmethod(obj) or
                inspect.isstaticmethod(obj, cls=self.parent, name=self.object_name)):
            # document class and static members before ordinary ones
            self.member_order = self.member_order - 1

        return ret

    def format_args(self, **kwargs: Any) -> str:
        if self.config.autodoc_typehints in ('none', 'description'):
            kwargs.setdefault('show_annotation', False)
        if self.config.autodoc_typehints_format == "short":
            kwargs.setdefault('unqualified_typehints', True)

        try:
            if self.object == object.__init__ and self.parent != object:
                # Classes not having own __init__() method are shown as no arguments.
                #
                # Note: The signature of object.__init__() is (self, /, *args, **kwargs).
                #       But it makes users confused.
                args = '()'
            else:
                if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name):
                    self.env.app.emit('autodoc-before-process-signature', self.object, False)
                    sig = inspect.signature(self.object, bound_method=False,
                                            type_aliases=self.config.autodoc_type_aliases)
                else:
                    self.env.app.emit('autodoc-before-process-signature', self.object, True)
                    sig = inspect.signature(self.object, bound_method=True,
                                            type_aliases=self.config.autodoc_type_aliases)
                args = stringify_signature(sig, **kwargs)
        except TypeError as exc:
            logger.warning(__("Failed to get a method signature for %s: %s"),
                           self.fullname, exc)
            return ''
        except ValueError:
            args = ''

        if self.config.strip_signature_backslash:
            # escape backslashes for reST
            args = args.replace('\\', '\\\\')
        return args

    def add_directive_header(self, sig: str) -> None:
        super().add_directive_header(sig)

        sourcename = self.get_sourcename()
        obj = self.parent.__dict__.get(self.object_name, self.object)
        if inspect.isabstractmethod(obj):
            self.add_line('   :abstractmethod:', sourcename)
        if inspect.iscoroutinefunction(obj) or inspect.isasyncgenfunction(obj):
            self.add_line('   :async:', sourcename)
        if inspect.isclassmethod(obj):
            self.add_line('   :classmethod:', sourcename)
        if inspect.isstaticmethod(obj, cls=self.parent, name=self.object_name):
            self.add_line('   :staticmethod:', sourcename)
        if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
            self.add_line('   :final:', sourcename)

    def document_members(self, all_members: bool = False) -> None:
        pass

    def format_signature(self, **kwargs: Any) -> str:
        if self.config.autodoc_typehints_format == "short":
            kwargs.setdefault('unqualified_typehints', True)

        sigs = []
        if (self.analyzer and
                '.'.join(self.objpath) in self.analyzer.overloads and
                self.config.autodoc_typehints != 'none'):
            # Use signatures for overloaded methods instead of the implementation method.
            overloaded = True
        else:
            overloaded = False
            sig = super().format_signature(**kwargs)
            sigs.append(sig)

        meth = self.parent.__dict__.get(self.objpath[-1])
        if inspect.is_singledispatch_method(meth):
            # append signature of singledispatch'ed functions
            for typ, func in meth.dispatcher.registry.items():
                if typ is object:
                    pass  # default implementation. skipped.
                else:
                    dispatchmeth = self.annotate_to_first_argument(func, typ)
                    if dispatchmeth:
                        documenter = MethodDocumenter(self.directive, '')
                        documenter.parent = self.parent
                        documenter.object = dispatchmeth
                        documenter.objpath = ['']
                        sigs.append(documenter.format_signature())
        if overloaded and self.analyzer is not None:
            if inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name):
                actual = inspect.signature(self.object, bound_method=False,
                                           type_aliases=self.config.autodoc_type_aliases)
            else:
                actual = inspect.signature(self.object, bound_method=True,
                                           type_aliases=self.config.autodoc_type_aliases)

            __globals__ = safe_getattr(self.object, '__globals__', {})
            for overload in self.analyzer.overloads['.'.join(self.objpath)]:
                overload = self.merge_default_value(actual, overload)
                overload = evaluate_signature(overload, __globals__,
                                              self.config.autodoc_type_aliases)

                if not inspect.isstaticmethod(self.object, cls=self.parent,
                                              name=self.object_name):
                    parameters = list(overload.parameters.values())
                    overload = overload.replace(parameters=parameters[1:])
                sig = stringify_signature(overload, **kwargs)
                sigs.append(sig)

        return "\n".join(sigs)

    def merge_default_value(self, actual: Signature, overload: Signature) -> Signature:
        """Merge default values of actual implementation to the overload variants."""
        parameters = list(overload.parameters.values())
        for i, param in enumerate(parameters):
            actual_param = actual.parameters.get(param.name)
            if actual_param and param.default == '...':
                parameters[i] = param.replace(default=actual_param.default)

        return overload.replace(parameters=parameters)

    def annotate_to_first_argument(self, func: Callable, typ: type) -> Callable | None:
        """Annotate type hint to the first argument of function if needed."""
        try:
            sig = inspect.signature(func, type_aliases=self.config.autodoc_type_aliases)
        except TypeError as exc:
            logger.warning(__("Failed to get a method signature for %s: %s"),
                           self.fullname, exc)
            return None
        except ValueError:
            return None

        if len(sig.parameters) == 1:
            return None

        def dummy():
            pass

        params = list(sig.parameters.values())
        if params[1].annotation is Parameter.empty:
            params[1] = params[1].replace(annotation=typ)
            try:
                dummy.__signature__ = sig.replace(  # type: ignore[attr-defined]
                    parameters=params)
                return dummy
            except (AttributeError, TypeError):
                # failed to update signature (ex. built-in or extension types)
                return None

        return func

    def get_doc(self) -> list[list[str]] | None:
        if self._new_docstrings is not None:
            # docstring already returned previously, then modified by
            # `DocstringSignatureMixin`.  Just return the previously-computed
            # result, so that we don't lose the processing done by
            # `DocstringSignatureMixin`.
            return self._new_docstrings
        if self.objpath[-1] == '__init__':
            docstring = getdoc(self.object, self.get_attr,
                               self.config.autodoc_inherit_docstrings,
                               self.parent, self.object_name)
            if (docstring is not None and
                (docstring == object.__init__.__doc__ or  # for pypy
                 docstring.strip() == object.__init__.__doc__)):  # for !pypy
                docstring = None
            if docstring:
                tab_width = self.directive.state.document.settings.tab_width
                return [prepare_docstring(docstring, tabsize=tab_width)]
            else:
                return []
        elif self.objpath[-1] == '__new__':
            docstring = getdoc(self.object, self.get_attr,
                               self.config.autodoc_inherit_docstrings,
                               self.parent, self.object_name)
            if (docstring is not None and
                (docstring == object.__new__.__doc__ or  # for pypy
                 docstring.strip() == object.__new__.__doc__)):  # for !pypy
                docstring = None
            if docstring:
                tab_width = self.directive.state.document.settings.tab_width
                return [prepare_docstring(docstring, tabsize=tab_width)]
            else:
                return []
        else:
            return super().get_doc()


class NonDataDescriptorMixin(DataDocumenterMixinBase):
    """
    Mixin for AttributeDocumenter to provide the feature for supporting non
    data-descriptors.

    .. note:: This mix-in must be inherited after other mix-ins.  Otherwise, docstring
              and :value: header will be suppressed unexpectedly.
    """

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)  # type: ignore[misc]
        if ret and not inspect.isattributedescriptor(self.object):
            self.non_data_descriptor = True
        else:
            self.non_data_descriptor = False

        return ret

    def should_suppress_value_header(self) -> bool:
        return (not getattr(self, 'non_data_descriptor', False) or
                super().should_suppress_directive_header())

    def get_doc(self) -> list[list[str]] | None:
        if getattr(self, 'non_data_descriptor', False):
            # the docstring of non datadescriptor is very probably the wrong thing
            # to display
            return None
        else:
            return super().get_doc()  # type: ignore[misc]


class SlotsMixin(DataDocumenterMixinBase):
    """
    Mixin for AttributeDocumenter to provide the feature for supporting __slots__.
    """

    def isslotsattribute(self) -> bool:
        """Check the subject is an attribute in __slots__."""
        try:
            if parent___slots__ := inspect.getslots(self.parent):
                return self.objpath[-1] in parent___slots__
            else:
                return False
        except (ValueError, TypeError):
            return False

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)  # type: ignore[misc]
        if self.isslotsattribute():
            self.object = SLOTSATTR

        return ret

    def should_suppress_value_header(self) -> bool:
        if self.object is SLOTSATTR:
            return True
        else:
            return super().should_suppress_value_header()

    def get_doc(self) -> list[list[str]] | None:
        if self.object is SLOTSATTR:
            try:
                parent___slots__ = inspect.getslots(self.parent)
                if parent___slots__ and parent___slots__.get(self.objpath[-1]):
                    docstring = prepare_docstring(parent___slots__[self.objpath[-1]])
                    return [docstring]
                else:
                    return []
            except ValueError as exc:
                logger.warning(__('Invalid __slots__ found on %s. Ignored.'),
                               (self.parent.__qualname__, exc), type='autodoc')
                return []
        else:
            return super().get_doc()  # type: ignore[misc]


class RuntimeInstanceAttributeMixin(DataDocumenterMixinBase):
    """
    Mixin for AttributeDocumenter to provide the feature for supporting runtime
    instance attributes (that are defined in __init__() methods with doc-comments).

    Example:

        class Foo:
            def __init__(self):
                self.attr = None  #: This is a target of this mix-in.
    """

    RUNTIME_INSTANCE_ATTRIBUTE = object()

    def is_runtime_instance_attribute(self, parent: Any) -> bool:
        """Check the subject is an attribute defined in __init__()."""
        # An instance variable defined in __init__().
        if self.get_attribute_comment(parent, self.objpath[-1]):  # type: ignore[attr-defined]
            return True
        if self.is_runtime_instance_attribute_not_commented(parent):
            return True
        return False

    def is_runtime_instance_attribute_not_commented(self, parent: Any) -> bool:
        """Check the subject is an attribute defined in __init__() without comment."""
        for cls in inspect.getmro(parent):
            try:
                module = safe_getattr(cls, '__module__')
                qualname = safe_getattr(cls, '__qualname__')

                analyzer = ModuleAnalyzer.for_module(module)
                analyzer.analyze()
                if qualname and self.objpath:
                    key = '.'.join([qualname, self.objpath[-1]])
                    if key in analyzer.tagorder:
                        return True
            except (AttributeError, PycodeError):
                pass

        return False

    def import_object(self, raiseerror: bool = False) -> bool:
        """Check the existence of runtime instance attribute after failing to import the
        attribute."""
        try:
            return super().import_object(raiseerror=True)  # type: ignore[misc]
        except ImportError as exc:
            try:
                with mock(self.config.autodoc_mock_imports):
                    ret = import_object(self.modname, self.objpath[:-1], 'class',
                                        attrgetter=self.get_attr,  # type: ignore[attr-defined]
                                        warningiserror=self.config.autodoc_warningiserror)
                    parent = ret[3]
                    if self.is_runtime_instance_attribute(parent):
                        self.object = self.RUNTIME_INSTANCE_ATTRIBUTE
                        self.parent = parent
                        return True
            except ImportError:
                pass

            if raiseerror:
                raise
            logger.warning(exc.args[0], type='autodoc', subtype='import_object')
            self.env.note_reread()
            return False

    def should_suppress_value_header(self) -> bool:
        return (self.object is self.RUNTIME_INSTANCE_ATTRIBUTE or
                super().should_suppress_value_header())

    def get_doc(self) -> list[list[str]] | None:
        if (self.object is self.RUNTIME_INSTANCE_ATTRIBUTE and
                self.is_runtime_instance_attribute_not_commented(self.parent)):
            return None
        else:
            return super().get_doc()  # type: ignore[misc]


class UninitializedInstanceAttributeMixin(DataDocumenterMixinBase):
    """
    Mixin for AttributeDocumenter to provide the feature for supporting uninitialized
    instance attributes (PEP-526 styled, annotation only attributes).

    Example:

        class Foo:
            attr: int  #: This is a target of this mix-in.
    """

    def is_uninitialized_instance_attribute(self, parent: Any) -> bool:
        """Check the subject is an annotation only attribute."""
        annotations = get_type_hints(parent, None, self.config.autodoc_type_aliases)
        return self.objpath[-1] in annotations

    def import_object(self, raiseerror: bool = False) -> bool:
        """Check the exisitence of uninitialized instance attribute when failed to import
        the attribute."""
        try:
            return super().import_object(raiseerror=True)  # type: ignore[misc]
        except ImportError as exc:
            try:
                ret = import_object(self.modname, self.objpath[:-1], 'class',
                                    attrgetter=self.get_attr,  # type: ignore[attr-defined]
                                    warningiserror=self.config.autodoc_warningiserror)
                parent = ret[3]
                if self.is_uninitialized_instance_attribute(parent):
                    self.object = UNINITIALIZED_ATTR
                    self.parent = parent
                    return True
            except ImportError:
                pass

            if raiseerror:
                raise
            logger.warning(exc.args[0], type='autodoc', subtype='import_object')
            self.env.note_reread()
            return False

    def should_suppress_value_header(self) -> bool:
        return (self.object is UNINITIALIZED_ATTR or
                super().should_suppress_value_header())

    def get_doc(self) -> list[list[str]] | None:
        if self.object is UNINITIALIZED_ATTR:
            return None
        return super().get_doc()  # type: ignore[misc]


class AttributeDocumenter(GenericAliasMixin, SlotsMixin,  # type: ignore[misc]
                          RuntimeInstanceAttributeMixin,
                          UninitializedInstanceAttributeMixin, NonDataDescriptorMixin,
                          DocstringStripSignatureMixin, ClassLevelDocumenter):
    """
    Specialized Documenter subclass for attributes.
    """
    objtype = 'attribute'
    member_order = 60
    option_spec: OptionSpec = dict(ModuleLevelDocumenter.option_spec)
    option_spec["annotation"] = annotation_option
    option_spec["no-value"] = bool_option

    # must be higher than the MethodDocumenter, else it will recognize
    # some non-data descriptors as methods
    priority = 10

    @staticmethod
    def is_function_or_method(obj: Any) -> bool:
        return inspect.isfunction(obj) or inspect.isbuiltin(obj) or inspect.ismethod(obj)

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        if isinstance(parent, ModuleDocumenter):
            return False
        if inspect.isattributedescriptor(member):
            return True
        if not inspect.isroutine(member) and not isinstance(member, type):
            return True
        return False

    def document_members(self, all_members: bool = False) -> None:
        pass

    def update_annotations(self, parent: Any) -> None:
        """Update __annotations__ to support type_comment and so on."""
        try:
            annotations = dict(inspect.getannotations(parent))
            parent.__annotations__ = annotations

            for cls in inspect.getmro(parent):
                try:
                    module = safe_getattr(cls, '__module__')
                    qualname = safe_getattr(cls, '__qualname__')

                    analyzer = ModuleAnalyzer.for_module(module)
                    analyzer.analyze()
                    for (classname, attrname), annotation in analyzer.annotations.items():
                        if classname == qualname and attrname not in annotations:
                            annotations[attrname] = annotation
                except (AttributeError, PycodeError):
                    pass
        except (AttributeError, TypeError):
            # Failed to set __annotations__ (built-in, extensions, etc.)
            pass

    def import_object(self, raiseerror: bool = False) -> bool:
        ret = super().import_object(raiseerror)
        if inspect.isenumattribute(self.object):
            self.object = self.object.value
        if self.parent:
            self.update_annotations(self.parent)

        return ret

    def get_real_modname(self) -> str:
        real_modname = self.get_attr(self.parent or self.object, '__module__', None)
        return real_modname or self.modname

    def should_suppress_value_header(self) -> bool:
        if super().should_suppress_value_header():
            return True
        else:
            doc = self.get_doc()
            if doc:
                docstring, metadata = separate_metadata('\n'.join(sum(doc, [])))
                if 'hide-value' in metadata:
                    return True

        return False

    def add_directive_header(self, sig: str) -> None:
        super().add_directive_header(sig)
        sourcename = self.get_sourcename()
        if self.options.annotation is SUPPRESS or self.should_suppress_directive_header():
            pass
        elif self.options.annotation:
            self.add_line('   :annotation: %s' % self.options.annotation, sourcename)
        else:
            if self.config.autodoc_typehints != 'none':
                # obtain type annotation for this attribute
                annotations = get_type_hints(self.parent, None,
                                             self.config.autodoc_type_aliases)
                if self.objpath[-1] in annotations:
                    if self.config.autodoc_typehints_format == "short":
                        objrepr = stringify_annotation(annotations.get(self.objpath[-1]),
                                                       "smart")
                    else:
                        objrepr = stringify_annotation(annotations.get(self.objpath[-1]),
                                                       "fully-qualified-except-typing")
                    self.add_line('   :type: ' + objrepr, sourcename)

            try:
                if (self.options.no_value or self.should_suppress_value_header() or
                        ismock(self.object)):
                    pass
                else:
                    objrepr = object_description(self.object)
                    self.add_line('   :value: ' + objrepr, sourcename)
            except ValueError:
                pass

    def get_attribute_comment(self, parent: Any, attrname: str) -> list[str] | None:
        for cls in inspect.getmro(parent):
            try:
                module = safe_getattr(cls, '__module__')
                qualname = safe_getattr(cls, '__qualname__')

                analyzer = ModuleAnalyzer.for_module(module)
                analyzer.analyze()
                if qualname and self.objpath:
                    key = (qualname, attrname)
                    if key in analyzer.attr_docs:
                        return list(analyzer.attr_docs[key])
            except (AttributeError, PycodeError):
                pass

        return None

    def get_doc(self) -> list[list[str]] | None:
        # Check the attribute has a docstring-comment
        comment = self.get_attribute_comment(self.parent, self.objpath[-1])
        if comment:
            return [comment]

        try:
            # Disable `autodoc_inherit_docstring` temporarily to avoid to obtain
            # a docstring from the value which descriptor returns unexpectedly.
            # ref: https://github.com/sphinx-doc/sphinx/issues/7805
            orig = self.config.autodoc_inherit_docstrings
            self.config.autodoc_inherit_docstrings = False  # type: ignore[attr-defined]
            return super().get_doc()
        finally:
            self.config.autodoc_inherit_docstrings = orig  # type: ignore[attr-defined]

    def add_content(self, more_content: StringList | None) -> None:
        # Disable analyzing attribute comment on Documenter.add_content() to control it on
        # AttributeDocumenter.add_content()
        self.analyzer = None

        if more_content is None:
            more_content = StringList()
        self.update_content(more_content)
        super().add_content(more_content)


class PropertyDocumenter(DocstringStripSignatureMixin,  # type: ignore[misc]
                         ClassLevelDocumenter):
    """
    Specialized Documenter subclass for properties.
    """
    objtype = 'property'
    member_order = 60

    # before AttributeDocumenter
    priority = AttributeDocumenter.priority + 1

    @classmethod
    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any,
                            ) -> bool:
        if isinstance(parent, ClassDocumenter):
            if inspect.isproperty(member):
                return True
            else:
                __dict__ = safe_getattr(parent.object, '__dict__', {})
                obj = __dict__.get(membername)
                return isinstance(obj, classmethod) and inspect.isproperty(obj.__func__)
        else:
            return False

    def import_object(self, raiseerror: bool = False) -> bool:
        """Check the exisitence of uninitialized instance attribute when failed to import
        the attribute."""
        ret = super().import_object(raiseerror)
        if ret and not inspect.isproperty(self.object):
            __dict__ = safe_getattr(self.parent, '__dict__', {})
            obj = __dict__.get(self.objpath[-1])
            if isinstance(obj, classmethod) and inspect.isproperty(obj.__func__):
                self.object = obj.__func__
                self.isclassmethod = True
                return True
            else:
                return False

        self.isclassmethod = False
        return ret

    def format_args(self, **kwargs: Any) -> str:
        func = self._get_property_getter()
        if func is None:
            return ''

        # update the annotations of the property getter
        self.env.app.emit('autodoc-before-process-signature', func, False)
        # correctly format the arguments for a property
        return super().format_args(**kwargs)

    def document_members(self, all_members: bool = False) -> None:
        pass

    def get_real_modname(self) -> str:
        real_modname = self.get_attr(self.parent or self.object, '__module__', None)
        return real_modname or self.modname

    def add_directive_header(self, sig: str) -> None:
        super().add_directive_header(sig)
        sourcename = self.get_sourcename()
        if inspect.isabstractmethod(self.object):
            self.add_line('   :abstractmethod:', sourcename)
        if self.isclassmethod:
            self.add_line('   :classmethod:', sourcename)

        func = self._get_property_getter()
        if func is None or self.config.autodoc_typehints == 'none':
            return

        try:
            signature = inspect.signature(func,
                                          type_aliases=self.config.autodoc_type_aliases)
            if signature.return_annotation is not Parameter.empty:
                if self.config.autodoc_typehints_format == "short":
                    objrepr = stringify_annotation(signature.return_annotation, "smart")
                else:
                    objrepr = stringify_annotation(signature.return_annotation,
                                                   "fully-qualified-except-typing")
                self.add_line('   :type: ' + objrepr, sourcename)
        except TypeError as exc:
            logger.warning(__("Failed to get a function signature for %s: %s"),
                           self.fullname, exc)
            pass
        except ValueError:
            pass

    def _get_property_getter(self):
        if safe_getattr(self.object, 'fget', None):  # property
            return self.object.fget
        if safe_getattr(self.object, 'func', None):  # cached_property
            return self.object.func
        return None


def autodoc_attrgetter(app: Sphinx, obj: Any, name: str, *defargs: Any) -> Any:
    """Alternative getattr() for types"""
    for typ, func in app.registry.autodoc_attrgettrs.items():
        if isinstance(obj, typ):
            return func(obj, name, *defargs)

    return safe_getattr(obj, name, *defargs)


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_autodocumenter(ModuleDocumenter)
    app.add_autodocumenter(ClassDocumenter)
    app.add_autodocumenter(ExceptionDocumenter)
    app.add_autodocumenter(DataDocumenter)
    app.add_autodocumenter(FunctionDocumenter)
    app.add_autodocumenter(DecoratorDocumenter)
    app.add_autodocumenter(MethodDocumenter)
    app.add_autodocumenter(AttributeDocumenter)
    app.add_autodocumenter(PropertyDocumenter)

    app.add_config_value('autoclass_content', 'class', True, ENUM('both', 'class', 'init'))
    app.add_config_value('autodoc_member_order', 'alphabetical', True,
                         ENUM('alphabetical', 'bysource', 'groupwise'))
    app.add_config_value('autodoc_class_signature', 'mixed', True, ENUM('mixed', 'separated'))
    app.add_config_value('autodoc_default_options', {}, True)
    app.add_config_value('autodoc_docstring_signature', True, True)
    app.add_config_value('autodoc_mock_imports', [], True)
    app.add_config_value('autodoc_typehints', "signature", True,
                         ENUM("signature", "description", "none", "both"))
    app.add_config_value('autodoc_typehints_description_target', 'all', True,
                         ENUM('all', 'documented', 'documented_params'))
    app.add_config_value('autodoc_type_aliases', {}, True)
    app.add_config_value('autodoc_typehints_format', "short", 'env',
                         ENUM("fully-qualified", "short"))
    app.add_config_value('autodoc_warningiserror', True, True)
    app.add_config_value('autodoc_inherit_docstrings', True, True)
    app.add_event('autodoc-before-process-signature')
    app.add_event('autodoc-process-docstring')
    app.add_event('autodoc-process-signature')
    app.add_event('autodoc-skip-member')
    app.add_event('autodoc-process-bases')

    app.setup_extension('sphinx.ext.autodoc.preserve_defaults')
    app.setup_extension('sphinx.ext.autodoc.type_comment')
    app.setup_extension('sphinx.ext.autodoc.typehints')

    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/autodoc/__init__.py
NikolaBlagojevic/pyrecodes
"""Importer utilities for autodoc"""

from __future__ import annotations

import importlib
import sys
import traceback
import typing
from typing import TYPE_CHECKING, Any, Callable, NamedTuple

from sphinx.ext.autodoc.mock import ismock, undecorate
from sphinx.pycode import ModuleAnalyzer, PycodeError
from sphinx.util import logging
from sphinx.util.inspect import (
    getannotations,
    getmro,
    getslots,
    isclass,
    isenumclass,
    safe_getattr,
)

if TYPE_CHECKING:
    from sphinx.ext.autodoc import ObjectMember

logger = logging.getLogger(__name__)


def mangle(subject: Any, name: str) -> str:
    """Mangle the given name."""
    try:
        if isclass(subject) and name.startswith('__') and not name.endswith('__'):
            return f"_{subject.__name__}{name}"
    except AttributeError:
        pass

    return name


def unmangle(subject: Any, name: str) -> str | None:
    """Unmangle the given name."""
    try:
        if isclass(subject) and not name.endswith('__'):
            prefix = "_%s__" % subject.__name__
            if name.startswith(prefix):
                return name.replace(prefix, "__", 1)
            else:
                for cls in subject.__mro__:
                    prefix = "_%s__" % cls.__name__
                    if name.startswith(prefix):
                        # mangled attribute defined in parent class
                        return None
    except AttributeError:
        pass

    return name


def import_module(modname: str, warningiserror: bool = False) -> Any:
    """
    Call importlib.import_module(modname), convert exceptions to ImportError
    """
    try:
        with logging.skip_warningiserror(not warningiserror):
            return importlib.import_module(modname)
    except BaseException as exc:
        # Importing modules may cause any side effects, including
        # SystemExit, so we need to catch all errors.
        raise ImportError(exc, traceback.format_exc()) from exc


def import_object(modname: str, objpath: list[str], objtype: str = '',
                  attrgetter: Callable[[Any, str], Any] = safe_getattr,
                  warningiserror: bool = False) -> Any:
    if objpath:
        logger.debug('[autodoc] from %s import %s', modname, '.'.join(objpath))
    else:
        logger.debug('[autodoc] import %s', modname)

    try:
        module = None
        exc_on_importing = None
        objpath = list(objpath)
        while module is None:
            try:
                orig_modules = frozenset(sys.modules)
                try:
                    # try importing with ``typing.TYPE_CHECKING == True``
                    typing.TYPE_CHECKING = True
                    module = import_module(modname, warningiserror=warningiserror)
                except ImportError:
                    # if that fails (e.g. circular import), retry with
                    # ``typing.TYPE_CHECKING == False`` after reverting
                    # changes made to ``sys.modules`` by the failed try
                    for m in [m for m in sys.modules if m not in orig_modules]:
                        sys.modules.pop(m)

                    typing.TYPE_CHECKING = False
                    module = import_module(modname, warningiserror=warningiserror)
                finally:
                    # ensure ``typing.TYPE_CHECKING == False``
                    typing.TYPE_CHECKING = False
                logger.debug('[autodoc] import %s => %r', modname, module)
            except ImportError as exc:
                logger.debug('[autodoc] import %s => failed', modname)
                exc_on_importing = exc
                if '.' in modname:
                    # retry with parent module
                    modname, name = modname.rsplit('.', 1)
                    objpath.insert(0, name)
                else:
                    raise

        obj = module
        parent = None
        object_name = None
        for attrname in objpath:
            parent = obj
            logger.debug('[autodoc] getattr(_, %r)', attrname)
            mangled_name = mangle(obj, attrname)
            obj = attrgetter(obj, mangled_name)

            try:
                logger.debug('[autodoc] => %r', obj)
            except TypeError:
                # fallback of failure on logging for broken object
                # refs: https://github.com/sphinx-doc/sphinx/issues/9095
                logger.debug('[autodoc] => %r', (obj,))

            object_name = attrname
        return [module, parent, object_name, obj]
    except (AttributeError, ImportError) as exc:
        if isinstance(exc, AttributeError) and exc_on_importing:
            # restore ImportError
            exc = exc_on_importing

        if objpath:
            errmsg = ('autodoc: failed to import %s %r from module %r' %
                      (objtype, '.'.join(objpath), modname))
        else:
            errmsg = f'autodoc: failed to import {objtype} {modname!r}'

        if isinstance(exc, ImportError):
            # import_module() raises ImportError having real exception obj and
            # traceback
            real_exc, traceback_msg = exc.args
            if isinstance(real_exc, SystemExit):
                errmsg += ('; the module executes module level statement '
                           'and it might call sys.exit().')
            elif isinstance(real_exc, ImportError) and real_exc.args:
                errmsg += '; the following exception was raised:\n%s' % real_exc.args[0]
            else:
                errmsg += '; the following exception was raised:\n%s' % traceback_msg
        else:
            errmsg += '; the following exception was raised:\n%s' % traceback.format_exc()

        logger.debug(errmsg)
        raise ImportError(errmsg) from exc


class Attribute(NamedTuple):
    name: str
    directly_defined: bool
    value: Any


def get_object_members(
    subject: Any,
    objpath: list[str],
    attrgetter: Callable,
    analyzer: ModuleAnalyzer | None = None,
) -> dict[str, Attribute]:
    """Get members and attributes of target object."""
    from sphinx.ext.autodoc import INSTANCEATTR

    # the members directly defined in the class
    obj_dict = attrgetter(subject, '__dict__', {})

    members: dict[str, Attribute] = {}

    # enum members
    if isenumclass(subject):
        for name, value in subject.__members__.items():
            if name not in members:
                members[name] = Attribute(name, True, value)

        superclass = subject.__mro__[1]
        for name in obj_dict:
            if name not in superclass.__dict__:
                value = safe_getattr(subject, name)
                members[name] = Attribute(name, True, value)

    # members in __slots__
    try:
        subject___slots__ = getslots(subject)
        if subject___slots__:
            from sphinx.ext.autodoc import SLOTSATTR

            for name in subject___slots__:
                members[name] = Attribute(name, True, SLOTSATTR)
    except (TypeError, ValueError):
        pass

    # other members
    for name in dir(subject):
        try:
            value = attrgetter(subject, name)
            directly_defined = name in obj_dict
            name = unmangle(subject, name)
            if name and name not in members:
                members[name] = Attribute(name, directly_defined, value)
        except AttributeError:
            continue

    # annotation only member (ex. attr: int)
    for i, cls in enumerate(getmro(subject)):
        for name in getannotations(cls):
            name = unmangle(cls, name)
            if name and name not in members:
                members[name] = Attribute(name, i == 0, INSTANCEATTR)

    if analyzer:
        # append instance attributes (cf. self.attr1) if analyzer knows
        namespace = '.'.join(objpath)
        for (ns, name) in analyzer.find_attr_docs():
            if namespace == ns and name not in members:
                members[name] = Attribute(name, True, INSTANCEATTR)

    return members


def get_class_members(subject: Any, objpath: Any, attrgetter: Callable,
                      inherit_docstrings: bool = True) -> dict[str, ObjectMember]:
    """Get members and attributes of target class."""
    from sphinx.ext.autodoc import INSTANCEATTR, ObjectMember

    # the members directly defined in the class
    obj_dict = attrgetter(subject, '__dict__', {})

    members: dict[str, ObjectMember] = {}

    # enum members
    if isenumclass(subject):
        for name, value in subject.__members__.items():
            if name not in members:
                members[name] = ObjectMember(name, value, class_=subject)

        superclass = subject.__mro__[1]
        for name in obj_dict:
            if name not in superclass.__dict__:
                value = safe_getattr(subject, name)
                members[name] = ObjectMember(name, value, class_=subject)

    # members in __slots__
    try:
        subject___slots__ = getslots(subject)
        if subject___slots__:
            from sphinx.ext.autodoc import SLOTSATTR

            for name, docstring in subject___slots__.items():
                members[name] = ObjectMember(name, SLOTSATTR, class_=subject,
                                             docstring=docstring)
    except (TypeError, ValueError):
        pass

    # other members
    for name in dir(subject):
        try:
            value = attrgetter(subject, name)
            if ismock(value):
                value = undecorate(value)

            unmangled = unmangle(subject, name)
            if unmangled and unmangled not in members:
                if name in obj_dict:
                    members[unmangled] = ObjectMember(unmangled, value, class_=subject)
                else:
                    members[unmangled] = ObjectMember(unmangled, value)
        except AttributeError:
            continue

    try:
        for cls in getmro(subject):
            try:
                modname = safe_getattr(cls, '__module__')
                qualname = safe_getattr(cls, '__qualname__')
                analyzer = ModuleAnalyzer.for_module(modname)
                analyzer.analyze()
            except AttributeError:
                qualname = None
                analyzer = None
            except PycodeError:
                analyzer = None

            # annotation only member (ex. attr: int)
            for name in getannotations(cls):
                name = unmangle(cls, name)
                if name and name not in members:
                    if analyzer and (qualname, name) in analyzer.attr_docs:
                        docstring = '\n'.join(analyzer.attr_docs[qualname, name])
                    else:
                        docstring = None

                    members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,
                                                 docstring=docstring)

            # append or complete instance attributes (cf. self.attr1) if analyzer knows
            if analyzer:
                for (ns, name), docstring in analyzer.attr_docs.items():
                    if ns == qualname and name not in members:
                        # otherwise unknown instance attribute
                        members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,
                                                     docstring='\n'.join(docstring))
                    elif (ns == qualname and docstring and
                          isinstance(members[name], ObjectMember) and
                          not members[name].docstring):
                        if cls != subject and not inherit_docstrings:
                            # If we are in the MRO of the class and not the class itself,
                            # and we do not want to inherit docstrings, then skip setting
                            # the docstring below
                            continue
                        # attribute is already known, because dir(subject) enumerates it.
                        # But it has no docstring yet
                        members[name].docstring = '\n'.join(docstring)
    except AttributeError:
        pass

    return members

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/autodoc/importer.py
NikolaBlagojevic/pyrecodes
"""Preserve function defaults.

Preserve the default argument values of function signatures in source code
and keep them not evaluated for readability.
"""

from __future__ import annotations

import ast
import inspect
import types
import warnings
from typing import TYPE_CHECKING

import sphinx
from sphinx.deprecation import RemovedInSphinx90Warning
from sphinx.locale import __
from sphinx.pycode.ast import unparse as ast_unparse
from sphinx.util import logging

if TYPE_CHECKING:
    from typing import Any

    from sphinx.application import Sphinx

logger = logging.getLogger(__name__)
_LAMBDA_NAME = (lambda: None).__name__


class DefaultValue:
    def __init__(self, name: str) -> None:
        self.name = name

    def __repr__(self) -> str:
        return self.name


def get_function_def(obj: Any) -> ast.FunctionDef | None:
    """Get FunctionDef object from living object.

    This tries to parse original code for living object and returns
    AST node for given *obj*.
    """
    warnings.warn('sphinx.ext.autodoc.preserve_defaults.get_function_def is'
                  ' deprecated and scheduled for removal in Sphinx 9.'
                  ' Use sphinx.ext.autodoc.preserve_defaults._get_arguments() to'
                  ' extract AST arguments objects from a lambda or regular'
                  ' function.', RemovedInSphinx90Warning, stacklevel=2)

    try:
        source = inspect.getsource(obj)
        if source.startswith((' ', '\t')):
            # subject is placed inside class or block.  To read its docstring,
            # this adds if-block before the declaration.
            module = ast.parse('if True:\n' + source)
            return module.body[0].body[0]  # type: ignore[attr-defined]
        else:
            module = ast.parse(source)
            return module.body[0]  # type: ignore[return-value]
    except (OSError, TypeError):  # failed to load source code
        return None


def _get_arguments(obj: Any, /) -> ast.arguments | None:
    """Parse 'ast.arguments' from an object.

    This tries to parse the original code for an object and returns
    an 'ast.arguments' node.
    """
    try:
        source = inspect.getsource(obj)
        if source.startswith((' ', '\t')):
            # 'obj' is in some indented block.
            module = ast.parse('if True:\n' + source)
            subject = module.body[0].body[0]  # type: ignore[attr-defined]
        else:
            module = ast.parse(source)
            subject = module.body[0]
    except (OSError, TypeError):
        # bail; failed to load source for 'obj'.
        return None
    except SyntaxError:
        if _is_lambda(obj):
            # Most likely a multi-line arising from detecting a lambda, e.g.:
            #
            # class Egg:
            #     x = property(
            #         lambda self: 1, doc="...")
            return None

        # Other syntax errors that are not due to the fact that we are
        # documenting a lambda function are propagated
        # (in particular if a lambda is renamed by the user).
        raise

    return _get_arguments_inner(subject)


def _is_lambda(x, /):
    return isinstance(x, types.LambdaType) and x.__name__ == _LAMBDA_NAME


def _get_arguments_inner(x: Any, /) -> ast.arguments | None:
    if isinstance(x, (ast.AsyncFunctionDef, ast.FunctionDef, ast.Lambda)):
        return x.args
    if isinstance(x, (ast.Assign, ast.AnnAssign)):
        return _get_arguments_inner(x.value)
    return None


def get_default_value(lines: list[str], position: ast.AST) -> str | None:
    try:
        if position.lineno == position.end_lineno:
            line = lines[position.lineno - 1]
            return line[position.col_offset:position.end_col_offset]
        else:
            # multiline value is not supported now
            return None
    except (AttributeError, IndexError):
        return None


def update_defvalue(app: Sphinx, obj: Any, bound_method: bool) -> None:
    """Update defvalue info of *obj* using type_comments."""
    if not app.config.autodoc_preserve_defaults:
        return

    try:
        lines = inspect.getsource(obj).splitlines()
        if lines[0].startswith((' ', '\t')):
            # insert a dummy line to follow what _get_arguments() does.
            lines.insert(0, '')
    except (OSError, TypeError):
        lines = []

    try:
        args = _get_arguments(obj)
    except SyntaxError:
        return
    if args is None:
        # If the object is a built-in, we won't be always able to recover
        # the function definition and its arguments. This happens if *obj*
        # is the `__init__` method generated automatically for dataclasses.
        return

    if not args.defaults and not args.kw_defaults:
        return

    try:
        if bound_method and inspect.ismethod(obj) and hasattr(obj, '__func__'):
            sig = inspect.signature(obj.__func__)
        else:
            sig = inspect.signature(obj)
        defaults = list(args.defaults)
        kw_defaults = list(args.kw_defaults)
        parameters = list(sig.parameters.values())
        for i, param in enumerate(parameters):
            if param.default is param.empty:
                if param.kind == param.KEYWORD_ONLY:
                    # Consume kw_defaults for kwonly args
                    kw_defaults.pop(0)
            else:
                if param.kind in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD):
                    default = defaults.pop(0)
                    value = get_default_value(lines, default)
                    if value is None:
                        value = ast_unparse(default)
                    parameters[i] = param.replace(default=DefaultValue(value))
                else:
                    default = kw_defaults.pop(0)  # type: ignore[assignment]
                    value = get_default_value(lines, default)
                    if value is None:
                        value = ast_unparse(default)
                    parameters[i] = param.replace(default=DefaultValue(value))

        sig = sig.replace(parameters=parameters)
        try:
            obj.__signature__ = sig
        except AttributeError:
            # __signature__ can't be set directly on bound methods.
            obj.__dict__['__signature__'] = sig
    except (AttributeError, TypeError):
        # Failed to update signature (e.g. built-in or extension types).
        # For user-defined functions, "obj" may not have __dict__,
        # e.g. when decorated with a class that defines __slots__.
        # In this case, we can't set __signature__.
        return
    except NotImplementedError as exc:  # failed to ast_unparse()
        logger.warning(__("Failed to parse a default argument value for %r: %s"), obj, exc)


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_config_value('autodoc_preserve_defaults', False, True)
    app.connect('autodoc-before-process-signature', update_defvalue)

    return {
        'version': sphinx.__display_version__,
        'parallel_read_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/autodoc/preserve_defaults.py
NikolaBlagojevic/pyrecodes
"""Generates reST source files for autosummary.

Usable as a library or script to generate automatic RST source files for
items referred to in autosummary:: directives.

Each generated RST file contains a single auto*:: directive which
extracts the docstring of the referred item.

Example Makefile rule::

   generate:
           sphinx-autogen -o source/generated source/*.rst
"""

from __future__ import annotations

import argparse
import importlib
import inspect
import locale
import os
import pkgutil
import pydoc
import re
import sys
from os import path
from typing import TYPE_CHECKING, Any, NamedTuple

from jinja2 import TemplateNotFound
from jinja2.sandbox import SandboxedEnvironment

import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.ext.autodoc.importer import import_module
from sphinx.ext.autosummary import (
    ImportExceptionGroup,
    get_documenter,
    import_by_name,
    import_ivar_by_name,
)
from sphinx.locale import __
from sphinx.pycode import ModuleAnalyzer, PycodeError
from sphinx.registry import SphinxComponentRegistry
from sphinx.util import logging, rst
from sphinx.util.inspect import getall, safe_getattr
from sphinx.util.osutil import ensuredir
from sphinx.util.template import SphinxTemplateLoader

if TYPE_CHECKING:
    from collections.abc import Sequence, Set
    from gettext import NullTranslations

    from sphinx.application import Sphinx
    from sphinx.ext.autodoc import Documenter

logger = logging.getLogger(__name__)


class DummyApplication:
    """Dummy Application class for sphinx-autogen command."""

    def __init__(self, translator: NullTranslations) -> None:
        self.config = Config()
        self.registry = SphinxComponentRegistry()
        self.messagelog: list[str] = []
        self.srcdir = "/"
        self.translator = translator
        self.verbosity = 0
        self._warncount = 0
        self.warningiserror = False

        self.config.add('autosummary_context', {}, True, None)
        self.config.add('autosummary_filename_map', {}, True, None)
        self.config.add('autosummary_ignore_module_all', True, 'env', bool)
        self.config.init_values()

    def emit_firstresult(self, *args: Any) -> None:
        pass


class AutosummaryEntry(NamedTuple):
    name: str
    path: str | None
    template: str
    recursive: bool


def setup_documenters(app: Any) -> None:
    from sphinx.ext.autodoc import (
        AttributeDocumenter,
        ClassDocumenter,
        DataDocumenter,
        DecoratorDocumenter,
        ExceptionDocumenter,
        FunctionDocumenter,
        MethodDocumenter,
        ModuleDocumenter,
        PropertyDocumenter,
    )
    documenters: list[type[Documenter]] = [
        ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter,
        FunctionDocumenter, MethodDocumenter,
        AttributeDocumenter, DecoratorDocumenter, PropertyDocumenter,
    ]
    for documenter in documenters:
        app.registry.add_documenter(documenter.objtype, documenter)


def _underline(title: str, line: str = '=') -> str:
    if '\n' in title:
        msg = 'Can only underline single lines'
        raise ValueError(msg)
    return title + '\n' + line * len(title)


class AutosummaryRenderer:
    """A helper class for rendering."""

    def __init__(self, app: Sphinx) -> None:
        if isinstance(app, Builder):
            msg = 'Expected a Sphinx application object!'
            raise ValueError(msg)

        system_templates_path = [os.path.join(package_dir, 'ext', 'autosummary', 'templates')]
        loader = SphinxTemplateLoader(app.srcdir, app.config.templates_path,
                                      system_templates_path)

        self.env = SandboxedEnvironment(loader=loader)
        self.env.filters['escape'] = rst.escape
        self.env.filters['e'] = rst.escape
        self.env.filters['underline'] = _underline

        if app.translator:
            self.env.add_extension("jinja2.ext.i18n")
            self.env.install_gettext_translations(app.translator)

    def render(self, template_name: str, context: dict) -> str:
        """Render a template file."""
        try:
            template = self.env.get_template(template_name)
        except TemplateNotFound:
            try:
                # objtype is given as template_name
                template = self.env.get_template('autosummary/%s.rst' % template_name)
            except TemplateNotFound:
                # fallback to base.rst
                template = self.env.get_template('autosummary/base.rst')

        return template.render(context)


def _split_full_qualified_name(name: str) -> tuple[str | None, str]:
    """Split full qualified name to a pair of modname and qualname.

    A qualname is an abbreviation for "Qualified name" introduced at PEP-3155
    (https://peps.python.org/pep-3155/).  It is a dotted path name
    from the module top-level.

    A "full" qualified name means a string containing both module name and
    qualified name.

    .. note:: This function actually imports the module to check its existence.
              Therefore you need to mock 3rd party modules if needed before
              calling this function.
    """
    parts = name.split('.')
    for i, _part in enumerate(parts, 1):
        try:
            modname = ".".join(parts[:i])
            importlib.import_module(modname)
        except ImportError:
            if parts[:i - 1]:
                return ".".join(parts[:i - 1]), ".".join(parts[i - 1:])
            else:
                return None, ".".join(parts)
        except IndexError:
            pass

    return name, ""


# -- Generating output ---------------------------------------------------------


class ModuleScanner:
    def __init__(self, app: Any, obj: Any) -> None:
        self.app = app
        self.object = obj

    def get_object_type(self, name: str, value: Any) -> str:
        return get_documenter(self.app, value, self.object).objtype

    def is_skipped(self, name: str, value: Any, objtype: str) -> bool:
        try:
            return self.app.emit_firstresult('autodoc-skip-member', objtype,
                                             name, value, False, {})
        except Exception as exc:
            logger.warning(__('autosummary: failed to determine %r to be documented, '
                              'the following exception was raised:\n%s'),
                           name, exc, type='autosummary')
            return False

    def scan(self, imported_members: bool) -> list[str]:
        members = []
        try:
            analyzer = ModuleAnalyzer.for_module(self.object.__name__)
            attr_docs = analyzer.find_attr_docs()
        except PycodeError:
            attr_docs = {}

        for name in members_of(self.object, self.app.config):
            try:
                value = safe_getattr(self.object, name)
            except AttributeError:
                value = None

            objtype = self.get_object_type(name, value)
            if self.is_skipped(name, value, objtype):
                continue

            try:
                if ('', name) in attr_docs:
                    imported = False
                elif inspect.ismodule(value):  # NoQA: SIM114
                    imported = True
                elif safe_getattr(value, '__module__') != self.object.__name__:
                    imported = True
                else:
                    imported = False
            except AttributeError:
                imported = False

            respect_module_all = not self.app.config.autosummary_ignore_module_all
            if (
                # list all members up
                imported_members
                # list not-imported members
                or imported is False
                # list members that have __all__ set
                or (respect_module_all and '__all__' in dir(self.object))
            ):
                members.append(name)

        return members


def members_of(obj: Any, conf: Config) -> Sequence[str]:
    """Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute

    Follows the ``conf.autosummary_ignore_module_all`` setting."""

    if conf.autosummary_ignore_module_all:
        return dir(obj)
    else:
        return getall(obj) or dir(obj)


def generate_autosummary_content(name: str, obj: Any, parent: Any,
                                 template: AutosummaryRenderer, template_name: str,
                                 imported_members: bool, app: Any,
                                 recursive: bool, context: dict,
                                 modname: str | None = None,
                                 qualname: str | None = None) -> str:
    doc = get_documenter(app, obj, parent)

    ns: dict[str, Any] = {}
    ns.update(context)

    if doc.objtype == 'module':
        scanner = ModuleScanner(app, obj)
        ns['members'] = scanner.scan(imported_members)

        respect_module_all = not app.config.autosummary_ignore_module_all
        imported_members = imported_members or ('__all__' in dir(obj) and respect_module_all)

        ns['functions'], ns['all_functions'] = \
            _get_members(doc, app, obj, {'function'}, imported=imported_members)
        ns['classes'], ns['all_classes'] = \
            _get_members(doc, app, obj, {'class'}, imported=imported_members)
        ns['exceptions'], ns['all_exceptions'] = \
            _get_members(doc, app, obj, {'exception'}, imported=imported_members)
        ns['attributes'], ns['all_attributes'] = \
            _get_module_attrs(name, ns['members'])
        ispackage = hasattr(obj, '__path__')
        if ispackage and recursive:
            # Use members that are not modules as skip list, because it would then mean
            # that module was overwritten in the package namespace
            skip = (
                ns["all_functions"]
                + ns["all_classes"]
                + ns["all_exceptions"]
                + ns["all_attributes"]
            )

            # If respect_module_all and module has a __all__ attribute, first get
            # modules that were explicitly imported. Next, find the rest with the
            # get_modules method, but only put in "public" modules that are in the
            # __all__ list
            #
            # Otherwise, use get_modules method normally
            if respect_module_all and '__all__' in dir(obj):
                imported_modules, all_imported_modules = \
                    _get_members(doc, app, obj, {'module'}, imported=True)
                skip += all_imported_modules
                imported_modules = [name + '.' + modname for modname in imported_modules]
                all_imported_modules = \
                    [name + '.' + modname for modname in all_imported_modules]
                public_members = getall(obj)
            else:
                imported_modules, all_imported_modules = [], []
                public_members = None

            modules, all_modules = _get_modules(obj, skip=skip, name=name,
                                                public_members=public_members)
            ns['modules'] = imported_modules + modules
            ns["all_modules"] = all_imported_modules + all_modules
    elif doc.objtype == 'class':
        ns['members'] = dir(obj)
        ns['inherited_members'] = \
            set(dir(obj)) - set(obj.__dict__.keys())
        ns['methods'], ns['all_methods'] = \
            _get_members(doc, app, obj, {'method'}, include_public={'__init__'})
        ns['attributes'], ns['all_attributes'] = \
            _get_members(doc, app, obj, {'attribute', 'property'})

    if modname is None or qualname is None:
        modname, qualname = _split_full_qualified_name(name)

    if doc.objtype in ('method', 'attribute', 'property'):
        ns['class'] = qualname.rsplit(".", 1)[0]

    if doc.objtype in ('class',):
        shortname = qualname
    else:
        shortname = qualname.rsplit(".", 1)[-1]

    ns['fullname'] = name
    ns['module'] = modname
    ns['objname'] = qualname
    ns['name'] = shortname

    ns['objtype'] = doc.objtype
    ns['underline'] = len(name) * '='

    if template_name:
        return template.render(template_name, ns)
    else:
        return template.render(doc.objtype, ns)


def _skip_member(app: Sphinx, obj: Any, name: str, objtype: str) -> bool:
    try:
        return app.emit_firstresult('autodoc-skip-member', objtype, name,
                                    obj, False, {})
    except Exception as exc:
        logger.warning(__('autosummary: failed to determine %r to be documented, '
                          'the following exception was raised:\n%s'),
                       name, exc, type='autosummary')
        return False


def _get_class_members(obj: Any) -> dict[str, Any]:
    members = sphinx.ext.autodoc.get_class_members(obj, None, safe_getattr)
    return {name: member.object for name, member in members.items()}


def _get_module_members(app: Sphinx, obj: Any) -> dict[str, Any]:
    members = {}
    for name in members_of(obj, app.config):
        try:
            members[name] = safe_getattr(obj, name)
        except AttributeError:
            continue
    return members


def _get_all_members(doc: type[Documenter], app: Sphinx, obj: Any) -> dict[str, Any]:
    if doc.objtype == 'module':
        return _get_module_members(app, obj)
    elif doc.objtype == 'class':
        return _get_class_members(obj)
    return {}


def _get_members(doc: type[Documenter], app: Sphinx, obj: Any, types: set[str], *,
                 include_public: Set[str] = frozenset(),
                 imported: bool = True) -> tuple[list[str], list[str]]:
    items: list[str] = []
    public: list[str] = []

    all_members = _get_all_members(doc, app, obj)
    for name, value in all_members.items():
        documenter = get_documenter(app, value, obj)
        if documenter.objtype in types:
            # skip imported members if expected
            if imported or getattr(value, '__module__', None) == obj.__name__:
                skipped = _skip_member(app, value, name, documenter.objtype)
                if skipped is True:
                    pass
                elif skipped is False:
                    # show the member forcedly
                    items.append(name)
                    public.append(name)
                else:
                    items.append(name)
                    if name in include_public or not name.startswith('_'):
                        # considers member as public
                        public.append(name)
    return public, items


def _get_module_attrs(name: str, members: Any) -> tuple[list[str], list[str]]:
    """Find module attributes with docstrings."""
    attrs, public = [], []
    try:
        analyzer = ModuleAnalyzer.for_module(name)
        attr_docs = analyzer.find_attr_docs()
        for namespace, attr_name in attr_docs:
            if namespace == '' and attr_name in members:
                attrs.append(attr_name)
                if not attr_name.startswith('_'):
                    public.append(attr_name)
    except PycodeError:
        pass    # give up if ModuleAnalyzer fails to parse code
    return public, attrs


def _get_modules(
        obj: Any,
        *,
        skip: Sequence[str],
        name: str,
        public_members: Sequence[str] | None = None) -> tuple[list[str], list[str]]:
    items: list[str] = []
    public: list[str] = []
    for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__):

        if modname in skip:
            # module was overwritten in __init__.py, so not accessible
            continue
        fullname = name + '.' + modname
        try:
            module = import_module(fullname)
            if module and hasattr(module, '__sphinx_mock__'):
                continue
        except ImportError:
            pass

        items.append(fullname)
        if public_members is not None:
            if modname in public_members:
                public.append(fullname)
        else:
            if not modname.startswith('_'):
                public.append(fullname)
    return public, items


def generate_autosummary_docs(sources: list[str],
                              output_dir: str | os.PathLike[str] | None = None,
                              suffix: str = '.rst',
                              base_path: str | os.PathLike[str] | None = None,
                              imported_members: bool = False, app: Any = None,
                              overwrite: bool = True, encoding: str = 'utf-8') -> None:
    showed_sources = sorted(sources)
    if len(showed_sources) > 20:
        showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:]
    logger.info(__('[autosummary] generating autosummary for: %s') %
                ', '.join(showed_sources))

    if output_dir:
        logger.info(__('[autosummary] writing to %s') % output_dir)

    if base_path is not None:
        sources = [os.path.join(base_path, filename) for filename in sources]

    template = AutosummaryRenderer(app)

    # read
    items = find_autosummary_in_files(sources)

    # keep track of new files
    new_files = []

    if app:
        filename_map = app.config.autosummary_filename_map
    else:
        filename_map = {}

    # write
    for entry in sorted(set(items), key=str):
        if entry.path is None:
            # The corresponding autosummary:: directive did not have
            # a :toctree: option
            continue

        path = output_dir or os.path.abspath(entry.path)
        ensuredir(path)

        try:
            name, obj, parent, modname = import_by_name(entry.name)
            qualname = name.replace(modname + ".", "")
        except ImportExceptionGroup as exc:
            try:
                # try to import as an instance attribute
                name, obj, parent, modname = import_ivar_by_name(entry.name)
                qualname = name.replace(modname + ".", "")
            except ImportError as exc2:
                if exc2.__cause__:
                    exceptions: list[BaseException] = exc.exceptions + [exc2.__cause__]
                else:
                    exceptions = exc.exceptions + [exc2]

                errors = list({f"* {type(e).__name__}: {e}" for e in exceptions})
                logger.warning(__('[autosummary] failed to import %s.\nPossible hints:\n%s'),
                               entry.name, '\n'.join(errors))
                continue

        context: dict[str, Any] = {}
        if app:
            context.update(app.config.autosummary_context)

        content = generate_autosummary_content(name, obj, parent, template, entry.template,
                                               imported_members, app, entry.recursive, context,
                                               modname, qualname)

        filename = os.path.join(path, filename_map.get(name, name) + suffix)
        if os.path.isfile(filename):
            with open(filename, encoding=encoding) as f:
                old_content = f.read()

            if content == old_content:
                continue
            if overwrite:  # content has changed
                with open(filename, 'w', encoding=encoding) as f:
                    f.write(content)
                new_files.append(filename)
        else:
            with open(filename, 'w', encoding=encoding) as f:
                f.write(content)
            new_files.append(filename)

    # descend recursively to new files
    if new_files:
        generate_autosummary_docs(new_files, output_dir=output_dir,
                                  suffix=suffix, base_path=base_path,
                                  imported_members=imported_members, app=app,
                                  overwrite=overwrite)


# -- Finding documented entries in files ---------------------------------------

def find_autosummary_in_files(filenames: list[str]) -> list[AutosummaryEntry]:
    """Find out what items are documented in source/*.rst.

    See `find_autosummary_in_lines`.
    """
    documented: list[AutosummaryEntry] = []
    for filename in filenames:
        with open(filename, encoding='utf-8', errors='ignore') as f:
            lines = f.read().splitlines()
            documented.extend(find_autosummary_in_lines(lines, filename=filename))
    return documented


def find_autosummary_in_docstring(
    name: str, filename: str | None = None,
) -> list[AutosummaryEntry]:
    """Find out what items are documented in the given object's docstring.

    See `find_autosummary_in_lines`.
    """
    try:
        real_name, obj, parent, modname = import_by_name(name)
        lines = pydoc.getdoc(obj).splitlines()
        return find_autosummary_in_lines(lines, module=name, filename=filename)
    except AttributeError:
        pass
    except ImportExceptionGroup as exc:
        errors = '\n'.join({f"* {type(e).__name__}: {e}" for e in exc.exceptions})
        logger.warning(f'Failed to import {name}.\nPossible hints:\n{errors}')  # NoQA: G004
    except SystemExit:
        logger.warning("Failed to import '%s'; the module executes module level "
                       'statement and it might call sys.exit().', name)
    return []


def find_autosummary_in_lines(
    lines: list[str], module: str | None = None, filename: str | None = None,
) -> list[AutosummaryEntry]:
    """Find out what items appear in autosummary:: directives in the
    given lines.

    Returns a list of (name, toctree, template) where *name* is a name
    of an object and *toctree* the :toctree: path of the corresponding
    autosummary directive (relative to the root of the file name), and
    *template* the value of the :template: option. *toctree* and
    *template* ``None`` if the directive does not have the
    corresponding options set.
    """
    autosummary_re = re.compile(r'^(\s*)\.\.\s+autosummary::\s*')
    automodule_re = re.compile(
        r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$')
    module_re = re.compile(
        r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$')
    autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?')
    recursive_arg_re = re.compile(r'^\s+:recursive:\s*$')
    toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$')
    template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$')

    documented: list[AutosummaryEntry] = []

    recursive = False
    toctree: str | None = None
    template = ''
    current_module = module
    in_autosummary = False
    base_indent = ""

    for line in lines:
        if in_autosummary:
            m = recursive_arg_re.match(line)
            if m:
                recursive = True
                continue

            m = toctree_arg_re.match(line)
            if m:
                toctree = m.group(1)
                if filename:
                    toctree = os.path.join(os.path.dirname(filename),
                                           toctree)
                continue

            m = template_arg_re.match(line)
            if m:
                template = m.group(1).strip()
                continue

            if line.strip().startswith(':'):
                continue  # skip options

            m = autosummary_item_re.match(line)
            if m:
                name = m.group(1).strip()
                if name.startswith('~'):
                    name = name[1:]
                if current_module and \
                   not name.startswith(current_module + '.'):
                    name = f"{current_module}.{name}"
                documented.append(AutosummaryEntry(name, toctree, template, recursive))
                continue

            if not line.strip() or line.startswith(base_indent + " "):
                continue

            in_autosummary = False

        m = autosummary_re.match(line)
        if m:
            in_autosummary = True
            base_indent = m.group(1)
            recursive = False
            toctree = None
            template = ''
            continue

        m = automodule_re.search(line)
        if m:
            current_module = m.group(1).strip()
            # recurse into the automodule docstring
            documented.extend(find_autosummary_in_docstring(
                current_module, filename=filename))
            continue

        m = module_re.match(line)
        if m:
            current_module = m.group(2)
            continue

    return documented


def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        usage='%(prog)s [OPTIONS] <SOURCE_FILE>...',
        epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'),
        description=__("""
Generate ReStructuredText using autosummary directives.

sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
the reStructuredText files from the autosummary directives contained in the
given input files.

The format of the autosummary directive is documented in the
``sphinx.ext.autosummary`` Python module and can be read using::

  pydoc sphinx.ext.autosummary
"""))

    parser.add_argument('--version', action='version', dest='show_version',
                        version='%%(prog)s %s' % __display_version__)

    parser.add_argument('source_file', nargs='+',
                        help=__('source files to generate rST files for'))

    parser.add_argument('-o', '--output-dir', action='store',
                        dest='output_dir',
                        help=__('directory to place all output in'))
    parser.add_argument('-s', '--suffix', action='store', dest='suffix',
                        default='rst',
                        help=__('default suffix for files (default: '
                                '%(default)s)'))
    parser.add_argument('-t', '--templates', action='store', dest='templates',
                        default=None,
                        help=__('custom template directory (default: '
                                '%(default)s)'))
    parser.add_argument('-i', '--imported-members', action='store_true',
                        dest='imported_members', default=False,
                        help=__('document imported members (default: '
                                '%(default)s)'))
    parser.add_argument('-a', '--respect-module-all', action='store_true',
                        dest='respect_module_all', default=False,
                        help=__('document exactly the members in module __all__ attribute. '
                                '(default: %(default)s)'))

    return parser


def main(argv: Sequence[str] = (), /) -> None:
    locale.setlocale(locale.LC_ALL, '')
    sphinx.locale.init_console()

    app = DummyApplication(sphinx.locale.get_translator())
    logging.setup(app, sys.stdout, sys.stderr)  # type: ignore[arg-type]
    setup_documenters(app)
    args = get_parser().parse_args(argv or sys.argv[1:])

    if args.templates:
        app.config.templates_path.append(path.abspath(args.templates))
    app.config.autosummary_ignore_module_all = (  # type: ignore[attr-defined]
        not args.respect_module_all
    )

    generate_autosummary_docs(args.source_file, args.output_dir,
                              '.' + args.suffix,
                              imported_members=args.imported_members,
                              app=app)


if __name__ == '__main__':
    main(sys.argv[1:])

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/autosummary/generate.py
NikolaBlagojevic/pyrecodes
"""Allow graphviz-formatted graphs to be included inline in generated documents.
"""

from __future__ import annotations

import posixpath
import re
import subprocess
import xml.etree.ElementTree as ET
from hashlib import sha1
from itertools import chain
from os import path
from subprocess import CalledProcessError
from typing import TYPE_CHECKING, Any
from urllib.parse import urlsplit, urlunsplit

from docutils import nodes
from docutils.parsers.rst import Directive, directives

import sphinx
from sphinx.errors import SphinxError
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.docutils import SphinxDirective
from sphinx.util.i18n import search_image_for_language
from sphinx.util.nodes import set_source_info
from sphinx.util.osutil import ensuredir

if TYPE_CHECKING:
    from docutils.nodes import Node

    from sphinx.application import Sphinx
    from sphinx.config import Config
    from sphinx.util.typing import OptionSpec
    from sphinx.writers.html import HTML5Translator
    from sphinx.writers.latex import LaTeXTranslator
    from sphinx.writers.manpage import ManualPageTranslator
    from sphinx.writers.texinfo import TexinfoTranslator
    from sphinx.writers.text import TextTranslator

logger = logging.getLogger(__name__)


class GraphvizError(SphinxError):
    category = 'Graphviz error'


class ClickableMapDefinition:
    """A manipulator for clickable map file of graphviz."""
    maptag_re = re.compile('<map id="(.*?)"')
    href_re = re.compile('href=".*?"')

    def __init__(self, filename: str, content: str, dot: str = '') -> None:
        self.id: str | None = None
        self.filename = filename
        self.content = content.splitlines()
        self.clickable: list[str] = []

        self.parse(dot=dot)

    def parse(self, dot: str) -> None:
        matched = self.maptag_re.match(self.content[0])
        if not matched:
            raise GraphvizError('Invalid clickable map file found: %s' % self.filename)

        self.id = matched.group(1)
        if self.id == '%3':
            # graphviz generates wrong ID if graph name not specified
            # https://gitlab.com/graphviz/graphviz/issues/1327
            hashed = sha1(dot.encode(), usedforsecurity=False).hexdigest()
            self.id = 'grapviz%s' % hashed[-10:]
            self.content[0] = self.content[0].replace('%3', self.id)

        for line in self.content:
            if self.href_re.search(line):
                self.clickable.append(line)

    def generate_clickable_map(self) -> str:
        """Generate clickable map tags if clickable item exists.

        If not exists, this only returns empty string.
        """
        if self.clickable:
            return '\n'.join([self.content[0]] + self.clickable + [self.content[-1]])
        else:
            return ''


class graphviz(nodes.General, nodes.Inline, nodes.Element):
    pass


def figure_wrapper(directive: Directive, node: graphviz, caption: str) -> nodes.figure:
    figure_node = nodes.figure('', node)
    if 'align' in node:
        figure_node['align'] = node.attributes.pop('align')

    inodes, messages = directive.state.inline_text(caption, directive.lineno)
    caption_node = nodes.caption(caption, '', *inodes)
    caption_node.extend(messages)
    set_source_info(directive, caption_node)
    figure_node += caption_node
    return figure_node


def align_spec(argument: Any) -> str:
    return directives.choice(argument, ('left', 'center', 'right'))


class Graphviz(SphinxDirective):
    """
    Directive to insert arbitrary dot markup.
    """
    has_content = True
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = False
    option_spec: OptionSpec = {
        'alt': directives.unchanged,
        'align': align_spec,
        'caption': directives.unchanged,
        'layout': directives.unchanged,
        'graphviz_dot': directives.unchanged,  # an old alias of `layout` option
        'name': directives.unchanged,
        'class': directives.class_option,
    }

    def run(self) -> list[Node]:
        if self.arguments:
            document = self.state.document
            if self.content:
                return [document.reporter.warning(
                    __('Graphviz directive cannot have both content and '
                       'a filename argument'), line=self.lineno)]
            argument = search_image_for_language(self.arguments[0], self.env)
            rel_filename, filename = self.env.relfn2path(argument)
            self.env.note_dependency(rel_filename)
            try:
                with open(filename, encoding='utf-8') as fp:
                    dotcode = fp.read()
            except OSError:
                return [document.reporter.warning(
                    __('External Graphviz file %r not found or reading '
                       'it failed') % filename, line=self.lineno)]
        else:
            dotcode = '\n'.join(self.content)
            rel_filename = None
            if not dotcode.strip():
                return [self.state_machine.reporter.warning(
                    __('Ignoring "graphviz" directive without content.'),
                    line=self.lineno)]
        node = graphviz()
        node['code'] = dotcode
        node['options'] = {'docname': self.env.docname}

        if 'graphviz_dot' in self.options:
            node['options']['graphviz_dot'] = self.options['graphviz_dot']
        if 'layout' in self.options:
            node['options']['graphviz_dot'] = self.options['layout']
        if 'alt' in self.options:
            node['alt'] = self.options['alt']
        if 'align' in self.options:
            node['align'] = self.options['align']
        if 'class' in self.options:
            node['classes'] = self.options['class']
        if rel_filename:
            node['filename'] = rel_filename

        if 'caption' not in self.options:
            self.add_name(node)
            return [node]
        else:
            figure = figure_wrapper(self, node, self.options['caption'])
            self.add_name(figure)
            return [figure]


class GraphvizSimple(SphinxDirective):
    """
    Directive to insert arbitrary dot markup.
    """
    has_content = True
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = False
    option_spec: OptionSpec = {
        'alt': directives.unchanged,
        'align': align_spec,
        'caption': directives.unchanged,
        'layout': directives.unchanged,
        'graphviz_dot': directives.unchanged,  # an old alias of `layout` option
        'name': directives.unchanged,
        'class': directives.class_option,
    }

    def run(self) -> list[Node]:
        node = graphviz()
        node['code'] = '%s %s {\n%s\n}\n' % \
                       (self.name, self.arguments[0], '\n'.join(self.content))
        node['options'] = {'docname': self.env.docname}
        if 'graphviz_dot' in self.options:
            node['options']['graphviz_dot'] = self.options['graphviz_dot']
        if 'layout' in self.options:
            node['options']['graphviz_dot'] = self.options['layout']
        if 'alt' in self.options:
            node['alt'] = self.options['alt']
        if 'align' in self.options:
            node['align'] = self.options['align']
        if 'class' in self.options:
            node['classes'] = self.options['class']

        if 'caption' not in self.options:
            self.add_name(node)
            return [node]
        else:
            figure = figure_wrapper(self, node, self.options['caption'])
            self.add_name(figure)
            return [figure]


def fix_svg_relative_paths(self: HTML5Translator | LaTeXTranslator | TexinfoTranslator,
                           filepath: str) -> None:
    """Change relative links in generated svg files to be relative to imgpath."""
    tree = ET.parse(filepath)  # NoQA: S314
    root = tree.getroot()
    ns = {'svg': 'http://www.w3.org/2000/svg', 'xlink': 'http://www.w3.org/1999/xlink'}
    href_name = '{http://www.w3.org/1999/xlink}href'
    modified = False

    for element in chain(
        root.findall('.//svg:image[@xlink:href]', ns),
        root.findall('.//svg:a[@xlink:href]', ns),
    ):
        scheme, hostname, rel_uri, query, fragment = urlsplit(element.attrib[href_name])
        if hostname:
            # not a relative link
            continue

        docname = self.builder.env.path2doc(self.document["source"])
        if docname is None:
            # This shouldn't happen!
            continue
        doc_dir = self.builder.app.outdir.joinpath(docname).resolve().parent

        old_path = doc_dir / rel_uri
        img_path = doc_dir / self.builder.imgpath
        new_path = path.relpath(old_path, start=img_path)
        modified_url = urlunsplit((scheme, hostname, new_path, query, fragment))

        element.set(href_name, modified_url)
        modified = True

    if modified:
        tree.write(filepath)


def render_dot(self: HTML5Translator | LaTeXTranslator | TexinfoTranslator,
               code: str, options: dict, format: str,
               prefix: str = 'graphviz', filename: str | None = None,
               ) -> tuple[str | None, str | None]:
    """Render graphviz code into a PNG or PDF output file."""
    graphviz_dot = options.get('graphviz_dot', self.builder.config.graphviz_dot)
    if not graphviz_dot:
        raise GraphvizError(
            __('graphviz_dot executable path must be set! %r') % graphviz_dot,
        )
    hashkey = (code + str(options) + str(graphviz_dot) +
               str(self.builder.config.graphviz_dot_args)).encode()

    fname = f'{prefix}-{sha1(hashkey, usedforsecurity=False).hexdigest()}.{format}'
    relfn = posixpath.join(self.builder.imgpath, fname)
    outfn = path.join(self.builder.outdir, self.builder.imagedir, fname)

    if path.isfile(outfn):
        return relfn, outfn

    if (hasattr(self.builder, '_graphviz_warned_dot') and
       self.builder._graphviz_warned_dot.get(graphviz_dot)):
        return None, None

    ensuredir(path.dirname(outfn))

    dot_args = [graphviz_dot]
    dot_args.extend(self.builder.config.graphviz_dot_args)
    dot_args.extend(['-T' + format, '-o' + outfn])

    docname = options.get('docname', 'index')
    if filename:
        cwd = path.dirname(path.join(self.builder.srcdir, filename))
    else:
        cwd = path.dirname(path.join(self.builder.srcdir, docname))

    if format == 'png':
        dot_args.extend(['-Tcmapx', '-o%s.map' % outfn])

    try:
        ret = subprocess.run(dot_args, input=code.encode(), capture_output=True,
                             cwd=cwd, check=True)
    except OSError:
        logger.warning(__('dot command %r cannot be run (needed for graphviz '
                          'output), check the graphviz_dot setting'), graphviz_dot)
        if not hasattr(self.builder, '_graphviz_warned_dot'):
            self.builder._graphviz_warned_dot = {}  # type: ignore[union-attr]
        self.builder._graphviz_warned_dot[graphviz_dot] = True
        return None, None
    except CalledProcessError as exc:
        raise GraphvizError(__('dot exited with error:\n[stderr]\n%r\n'
                               '[stdout]\n%r') % (exc.stderr, exc.stdout)) from exc
    if not path.isfile(outfn):
        raise GraphvizError(__('dot did not produce an output file:\n[stderr]\n%r\n'
                               '[stdout]\n%r') % (ret.stderr, ret.stdout))

    if format == 'svg':
        fix_svg_relative_paths(self, outfn)

    return relfn, outfn


def render_dot_html(self: HTML5Translator, node: graphviz, code: str, options: dict,
                    prefix: str = 'graphviz', imgcls: str | None = None,
                    alt: str | None = None, filename: str | None = None,
                    ) -> tuple[str, str]:
    format = self.builder.config.graphviz_output_format
    try:
        if format not in ('png', 'svg'):
            raise GraphvizError(__("graphviz_output_format must be one of 'png', "
                                   "'svg', but is %r") % format)
        fname, outfn = render_dot(self, code, options, format, prefix, filename)
    except GraphvizError as exc:
        logger.warning(__('dot code %r: %s'), code, exc)
        raise nodes.SkipNode from exc

    classes = [imgcls, 'graphviz'] + node.get('classes', [])
    imgcls = ' '.join(filter(None, classes))

    if fname is None:
        self.body.append(self.encode(code))
    else:
        if alt is None:
            alt = node.get('alt', self.encode(code).strip())
        if 'align' in node:
            self.body.append('<div align="%s" class="align-%s">' %
                             (node['align'], node['align']))
        if format == 'svg':
            self.body.append('<div class="graphviz">')
            self.body.append('<object data="%s" type="image/svg+xml" class="%s">\n' %
                             (fname, imgcls))
            self.body.append('<p class="warning">%s</p>' % alt)
            self.body.append('</object></div>\n')
        else:
            assert outfn is not None
            with open(outfn + '.map', encoding='utf-8') as mapfile:
                imgmap = ClickableMapDefinition(outfn + '.map', mapfile.read(), dot=code)
                if imgmap.clickable:
                    # has a map
                    self.body.append('<div class="graphviz">')
                    self.body.append('<img src="%s" alt="%s" usemap="#%s" class="%s" />' %
                                     (fname, alt, imgmap.id, imgcls))
                    self.body.append('</div>\n')
                    self.body.append(imgmap.generate_clickable_map())
                else:
                    # nothing in image map
                    self.body.append('<div class="graphviz">')
                    self.body.append('<img src="%s" alt="%s" class="%s" />' %
                                     (fname, alt, imgcls))
                    self.body.append('</div>\n')
        if 'align' in node:
            self.body.append('</div>\n')

    raise nodes.SkipNode


def html_visit_graphviz(self: HTML5Translator, node: graphviz) -> None:
    render_dot_html(self, node, node['code'], node['options'], filename=node.get('filename'))


def render_dot_latex(self: LaTeXTranslator, node: graphviz, code: str,
                     options: dict, prefix: str = 'graphviz', filename: str | None = None,
                     ) -> None:
    try:
        fname, outfn = render_dot(self, code, options, 'pdf', prefix, filename)
    except GraphvizError as exc:
        logger.warning(__('dot code %r: %s'), code, exc)
        raise nodes.SkipNode from exc

    is_inline = self.is_inline(node)

    if not is_inline:
        pre = ''
        post = ''
        if 'align' in node:
            if node['align'] == 'left':
                pre = '{'
                post = r'\hspace*{\fill}}'
            elif node['align'] == 'right':
                pre = r'{\hspace*{\fill}'
                post = '}'
            elif node['align'] == 'center':
                pre = r'{\hfill'
                post = r'\hspace*{\fill}}'
        self.body.append('\n%s' % pre)

    self.body.append(r'\sphinxincludegraphics[]{%s}' % fname)

    if not is_inline:
        self.body.append('%s\n' % post)

    raise nodes.SkipNode


def latex_visit_graphviz(self: LaTeXTranslator, node: graphviz) -> None:
    render_dot_latex(self, node, node['code'], node['options'], filename=node.get('filename'))


def render_dot_texinfo(self: TexinfoTranslator, node: graphviz, code: str,
                       options: dict, prefix: str = 'graphviz') -> None:
    try:
        fname, outfn = render_dot(self, code, options, 'png', prefix)
    except GraphvizError as exc:
        logger.warning(__('dot code %r: %s'), code, exc)
        raise nodes.SkipNode from exc
    if fname is not None:
        self.body.append('@image{%s,,,[graphviz],png}\n' % fname[:-4])
    raise nodes.SkipNode


def texinfo_visit_graphviz(self: TexinfoTranslator, node: graphviz) -> None:
    render_dot_texinfo(self, node, node['code'], node['options'])


def text_visit_graphviz(self: TextTranslator, node: graphviz) -> None:
    if 'alt' in node.attributes:
        self.add_text(_('[graph: %s]') % node['alt'])
    else:
        self.add_text(_('[graph]'))
    raise nodes.SkipNode


def man_visit_graphviz(self: ManualPageTranslator, node: graphviz) -> None:
    if 'alt' in node.attributes:
        self.body.append(_('[graph: %s]') % node['alt'])
    else:
        self.body.append(_('[graph]'))
    raise nodes.SkipNode


def on_config_inited(_app: Sphinx, config: Config) -> None:
    css_path = path.join(sphinx.package_dir, 'templates', 'graphviz', 'graphviz.css')
    config.html_static_path.append(css_path)


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_node(graphviz,
                 html=(html_visit_graphviz, None),
                 latex=(latex_visit_graphviz, None),
                 texinfo=(texinfo_visit_graphviz, None),
                 text=(text_visit_graphviz, None),
                 man=(man_visit_graphviz, None))
    app.add_directive('graphviz', Graphviz)
    app.add_directive('graph', GraphvizSimple)
    app.add_directive('digraph', GraphvizSimple)
    app.add_config_value('graphviz_dot', 'dot', 'html')
    app.add_config_value('graphviz_dot_args', [], 'html')
    app.add_config_value('graphviz_output_format', 'png', 'html')
    app.add_css_file('graphviz.css')
    app.connect('config-inited', on_config_inited)
    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/graphviz.py
NikolaBlagojevic/pyrecodes
r"""Defines a docutils directive for inserting inheritance diagrams.

Provide the directive with one or more classes or modules (separated
by whitespace).  For modules, all of the classes in that module will
be used.

Example::

   Given the following classes:

   class A: pass
   class B(A): pass
   class C(A): pass
   class D(B, C): pass
   class E(B): pass

   .. inheritance-diagram: D E

   Produces a graph like the following:

               A
              / \
             B   C
            / \ /
           E   D

The graph is inserted as a PNG+image map into HTML and a PDF in
LaTeX.
"""

from __future__ import annotations

import builtins
import hashlib
import inspect
import re
from collections.abc import Iterable, Sequence
from importlib import import_module
from os import path
from typing import TYPE_CHECKING, Any, cast

from docutils import nodes
from docutils.parsers.rst import directives

import sphinx
from sphinx import addnodes
from sphinx.ext.graphviz import (
    figure_wrapper,
    graphviz,
    render_dot_html,
    render_dot_latex,
    render_dot_texinfo,
)
from sphinx.util.docutils import SphinxDirective

if TYPE_CHECKING:
    from docutils.nodes import Node

    from sphinx.application import Sphinx
    from sphinx.environment import BuildEnvironment
    from sphinx.util.typing import OptionSpec
    from sphinx.writers.html import HTML5Translator
    from sphinx.writers.latex import LaTeXTranslator
    from sphinx.writers.texinfo import TexinfoTranslator

module_sig_re = re.compile(r'''^(?:([\w.]*)\.)?  # module names
                           (\w+)  \s* $          # class/final module name
                           ''', re.VERBOSE)


py_builtins = [obj for obj in vars(builtins).values()
               if inspect.isclass(obj)]


def try_import(objname: str) -> Any:
    """Import a object or module using *name* and *currentmodule*.
    *name* should be a relative name from *currentmodule* or
    a fully-qualified name.

    Returns imported object or module.  If failed, returns None value.
    """
    try:
        return import_module(objname)
    except TypeError:
        # Relative import
        return None
    except ImportError:
        matched = module_sig_re.match(objname)

        if not matched:
            return None

        modname, attrname = matched.groups()

        if modname is None:
            return None
        try:
            module = import_module(modname)
            return getattr(module, attrname, None)
        except ImportError:
            return None


def import_classes(name: str, currmodule: str) -> Any:
    """Import a class using its fully-qualified *name*."""
    target = None

    # import class or module using currmodule
    if currmodule:
        target = try_import(currmodule + '.' + name)

    # import class or module without currmodule
    if target is None:
        target = try_import(name)

    if target is None:
        raise InheritanceException(
            'Could not import class or module %r specified for '
            'inheritance diagram' % name)

    if inspect.isclass(target):
        # If imported object is a class, just return it
        return [target]
    elif inspect.ismodule(target):
        # If imported object is a module, return classes defined on it
        classes = []
        for cls in target.__dict__.values():
            if inspect.isclass(cls) and cls.__module__ == target.__name__:
                classes.append(cls)
        return classes
    raise InheritanceException('%r specified for inheritance diagram is '
                               'not a class or module' % name)


class InheritanceException(Exception):
    pass


class InheritanceGraph:
    """
    Given a list of classes, determines the set of classes that they inherit
    from all the way to the root "object", and then is able to generate a
    graphviz dot graph from them.
    """
    def __init__(self, class_names: list[str], currmodule: str, show_builtins: bool = False,
                 private_bases: bool = False, parts: int = 0,
                 aliases: dict[str, str] | None = None, top_classes: Sequence[Any] = (),
                 ) -> None:
        """*class_names* is a list of child classes to show bases from.

        If *show_builtins* is True, then Python builtins will be shown
        in the graph.
        """
        self.class_names = class_names
        classes = self._import_classes(class_names, currmodule)
        self.class_info = self._class_info(classes, show_builtins,
                                           private_bases, parts, aliases, top_classes)
        if not self.class_info:
            msg = 'No classes found for inheritance diagram'
            raise InheritanceException(msg)

    def _import_classes(self, class_names: list[str], currmodule: str) -> list[Any]:
        """Import a list of classes."""
        classes: list[Any] = []
        for name in class_names:
            classes.extend(import_classes(name, currmodule))
        return classes

    def _class_info(self, classes: list[Any], show_builtins: bool, private_bases: bool,
                    parts: int, aliases: dict[str, str] | None, top_classes: Sequence[Any],
                    ) -> list[tuple[str, str, list[str], str]]:
        """Return name and bases for all classes that are ancestors of
        *classes*.

        *parts* gives the number of dotted name parts to include in the
        displayed node names, from right to left. If given as a negative, the
        number of parts to drop from the left. A value of 0 displays the full
        dotted name. E.g. ``sphinx.ext.inheritance_diagram.InheritanceGraph``
        with ``parts=2`` or ``parts=-2`` gets displayed as
        ``inheritance_diagram.InheritanceGraph``, and as
        ``ext.inheritance_diagram.InheritanceGraph`` with ``parts=3`` or
        ``parts=-1``.

        *top_classes* gives the name(s) of the top most ancestor class to
        traverse to. Multiple names can be specified separated by comma.
        """
        all_classes = {}

        def recurse(cls: Any) -> None:
            if not show_builtins and cls in py_builtins:
                return
            if not private_bases and cls.__name__.startswith('_'):
                return

            nodename = self.class_name(cls, parts, aliases)
            fullname = self.class_name(cls, 0, aliases)

            # Use first line of docstring as tooltip, if available
            tooltip = None
            try:
                if cls.__doc__:
                    doc = cls.__doc__.strip().split("\n")[0]
                    if doc:
                        tooltip = '"%s"' % doc.replace('"', '\\"')
            except Exception:  # might raise AttributeError for strange classes
                pass

            baselist: list[str] = []
            all_classes[cls] = (nodename, fullname, baselist, tooltip)

            if fullname in top_classes:
                return

            for base in cls.__bases__:
                if not show_builtins and base in py_builtins:
                    continue
                if not private_bases and base.__name__.startswith('_'):
                    continue
                baselist.append(self.class_name(base, parts, aliases))
                if base not in all_classes:
                    recurse(base)

        for cls in classes:
            recurse(cls)

        return list(all_classes.values())  # type: ignore[arg-type]

    def class_name(
        self, cls: Any, parts: int = 0, aliases: dict[str, str] | None = None,
    ) -> str:
        """Given a class object, return a fully-qualified name.

        This works for things I've tested in matplotlib so far, but may not be
        completely general.
        """
        module = cls.__module__
        if module in ('__builtin__', 'builtins'):
            fullname = cls.__name__
        else:
            fullname = f'{module}.{cls.__qualname__}'
        if parts == 0:
            result = fullname
        else:
            name_parts = fullname.split('.')
            result = '.'.join(name_parts[-parts:])
        if aliases is not None and result in aliases:
            return aliases[result]
        return result

    def get_all_class_names(self) -> list[str]:
        """Get all of the class names involved in the graph."""
        return [fullname for (_, fullname, _, _) in self.class_info]

    # These are the default attrs for graphviz
    default_graph_attrs = {
        'rankdir': 'LR',
        'size': '"8.0, 12.0"',
        'bgcolor': 'transparent',
    }
    default_node_attrs = {
        'shape': 'box',
        'fontsize': 10,
        'height': 0.25,
        'fontname': '"Vera Sans, DejaVu Sans, Liberation Sans, '
                    'Arial, Helvetica, sans"',
        'style': '"setlinewidth(0.5),filled"',
        'fillcolor': 'white',
    }
    default_edge_attrs = {
        'arrowsize': 0.5,
        'style': '"setlinewidth(0.5)"',
    }

    def _format_node_attrs(self, attrs: dict[str, Any]) -> str:
        return ','.join(['%s=%s' % x for x in sorted(attrs.items())])

    def _format_graph_attrs(self, attrs: dict[str, Any]) -> str:
        return ''.join(['%s=%s;\n' % x for x in sorted(attrs.items())])

    def generate_dot(self, name: str, urls: dict[str, str] | None = None,
                     env: BuildEnvironment | None = None,
                     graph_attrs: dict | None = None,
                     node_attrs: dict | None = None,
                     edge_attrs: dict | None = None,
                     ) -> str:
        """Generate a graphviz dot graph from the classes that were passed in
        to __init__.

        *name* is the name of the graph.

        *urls* is a dictionary mapping class names to HTTP URLs.

        *graph_attrs*, *node_attrs*, *edge_attrs* are dictionaries containing
        key/value pairs to pass on as graphviz properties.
        """
        if urls is None:
            urls = {}
        g_attrs = self.default_graph_attrs.copy()
        n_attrs = self.default_node_attrs.copy()
        e_attrs = self.default_edge_attrs.copy()
        if graph_attrs is not None:
            g_attrs.update(graph_attrs)
        if node_attrs is not None:
            n_attrs.update(node_attrs)
        if edge_attrs is not None:
            e_attrs.update(edge_attrs)
        if env:
            g_attrs.update(env.config.inheritance_graph_attrs)
            n_attrs.update(env.config.inheritance_node_attrs)
            e_attrs.update(env.config.inheritance_edge_attrs)

        res: list[str] = []
        res.append('digraph %s {\n' % name)
        res.append(self._format_graph_attrs(g_attrs))

        for name, fullname, bases, tooltip in sorted(self.class_info):
            # Write the node
            this_node_attrs = n_attrs.copy()
            if fullname in urls:
                this_node_attrs['URL'] = '"%s"' % urls[fullname]
                this_node_attrs['target'] = '"_top"'
            if tooltip:
                this_node_attrs['tooltip'] = tooltip
            res.append('  "%s" [%s];\n' %
                       (name, self._format_node_attrs(this_node_attrs)))

            # Write the edges
            for base_name in bases:
                res.append('  "%s" -> "%s" [%s];\n' %
                           (base_name, name,
                            self._format_node_attrs(e_attrs)))
        res.append('}\n')
        return ''.join(res)


class inheritance_diagram(graphviz):
    """
    A docutils node to use as a placeholder for the inheritance diagram.
    """
    pass


class InheritanceDiagram(SphinxDirective):
    """
    Run when the inheritance_diagram directive is first encountered.
    """
    has_content = False
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    option_spec: OptionSpec = {
        'parts': int,
        'private-bases': directives.flag,
        'caption': directives.unchanged,
        'top-classes': directives.unchanged_required,
    }

    def run(self) -> list[Node]:
        node = inheritance_diagram()
        node.document = self.state.document
        class_names = self.arguments[0].split()
        class_role = self.env.get_domain('py').role('class')
        # Store the original content for use as a hash
        node['parts'] = self.options.get('parts', 0)
        node['content'] = ', '.join(class_names)
        node['top-classes'] = []
        for cls in self.options.get('top-classes', '').split(','):
            cls = cls.strip()
            if cls:
                node['top-classes'].append(cls)

        # Create a graph starting with the list of classes
        try:
            graph = InheritanceGraph(
                class_names, self.env.ref_context.get('py:module'),  # type: ignore[arg-type]
                parts=node['parts'],
                private_bases='private-bases' in self.options,
                aliases=self.config.inheritance_alias,
                top_classes=node['top-classes'])
        except InheritanceException as err:
            return [node.document.reporter.warning(err, line=self.lineno)]

        # Create xref nodes for each target of the graph's image map and
        # add them to the doc tree so that Sphinx can resolve the
        # references to real URLs later.  These nodes will eventually be
        # removed from the doctree after we're done with them.
        for name in graph.get_all_class_names():
            refnodes, x = class_role(  # type: ignore[call-arg,misc]
                'class', ':class:`%s`' % name, name, 0, self.state)  # type: ignore[arg-type]
            node.extend(refnodes)
        # Store the graph object so we can use it to generate the
        # dot file later
        node['graph'] = graph

        if 'caption' not in self.options:
            self.add_name(node)
            return [node]
        else:
            figure = figure_wrapper(self, node, self.options['caption'])
            self.add_name(figure)
            return [figure]


def get_graph_hash(node: inheritance_diagram) -> str:
    encoded = (node['content'] + str(node['parts'])).encode()
    return hashlib.md5(encoded, usedforsecurity=False).hexdigest()[-10:]


def html_visit_inheritance_diagram(self: HTML5Translator, node: inheritance_diagram) -> None:
    """
    Output the graph for HTML.  This will insert a PNG with clickable
    image map.
    """
    graph = node['graph']

    graph_hash = get_graph_hash(node)
    name = 'inheritance%s' % graph_hash

    # Create a mapping from fully-qualified class names to URLs.
    graphviz_output_format = self.builder.env.config.graphviz_output_format.upper()
    current_filename = path.basename(self.builder.current_docname + self.builder.out_suffix)
    urls = {}
    pending_xrefs = cast(Iterable[addnodes.pending_xref], node)
    for child in pending_xrefs:
        if child.get('refuri') is not None:
            # Construct the name from the URI if the reference is external via intersphinx
            if not child.get('internal', True):
                refname = child['refuri'].rsplit('#', 1)[-1]
            else:
                refname = child['reftitle']

            urls[refname] = child.get('refuri')
        elif child.get('refid') is not None:
            if graphviz_output_format == 'SVG':
                urls[child['reftitle']] = current_filename + '#' + child.get('refid')
            else:
                urls[child['reftitle']] = '#' + child.get('refid')

    dotcode = graph.generate_dot(name, urls, env=self.builder.env)
    render_dot_html(self, node, dotcode, {}, 'inheritance', 'inheritance',
                    alt='Inheritance diagram of ' + node['content'])
    raise nodes.SkipNode


def latex_visit_inheritance_diagram(self: LaTeXTranslator, node: inheritance_diagram) -> None:
    """
    Output the graph for LaTeX.  This will insert a PDF.
    """
    graph = node['graph']

    graph_hash = get_graph_hash(node)
    name = 'inheritance%s' % graph_hash

    dotcode = graph.generate_dot(name, env=self.builder.env,
                                 graph_attrs={'size': '"6.0,6.0"'})
    render_dot_latex(self, node, dotcode, {}, 'inheritance')
    raise nodes.SkipNode


def texinfo_visit_inheritance_diagram(self: TexinfoTranslator, node: inheritance_diagram,
                                      ) -> None:
    """
    Output the graph for Texinfo.  This will insert a PNG.
    """
    graph = node['graph']

    graph_hash = get_graph_hash(node)
    name = 'inheritance%s' % graph_hash

    dotcode = graph.generate_dot(name, env=self.builder.env,
                                 graph_attrs={'size': '"6.0,6.0"'})
    render_dot_texinfo(self, node, dotcode, {}, 'inheritance')
    raise nodes.SkipNode


def skip(self: nodes.NodeVisitor, node: inheritance_diagram) -> None:
    raise nodes.SkipNode


def setup(app: Sphinx) -> dict[str, Any]:
    app.setup_extension('sphinx.ext.graphviz')
    app.add_node(
        inheritance_diagram,
        latex=(latex_visit_inheritance_diagram, None),
        html=(html_visit_inheritance_diagram, None),
        text=(skip, None),
        man=(skip, None),
        texinfo=(texinfo_visit_inheritance_diagram, None))
    app.add_directive('inheritance-diagram', InheritanceDiagram)
    app.add_config_value('inheritance_graph_attrs', {}, False)
    app.add_config_value('inheritance_node_attrs', {}, False)
    app.add_config_value('inheritance_edge_attrs', {}, False)
    app.add_config_value('inheritance_alias', {}, False)
    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/inheritance_diagram.py
NikolaBlagojevic/pyrecodes
"""Insert links to objects documented in remote Sphinx documentation.

This works as follows:

* Each Sphinx HTML build creates a file named "objects.inv" that contains a
  mapping from object names to URIs relative to the HTML set's root.

* Projects using the Intersphinx extension can specify links to such mapping
  files in the `intersphinx_mapping` config value.  The mapping will then be
  used to resolve otherwise missing references to objects into links to the
  other documentation.

* By default, the mapping file is assumed to be at the same location as the
  rest of the documentation; however, the location of the mapping file can
  also be specified individually, e.g. if the docs should be buildable
  without Internet access.
"""

from __future__ import annotations

import concurrent.futures
import functools
import posixpath
import re
import sys
import time
from os import path
from typing import TYPE_CHECKING, cast
from urllib.parse import urlsplit, urlunsplit

from docutils import nodes
from docutils.utils import relative_path

import sphinx
from sphinx.addnodes import pending_xref
from sphinx.builders.html import INVENTORY_FILENAME
from sphinx.errors import ExtensionError
from sphinx.locale import _, __
from sphinx.transforms.post_transforms import ReferencesResolver
from sphinx.util import logging, requests
from sphinx.util.docutils import CustomReSTDispatcher, SphinxRole
from sphinx.util.inventory import InventoryFile

if TYPE_CHECKING:
    from collections.abc import Iterable
    from types import ModuleType
    from typing import IO, Any, Union

    from docutils.nodes import Node, TextElement, system_message
    from docutils.utils import Reporter

    from sphinx.application import Sphinx
    from sphinx.config import Config
    from sphinx.domains import Domain
    from sphinx.environment import BuildEnvironment
    from sphinx.util.typing import Inventory, InventoryItem, RoleFunction

    InventoryCacheEntry = tuple[Union[str, None], int, Inventory]

logger = logging.getLogger(__name__)


class InventoryAdapter:
    """Inventory adapter for environment"""

    def __init__(self, env: BuildEnvironment) -> None:
        self.env = env

        if not hasattr(env, 'intersphinx_cache'):
            # initial storage when fetching inventories before processing
            self.env.intersphinx_cache = {}  # type: ignore[attr-defined]

            self.env.intersphinx_inventory = {}  # type: ignore[attr-defined]
            self.env.intersphinx_named_inventory = {}  # type: ignore[attr-defined]

    @property
    def cache(self) -> dict[str, InventoryCacheEntry]:
        """Intersphinx cache.

        - Key is the URI of the remote inventory
        - Element one is the key given in the Sphinx intersphinx_mapping
          configuration value
        - Element two is a time value for cache invalidation, a float
        - Element three is the loaded remote inventory, type Inventory
        """
        return self.env.intersphinx_cache  # type: ignore[attr-defined]

    @property
    def main_inventory(self) -> Inventory:
        return self.env.intersphinx_inventory  # type: ignore[attr-defined]

    @property
    def named_inventory(self) -> dict[str, Inventory]:
        return self.env.intersphinx_named_inventory  # type: ignore[attr-defined]

    def clear(self) -> None:
        self.env.intersphinx_inventory.clear()  # type: ignore[attr-defined]
        self.env.intersphinx_named_inventory.clear()  # type: ignore[attr-defined]


def _strip_basic_auth(url: str) -> str:
    """Returns *url* with basic auth credentials removed. Also returns the
    basic auth username and password if they're present in *url*.

    E.g.: https://user:pass@example.com => https://example.com

    *url* need not include basic auth credentials.

    :param url: url which may or may not contain basic auth credentials
    :type url: ``str``

    :return: *url* with any basic auth creds removed
    :rtype: ``str``
    """
    frags = list(urlsplit(url))
    # swap out "user[:pass]@hostname" for "hostname"
    if '@' in frags[1]:
        frags[1] = frags[1].split('@')[1]
    return urlunsplit(frags)


def _read_from_url(url: str, *, config: Config) -> IO:
    """Reads data from *url* with an HTTP *GET*.

    This function supports fetching from resources which use basic HTTP auth as
    laid out by RFC1738 § 3.1. See § 5 for grammar definitions for URLs.

    .. seealso:

       https://www.ietf.org/rfc/rfc1738.txt

    :param url: URL of an HTTP resource
    :type url: ``str``

    :return: data read from resource described by *url*
    :rtype: ``file``-like object
    """
    r = requests.get(url, stream=True, timeout=config.intersphinx_timeout,
                     _user_agent=config.user_agent,
                     _tls_info=(config.tls_verify, config.tls_cacerts))
    r.raise_for_status()
    r.raw.url = r.url
    # decode content-body based on the header.
    # ref: https://github.com/psf/requests/issues/2155
    r.raw.read = functools.partial(r.raw.read, decode_content=True)
    return r.raw


def _get_safe_url(url: str) -> str:
    """Gets version of *url* with basic auth passwords obscured. This function
    returns results suitable for printing and logging.

    E.g.: https://user:12345@example.com => https://user@example.com

    :param url: a url
    :type url: ``str``

    :return: *url* with password removed
    :rtype: ``str``
    """
    parts = urlsplit(url)
    if parts.username is None:
        return url
    else:
        frags = list(parts)
        if parts.port:
            frags[1] = f'{parts.username}@{parts.hostname}:{parts.port}'
        else:
            frags[1] = f'{parts.username}@{parts.hostname}'

        return urlunsplit(frags)


def fetch_inventory(app: Sphinx, uri: str, inv: str) -> Inventory:
    """Fetch, parse and return an intersphinx inventory file."""
    # both *uri* (base URI of the links to generate) and *inv* (actual
    # location of the inventory file) can be local or remote URIs
    if '://' in uri:
        # case: inv URI points to remote resource; strip any existing auth
        uri = _strip_basic_auth(uri)
    try:
        if '://' in inv:
            f = _read_from_url(inv, config=app.config)
        else:
            f = open(path.join(app.srcdir, inv), 'rb')  # NoQA: SIM115
    except Exception as err:
        err.args = ('intersphinx inventory %r not fetchable due to %s: %s',
                    inv, err.__class__, str(err))
        raise
    try:
        if hasattr(f, 'url'):
            newinv = f.url
            if inv != newinv:
                logger.info(__('intersphinx inventory has moved: %s -> %s'), inv, newinv)

                if uri in (inv, path.dirname(inv), path.dirname(inv) + '/'):
                    uri = path.dirname(newinv)
        with f:
            try:
                invdata = InventoryFile.load(f, uri, posixpath.join)
            except ValueError as exc:
                raise ValueError('unknown or unsupported inventory version: %r' % exc) from exc
    except Exception as err:
        err.args = ('intersphinx inventory %r not readable due to %s: %s',
                    inv, err.__class__.__name__, str(err))
        raise
    else:
        return invdata


def fetch_inventory_group(
    name: str | None,
    uri: str,
    invs: tuple[str | None, ...],
    cache: dict[str, InventoryCacheEntry],
    app: Sphinx,
    now: int,
) -> bool:
    cache_time = now - app.config.intersphinx_cache_limit * 86400
    failures = []
    try:
        for inv in invs:
            if not inv:
                inv = posixpath.join(uri, INVENTORY_FILENAME)
            # decide whether the inventory must be read: always read local
            # files; remote ones only if the cache time is expired
            if '://' not in inv or uri not in cache or cache[uri][1] < cache_time:
                safe_inv_url = _get_safe_url(inv)
                logger.info(__('loading intersphinx inventory from %s...'), safe_inv_url)
                try:
                    invdata = fetch_inventory(app, uri, inv)
                except Exception as err:
                    failures.append(err.args)
                    continue
                if invdata:
                    cache[uri] = name, now, invdata
                    return True
        return False
    finally:
        if failures == []:
            pass
        elif len(failures) < len(invs):
            logger.info(__("encountered some issues with some of the inventories,"
                           " but they had working alternatives:"))
            for fail in failures:
                logger.info(*fail)
        else:
            issues = '\n'.join([f[0] % f[1:] for f in failures])
            logger.warning(__("failed to reach any of the inventories "
                              "with the following issues:") + "\n" + issues)


def load_mappings(app: Sphinx) -> None:
    """Load all intersphinx mappings into the environment."""
    now = int(time.time())
    inventories = InventoryAdapter(app.builder.env)
    intersphinx_cache: dict[str, InventoryCacheEntry] = inventories.cache

    with concurrent.futures.ThreadPoolExecutor() as pool:
        futures = []
        name: str | None
        uri: str
        invs: tuple[str | None, ...]
        for name, (uri, invs) in app.config.intersphinx_mapping.values():
            futures.append(pool.submit(
                fetch_inventory_group, name, uri, invs, intersphinx_cache, app, now,
            ))
        updated = [f.result() for f in concurrent.futures.as_completed(futures)]

    if any(updated):
        inventories.clear()

        # Duplicate values in different inventories will shadow each
        # other; which one will override which can vary between builds
        # since they are specified using an unordered dict.  To make
        # it more consistent, we sort the named inventories and then
        # add the unnamed inventories last.  This means that the
        # unnamed inventories will shadow the named ones but the named
        # ones can still be accessed when the name is specified.
        named_vals = []
        unnamed_vals = []
        for name, _expiry, invdata in intersphinx_cache.values():
            if name:
                named_vals.append((name, invdata))
            else:
                unnamed_vals.append((name, invdata))
        for name, invdata in sorted(named_vals) + unnamed_vals:
            if name:
                inventories.named_inventory[name] = invdata
            for type, objects in invdata.items():
                inventories.main_inventory.setdefault(type, {}).update(objects)


def _create_element_from_result(domain: Domain, inv_name: str | None,
                                data: InventoryItem,
                                node: pending_xref, contnode: TextElement) -> nodes.reference:
    proj, version, uri, dispname = data
    if '://' not in uri and node.get('refdoc'):
        # get correct path in case of subdirectories
        uri = posixpath.join(relative_path(node['refdoc'], '.'), uri)
    if version:
        reftitle = _('(in %s v%s)') % (proj, version)
    else:
        reftitle = _('(in %s)') % (proj,)
    newnode = nodes.reference('', '', internal=False, refuri=uri, reftitle=reftitle)
    if node.get('refexplicit'):
        # use whatever title was given
        newnode.append(contnode)
    elif dispname == '-' or \
            (domain.name == 'std' and node['reftype'] == 'keyword'):
        # use whatever title was given, but strip prefix
        title = contnode.astext()
        if inv_name is not None and title.startswith(inv_name + ':'):
            newnode.append(contnode.__class__(title[len(inv_name) + 1:],
                                              title[len(inv_name) + 1:]))
        else:
            newnode.append(contnode)
    else:
        # else use the given display name (used for :ref:)
        newnode.append(contnode.__class__(dispname, dispname))
    return newnode


def _resolve_reference_in_domain_by_target(
        inv_name: str | None, inventory: Inventory,
        domain: Domain, objtypes: Iterable[str],
        target: str,
        node: pending_xref, contnode: TextElement) -> nodes.reference | None:
    for objtype in objtypes:
        if objtype not in inventory:
            # Continue if there's nothing of this kind in the inventory
            continue

        if target in inventory[objtype]:
            # Case sensitive match, use it
            data = inventory[objtype][target]
        elif objtype == 'std:term':
            # Check for potential case insensitive matches for terms only
            target_lower = target.lower()
            insensitive_matches = list(filter(lambda k: k.lower() == target_lower,
                                              inventory[objtype].keys()))
            if insensitive_matches:
                data = inventory[objtype][insensitive_matches[0]]
            else:
                # No case insensitive match either, continue to the next candidate
                continue
        else:
            # Could reach here if we're not a term but have a case insensitive match.
            # This is a fix for terms specifically, but potentially should apply to
            # other types.
            continue
        return _create_element_from_result(domain, inv_name, data, node, contnode)
    return None


def _resolve_reference_in_domain(env: BuildEnvironment,
                                 inv_name: str | None, inventory: Inventory,
                                 honor_disabled_refs: bool,
                                 domain: Domain, objtypes: Iterable[str],
                                 node: pending_xref, contnode: TextElement,
                                 ) -> nodes.reference | None:
    obj_types: dict[str, None] = {}.fromkeys(objtypes)

    # we adjust the object types for backwards compatibility
    if domain.name == 'std' and 'cmdoption' in obj_types:
        # cmdoptions were stored as std:option until Sphinx 1.6
        obj_types['option'] = None
    if domain.name == 'py' and 'attribute' in obj_types:
        # properties are stored as py:method since Sphinx 2.1
        obj_types['method'] = None

    # the inventory contains domain:type as objtype
    domain_name = domain.name
    obj_types = {f"{domain_name}:{obj_type}": None for obj_type in obj_types}

    # now that the objtypes list is complete we can remove the disabled ones
    if honor_disabled_refs:
        disabled = set(env.config.intersphinx_disabled_reftypes)
        obj_types = {obj_type: None
                     for obj_type in obj_types
                     if obj_type not in disabled}

    objtypes = [*obj_types.keys()]

    # without qualification
    res = _resolve_reference_in_domain_by_target(inv_name, inventory, domain, objtypes,
                                                 node['reftarget'], node, contnode)
    if res is not None:
        return res

    # try with qualification of the current scope instead
    full_qualified_name = domain.get_full_qualified_name(node)
    if full_qualified_name is None:
        return None
    return _resolve_reference_in_domain_by_target(inv_name, inventory, domain, objtypes,
                                                  full_qualified_name, node, contnode)


def _resolve_reference(env: BuildEnvironment, inv_name: str | None, inventory: Inventory,
                       honor_disabled_refs: bool,
                       node: pending_xref, contnode: TextElement) -> nodes.reference | None:
    # disabling should only be done if no inventory is given
    honor_disabled_refs = honor_disabled_refs and inv_name is None

    if honor_disabled_refs and '*' in env.config.intersphinx_disabled_reftypes:
        return None

    typ = node['reftype']
    if typ == 'any':
        for domain_name, domain in env.domains.items():
            if (honor_disabled_refs
                    and (domain_name + ":*") in env.config.intersphinx_disabled_reftypes):
                continue
            objtypes: Iterable[str] = domain.object_types.keys()
            res = _resolve_reference_in_domain(env, inv_name, inventory,
                                               honor_disabled_refs,
                                               domain, objtypes,
                                               node, contnode)
            if res is not None:
                return res
        return None
    else:
        domain_name = node.get('refdomain')
        if not domain_name:
            # only objects in domains are in the inventory
            return None
        if honor_disabled_refs \
                and (domain_name + ":*") in env.config.intersphinx_disabled_reftypes:
            return None
        domain = env.get_domain(domain_name)
        objtypes = domain.objtypes_for_role(typ) or ()
        if not objtypes:
            return None
        return _resolve_reference_in_domain(env, inv_name, inventory,
                                            honor_disabled_refs,
                                            domain, objtypes,
                                            node, contnode)


def inventory_exists(env: BuildEnvironment, inv_name: str) -> bool:
    return inv_name in InventoryAdapter(env).named_inventory


def resolve_reference_in_inventory(env: BuildEnvironment,
                                   inv_name: str,
                                   node: pending_xref, contnode: TextElement,
                                   ) -> nodes.reference | None:
    """Attempt to resolve a missing reference via intersphinx references.

    Resolution is tried in the given inventory with the target as is.

    Requires ``inventory_exists(env, inv_name)``.
    """
    assert inventory_exists(env, inv_name)
    return _resolve_reference(env, inv_name, InventoryAdapter(env).named_inventory[inv_name],
                              False, node, contnode)


def resolve_reference_any_inventory(env: BuildEnvironment,
                                    honor_disabled_refs: bool,
                                    node: pending_xref, contnode: TextElement,
                                    ) -> nodes.reference | None:
    """Attempt to resolve a missing reference via intersphinx references.

    Resolution is tried with the target as is in any inventory.
    """
    return _resolve_reference(env, None, InventoryAdapter(env).main_inventory,
                              honor_disabled_refs,
                              node, contnode)


def resolve_reference_detect_inventory(env: BuildEnvironment,
                                       node: pending_xref, contnode: TextElement,
                                       ) -> nodes.reference | None:
    """Attempt to resolve a missing reference via intersphinx references.

    Resolution is tried first with the target as is in any inventory.
    If this does not succeed, then the target is split by the first ``:``,
    to form ``inv_name:newtarget``. If ``inv_name`` is a named inventory, then resolution
    is tried in that inventory with the new target.
    """

    # ordinary direct lookup, use data as is
    res = resolve_reference_any_inventory(env, True, node, contnode)
    if res is not None:
        return res

    # try splitting the target into 'inv_name:target'
    target = node['reftarget']
    if ':' not in target:
        return None
    inv_name, newtarget = target.split(':', 1)
    if not inventory_exists(env, inv_name):
        return None
    node['reftarget'] = newtarget
    res_inv = resolve_reference_in_inventory(env, inv_name, node, contnode)
    node['reftarget'] = target
    return res_inv


def missing_reference(app: Sphinx, env: BuildEnvironment, node: pending_xref,
                      contnode: TextElement) -> nodes.reference | None:
    """Attempt to resolve a missing reference via intersphinx references."""

    return resolve_reference_detect_inventory(env, node, contnode)


class IntersphinxDispatcher(CustomReSTDispatcher):
    """Custom dispatcher for external role.

    This enables :external:***:/:external+***: roles on parsing reST document.
    """

    def role(
        self, role_name: str, language_module: ModuleType, lineno: int, reporter: Reporter,
    ) -> tuple[RoleFunction, list[system_message]]:
        if len(role_name) > 9 and role_name.startswith(('external:', 'external+')):
            return IntersphinxRole(role_name), []
        else:
            return super().role(role_name, language_module, lineno, reporter)


class IntersphinxRole(SphinxRole):
    # group 1: just for the optionality of the inventory name
    # group 2: the inventory name (optional)
    # group 3: the domain:role or role part
    _re_inv_ref = re.compile(r"(\+([^:]+))?:(.*)")

    def __init__(self, orig_name: str) -> None:
        self.orig_name = orig_name

    def run(self) -> tuple[list[Node], list[system_message]]:
        assert self.name == self.orig_name.lower()
        inventory, name_suffix = self.get_inventory_and_name_suffix(self.orig_name)
        if inventory and not inventory_exists(self.env, inventory):
            logger.warning(__('inventory for external cross-reference not found: %s'),
                           inventory, location=(self.env.docname, self.lineno))
            return [], []

        role_name = self.get_role_name(name_suffix)
        if role_name is None:
            logger.warning(__('role for external cross-reference not found: %s'), name_suffix,
                           location=(self.env.docname, self.lineno))
            return [], []

        result, messages = self.invoke_role(role_name)
        for node in result:
            if isinstance(node, pending_xref):
                node['intersphinx'] = True
                node['inventory'] = inventory

        return result, messages

    def get_inventory_and_name_suffix(self, name: str) -> tuple[str | None, str]:
        assert name.startswith('external'), name
        # either we have an explicit inventory name, i.e,
        # :external+inv:role:        or
        # :external+inv:domain:role:
        # or we look in all inventories, i.e.,
        # :external:role:            or
        # :external:domain:role:
        suffix = name[9:]
        if name[8] == '+':
            inv_name, suffix = suffix.split(':', 1)
            return inv_name, suffix
        elif name[8] == ':':
            return None, suffix
        else:
            msg = f'Malformed :external: role name: {name}'
            raise ValueError(msg)

    def get_role_name(self, name: str) -> tuple[str, str] | None:
        names = name.split(':')
        if len(names) == 1:
            # role
            default_domain = self.env.temp_data.get('default_domain')
            domain = default_domain.name if default_domain else None
            role = names[0]
        elif len(names) == 2:
            # domain:role:
            domain = names[0]
            role = names[1]
        else:
            return None

        if domain and self.is_existent_role(domain, role):
            return (domain, role)
        elif self.is_existent_role('std', role):
            return ('std', role)
        else:
            return None

    def is_existent_role(self, domain_name: str, role_name: str) -> bool:
        try:
            domain = self.env.get_domain(domain_name)
            return role_name in domain.roles
        except ExtensionError:
            return False

    def invoke_role(self, role: tuple[str, str]) -> tuple[list[Node], list[system_message]]:
        domain = self.env.get_domain(role[0])
        if domain:
            role_func = domain.role(role[1])
            assert role_func is not None

            return role_func(':'.join(role), self.rawtext, self.text, self.lineno,
                             self.inliner, self.options, self.content)
        else:
            return [], []


class IntersphinxRoleResolver(ReferencesResolver):
    """pending_xref node resolver for intersphinx role.

    This resolves pending_xref nodes generated by :intersphinx:***: role.
    """

    default_priority = ReferencesResolver.default_priority - 1

    def run(self, **kwargs: Any) -> None:
        for node in self.document.findall(pending_xref):
            if 'intersphinx' not in node:
                continue
            contnode = cast(nodes.TextElement, node[0].deepcopy())
            inv_name = node['inventory']
            if inv_name is not None:
                assert inventory_exists(self.env, inv_name)
                newnode = resolve_reference_in_inventory(self.env, inv_name, node, contnode)
            else:
                newnode = resolve_reference_any_inventory(self.env, False, node, contnode)
            if newnode is None:
                typ = node['reftype']
                msg = (__('external %s:%s reference target not found: %s') %
                       (node['refdomain'], typ, node['reftarget']))
                logger.warning(msg, location=node, type='ref', subtype=typ)
                node.replace_self(contnode)
            else:
                node.replace_self(newnode)


def install_dispatcher(app: Sphinx, docname: str, source: list[str]) -> None:
    """Enable IntersphinxDispatcher.

    .. note:: The installed dispatcher will be uninstalled on disabling sphinx_domain
              automatically.
    """
    dispatcher = IntersphinxDispatcher()
    dispatcher.enable()


def normalize_intersphinx_mapping(app: Sphinx, config: Config) -> None:
    for key, value in config.intersphinx_mapping.copy().items():
        try:
            if isinstance(value, (list, tuple)):
                # new format
                name, (uri, inv) = key, value
                if not isinstance(name, str):
                    logger.warning(__('intersphinx identifier %r is not string. Ignored'),
                                   name)
                    config.intersphinx_mapping.pop(key)
                    continue
            else:
                # old format, no name
                # xref RemovedInSphinx80Warning
                name, uri, inv = None, key, value
                msg = (
                    "The pre-Sphinx 1.0 'intersphinx_mapping' format is "
                    "deprecated and will be removed in Sphinx 8. Update to the "
                    "current format as described in the documentation. "
                    f"Hint: \"intersphinx_mapping = {{'<name>': {(uri, inv)!r}}}\"."
                    "https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping"  # NoQA: E501
                )
                logger.warning(msg)

            if not isinstance(inv, tuple):
                config.intersphinx_mapping[key] = (name, (uri, (inv,)))
            else:
                config.intersphinx_mapping[key] = (name, (uri, inv))
        except Exception as exc:
            logger.warning(__('Failed to read intersphinx_mapping[%s], ignored: %r'), key, exc)
            config.intersphinx_mapping.pop(key)


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_config_value('intersphinx_mapping', {}, True)
    app.add_config_value('intersphinx_cache_limit', 5, False)
    app.add_config_value('intersphinx_timeout', None, False)
    app.add_config_value('intersphinx_disabled_reftypes', ['std:doc'], True)
    app.connect('config-inited', normalize_intersphinx_mapping, priority=800)
    app.connect('builder-inited', load_mappings)
    app.connect('source-read', install_dispatcher)
    app.connect('missing-reference', missing_reference)
    app.add_post_transform(IntersphinxRoleResolver)
    return {
        'version': sphinx.__display_version__,
        'env_version': 1,
        'parallel_read_safe': True,
    }


def inspect_main(argv: list[str], /) -> int:
    """Debug functionality to print out an inventory"""
    if len(argv) < 1:
        print("Print out an inventory file.\n"
              "Error: must specify local path or URL to an inventory file.",
              file=sys.stderr)
        return 1

    class MockConfig:
        intersphinx_timeout: int | None = None
        tls_verify = False
        tls_cacerts: str | dict[str, str] | None = None
        user_agent: str = ''

    class MockApp:
        srcdir = ''
        config = MockConfig()

    try:
        filename = argv[0]
        inv_data = fetch_inventory(MockApp(), '', filename)  # type: ignore[arg-type]
        for key in sorted(inv_data or {}):
            print(key)
            inv_entries = sorted(inv_data[key].items())
            for entry, (_proj, _ver, url_path, display_name) in inv_entries:
                display_name = display_name * (display_name != '-')
                print(f'    {entry:<40} {display_name:<40}: {url_path}')
    except ValueError as exc:
        print(exc.args[0] % exc.args[1:], file=sys.stderr)
        return 1
    except Exception as exc:
        print(f'Unknown error: {exc!r}', file=sys.stderr)
        return 1
    else:
        return 0


if __name__ == '__main__':
    import logging as _logging
    _logging.basicConfig()

    raise SystemExit(inspect_main(sys.argv[1:]))

env_pyrecodes/lib/python3.9/site-packages/sphinx/ext/intersphinx.py
NikolaBlagojevic/pyrecodes
"""Glue code for the jinja2 templating engine."""

from __future__ import annotations

from os import path
from pprint import pformat
from typing import TYPE_CHECKING, Any, Callable

from jinja2 import BaseLoader, FileSystemLoader, TemplateNotFound
from jinja2.sandbox import SandboxedEnvironment
from jinja2.utils import open_if_exists

from sphinx.application import TemplateBridge
from sphinx.util import logging
from sphinx.util.osutil import mtimes_of_files

try:
    from jinja2.utils import pass_context
except ImportError:
    from jinja2 import contextfunction as pass_context

if TYPE_CHECKING:
    from collections.abc import Iterator

    from jinja2.environment import Environment

    from sphinx.builders import Builder
    from sphinx.theming import Theme


def _tobool(val: str) -> bool:
    if isinstance(val, str):
        return val.lower() in ('true', '1', 'yes', 'on')
    return bool(val)


def _toint(val: str) -> int:
    try:
        return int(val)
    except ValueError:
        return 0


def _todim(val: int | str) -> str:
    """
    Make val a css dimension. In particular the following transformations
    are performed:

    - None -> 'initial' (default CSS value)
    - 0 -> '0'
    - ints and string representations of ints are interpreted as pixels.

    Everything else is returned unchanged.
    """
    if val is None:
        return 'initial'
    elif str(val).isdigit():
        return '0' if int(val) == 0 else '%spx' % val
    return val  # type: ignore[return-value]


def _slice_index(values: list, slices: int) -> Iterator[list]:
    seq = list(values)
    length = 0
    for value in values:
        length += 1 + len(value[1][1])  # count includes subitems
    items_per_slice = length // slices
    offset = 0
    for slice_number in range(slices):
        count = 0
        start = offset
        if slices == slice_number + 1:  # last column
            offset = len(seq)  # noqa: SIM113
        else:
            for value in values[offset:]:
                count += 1 + len(value[1][1])
                offset += 1
                if count >= items_per_slice:
                    break
        yield seq[start:offset]


def accesskey(context: Any, key: str) -> str:
    """Helper to output each access key only once."""
    if '_accesskeys' not in context:
        context.vars['_accesskeys'] = {}
    if key and key not in context.vars['_accesskeys']:
        context.vars['_accesskeys'][key] = 1
        return 'accesskey="%s"' % key
    return ''


class idgen:
    def __init__(self) -> None:
        self.id = 0

    def current(self) -> int:
        return self.id

    def __next__(self) -> int:
        self.id += 1
        return self.id
    next = __next__  # Python 2/Jinja compatibility


@pass_context
def warning(context: dict, message: str, *args: Any, **kwargs: Any) -> str:
    if 'pagename' in context:
        filename = context.get('pagename') + context.get('file_suffix', '')
        message = f'in rendering {filename}: {message}'
    logger = logging.getLogger('sphinx.themes')
    logger.warning(message, *args, **kwargs)
    return ''  # return empty string not to output any values


class SphinxFileSystemLoader(FileSystemLoader):
    """
    FileSystemLoader subclass that is not so strict about '..'  entries in
    template names.
    """

    def get_source(self, environment: Environment, template: str) -> tuple[str, str, Callable]:
        for searchpath in self.searchpath:
            filename = path.join(searchpath, template)
            f = open_if_exists(filename)
            if f is not None:
                break
        else:
            raise TemplateNotFound(template)

        with f:
            contents = f.read().decode(self.encoding)

        mtime = path.getmtime(filename)

        def uptodate() -> bool:
            try:
                return path.getmtime(filename) == mtime
            except OSError:
                return False
        return contents, filename, uptodate


class BuiltinTemplateLoader(TemplateBridge, BaseLoader):
    """
    Interfaces the rendering environment of jinja2 for use in Sphinx.
    """

    # TemplateBridge interface

    def init(
        self,
        builder: Builder,
        theme: Theme | None = None,
        dirs: list[str] | None = None,
    ) -> None:
        # create a chain of paths to search
        if theme:
            # the theme's own dir and its bases' dirs
            pathchain = theme.get_theme_dirs()
            # the loader dirs: pathchain + the parent directories for all themes
            loaderchain = pathchain + [path.join(p, '..') for p in pathchain]
        elif dirs:
            pathchain = list(dirs)
            loaderchain = list(dirs)
        else:
            pathchain = []
            loaderchain = []

        # prepend explicit template paths
        self.templatepathlen = len(builder.config.templates_path)
        if builder.config.templates_path:
            cfg_templates_path = [path.join(builder.confdir, tp)
                                  for tp in builder.config.templates_path]
            pathchain[0:0] = cfg_templates_path
            loaderchain[0:0] = cfg_templates_path

        # store it for use in newest_template_mtime
        self.pathchain = pathchain

        # make the paths into loaders
        self.loaders = [SphinxFileSystemLoader(x) for x in loaderchain]

        use_i18n = builder.app.translator is not None
        extensions = ['jinja2.ext.i18n'] if use_i18n else []
        self.environment = SandboxedEnvironment(loader=self,
                                                extensions=extensions)
        self.environment.filters['tobool'] = _tobool
        self.environment.filters['toint'] = _toint
        self.environment.filters['todim'] = _todim
        self.environment.filters['slice_index'] = _slice_index
        self.environment.globals['debug'] = pass_context(pformat)
        self.environment.globals['warning'] = warning
        self.environment.globals['accesskey'] = pass_context(accesskey)
        self.environment.globals['idgen'] = idgen
        if use_i18n:
            self.environment.install_gettext_translations(builder.app.translator)

    def render(self, template: str, context: dict) -> str:  # type: ignore[override]
        return self.environment.get_template(template).render(context)

    def render_string(self, source: str, context: dict) -> str:
        return self.environment.from_string(source).render(context)

    def newest_template_mtime(self) -> float:
        return max(mtimes_of_files(self.pathchain, '.html'))

    # Loader interface

    def get_source(self, environment: Environment, template: str) -> tuple[str, str, Callable]:
        loaders = self.loaders
        # exclamation mark starts search from theme
        if template.startswith('!'):
            loaders = loaders[self.templatepathlen:]
            template = template[1:]
        for loader in loaders:
            try:
                return loader.get_source(environment, template)
            except TemplateNotFound:
                pass
        raise TemplateNotFound(template)

env_pyrecodes/lib/python3.9/site-packages/sphinx/jinja2glue.py
NikolaBlagojevic/pyrecodes
"""Docutils transforms used by Sphinx when reading documents."""

from __future__ import annotations

import re
import unicodedata
from typing import TYPE_CHECKING, Any, cast

from docutils import nodes
from docutils.transforms import Transform, Transformer
from docutils.transforms.parts import ContentsFilter
from docutils.transforms.universal import SmartQuotes
from docutils.utils import normalize_language_tag
from docutils.utils.smartquotes import smartchars

from sphinx import addnodes
from sphinx.locale import _, __
from sphinx.util import logging
from sphinx.util.docutils import new_document
from sphinx.util.i18n import format_date
from sphinx.util.nodes import apply_source_workaround, is_smartquotable

if TYPE_CHECKING:
    from collections.abc import Generator

    from docutils.nodes import Node, Text

    from sphinx.application import Sphinx
    from sphinx.config import Config
    from sphinx.domains.std import StandardDomain
    from sphinx.environment import BuildEnvironment


logger = logging.getLogger(__name__)

default_substitutions = {
    'version',
    'release',
    'today',
    'translation progress',
}


class SphinxTransform(Transform):
    """A base class of Transforms.

    Compared with ``docutils.transforms.Transform``, this class improves accessibility to
    Sphinx APIs.
    """

    @property
    def app(self) -> Sphinx:
        """Reference to the :class:`.Sphinx` object."""
        return self.env.app

    @property
    def env(self) -> BuildEnvironment:
        """Reference to the :class:`.BuildEnvironment` object."""
        return self.document.settings.env

    @property
    def config(self) -> Config:
        """Reference to the :class:`.Config` object."""
        return self.env.config


class SphinxTransformer(Transformer):
    """
    A transformer for Sphinx.
    """

    document: nodes.document
    env: BuildEnvironment | None = None

    def set_environment(self, env: BuildEnvironment) -> None:
        self.env = env

    def apply_transforms(self) -> None:
        if isinstance(self.document, nodes.document):
            if not hasattr(self.document.settings, 'env') and self.env:
                self.document.settings.env = self.env

            super().apply_transforms()
        else:
            # wrap the target node by document node during transforming
            try:
                document = new_document('')
                if self.env:
                    document.settings.env = self.env
                document += self.document
                self.document = document
                super().apply_transforms()
            finally:
                self.document = self.document[0]


class DefaultSubstitutions(SphinxTransform):
    """
    Replace some substitutions if they aren't defined in the document.
    """
    # run before the default Substitutions
    default_priority = 210

    def apply(self, **kwargs: Any) -> None:
        # only handle those not otherwise defined in the document
        to_handle = default_substitutions - set(self.document.substitution_defs)
        for ref in self.document.findall(nodes.substitution_reference):
            refname = ref['refname']
            if refname in to_handle:
                if refname == 'translation progress':
                    # special handling: calculate translation progress
                    text = _calculate_translation_progress(self.document)
                else:
                    text = self.config[refname]
                if refname == 'today' and not text:
                    # special handling: can also specify a strftime format
                    text = format_date(self.config.today_fmt or _('%b %d, %Y'),
                                       language=self.config.language)
                ref.replace_self(nodes.Text(text))


def _calculate_translation_progress(document: nodes.document) -> str:
    try:
        translation_progress = document['translation_progress']
    except KeyError:
        return _('could not calculate translation progress!')

    total = translation_progress['total']
    translated = translation_progress['translated']
    if total <= 0:
        return _('no translated elements!')
    return f'{translated / total:.2%}'


class MoveModuleTargets(SphinxTransform):
    """
    Move module targets that are the first thing in a section to the section
    title.

    XXX Python specific
    """
    default_priority = 210

    def apply(self, **kwargs: Any) -> None:
        for node in list(self.document.findall(nodes.target)):
            if not node['ids']:
                continue
            if (
                'ismod' in node
                and type(node.parent) is nodes.section
                # index 0: section title node
                # index 1: index node
                # index 2: target node
                and node.parent.index(node) == 2
            ):
                node.parent['ids'][0:0] = node['ids']
                node.parent.remove(node)


class HandleCodeBlocks(SphinxTransform):
    """
    Several code block related transformations.
    """
    default_priority = 210

    def apply(self, **kwargs: Any) -> None:
        # move doctest blocks out of blockquotes
        for node in self.document.findall(nodes.block_quote):
            if all(isinstance(child, nodes.doctest_block) for child
                   in node.children):
                node.replace_self(node.children)
        # combine successive doctest blocks
        # for node in self.document.findall(nodes.doctest_block):
        #    if node not in node.parent.children:
        #        continue
        #    parindex = node.parent.index(node)
        #    while len(node.parent) > parindex+1 and \
        #            isinstance(node.parent[parindex+1], nodes.doctest_block):
        #        node[0] = nodes.Text(node[0] + '\n\n' +
        #                             node.parent[parindex+1][0])
        #        del node.parent[parindex+1]


class AutoNumbering(SphinxTransform):
    """
    Register IDs of tables, figures and literal_blocks to assign numbers.
    """
    default_priority = 210

    def apply(self, **kwargs: Any) -> None:
        domain: StandardDomain = self.env.domains['std']

        for node in self.document.findall(nodes.Element):
            if (domain.is_enumerable_node(node) and
                    domain.get_numfig_title(node) is not None and
                    node['ids'] == []):
                self.document.note_implicit_target(node)


class SortIds(SphinxTransform):
    """
    Sort section IDs so that the "id[0-9]+" one comes last.
    """
    default_priority = 261

    def apply(self, **kwargs: Any) -> None:
        for node in self.document.findall(nodes.section):
            if len(node['ids']) > 1 and node['ids'][0].startswith('id'):
                node['ids'] = node['ids'][1:] + [node['ids'][0]]


TRANSLATABLE_NODES = {
    'literal-block': nodes.literal_block,
    'doctest-block': nodes.doctest_block,
    'raw': nodes.raw,
    'index': addnodes.index,
    'image': nodes.image,
}


class ApplySourceWorkaround(SphinxTransform):
    """
    Update source and rawsource attributes
    """
    default_priority = 10

    def apply(self, **kwargs: Any) -> None:
        for node in self.document.findall():  # type: Node
            if isinstance(node, (nodes.TextElement, nodes.image, nodes.topic)):
                apply_source_workaround(node)


class AutoIndexUpgrader(SphinxTransform):
    """
    Detect old style (4 column based indices) and automatically upgrade to new style.
    """
    default_priority = 210

    def apply(self, **kwargs: Any) -> None:
        for node in self.document.findall(addnodes.index):
            if 'entries' in node and any(len(entry) == 4 for entry in node['entries']):
                msg = __('4 column based index found. '
                         'It might be a bug of extensions you use: %r') % node['entries']
                logger.warning(msg, location=node)
                for i, entry in enumerate(node['entries']):
                    if len(entry) == 4:
                        node['entries'][i] = entry + (None,)


class ExtraTranslatableNodes(SphinxTransform):
    """
    Make nodes translatable
    """
    default_priority = 10

    def apply(self, **kwargs: Any) -> None:
        targets = self.config.gettext_additional_targets
        target_nodes = [v for k, v in TRANSLATABLE_NODES.items() if k in targets]
        if not target_nodes:
            return

        def is_translatable_node(node: Node) -> bool:
            return isinstance(node, tuple(target_nodes))

        for node in self.document.findall(is_translatable_node):  # type: nodes.Element
            node['translatable'] = True


class UnreferencedFootnotesDetector(SphinxTransform):
    """
    Detect unreferenced footnotes and emit warnings
    """
    default_priority = 200

    def apply(self, **kwargs: Any) -> None:
        for node in self.document.footnotes:
            if node['names'] == []:
                # footnote having duplicated number.  It is already warned at parser.
                pass
            elif node['names'][0] not in self.document.footnote_refs:
                logger.warning(__('Footnote [%s] is not referenced.'), node['names'][0],
                               type='ref', subtype='footnote',
                               location=node)

        for node in self.document.autofootnotes:
            if not any(ref['auto'] == node['auto'] for ref in self.document.autofootnote_refs):
                logger.warning(__('Footnote [#] is not referenced.'),
                               type='ref', subtype='footnote',
                               location=node)


class DoctestTransform(SphinxTransform):
    """Set "doctest" style to each doctest_block node"""
    default_priority = 500

    def apply(self, **kwargs: Any) -> None:
        for node in self.document.findall(nodes.doctest_block):
            node['classes'].append('doctest')


class FilterSystemMessages(SphinxTransform):
    """Filter system messages from a doctree."""
    default_priority = 999

    def apply(self, **kwargs: Any) -> None:
        filterlevel = 2 if self.config.keep_warnings else 5
        for node in list(self.document.findall(nodes.system_message)):
            if node['level'] < filterlevel:
                logger.debug('%s [filtered system message]', node.astext())
                node.parent.remove(node)


class SphinxContentsFilter(ContentsFilter):
    """
    Used with BuildEnvironment.add_toc_from() to discard cross-file links
    within table-of-contents link nodes.
    """
    visit_pending_xref = ContentsFilter.ignore_node_but_process_children

    def visit_image(self, node: nodes.image) -> None:
        raise nodes.SkipNode


class SphinxSmartQuotes(SmartQuotes, SphinxTransform):
    """
    Customized SmartQuotes to avoid transform for some extra node types.

    refs: sphinx.parsers.RSTParser
    """
    default_priority = 750

    def apply(self, **kwargs: Any) -> None:
        if not self.is_available():
            return

        # override default settings with :confval:`smartquotes_action`
        self.smartquotes_action = self.config.smartquotes_action

        super().apply()

    def is_available(self) -> bool:
        builders = self.config.smartquotes_excludes.get('builders', [])
        languages = self.config.smartquotes_excludes.get('languages', [])

        if self.document.settings.smart_quotes is False:
            # disabled by 3rd party extension (workaround)
            return False
        if self.config.smartquotes is False:
            # disabled by confval smartquotes
            return False
        if self.app.builder.name in builders:
            # disabled by confval smartquotes_excludes['builders']
            return False
        if self.config.language in languages:
            # disabled by confval smartquotes_excludes['languages']
            return False

        # confirm selected language supports smart_quotes or not
        language = self.env.settings['language_code']
        return any(
            tag in smartchars.quotes
            for tag in normalize_language_tag(language)
        )

    def get_tokens(self, txtnodes: list[Text]) -> Generator[tuple[str, str], None, None]:
        # A generator that yields ``(texttype, nodetext)`` tuples for a list
        # of "Text" nodes (interface to ``smartquotes.educate_tokens()``).
        for txtnode in txtnodes:
            if is_smartquotable(txtnode):
                # SmartQuotes uses backslash escapes instead of null-escapes
                text = re.sub(r'(?<=\x00)([-\\\'".`])', r'\\\1', str(txtnode))
                yield 'plain', text
            else:
                # skip smart quotes
                yield 'literal', txtnode.astext()


class DoctreeReadEvent(SphinxTransform):
    """Emit :event:`doctree-read` event."""
    default_priority = 880

    def apply(self, **kwargs: Any) -> None:
        self.app.emit('doctree-read', self.document)


class ManpageLink(SphinxTransform):
    """Find manpage section numbers and names"""
    default_priority = 999

    def apply(self, **kwargs: Any) -> None:
        for node in self.document.findall(addnodes.manpage):
            manpage = ' '.join([str(x) for x in node.children
                                if isinstance(x, nodes.Text)])
            pattern = r'^(?P<path>(?P<page>.+)[\(\.](?P<section>[1-9]\w*)?\)?)$'
            info = {'path': manpage,
                    'page': manpage,
                    'section': ''}
            r = re.match(pattern, manpage)
            if r:
                info = r.groupdict()
            node.attributes.update(info)


class GlossarySorter(SphinxTransform):
    """Sort glossaries that have the ``sorted`` flag."""
    # This must be done after i18n, therefore not right
    # away in the glossary directive.
    default_priority = 500

    def apply(self, **kwargs: Any) -> None:
        for glossary in self.document.findall(addnodes.glossary):
            if glossary["sorted"]:
                definition_list = cast(nodes.definition_list, glossary[0])
                definition_list[:] = sorted(
                    definition_list,
                    key=lambda item: unicodedata.normalize(
                        'NFD',
                        cast(nodes.term, item)[0].astext().lower()),
                )


class ReorderConsecutiveTargetAndIndexNodes(SphinxTransform):
    """Index nodes interspersed between target nodes prevent other
    Transformations from combining those target nodes,
    e.g. ``PropagateTargets``.  This transformation reorders them:

    Given the following ``document`` as input::

        <document>
            <target ids="id1" ...>
            <index entries="...1...">
            <target ids="id2" ...>
            <target ids="id3" ...>
            <index entries="...2...">
            <target ids="id4" ...>

    The transformed result will be::

        <document>
            <index entries="...1...">
            <index entries="...2...">
            <target ids="id1" ...>
            <target ids="id2" ...>
            <target ids="id3" ...>
            <target ids="id4" ...>
    """

    # This transform MUST run before ``PropagateTargets``.
    default_priority = 220

    def apply(self, **kwargs: Any) -> None:
        for target in self.document.findall(nodes.target):
            _reorder_index_target_nodes(target)


def _reorder_index_target_nodes(start_node: nodes.target) -> None:
    """Sort target and index nodes.

    Find all consecutive target and index nodes starting from ``start_node``,
    and move all index nodes to before the first target node.
    """
    nodes_to_reorder: list[nodes.target | addnodes.index] = []

    # Note that we cannot use 'condition' to filter,
    # as we want *consecutive* target & index nodes.
    node: nodes.Node
    for node in start_node.findall(descend=False, siblings=True):
        if isinstance(node, (nodes.target, addnodes.index)):
            nodes_to_reorder.append(node)
            continue
        break  # must be a consecutive run of target or index nodes

    if len(nodes_to_reorder) < 2:
        return  # Nothing to reorder

    parent = nodes_to_reorder[0].parent
    if parent == nodes_to_reorder[-1].parent:
        first_idx = parent.index(nodes_to_reorder[0])
        last_idx = parent.index(nodes_to_reorder[-1])
        if first_idx + len(nodes_to_reorder) - 1 == last_idx:
            parent[first_idx:last_idx + 1] = sorted(nodes_to_reorder, key=_sort_key)


def _sort_key(node: nodes.Node) -> int:
    # Must be a stable sort.
    if isinstance(node, addnodes.index):
        return 0
    if isinstance(node, nodes.target):
        return 1
    msg = f'_sort_key called with unexpected node type {type(node)!r}'
    raise ValueError(msg)


def setup(app: Sphinx) -> dict[str, Any]:
    app.add_transform(ApplySourceWorkaround)
    app.add_transform(ExtraTranslatableNodes)
    app.add_transform(DefaultSubstitutions)
    app.add_transform(MoveModuleTargets)
    app.add_transform(HandleCodeBlocks)
    app.add_transform(SortIds)
    app.add_transform(DoctestTransform)
    app.add_transform(AutoNumbering)
    app.add_transform(AutoIndexUpgrader)
    app.add_transform(FilterSystemMessages)
    app.add_transform(UnreferencedFootnotesDetector)
    app.add_transform(SphinxSmartQuotes)
    app.add_transform(DoctreeReadEvent)
    app.add_transform(ManpageLink)
    app.add_transform(GlossarySorter)
    app.add_transform(ReorderConsecutiveTargetAndIndexNodes)

    return {
        'version': 'builtin',
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx/transforms/__init__.py
NikolaBlagojevic/pyrecodes
"""Utility functions for Sphinx."""

from __future__ import annotations

import hashlib
import os
import posixpath
import re
from importlib import import_module
from os import path
from typing import IO, Any
from urllib.parse import parse_qsl, quote_plus, urlencode, urlsplit, urlunsplit

from sphinx.errors import ExtensionError, FiletypeNotFoundError
from sphinx.locale import __
from sphinx.util import display as _display
from sphinx.util import exceptions as _exceptions
from sphinx.util import http_date as _http_date
from sphinx.util import index_entries as _index_entries
from sphinx.util import logging
from sphinx.util import osutil as _osutil
from sphinx.util.console import strip_colors  # NoQA: F401
from sphinx.util.matching import patfilter  # noqa: F401
from sphinx.util.nodes import (  # noqa: F401
    caption_ref_re,
    explicit_title_re,
    nested_parse_with_titles,
    split_explicit_title,
)

# import other utilities; partly for backwards compatibility, so don't
# prune unused ones indiscriminately
from sphinx.util.osutil import (  # noqa: F401
    SEP,
    copyfile,
    copytimes,
    ensuredir,
    make_filename,
    mtimes_of_files,
    os_path,
    relative_uri,
)

logger = logging.getLogger(__name__)

# Generally useful regular expressions.
ws_re: re.Pattern[str] = re.compile(r'\s+')
url_re: re.Pattern[str] = re.compile(r'(?P<schema>.+)://.*')


# High-level utility functions.

def docname_join(basedocname: str, docname: str) -> str:
    return posixpath.normpath(posixpath.join('/' + basedocname, '..', docname))[1:]


def get_filetype(source_suffix: dict[str, str], filename: str) -> str:
    for suffix, filetype in source_suffix.items():
        if filename.endswith(suffix):
            # If default filetype (None), considered as restructuredtext.
            return filetype or 'restructuredtext'
    raise FiletypeNotFoundError


class FilenameUniqDict(dict):
    """
    A dictionary that automatically generates unique names for its keys,
    interpreted as filenames, and keeps track of a set of docnames they
    appear in.  Used for images and downloadable files in the environment.
    """
    def __init__(self) -> None:
        self._existing: set[str] = set()

    def add_file(self, docname: str, newfile: str) -> str:
        if newfile in self:
            self[newfile][0].add(docname)
            return self[newfile][1]
        uniquename = path.basename(newfile)
        base, ext = path.splitext(uniquename)
        i = 0
        while uniquename in self._existing:
            i += 1
            uniquename = f'{base}{i}{ext}'
        self[newfile] = ({docname}, uniquename)
        self._existing.add(uniquename)
        return uniquename

    def purge_doc(self, docname: str) -> None:
        for filename, (docs, unique) in list(self.items()):
            docs.discard(docname)
            if not docs:
                del self[filename]
                self._existing.discard(unique)

    def merge_other(self, docnames: set[str], other: dict[str, tuple[set[str], Any]]) -> None:
        for filename, (docs, _unique) in other.items():
            for doc in docs & set(docnames):
                self.add_file(doc, filename)

    def __getstate__(self) -> set[str]:
        return self._existing

    def __setstate__(self, state: set[str]) -> None:
        self._existing = state


def _md5(data=b'', **_kw):
    """Deprecated wrapper around hashlib.md5

    To be removed in Sphinx 9.0
    """
    return hashlib.md5(data, usedforsecurity=False)


def _sha1(data=b'', **_kw):
    """Deprecated wrapper around hashlib.sha1

    To be removed in Sphinx 9.0
    """
    return hashlib.sha1(data, usedforsecurity=False)


class DownloadFiles(dict):
    """A special dictionary for download files.

    .. important:: This class would be refactored in nearly future.
                   Hence don't hack this directly.
    """

    def add_file(self, docname: str, filename: str) -> str:
        if filename not in self:
            digest = hashlib.md5(filename.encode(), usedforsecurity=False).hexdigest()
            dest = f'{digest}/{os.path.basename(filename)}'
            self[filename] = (set(), dest)

        self[filename][0].add(docname)
        return self[filename][1]

    def purge_doc(self, docname: str) -> None:
        for filename, (docs, _dest) in list(self.items()):
            docs.discard(docname)
            if not docs:
                del self[filename]

    def merge_other(self, docnames: set[str], other: dict[str, tuple[set[str], Any]]) -> None:
        for filename, (docs, _dest) in other.items():
            for docname in docs & set(docnames):
                self.add_file(docname, filename)


# a regex to recognize coding cookies
_coding_re = re.compile(r'coding[:=]\s*([-\w.]+)')


class UnicodeDecodeErrorHandler:
    """Custom error handler for open() that warns and replaces."""

    def __init__(self, docname: str) -> None:
        self.docname = docname

    def __call__(self, error: UnicodeDecodeError) -> tuple[str, int]:
        linestart = error.object.rfind(b'\n', 0, error.start)
        lineend = error.object.find(b'\n', error.start)
        if lineend == -1:
            lineend = len(error.object)
        lineno = error.object.count(b'\n', 0, error.start) + 1
        logger.warning(__('undecodable source characters, replacing with "?": %r'),
                       (error.object[linestart + 1:error.start] + b'>>>' +
                        error.object[error.start:error.end] + b'<<<' +
                        error.object[error.end:lineend]),
                       location=(self.docname, lineno))
        return ('?', error.end)


# Low-level utility functions and classes.

class Tee:
    """
    File-like object writing to two streams.
    """
    def __init__(self, stream1: IO, stream2: IO) -> None:
        self.stream1 = stream1
        self.stream2 = stream2

    def write(self, text: str) -> None:
        self.stream1.write(text)
        self.stream2.write(text)

    def flush(self) -> None:
        if hasattr(self.stream1, 'flush'):
            self.stream1.flush()
        if hasattr(self.stream2, 'flush'):
            self.stream2.flush()


def parselinenos(spec: str, total: int) -> list[int]:
    """Parse a line number spec (such as "1,2,4-6") and return a list of
    wanted line numbers.
    """
    items = []
    parts = spec.split(',')
    for part in parts:
        try:
            begend = part.strip().split('-')
            if ['', ''] == begend:
                raise ValueError
            if len(begend) == 1:
                items.append(int(begend[0]) - 1)
            elif len(begend) == 2:
                start = int(begend[0] or 1)  # left half open (cf. -10)
                end = int(begend[1] or max(start, total))  # right half open (cf. 10-)
                if start > end:  # invalid range (cf. 10-1)
                    raise ValueError
                items.extend(range(start - 1, end))
            else:
                raise ValueError
        except ValueError as exc:
            msg = f'invalid line number spec: {spec!r}'
            raise ValueError(msg) from exc

    return items


def import_object(objname: str, source: str | None = None) -> Any:
    """Import python object by qualname."""
    try:
        objpath = objname.split('.')
        modname = objpath.pop(0)
        obj = import_module(modname)
        for name in objpath:
            modname += '.' + name
            try:
                obj = getattr(obj, name)
            except AttributeError:
                obj = import_module(modname)

        return obj
    except (AttributeError, ImportError) as exc:
        if source:
            raise ExtensionError('Could not import %s (needed for %s)' %
                                 (objname, source), exc) from exc
        raise ExtensionError('Could not import %s' % objname, exc) from exc


def encode_uri(uri: str) -> str:
    split = list(urlsplit(uri))
    split[1] = split[1].encode('idna').decode('ascii')
    split[2] = quote_plus(split[2].encode(), '/')
    query = [(q, v.encode()) for (q, v) in parse_qsl(split[3])]
    split[3] = urlencode(query)
    return urlunsplit(split)


def isurl(url: str) -> bool:
    """Check *url* is URL or not."""
    return bool(url) and '://' in url


def _xml_name_checker():
    # to prevent import cycles
    from sphinx.builders.epub3 import _XML_NAME_PATTERN

    return _XML_NAME_PATTERN


# deprecated name -> (object to return, canonical path or empty string)
_DEPRECATED_OBJECTS = {
    'path_stabilize': (_osutil.path_stabilize, 'sphinx.util.osutil.path_stabilize'),
    'display_chunk': (_display.display_chunk, 'sphinx.util.display.display_chunk'),
    'status_iterator': (_display.status_iterator, 'sphinx.util.display.status_iterator'),
    'SkipProgressMessage': (_display.SkipProgressMessage,
                            'sphinx.util.display.SkipProgressMessage'),
    'progress_message': (_display.progress_message, 'sphinx.util.display.progress_message'),
    'epoch_to_rfc1123': (_http_date.epoch_to_rfc1123, 'sphinx.http_date.epoch_to_rfc1123'),
    'rfc1123_to_epoch': (_http_date.rfc1123_to_epoch, 'sphinx.http_date.rfc1123_to_epoch'),
    'save_traceback': (_exceptions.save_traceback, 'sphinx.exceptions.save_traceback'),
    'format_exception_cut_frames': (_exceptions.format_exception_cut_frames,
                                    'sphinx.exceptions.format_exception_cut_frames'),
    'xmlname_checker': (_xml_name_checker, 'sphinx.builders.epub3._XML_NAME_PATTERN'),
    'split_index_msg': (_index_entries.split_index_msg,
                        'sphinx.util.index_entries.split_index_msg'),
    'split_into': (_index_entries.split_index_msg, 'sphinx.util.index_entries.split_into'),
    'md5': (_md5, ''),
    'sha1': (_sha1, ''),
}


def __getattr__(name):
    if name not in _DEPRECATED_OBJECTS:
        msg = f'module {__name__!r} has no attribute {name!r}'
        raise AttributeError(msg)

    from sphinx.deprecation import _deprecation_warning

    deprecated_object, canonical_name = _DEPRECATED_OBJECTS[name]
    _deprecation_warning(__name__, name, canonical_name, remove=(8, 0))
    return deprecated_object

env_pyrecodes/lib/python3.9/site-packages/sphinx/util/__init__.py
NikolaBlagojevic/pyrecodes
"""What follows is awful and will be gone in Sphinx 8"""

from __future__ import annotations

import sys
import warnings
from pathlib import Path, PosixPath, PurePath, WindowsPath

from sphinx.deprecation import RemovedInSphinx80Warning

_STR_METHODS = frozenset(str.__dict__)
_PATH_NAME = Path().__class__.__name__

_MSG = (
    'Sphinx 8 will drop support for representing paths as strings. '
    'Use "pathlib.Path" or "os.fspath" instead.'
)

# https://docs.python.org/3/library/stdtypes.html#typesseq-common
# https://docs.python.org/3/library/stdtypes.html#string-methods

if sys.platform == 'win32':
    class _StrPath(WindowsPath):
        def replace(self, old, new, count=-1, /):
            # replace exists in both Path and str;
            # in Path it makes filesystem changes, so we use the safer str version
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return self.__str__().replace(old, new, count)

        def __getattr__(self, item):
            if item in _STR_METHODS:
                warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
                return getattr(self.__str__(), item)
            msg = f'{_PATH_NAME!r} has no attribute {item!r}'
            raise AttributeError(msg)

        def __add__(self, other):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return self.__str__() + other

        def __bool__(self):
            if not self.__str__():
                warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
                return False
            return True

        def __contains__(self, item):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return item in self.__str__()

        def __eq__(self, other):
            if isinstance(other, PurePath):
                return super().__eq__(other)
            if isinstance(other, str):
                warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
                return self.__str__() == other
            return NotImplemented

        def __hash__(self):
            return super().__hash__()

        def __getitem__(self, item):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return self.__str__()[item]

        def __len__(self):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return len(self.__str__())
else:
    class _StrPath(PosixPath):
        def replace(self, old, new, count=-1, /):
            # replace exists in both Path and str;
            # in Path it makes filesystem changes, so we use the safer str version
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return self.__str__().replace(old, new, count)

        def __getattr__(self, item):
            if item in _STR_METHODS:
                warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
                return getattr(self.__str__(), item)
            msg = f'{_PATH_NAME!r} has no attribute {item!r}'
            raise AttributeError(msg)

        def __add__(self, other):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return self.__str__() + other

        def __bool__(self):
            if not self.__str__():
                warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
                return False
            return True

        def __contains__(self, item):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return item in self.__str__()

        def __eq__(self, other):
            if isinstance(other, PurePath):
                return super().__eq__(other)
            if isinstance(other, str):
                warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
                return self.__str__() == other
            return NotImplemented

        def __hash__(self):
            return super().__hash__()

        def __getitem__(self, item):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return self.__str__()[item]

        def __len__(self):
            warnings.warn(_MSG, RemovedInSphinx80Warning, stacklevel=2)
            return len(self.__str__())

env_pyrecodes/lib/python3.9/site-packages/sphinx/util/_pathlib.py
NikolaBlagojevic/pyrecodes
"""Convert times to and from HTTP-date serialisations.

Reference: https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1
"""

import time
import warnings
from email.utils import formatdate, parsedate_tz

from sphinx.deprecation import RemovedInSphinx90Warning

_GMT_OFFSET = float(time.localtime().tm_gmtoff)


def epoch_to_rfc1123(epoch: float) -> str:
    """Return HTTP-date string from epoch offset."""
    return formatdate(epoch, usegmt=True)


def rfc1123_to_epoch(rfc1123: str) -> float:
    """Return epoch offset from HTTP-date string."""
    t = parsedate_tz(rfc1123)
    if t is None:
        raise ValueError
    if not rfc1123.endswith(" GMT"):
        warnings.warn(
            "HTTP-date string does not meet RFC 7231 requirements "
            f"(must end with 'GMT'): {rfc1123!r}",
            RemovedInSphinx90Warning, stacklevel=3,
        )
    epoch_secs = time.mktime(time.struct_time(t[:9])) + _GMT_OFFSET
    if (gmt_offset := t[9]) != 0:
        warnings.warn(
            "HTTP-date string does not meet RFC 7231 requirements "
            f"(must be GMT time): {rfc1123!r}",
            RemovedInSphinx90Warning, stacklevel=3,
        )
        return epoch_secs - (gmt_offset or 0)
    return epoch_secs

env_pyrecodes/lib/python3.9/site-packages/sphinx/util/http_date.py
NikolaBlagojevic/pyrecodes
"""Sphinx autodoc type hints."""
from __future__ import annotations

import ast
import inspect
import re
import sys
import textwrap
import types
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, AnyStr, Callable, ForwardRef, NewType, TypeVar, get_type_hints

from docutils.frontend import OptionParser
from docutils.parsers.rst import Parser as RstParser
from docutils.utils import new_document
from sphinx.ext.autodoc.mock import mock
from sphinx.util import logging
from sphinx.util.inspect import signature as sphinx_signature
from sphinx.util.inspect import stringify_signature

from .patches import install_patches
from .version import __version__

if TYPE_CHECKING:
    from ast import FunctionDef, Module, stmt

    from docutils.nodes import Node
    from sphinx.application import Sphinx
    from sphinx.config import Config
    from sphinx.environment import BuildEnvironment
    from sphinx.ext.autodoc import Options

_LOGGER = logging.getLogger(__name__)
_PYDATA_ANNOTATIONS = {"Any", "AnyStr", "Callable", "ClassVar", "Literal", "NoReturn", "Optional", "Tuple", "Union"}

# types has a bunch of things like ModuleType where ModuleType.__module__ is
# "builtins" and ModuleType.__name__ is "module", so we have to check for this.
_TYPES_DICT = {getattr(types, name): name for name in types.__all__}
# Prefer FunctionType to LambdaType (they are synonymous)
_TYPES_DICT[types.FunctionType] = "FunctionType"


def _get_types_type(obj: Any) -> str | None:
    try:
        return _TYPES_DICT.get(obj)
    except Exception:  # noqa: BLE001
        # e.g. exception: unhashable type
        return None


def get_annotation_module(annotation: Any) -> str:
    """
    Get module for an annotation.

    :param annotation:
    :return:
    """
    if annotation is None:
        return "builtins"
    if _get_types_type(annotation) is not None:
        return "types"
    is_new_type = sys.version_info >= (3, 10) and isinstance(annotation, NewType)
    if (
        is_new_type
        or isinstance(annotation, TypeVar)
        or type(annotation).__name__ in ("ParamSpec", "ParamSpecArgs", "ParamSpecKwargs")
    ):
        return "typing"
    if hasattr(annotation, "__module__"):
        return annotation.__module__  # type: ignore[no-any-return]
    if hasattr(annotation, "__origin__"):
        return annotation.__origin__.__module__  # type: ignore[no-any-return]
    msg = f"Cannot determine the module of {annotation}"
    raise ValueError(msg)


def _is_newtype(annotation: Any) -> bool:
    if sys.version_info < (3, 10):
        return inspect.isfunction(annotation) and hasattr(annotation, "__supertype__")
    return isinstance(annotation, NewType)


def get_annotation_class_name(annotation: Any, module: str) -> str:  # noqa: C901, PLR0911
    """
    Get class name for annotation.

    :param annotation:
    :param module:
    :return:
    """
    # Special cases
    if annotation is None:
        return "None"
    if annotation is AnyStr:
        return "AnyStr"
    val = _get_types_type(annotation)
    if val is not None:
        return val
    if _is_newtype(annotation):
        return "NewType"

    if getattr(annotation, "__qualname__", None):
        return annotation.__qualname__  # type: ignore[no-any-return]
    if getattr(annotation, "_name", None):  # Required for generic aliases on Python 3.7+
        return annotation._name  # type: ignore[no-any-return]  # noqa: SLF001
    if module in ("typing", "typing_extensions") and isinstance(getattr(annotation, "name", None), str):
        # Required for at least Pattern and Match
        return annotation.name  # type: ignore[no-any-return]

    origin = getattr(annotation, "__origin__", None)
    if origin:
        if getattr(origin, "__qualname__", None):  # Required for Protocol subclasses
            return origin.__qualname__  # type: ignore[no-any-return]
        if getattr(origin, "_name", None):  # Required for Union on Python 3.7+
            return origin._name  # type: ignore[no-any-return]  # noqa: SLF001

    annotation_cls = annotation if inspect.isclass(annotation) else type(annotation)
    return annotation_cls.__qualname__.lstrip("_")


def get_annotation_args(annotation: Any, module: str, class_name: str) -> tuple[Any, ...]:  # noqa: PLR0911
    """
    Get annotation arguments.

    :param annotation:
    :param module:
    :param class_name:
    :return:
    """
    try:
        original = getattr(sys.modules[module], class_name)
    except (KeyError, AttributeError):
        pass
    else:
        if annotation is original:
            return ()  # This is the original, not parametrized type

    # Special cases
    if class_name in ("Pattern", "Match") and hasattr(annotation, "type_var"):  # Python < 3.7
        return (annotation.type_var,)
    if class_name == "ClassVar" and hasattr(annotation, "__type__"):  # ClassVar on Python < 3.7
        return (annotation.__type__,)
    if class_name == "TypeVar" and hasattr(annotation, "__constraints__"):
        return annotation.__constraints__  # type: ignore[no-any-return]
    if class_name == "NewType" and hasattr(annotation, "__supertype__"):
        return (annotation.__supertype__,)
    if class_name == "Literal" and hasattr(annotation, "__values__"):
        return annotation.__values__  # type: ignore[no-any-return]
    if class_name == "Generic":
        return annotation.__parameters__  # type: ignore[no-any-return]
    result = getattr(annotation, "__args__", ())
    # 3.10 and earlier Tuple[()] returns ((), ) instead of () the tuple does
    return () if len(result) == 1 and result[0] == () else result  # type: ignore[misc]


def format_internal_tuple(t: tuple[Any, ...], config: Config) -> str:
    # An annotation can be a tuple, e.g., for nptyping:
    # In this case, format_annotation receives:
    # This solution should hopefully be general for *any* type that allows tuples in annotations
    fmt = [format_annotation(a, config) for a in t]
    if len(fmt) == 0:
        return "()"
    if len(fmt) == 1:
        return f"({fmt[0]}, )"
    return f"({', '.join(fmt)})"


def format_annotation(annotation: Any, config: Config) -> str:  # noqa: C901, PLR0911, PLR0912, PLR0915
    """
    Format the annotation.

    :param annotation:
    :param config:
    :return:
    """
    typehints_formatter: Callable[..., str] | None = getattr(config, "typehints_formatter", None)
    if typehints_formatter is not None:
        formatted = typehints_formatter(annotation, config)
        if formatted is not None:
            return formatted

    # Special cases
    if isinstance(annotation, ForwardRef):
        return annotation.__forward_arg__
    if annotation is None or annotation is type(None):
        return ":py:obj:`None`"
    if annotation is Ellipsis:
        return ":py:data:`...<Ellipsis>`"

    if isinstance(annotation, tuple):
        return format_internal_tuple(annotation, config)

    try:
        module = get_annotation_module(annotation)
        class_name = get_annotation_class_name(annotation, module)
        args = get_annotation_args(annotation, module, class_name)
    except ValueError:
        return str(annotation).strip("'")

    # Redirect all typing_extensions types to the stdlib typing module
    if module == "typing_extensions":
        module = "typing"

    if module == "_io":
        module = "io"

    full_name = f"{module}.{class_name}" if module != "builtins" else class_name
    fully_qualified: bool = getattr(config, "typehints_fully_qualified", False)
    prefix = "" if fully_qualified or full_name == class_name else "~"
    role = "data" if module == "typing" and class_name in _PYDATA_ANNOTATIONS else "class"
    args_format = "\\[{}]"
    formatted_args: str | None = ""

    # Some types require special handling
    if full_name == "typing.NewType":
        args_format = f"\\(``{annotation.__name__}``, {{}})"
        role = "class" if sys.version_info >= (3, 10) else "func"
    elif full_name in {"typing.TypeVar", "typing.ParamSpec"}:
        params = {k: getattr(annotation, f"__{k}__") for k in ("bound", "covariant", "contravariant")}
        params = {k: v for k, v in params.items() if v}
        if "bound" in params:
            params["bound"] = f" {format_annotation(params['bound'], config)}"
        args_format = f"\\(``{annotation.__name__}``{', {}' if args else ''}"
        if params:
            args_format += "".join(f", {k}={v}" for k, v in params.items())
        args_format += ")"
        formatted_args = None if args else args_format
    elif full_name == "typing.Optional":
        args = tuple(x for x in args if x is not type(None))
    elif full_name in ("typing.Union", "types.UnionType") and type(None) in args:
        if len(args) == 2:  # noqa: PLR2004
            full_name = "typing.Optional"
            role = "data"
            args = tuple(x for x in args if x is not type(None))
        else:
            simplify_optional_unions: bool = getattr(config, "simplify_optional_unions", True)
            if not simplify_optional_unions:
                full_name = "typing.Optional"
                role = "data"
                args_format = f"\\[:py:data:`{prefix}typing.Union`\\[{{}}]]"
                args = tuple(x for x in args if x is not type(None))
    elif full_name in ("typing.Callable", "collections.abc.Callable") and args and args[0] is not ...:
        fmt = [format_annotation(arg, config) for arg in args]
        formatted_args = f"\\[\\[{', '.join(fmt[:-1])}], {fmt[-1]}]"
    elif full_name == "typing.Literal":
        formatted_args = f"\\[{', '.join(f'``{arg!r}``' for arg in args)}]"
    elif full_name == "types.UnionType":
        return " | ".join([format_annotation(arg, config) for arg in args])

    if args and not formatted_args:
        try:
            iter(args)
        except TypeError:
            fmt = [format_annotation(args, config)]
        else:
            fmt = [format_annotation(arg, config) for arg in args]
        formatted_args = args_format.format(", ".join(fmt))

    return f":py:{role}:`{prefix}{full_name}`{formatted_args}"


# reference: https://github.com/pytorch/pytorch/pull/46548/files
def normalize_source_lines(source_lines: str) -> str:
    """
    Normalize the source lines.

    It finds the indentation level of the function definition (`def`), then it indents all lines in the function body to
    a point at or greater than that level. This allows for comments and continued string literals that are at a lower
    indentation than the rest of the code.

    :param source_lines: source code
    :return: source lines that have been correctly aligned
    """
    lines = source_lines.split("\n")

    def remove_prefix(text: str, prefix: str) -> str:
        return text[text.startswith(prefix) and len(prefix) :]

    # Find the line and line number containing the function definition
    for pos, line in enumerate(lines):
        if line.lstrip().startswith("def "):
            idx = pos
            whitespace_separator = "def"
            break
        if line.lstrip().startswith("async def"):
            idx = pos
            whitespace_separator = "async def"
            break

    else:
        return "\n".join(lines)
    fn_def = lines[idx]

    # Get a string representing the amount of leading whitespace
    whitespace = fn_def.split(whitespace_separator)[0]

    # Add this leading whitespace to all lines before and after the `def`
    aligned_prefix = [whitespace + remove_prefix(s, whitespace) for s in lines[:idx]]
    aligned_suffix = [whitespace + remove_prefix(s, whitespace) for s in lines[idx + 1 :]]

    # Put it together again
    aligned_prefix.append(fn_def)
    return "\n".join(aligned_prefix + aligned_suffix)


def process_signature(  # noqa: C901, PLR0913
    app: Sphinx,
    what: str,
    name: str,
    obj: Any,
    options: Options,  # noqa: ARG001
    signature: str,  # noqa: ARG001
    return_annotation: str,  # noqa: ARG001
) -> tuple[str, None] | None:
    """
    Process the signature.

    :param app:
    :param what:
    :param name:
    :param obj:
    :param options:
    :param signature:
    :param return_annotation:
    :return:
    """
    if not callable(obj):
        return None

    original_obj = obj
    obj = getattr(obj, "__init__", getattr(obj, "__new__", None)) if inspect.isclass(obj) else obj
    if not getattr(obj, "__annotations__", None):  # when has no annotation we cannot autodoc typehints so bail
        return None

    obj = inspect.unwrap(obj)
    sph_signature = sphinx_signature(obj)

    if app.config.typehints_use_signature:
        parameters = list(sph_signature.parameters.values())
    else:
        parameters = [param.replace(annotation=inspect.Parameter.empty) for param in sph_signature.parameters.values()]

    # if we have parameters we may need to delete first argument that's not documented, e.g. self
    start = 0
    if parameters:
        if inspect.isclass(original_obj) or (what == "method" and name.endswith(".__init__")):
            start = 1
        elif what == "method":
            # bail if it is a local method as we cannot determine if first argument needs to be deleted or not
            if "<locals>" in obj.__qualname__ and not _is_dataclass(name, what, obj.__qualname__):
                _LOGGER.warning('Cannot handle as a local function: "%s" (use @functools.wraps)', name)
                return None
            outer = inspect.getmodule(obj)
            for class_name in obj.__qualname__.split(".")[:-1]:
                outer = getattr(outer, class_name)
            method_name = obj.__name__
            if method_name.startswith("__") and not method_name.endswith("__"):
                # when method starts with double underscore Python applies mangling -> prepend the class name
                method_name = f"_{obj.__qualname__.split('.')[-2]}{method_name}"
            method_object = outer.__dict__[method_name] if outer else obj
            if not isinstance(method_object, (classmethod, staticmethod)):
                start = 1

    sph_signature = sph_signature.replace(parameters=parameters[start:])
    if not app.config.typehints_use_signature_return:
        sph_signature = sph_signature.replace(return_annotation=inspect.Signature.empty)

    return stringify_signature(sph_signature).replace("\\", "\\\\"), None


def _is_dataclass(name: str, what: str, qualname: str) -> bool:
    # generated dataclass __init__() and class need extra checks, as the function operates on the generated class
    # and methods (not an instantiated dataclass object) it cannot be replaced by a call to
    # `dataclasses.is_dataclass()` => check manually for either generated __init__ or generated class
    return (what == "method" and name.endswith(".__init__")) or (what == "class" and qualname.endswith(".__init__"))


def _future_annotations_imported(obj: Any) -> bool:
    _annotations = getattr(inspect.getmodule(obj), "annotations", None)
    if _annotations is None:
        return False

    # Make sure that annotations is imported from __future__ - defined in cpython/Lib/__future__.py
    # annotations become strings at runtime
    future_annotations = 0x100000 if sys.version_info[0:2] == (3, 7) else 0x1000000
    return bool(_annotations.compiler_flag == future_annotations)


def get_all_type_hints(autodoc_mock_imports: list[str], obj: Any, name: str) -> dict[str, Any]:
    result = _get_type_hint(autodoc_mock_imports, name, obj)
    if not result:
        result = backfill_type_hints(obj, name)
        try:
            obj.__annotations__ = result
        except (AttributeError, TypeError):
            pass
        else:
            result = _get_type_hint(autodoc_mock_imports, name, obj)
    return result


_TYPE_GUARD_IMPORT_RE = re.compile(r"\nif (typing.)?TYPE_CHECKING:[^\n]*([\s\S]*?)(?=\n\S)")
_TYPE_GUARD_IMPORTS_RESOLVED = set()
_TYPE_GUARD_IMPORTS_RESOLVED_GLOBALS_ID = set()


def _resolve_type_guarded_imports(autodoc_mock_imports: list[str], obj: Any) -> None:  # noqa: C901
    if hasattr(obj, "__module__") and obj.__module__ in _TYPE_GUARD_IMPORTS_RESOLVED:
        return  # already processed module
    if not hasattr(obj, "__globals__"):  # classes with __slots__ do not have this
        return  # if lacks globals nothing we can do
    if id(obj.__globals__) in _TYPE_GUARD_IMPORTS_RESOLVED_GLOBALS_ID:
        return  # already processed object
    _TYPE_GUARD_IMPORTS_RESOLVED.add(obj.__module__)
    if obj.__module__ not in sys.builtin_module_names:
        if hasattr(obj, "__globals__"):
            _TYPE_GUARD_IMPORTS_RESOLVED_GLOBALS_ID.add(id(obj.__globals__))

        module = inspect.getmodule(obj)
        if module:
            try:
                module_code = inspect.getsource(module)
            except (TypeError, OSError):
                ...  # no source code => no type guards
            else:
                for _, part in _TYPE_GUARD_IMPORT_RE.findall(module_code):
                    guarded_code = textwrap.dedent(part)
                    try:
                        with mock(autodoc_mock_imports):
                            exec(guarded_code, obj.__globals__)  # noqa: S102
                    except Exception as exc:  # noqa: BLE001, PERF203
                        _LOGGER.warning(f"Failed guarded type import with {exc!r}")


def _get_type_hint(autodoc_mock_imports: list[str], name: str, obj: Any) -> dict[str, Any]:
    _resolve_type_guarded_imports(autodoc_mock_imports, obj)
    try:
        result = get_type_hints(obj)
    except (AttributeError, TypeError, RecursionError) as exc:
        # TypeError - slot wrapper, PEP-563 when part of new syntax not supported
        # RecursionError - some recursive type definitions https://github.com/python/typing/issues/574
        if isinstance(exc, TypeError) and _future_annotations_imported(obj) and "unsupported operand type" in str(exc):
            result = obj.__annotations__
        else:
            result = {}
    except NameError as exc:
        _LOGGER.warning('Cannot resolve forward reference in type annotations of "%s": %s', name, exc)
        result = obj.__annotations__
    return result


def backfill_type_hints(obj: Any, name: str) -> dict[str, Any]:  # noqa: C901, PLR0911, PLR0912
    """
    Backfill type hints.

    :param obj: the object
    :param name: the name
    :return: backfilled value
    """
    parse_kwargs = {"type_comments": True}

    def _one_child(module: Module) -> stmt | None:
        children = module.body  # use the body to ignore type comments
        if len(children) != 1:
            _LOGGER.warning('Did not get exactly one node from AST for "%s", got %s', name, len(children))
            return None
        return children[0]

    try:
        code = textwrap.dedent(normalize_source_lines(inspect.getsource(obj)))
        obj_ast = ast.parse(code, **parse_kwargs)  # type: ignore[call-overload]  # dynamic kwargs
    except (OSError, TypeError, SyntaxError):
        return {}

    obj_ast = _one_child(obj_ast)
    if obj_ast is None:
        return {}

    try:
        type_comment = obj_ast.type_comment
    except AttributeError:
        return {}

    if not type_comment:
        return {}

    try:
        comment_args_str, comment_returns = type_comment.split(" -> ")
    except ValueError:
        _LOGGER.warning('Unparseable type hint comment for "%s": Expected to contain ` -> `', name)
        return {}

    rv = {}
    if comment_returns:
        rv["return"] = comment_returns

    args = load_args(obj_ast)
    comment_args = split_type_comment_args(comment_args_str)
    is_inline = len(comment_args) == 1 and comment_args[0] == "..."
    if not is_inline:
        if args and args[0].arg in ("self", "cls") and len(comment_args) != len(args):
            comment_args.insert(0, None)  # self/cls may be omitted in type comments, insert blank

        if len(args) != len(comment_args):
            _LOGGER.warning('Not enough type comments found on "%s"', name)
            return rv

    for at, arg in enumerate(args):
        arg_key = getattr(arg, "arg", None)
        if arg_key is None:
            continue

        value = getattr(arg, "type_comment", None) if is_inline else comment_args[at]

        if value is not None:
            rv[arg_key] = value

    return rv


def load_args(obj_ast: FunctionDef) -> list[Any]:
    func_args = obj_ast.args
    args = []
    pos_only = getattr(func_args, "posonlyargs", None)
    if pos_only:
        args.extend(pos_only)

    args.extend(func_args.args)
    if func_args.vararg:
        args.append(func_args.vararg)

    args.extend(func_args.kwonlyargs)
    if func_args.kwarg:
        args.append(func_args.kwarg)

    return args


def split_type_comment_args(comment: str) -> list[str | None]:
    def add(val: str) -> None:
        result.append(val.strip().lstrip("*"))  # remove spaces, and var/kw arg marker

    comment = comment.strip().lstrip("(").rstrip(")")
    result: list[str | None] = []
    if not comment:
        return result

    brackets, start_arg_at, at = 0, 0, 0
    for at, char in enumerate(comment):
        if char in ("[", "("):
            brackets += 1
        elif char in ("]", ")"):
            brackets -= 1
        elif char == "," and brackets == 0:
            add(comment[start_arg_at:at])
            start_arg_at = at + 1

    add(comment[start_arg_at : at + 1])
    return result


def format_default(app: Sphinx, default: Any, is_annotated: bool) -> str | None:  # noqa: FBT001
    if default is inspect.Parameter.empty:
        return None
    formatted = repr(default).replace("\\", "\\\\")

    if is_annotated:
        if app.config.typehints_defaults.startswith("braces"):
            return f" (default: ``{formatted}``)"
        return f", default: ``{formatted}``"
    if app.config.typehints_defaults == "braces-after":
        return f" (default: ``{formatted}``)"
    return f"default: ``{formatted}``"


def process_docstring(  # noqa: PLR0913
    app: Sphinx,
    what: str,
    name: str,
    obj: Any,
    options: Options | None,  # noqa: ARG001
    lines: list[str],
) -> None:
    """
    Process the docstring for an entry.

    :param app: the Sphinx app
    :param what: the target
    :param name: the name
    :param obj: the object
    :param options: the options
    :param lines: the lines
    :return:
    """
    original_obj = obj
    obj = obj.fget if isinstance(obj, property) else obj
    if not callable(obj):
        return
    obj = obj.__init__ if inspect.isclass(obj) else obj
    obj = inspect.unwrap(obj)

    try:
        signature = sphinx_signature(obj)
    except (ValueError, TypeError):
        signature = None
    type_hints = get_all_type_hints(app.config.autodoc_mock_imports, obj, name)
    app.config._annotation_globals = getattr(obj, "__globals__", {})  # type: ignore[attr-defined]  # noqa: SLF001
    try:
        _inject_types_to_docstring(type_hints, signature, original_obj, app, what, name, lines)
    finally:
        delattr(app.config, "_annotation_globals")


def _get_sphinx_line_keyword_and_argument(line: str) -> tuple[str, str | None] | None:
    """
    Extract a keyword, and its optional argument out of a sphinx field option line.

    For example
    >>> _get_sphinx_line_keyword_and_argument(":param parameter:")
    ("param", "parameter")
    >>> _get_sphinx_line_keyword_and_argument(":return:")
    ("return", None)
    >>> _get_sphinx_line_keyword_and_argument("some invalid line")
    None
    """
    param_line_without_description = line.split(":", maxsplit=2)
    if len(param_line_without_description) != 3:  # noqa: PLR2004
        return None

    split_directive_and_name = param_line_without_description[1].split(maxsplit=1)
    if len(split_directive_and_name) != 2:  # noqa: PLR2004
        if not len(split_directive_and_name):
            return None
        return split_directive_and_name[0], None

    return tuple(split_directive_and_name)  # type: ignore[return-value]


def _line_is_param_line_for_arg(line: str, arg_name: str) -> bool:
    """Return True if `line` is a valid parameter line for `arg_name`, false otherwise."""
    keyword_and_name = _get_sphinx_line_keyword_and_argument(line)
    if keyword_and_name is None:
        return False

    keyword, doc_name = keyword_and_name
    if doc_name is None:
        return False

    if keyword not in {"param", "parameter", "arg", "argument"}:
        return False

    return any(doc_name == prefix + arg_name for prefix in ("", "\\*", "\\**", "\\*\\*"))


def _inject_types_to_docstring(  # noqa: PLR0913
    type_hints: dict[str, Any],
    signature: inspect.Signature | None,
    original_obj: Any,
    app: Sphinx,
    what: str,
    name: str,
    lines: list[str],
) -> None:
    if signature is not None:
        _inject_signature(type_hints, signature, app, lines)
    if "return" in type_hints:
        _inject_rtype(type_hints, original_obj, app, what, name, lines)


def _inject_signature(  # noqa: C901
    type_hints: dict[str, Any],
    signature: inspect.Signature,
    app: Sphinx,
    lines: list[str],
) -> None:
    for arg_name in signature.parameters:
        annotation = type_hints.get(arg_name, None)

        default = signature.parameters[arg_name].default

        if arg_name.endswith("_"):
            arg_name = f"{arg_name[:-1]}\\_"  # noqa: PLW2901

        insert_index = None
        for at, line in enumerate(lines):
            if _line_is_param_line_for_arg(line, arg_name):
                # Get the arg_name from the doc to match up for type in case it has a star prefix.
                # Line is in the correct format so this is guaranteed to return tuple[str, str].
                func = _get_sphinx_line_keyword_and_argument
                _, arg_name = func(line)  # type: ignore[assignment, misc] # noqa: PLW2901
                insert_index = at
                break

        if annotation is not None and insert_index is None and app.config.always_document_param_types:
            lines.append(f":param {arg_name}:")
            insert_index = len(lines)

        if insert_index is not None:
            if annotation is None:
                type_annotation = f":type {arg_name}: "
            else:
                formatted_annotation = format_annotation(annotation, app.config)
                type_annotation = f":type {arg_name}: {formatted_annotation}"

            if app.config.typehints_defaults:
                formatted_default = format_default(app, default, annotation is not None)
                if formatted_default:
                    if app.config.typehints_defaults.endswith("after"):
                        lines[insert_index] += formatted_default
                    else:  # add to last param doc line
                        type_annotation += formatted_default

            lines.insert(insert_index, type_annotation)


@dataclass
class InsertIndexInfo:
    insert_index: int
    found_param: bool = False
    found_return: bool = False
    found_directive: bool = False


# Sphinx allows so many synonyms...
# See sphinx.domains.python.PyObject
PARAM_SYNONYMS = ("param ", "parameter ", "arg ", "argument ", "keyword ", "kwarg ", "kwparam ")


def node_line_no(node: Node) -> int | None:
    """
    Get the 1-indexed line on which the node starts if possible. If not, return None.

    Descend through the first children until we locate one with a line number or return None if None of them have one.

    I'm not aware of any rst on which this returns None, to find out would require a more detailed analysis of the
    docutils rst parser source code. An example where the node doesn't have a line number but the first child does is
    all `definition_list` nodes. It seems like bullet_list and option_list get line numbers, but enum_list also doesn't.
    """
    while node.line is None and node.children:
        node = node.children[0]
    return node.line


def tag_name(node: Node) -> str:
    return node.tagname  # type:ignore[attr-defined,no-any-return]


def get_insert_index(app: Sphinx, lines: list[str]) -> InsertIndexInfo | None:
    # 1. If there is an existing :rtype: anywhere, don't insert anything.
    if any(line.startswith(":rtype:") for line in lines):
        return None

    # 2. If there is a :returns: anywhere, either modify that line or insert
    #    just before it.
    for at, line in enumerate(lines):
        if line.startswith((":return:", ":returns:")):
            return InsertIndexInfo(insert_index=at, found_return=True)

    # 3. Insert after the parameters.
    # To find the parameters, parse as a docutils tree.
    settings = OptionParser(components=(RstParser,)).get_default_values()
    settings.env = app.env
    doc = new_document("", settings=settings)
    RstParser().parse("\n".join(lines), doc)

    # Find a top level child which is a field_list that contains a field whose
    # name starts with one of the PARAM_SYNONYMS. This is the parameter list. We
    # hope there is at most of these.
    for child in doc.children:
        if tag_name(child) != "field_list":
            continue

        if not any(c.children[0].astext().startswith(PARAM_SYNONYMS) for c in child.children):
            continue

        # Found it! Try to insert before the next sibling. If there is no next
        # sibling, insert at end.
        # If there is a next sibling but we can't locate a line number, insert
        # at end. (I don't know of any input where this happens.)
        next_sibling = child.next_node(descend=False, siblings=True)
        line_no = node_line_no(next_sibling) if next_sibling else None
        at = line_no - 2 if line_no else len(lines)
        return InsertIndexInfo(insert_index=at, found_param=True)

    # 4. Insert before examples
    for child in doc.children:
        if tag_name(child) in ["literal_block", "paragraph", "field_list"]:
            continue
        line_no = node_line_no(child)
        at = line_no - 2 if line_no else len(lines)
        return InsertIndexInfo(insert_index=at, found_directive=True)

    # 5. Otherwise, insert at end
    return InsertIndexInfo(insert_index=len(lines))


def _inject_rtype(  # noqa: PLR0913
    type_hints: dict[str, Any],
    original_obj: Any,
    app: Sphinx,
    what: str,
    name: str,
    lines: list[str],
) -> None:
    if inspect.isclass(original_obj) or inspect.isdatadescriptor(original_obj):
        return
    if what == "method" and name.endswith(".__init__"):  # avoid adding a return type for data class __init__
        return
    if not app.config.typehints_document_rtype:
        return

    r = get_insert_index(app, lines)
    if r is None:
        return

    insert_index = r.insert_index

    if not app.config.typehints_use_rtype and r.found_return and " -- " in lines[insert_index]:
        return

    formatted_annotation = format_annotation(type_hints["return"], app.config)

    if r.found_param and insert_index < len(lines) and lines[insert_index].strip():
        insert_index -= 1

    if insert_index == len(lines) and not r.found_param:
        # ensure that :rtype: doesn't get joined with a paragraph of text
        lines.append("")
        insert_index += 1
    if app.config.typehints_use_rtype or not r.found_return:
        line = f":rtype: {formatted_annotation}"
        lines.insert(insert_index, line)
        if r.found_directive:
            lines.insert(insert_index + 1, "")
    else:
        line = lines[insert_index]
        lines[insert_index] = f":return: {formatted_annotation} --{line[line.find(' '):]}"


def validate_config(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:  # noqa: ARG001
    valid = {None, "comma", "braces", "braces-after"}
    if app.config.typehints_defaults not in valid | {False}:
        msg = f"typehints_defaults needs to be one of {valid!r}, not {app.config.typehints_defaults!r}"
        raise ValueError(msg)

    formatter = app.config.typehints_formatter
    if formatter is not None and not callable(formatter):
        msg = f"typehints_formatter needs to be callable or `None`, not {formatter}"
        raise ValueError(msg)


def setup(app: Sphinx) -> dict[str, bool]:
    app.add_config_value("always_document_param_types", False, "html")  # noqa: FBT003
    app.add_config_value("typehints_fully_qualified", False, "env")  # noqa: FBT003
    app.add_config_value("typehints_document_rtype", True, "env")  # noqa: FBT003
    app.add_config_value("typehints_use_rtype", True, "env")  # noqa: FBT003
    app.add_config_value("typehints_defaults", None, "env")
    app.add_config_value("simplify_optional_unions", True, "env")  # noqa: FBT003
    app.add_config_value("typehints_formatter", None, "env")
    app.add_config_value("typehints_use_signature", False, "env")  # noqa: FBT003
    app.add_config_value("typehints_use_signature_return", False, "env")  # noqa: FBT003
    app.connect("env-before-read-docs", validate_config)  # config may be changed after “config-inited” event
    app.connect("autodoc-process-signature", process_signature)
    app.connect("autodoc-process-docstring", process_docstring)
    install_patches(app)
    return {"parallel_read_safe": True, "parallel_write_safe": True}


__all__ = [
    "__version__",
    "format_annotation",
    "get_annotation_args",
    "get_annotation_class_name",
    "get_annotation_module",
    "normalize_source_lines",
    "process_docstring",
    "process_signature",
    "backfill_type_hints",
]

env_pyrecodes/lib/python3.9/site-packages/sphinx_autodoc_typehints/__init__.py
NikolaBlagojevic/pyrecodes
"""Patch for attributes."""
from __future__ import annotations

from functools import partial
from typing import TYPE_CHECKING, Any
from unittest.mock import patch

import sphinx.domains.python
import sphinx.ext.autodoc
from docutils.parsers.rst import Parser as RstParser
from docutils.utils import new_document
from sphinx.domains.python import PyAttribute
from sphinx.ext.autodoc import AttributeDocumenter

if TYPE_CHECKING:
    from optparse import Values

    from sphinx.addnodes import desc_signature
    from sphinx.application import Sphinx

# Defensively check for the things we want to patch
_parse_annotation = getattr(sphinx.domains.python, "_parse_annotation", None)

# We want to patch:
# * sphinx.ext.autodoc.stringify_typehint (in sphinx < 6.1)
# * sphinx.ext.autodoc.stringify_annotation (in sphinx >= 6.1)
STRINGIFY_PATCH_TARGET = ""
for target in ["stringify_typehint", "stringify_annotation"]:
    if hasattr(sphinx.ext.autodoc, target):
        STRINGIFY_PATCH_TARGET = f"sphinx.ext.autodoc.{target}"
        break

# If we didn't locate both patch targets, we will just do nothing.
OKAY_TO_PATCH = bool(_parse_annotation and STRINGIFY_PATCH_TARGET)

# A label we inject to the type string so we know not to try to treat it as a
# type annotation
TYPE_IS_RST_LABEL = "--is-rst--"


orig_add_directive_header = AttributeDocumenter.add_directive_header
orig_handle_signature = PyAttribute.handle_signature


def _stringify_annotation(app: Sphinx, annotation: Any, mode: str = "") -> str:  # noqa: ARG001
    # Format the annotation with sphinx-autodoc-typehints and inject our magic prefix to tell our patched
    # PyAttribute.handle_signature to treat it as rst.
    from . import format_annotation

    return TYPE_IS_RST_LABEL + format_annotation(annotation, app.config)


def patch_attribute_documenter(app: Sphinx) -> None:
    """Instead of using stringify_typehint in `AttributeDocumenter.add_directive_header`, use `format_annotation`."""

    def add_directive_header(*args: Any, **kwargs: Any) -> Any:
        with patch(STRINGIFY_PATCH_TARGET, partial(_stringify_annotation, app)):
            return orig_add_directive_header(*args, **kwargs)

    AttributeDocumenter.add_directive_header = add_directive_header  # type:ignore[method-assign]


def rst_to_docutils(settings: Values, rst: str) -> Any:
    """Convert rst to a sequence of docutils nodes."""
    doc = new_document("", settings)
    RstParser().parse(rst, doc)
    # Remove top level paragraph node so that there is no line break.
    return doc.children[0].children


def patched_parse_annotation(settings: Values, typ: str, env: Any) -> Any:
    # if typ doesn't start with our label, use original function
    if not typ.startswith(TYPE_IS_RST_LABEL):
        assert _parse_annotation is not None  # noqa: S101
        return _parse_annotation(typ, env)
    # Otherwise handle as rst
    typ = typ[len(TYPE_IS_RST_LABEL) :]
    return rst_to_docutils(settings, typ)


def patched_handle_signature(self: PyAttribute, sig: str, signode: desc_signature) -> tuple[str, str]:
    target = "sphinx.domains.python._parse_annotation"
    new_func = partial(patched_parse_annotation, self.state.document.settings)
    with patch(target, new_func):
        return orig_handle_signature(self, sig, signode)


def patch_attribute_handling(app: Sphinx) -> None:
    """Use format_signature to format class attribute type annotations."""
    if not OKAY_TO_PATCH:
        return
    PyAttribute.handle_signature = patched_handle_signature  # type:ignore[method-assign]
    patch_attribute_documenter(app)


__all__ = ["patch_attribute_handling"]

env_pyrecodes/lib/python3.9/site-packages/sphinx_autodoc_typehints/attributes_patch.py
NikolaBlagojevic/pyrecodes
"""Custom patches to make the world work."""
from __future__ import annotations

from functools import lru_cache
from typing import TYPE_CHECKING, Any

from docutils.parsers.rst.directives.admonitions import BaseAdmonition
from docutils.parsers.rst.states import Text
from sphinx.ext.napoleon.docstring import GoogleDocstring

from .attributes_patch import patch_attribute_handling

if TYPE_CHECKING:
    from sphinx.application import Sphinx
    from sphinx.ext.autodoc import Options


@lru_cache  # A cute way to make sure the function only runs once.
def fix_autodoc_typehints_for_overloaded_methods() -> None:
    """
    sphinx-autodoc-typehints responds to the "autodoc-process-signature" event to remove types from the signature line.

    Normally, `FunctionDocumenter.format_signature` and `MethodDocumenter.format_signature` call
    `super().format_signature` which ends up going to `Documenter.format_signature`, and this last method emits the
    `autodoc-process-signature` event. However, if there are overloads, `FunctionDocumenter.format_signature` does
    something else and the event never occurs. Here we remove this alternative code path by brute force.

    See https://github.com/tox-dev/sphinx-autodoc-typehints/issues/296
    """
    from sphinx.ext.autodoc import FunctionDocumenter, MethodDocumenter

    del FunctionDocumenter.format_signature
    del MethodDocumenter.format_signature


def napoleon_numpy_docstring_return_type_processor(  # noqa: PLR0913
    app: Sphinx,
    what: str,
    name: str,  # noqa: ARG001
    obj: Any,  # noqa: ARG001
    options: Options | None,  # noqa: ARG001
    lines: list[str],
) -> None:
    """Insert a : under Returns: to tell napoleon not to look for a return type."""
    if what not in ["function", "method"]:
        return
    if not getattr(app.config, "napoleon_numpy_docstring", False):
        return

    # Search for the returns header:
    # Returns:
    # --------
    for pos, line in enumerate(lines[:-2]):
        if line.lower().strip(":") not in ["return", "returns"]:
            continue
        # Underline detection.
        chars = set(lines[pos + 1].strip())
        # Napoleon allows the underline to consist of a bunch of weirder things...
        if len(chars) != 1 or next(iter(chars)) not in "=-~_*+#":
            continue
        pos = pos + 2  # noqa: PLW2901
        break
    else:
        return

    lines.insert(pos, ":")


def fix_napoleon_numpy_docstring_return_type(app: Sphinx) -> None:
    """If no return type is explicitly set, numpy docstrings will use the return type text as return types."""
    # standard priority is 500. Setting priority to 499 ensures this runs before
    # napoleon's docstring processor.
    app.connect("autodoc-process-docstring", napoleon_numpy_docstring_return_type_processor, priority=499)


def _patched_lookup_annotation(*_args: Any) -> str:
    """
    GoogleDocstring._lookup_annotation sometimes adds incorrect type annotations to constructor parameters.

    Disable it so we can handle this on our own.
    """
    return ""


def _patch_google_docstring_lookup_annotation() -> None:
    """Fix issue https://github.com/tox-dev/sphinx-autodoc-typehints/issues/308."""
    GoogleDocstring._lookup_annotation = _patched_lookup_annotation  # type: ignore[assignment] # noqa: SLF001


orig_base_admonition_run = BaseAdmonition.run


def _patched_base_admonition_run(self: BaseAdmonition) -> Any:
    result = orig_base_admonition_run(self)
    result[0].line = self.lineno
    return result


orig_text_indent = Text.indent


def _patched_text_indent(self: Text, *args: Any) -> Any:
    _, line = self.state_machine.get_source_and_line()
    result = orig_text_indent(self, *args)
    node = self.parent[-1]
    if node.tagname == "system_message":
        node = self.parent[-2]
    node.line = line
    return result


def _patch_line_numbers() -> None:
    """
    Make the rst parser put line numbers on more nodes.

    When the line numbers are missing, we have a hard time placing the :rtype:.
    """
    Text.indent = _patched_text_indent
    BaseAdmonition.run = _patched_base_admonition_run


def install_patches(app: Sphinx) -> None:
    """
    Install the patches.

    :param app: the Sphinx app
    """
    fix_autodoc_typehints_for_overloaded_methods()
    patch_attribute_handling(app)
    _patch_google_docstring_lookup_annotation()
    fix_napoleon_numpy_docstring_return_type(app)
    _patch_line_numbers()


___all__ = [
    "install_patches",
]

env_pyrecodes/lib/python3.9/site-packages/sphinx_autodoc_typehints/patches.py
NikolaBlagojevic/pyrecodes
"""A modern skeleton for Sphinx themes."""

__version__ = "1.0.0.beta1"

from pathlib import Path
from typing import Any, Dict

from sphinx.application import Sphinx

_THEME_PATH = (Path(__file__).parent / "theme" / "basic-ng").resolve()


def setup(app: Sphinx) -> Dict[str, Any]:
    """Entry point for sphinx theming."""
    app.require_sphinx("4.0")

    app.add_html_theme("basic-ng", str(_THEME_PATH))

    return {
        "parallel_read_safe": True,
        "parallel_write_safe": True,
        "version": __version__,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx_basic_ng/__init__.py
NikolaBlagojevic/pyrecodes
"""Sphinx Carousel."""
__author__ = "Robpol86"
__license__ = "BSD-2-Clause"
__version__ = "1.2.0"

env_pyrecodes/lib/python3.9/site-packages/sphinx_carousel/__init__.py
NikolaBlagojevic/pyrecodes
"""A Sphinx extension for creating slideshows using Bootstrap Carousels.

https://sphinx-carousel.readthedocs.io
https://github.com/Robpol86/sphinx-carousel
https://pypi.org/project/sphinx-carousel
"""
from pathlib import Path
from typing import Dict

from docutils.nodes import document
from sphinx.application import Sphinx
from sphinx.util.fileutil import copy_asset_file

from sphinx_carousel import __version__, nodes
from sphinx_carousel.directives import Carousel

CONF_DEFAULT_BOOTSTRAP_ADD_CSS_JS = True
CONF_DEFAULT_BOOTSTRAP_PREFIX = "scbs-"
CONF_DEFAULT_SHOW_BUTTONS_ON_TOP = False
CONF_DEFAULT_SHOW_CAPTIONS_BELOW = False
CONF_DEFAULT_SHOW_CONTROLS = False
CONF_DEFAULT_SHOW_DARK = False
CONF_DEFAULT_SHOW_FADE = False
CONF_DEFAULT_SHOW_INDICATORS = False
CONF_DEFAULT_SHOW_SHADOWS = False


def copy_static(app: Sphinx):
    """Install CSS and JS files into the output directory.

    :param app: Sphinx application object.
    """
    if app.builder.format != "html":
        return

    static_in = Path(__file__).parent / "_static"
    static_out = Path(app.builder.outdir) / "_static"
    static_out.mkdir(exist_ok=True)

    if app.config.carousel_bootstrap_add_css_js:
        copy_asset_file(str(static_in / "bootstrap-carousel.min.css"), str(static_out))
        copy_asset_file(str(static_in / "bootstrap-carousel.min.js"), str(static_out))

    context = {
        "bootstrap_prefix": app.config["carousel_bootstrap_prefix"],
    }
    copy_asset_file(str(static_in / "carousel-custom.css_t"), str(static_out), context)


def include_static_on_demand(app: Sphinx, _: str, __: str, ___: dict, doctree: document):
    """Add CSS and JS files to <head /> only on specific pages that use the directive.

    :param app: Sphinx application object.
    :param _: Unused.
    :param __: Unused.
    :param ___: Unused.
    :param doctree: Tree of docutils nodes.
    """
    if not doctree or not doctree.traverse(nodes.CarouselMainNode):
        return

    if app.config.carousel_bootstrap_add_css_js:
        app.add_css_file("bootstrap-carousel.min.css")
        app.add_js_file("bootstrap-carousel.min.js")

    app.add_css_file("carousel-custom.css")


def setup(app: Sphinx) -> Dict[str, str]:
    """Called by Sphinx during phase 0 (initialization).

    :param app: Sphinx application object.

    :returns: Extension version.
    """
    app.add_config_value("carousel_bootstrap_add_css_js", CONF_DEFAULT_BOOTSTRAP_ADD_CSS_JS, "html")
    app.add_config_value("carousel_bootstrap_prefix", CONF_DEFAULT_BOOTSTRAP_PREFIX, "html")
    app.add_config_value("carousel_show_buttons_on_top", CONF_DEFAULT_SHOW_BUTTONS_ON_TOP, "html")
    app.add_config_value("carousel_show_captions_below", CONF_DEFAULT_SHOW_CAPTIONS_BELOW, "html")
    app.add_config_value("carousel_show_controls", CONF_DEFAULT_SHOW_CONTROLS, "html")
    app.add_config_value("carousel_show_dark", CONF_DEFAULT_SHOW_DARK, "html")
    app.add_config_value("carousel_show_fade", CONF_DEFAULT_SHOW_FADE, "html")
    app.add_config_value("carousel_show_indicators", CONF_DEFAULT_SHOW_INDICATORS, "html")
    app.add_config_value("carousel_show_shadows", CONF_DEFAULT_SHOW_SHADOWS, "html")
    app.add_directive("carousel", Carousel)
    app.connect("builder-inited", copy_static)
    app.connect("html-page-context", include_static_on_demand)
    nodes.CarouselCaptionNode.add_node(app)
    nodes.CarouselControlNode.add_node(app)
    nodes.CarouselIndicatorsNode.add_node(app)
    nodes.CarouselInnerNode.add_node(app)
    nodes.CarouselItemNode.add_node(app)
    nodes.CarouselMainNode.add_node(app)
    return dict(version=__version__)

env_pyrecodes/lib/python3.9/site-packages/sphinx_carousel/carousel.py
NikolaBlagojevic/pyrecodes
"""Sphinx directives."""
from typing import List, Optional, Tuple

from docutils.nodes import caption, Element, image as docutils_image, legend, reference
from docutils.parsers.rst.directives import flag, unchanged
from sphinx.util import logging as sphinx_logging
from sphinx.util.docutils import SphinxDirective

from sphinx_carousel import nodes

ImageTuple = Tuple[docutils_image, Optional[reference], str, str]


class Carousel(SphinxDirective):
    """Sphinx directive implementing `.. carousel::` in documents."""

    has_content = True
    option_spec = {
        # Div data attributes.
        "data-bs-interval": unchanged,
        "data-bs-keyboard": unchanged,
        "data-bs-pause": unchanged,
        "data-bs-ride": unchanged,
        "data-bs-wrap": unchanged,
        "data-bs-touch": unchanged,
        # Crossfade/dark.
        "no_fade": flag,
        "show_fade": flag,
        "no_dark": flag,
        "show_dark": flag,
        # Controls/indicators.
        "no_buttons_on_top": flag,
        "show_buttons_on_top": flag,
        "no_controls": flag,
        "show_controls": flag,
        "no_indicators": flag,
        "show_indicators": flag,
        "no_shadows": flag,
        "show_shadows": flag,
        # Captions.
        "no_captions_below": flag,
        "show_captions_below": flag,
    }

    def images(self) -> List[ImageTuple]:
        """Return list of image/figure nodes along with other associated data.

        :return: Image node, parent reference node if :target: was specified, and figure's title/description if available.
        """
        # Create temporary empty document to dump nested image/figure directives into.
        directive_content = Element()
        directive_content.document = self.state.document
        self.state.nested_parse(self.content, self.content_offset, directive_content)

        images = []
        for image in directive_content.traverse(docutils_image):
            # Handle linked images.
            linked_image = image.parent if image.parent.hasattr("refuri") else None
            # Handle captions.
            node = (linked_image or image).next_node(caption, siblings=True, ascend=False, descend=False)
            title = node.astext() if node else ""
            node = (linked_image or image).next_node(legend, siblings=True, ascend=False, descend=False)
            description = node.astext() if node else ""
            # Done with image.
            images.append((image, linked_image, title, description))

        return images

    def config_read_flag(self, name: str) -> bool:
        """Evaluate a directive flag option and the corresponding Sphinx conf.py entry as fallback.

        :param name: Suffix.
        """
        if f"show_{name}" in self.options:
            return True
        if f"no_{name}" in self.options:
            return False
        if self.config[f"carousel_show_{name}"] is True:
            return True
        return False

    def create_inner_node(self, images: List[ImageTuple]) -> Element:
        """Return carousel-inner div node along with child nodes such as images and captions.

        :param images: Output of self.images().
        """
        prefix = self.config["carousel_bootstrap_prefix"]
        captions_below = self.config_read_flag("captions_below")
        items = []

        for idx, (image, linked_image, title, description) in enumerate(images):
            image["classes"] += [f"{prefix}d-block", f"{prefix}w-100"]
            child_nodes = [linked_image or image]
            if title or description:
                child_nodes.append(nodes.CarouselCaptionNode(title=title, description=description, below=captions_below))
            items.append(nodes.CarouselItemNode("", *child_nodes, active=idx == 0))

        return nodes.CarouselInnerNode("", *items)

    def run(self) -> List[Element]:
        """Main method."""
        images = self.images()
        if not images:
            log = sphinx_logging.getLogger(__name__)
            log.warning("No images specified in carousel.", location=(self.env.docname, self.lineno))
            return []

        main_div = nodes.CarouselMainNode(
            div_id=f"carousel-{self.env.new_serialno('carousel')}",
            prefix=self.config["carousel_bootstrap_prefix"],
            attributes={k: v for k, v in self.options.items() if k.startswith("data-")},
            fade=self.config_read_flag("fade"),
            dark=self.config_read_flag("dark"),
        )
        buttons_on_top = self.config_read_flag("buttons_on_top")
        shadows = self.config_read_flag("shadows")

        # Build indicators.
        if self.config_read_flag("indicators"):
            main_div.append(nodes.CarouselIndicatorsNode(count=len(images), top=buttons_on_top, shadow=shadows))

        # Build carousel-inner div.
        main_div.append(self.create_inner_node(images))

        # Build control buttons.
        if self.config_read_flag("controls"):
            main_div.append(nodes.CarouselControlNode(top=buttons_on_top, shadow=shadows, prev=True))
            main_div.append(nodes.CarouselControlNode(top=buttons_on_top, shadow=shadows))

        return [main_div]

env_pyrecodes/lib/python3.9/site-packages/sphinx_carousel/directives.py
NikolaBlagojevic/pyrecodes
"""Docutils nodes."""
# pylint: disable=keyword-arg-before-vararg
from abc import ABCMeta, abstractmethod
from typing import Dict, List, Optional, Union

from docutils import nodes
from sphinx.application import Sphinx
from sphinx.writers.html5 import HTML5Translator


class BaseNode(nodes.Element, nodes.General):
    """Base node class."""

    CLASSES = []

    def classes(self, *classes_in: Optional[List[str]], join: bool = True) -> Union[str, List[str]]:
        """Return prefixed classes.

        1. If a class starts with "*" then it will not be prefixed and the leading character will be removed.
        2. If no input classes are provided then this defaults to self.CLASSES.
        3. Multiple groups of classes will be flattened into one list.

        :param join: Return a space delimited joined string instead of list of strings.
        """
        prefix = getattr(self, "prefix", "")
        flattened = [
            f"{c[1:]}" if c.startswith("*") else f"{prefix}{c}" for g in (classes_in or [self.CLASSES]) if g for c in g
        ]
        return " ".join(flattened) if join else flattened

    @staticmethod
    @abstractmethod
    def html_visit(writer: HTML5Translator, node: "BaseNode"):
        """Append opening tags to document body list."""
        raise NotImplementedError

    @staticmethod
    @abstractmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        raise NotImplementedError

    @classmethod
    def add_node(cls, app: Sphinx):
        """Convenience method that adds the subclass node to the Sphinx app.."""
        app.add_node(
            cls,
            html=(cls.html_visit, cls.html_depart),
            latex=(lambda *_: None, lambda *_: None),  # TODO https://github.com/Robpol86/sphinx-carousel/issues/50
            man=(lambda *_: None, lambda *_: None),  # TODO https://github.com/Robpol86/sphinx-carousel/issues/51
            texinfo=(lambda *_: None, lambda *_: None),  # TODO https://github.com/Robpol86/sphinx-carousel/issues/51
            text=(lambda *_: None, lambda *_: None),  # TODO https://github.com/Robpol86/sphinx-carousel/issues/51
        )


class CarouselMainNode(BaseNode):
    """Main div."""

    CLASSES = ["carousel", "slide"]
    CLASSES_DARK = ["carousel-dark"]
    CLASSES_FADE = ["carousel-fade"]

    def __init__(
        self,
        *args,
        div_id: str = "",
        prefix: str = "",
        attributes: Dict[str, str] = None,
        fade: bool = False,
        dark: bool = False,
        **kwargs,
    ):
        """Constructor.

        :param args: Passed to parent class.
        :param div_id: <div id="...">.
        :param prefix: Bootstrap class' prefix.
        :param attributes: Div attributes (e.g. {"data-bs-ride": "carousel", ...}.
        :param fade: Fade between images instead of using a slide transition.
        :param dark: Carousel dark variant.
        :param kwargs: Passed to parent class.
        """
        super().__init__(*args, **kwargs)
        self.div_id = div_id
        self.prefix = prefix
        self.attributes = attributes
        self.fade = fade
        self.dark = dark

    @staticmethod
    def html_visit(writer: HTML5Translator, node: "CarouselMainNode"):
        """Append opening tags to document body list."""
        writer.body.append(
            writer.starttag(
                node,
                "div",
                CLASS=node.classes(
                    node.CLASSES,
                    node.CLASSES_FADE if node.fade else [],
                    node.CLASSES_DARK if node.dark else [],
                ),
                ids=[node.div_id],
                **node.attributes,
            )
        )

    @staticmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        writer.body.append("</div>\n")


class SubNode(BaseNode, metaclass=ABCMeta):
    """Base class for child nodes under main node."""

    @property
    def main_node(self) -> CarouselMainNode:
        """Return the main carousel node instance."""
        return self.parent  # noqa

    @property
    def prefix(self) -> str:
        """Return Bootstrap class' prefix."""
        return self.main_node.prefix


class CarouselInnerNode(SubNode):
    """Secondary div that contains the image divs."""

    CLASSES = ["carousel-inner"]

    @staticmethod
    def html_visit(writer: HTML5Translator, node: "CarouselInnerNode"):
        """Append opening tags to document body list."""
        writer.body.append(writer.starttag(node, "div", CLASS=node.classes()))

    @staticmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        writer.body.append("</div>\n")


class CarouselItemNode(SubNode):
    """Div that contains an image."""

    CLASSES = ["carousel-item"]
    CLASSES_ACTIVE = ["active"]

    def __init__(self, *args, active: bool = False, **kwargs):
        """Constructor.

        :param args: Passed to parent class.
        :param active: Append active class to div, needed for first image.
        :param kwargs: Passed to parent class.
        """
        super().__init__(*args, **kwargs)
        self.active = active

    @property
    def main_node(self) -> CarouselMainNode:
        """Return the main carousel node instance."""
        return self.parent.parent  # noqa

    @staticmethod
    def html_visit(writer: HTML5Translator, node: "CarouselItemNode"):
        """Append opening tags to document body list."""
        writer.body.append(
            writer.starttag(
                node,
                "div",
                CLASS=node.classes(node.CLASSES, node.CLASSES_ACTIVE if node.active else []),
            )
        )

    @staticmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        writer.body.append("</div>\n")


class CarouselControlNode(SubNode):
    """Previous/next buttons."""

    CLASSES_NEXT = ["carousel-control-next"]
    CLASSES_NEXT_ICON = ["carousel-control-next-icon"]
    CLASSES_PREV = ["carousel-control-prev"]
    CLASSES_PREV_ICON = ["carousel-control-prev-icon"]
    CLASSES_TOP = ["my-4", "*scc-top-control"]
    CLASSES_SHADOW = ["*scc-shadow-control"]
    CLASSES_HIDDEN = ["visually-hidden"]

    def __init__(self, *args, prev: bool = False, top: bool = False, shadow: bool = False, **kwargs):
        """Constructor.

        :param args: Passed to parent class.
        :param prev: Previous button if True, else Next button.
        :param top: Display controls at the top of the image instead of the middle.
        :param shadow: Show a shadow around the icons for better visibility when an image is a similar color.
        :param kwargs: Passed to parent class.
        """
        super().__init__(*args, **kwargs)
        self.prev = prev
        self.top = top
        self.shadow = shadow

    @staticmethod
    def html_visit(writer: HTML5Translator, node: "CarouselControlNode"):
        """Append opening tags to document body list."""
        writer.body.append(
            writer.starttag(
                node,
                "button",
                CLASS=node.classes(
                    node.CLASSES_PREV if node.prev else node.CLASSES_NEXT,
                    node.CLASSES_TOP if node.top else [],
                    node.CLASSES_SHADOW if node.shadow else [],
                ),
                type="button",
                **{"data-bs-target": f"#{node.main_node.div_id}", "data-bs-slide": "prev" if node.prev else "next"},
            )
        )

        # Nested icon.
        writer.body.append(
            writer.starttag(
                node,
                "span",
                "",
                CLASS=node.classes(node.CLASSES_PREV_ICON if node.prev else node.CLASSES_NEXT_ICON),
                **{"aria-hidden": "true"},
            )
        )
        writer.body.append("</span>\n")

        # Nested hidden text for screen readers.
        writer.body.append(writer.starttag(node, "span", "", CLASS=node.classes(node.CLASSES_HIDDEN)))
        writer.body.append("Previous" if node.prev else "Next")
        writer.body.append("</span>\n")

    @staticmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        writer.body.append("</button>\n")


class CarouselIndicatorsNode(SubNode):
    """Indicators."""

    CLASSES = ["carousel-indicators"]
    CLASSES_ACTIVE = ["active"]
    CLASSES_TOP = ["my-4", "*scc-top-indicator"]
    CLASSES_SHADOW = ["*scc-shadow-indicator"]

    def __init__(self, *args, count: int = 0, top: bool = False, shadow: bool = False, **kwargs):
        """Constructor.

        :param args: Passed to parent class.
        :param count: Number of images.
        :param top: Display indicators at the top of the image instead of the middle.
        :param shadow: Show a shadow around the icons for better visibility when an image is a similar color.
        :param kwargs: Passed to parent class.
        """
        super().__init__(*args, **kwargs)
        self.count = count
        self.top = top
        self.shadow = shadow

    @staticmethod
    def html_visit(writer: HTML5Translator, node: "CarouselIndicatorsNode"):
        """Append opening tags to document body list."""
        writer.body.append(
            writer.starttag(node, "div", CLASS=node.classes(node.CLASSES, node.CLASSES_TOP if node.top else []))
        )

        # Add indicator buttons.
        for i in range(node.count):
            attributes = {
                "data-bs-target": f"#{node.main_node.div_id}",
                "data-bs-slide-to": f"{i}",
                "aria-label": f"Slide {i+1}",
            }
            if i == 0:
                attributes["aria-current"] = "true"
            writer.body.append(
                writer.starttag(
                    node,
                    "button",
                    "",
                    type="button",
                    CLASS=node.classes(
                        node.CLASSES_ACTIVE if i == 0 else [],
                        node.CLASSES_SHADOW if node.shadow else [],
                    ),
                    **attributes,
                )
            )
            writer.body.append("</button>\n")

    @staticmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        writer.body.append("</div>\n")


class CarouselCaptionNode(SubNode):
    """Captions."""

    CLASSES = ["carousel-caption"]
    CLASSES_BG_DARK = ["bg-dark"]
    CLASSES_BG_LIGHT = ["bg-light"]
    CLASSES_BELOW = ["d-sm-block", "*scc-below-control"]
    CLASSES_NOT_BELOW = ["d-none", "d-md-block"]

    def __init__(self, *args, title: str = "", description: str = "", below: bool = False, **kwargs):
        """Constructor.

        :param args: Passed to parent class.
        :param title: Caption heading.
        :param description: Caption paragraph.
        :param below: Display caption below image instead of overlayed on top.
        :param kwargs: Passed to parent class.
        """
        super().__init__(*args, **kwargs)
        self.title = title
        self.description = description
        self.below = below

    @property
    def main_node(self) -> CarouselMainNode:
        """Return the main carousel node instance."""
        return self.parent.parent.parent  # noqa

    @staticmethod
    def html_visit(writer: HTML5Translator, node: "CarouselCaptionNode"):
        """Append opening tags to document body list."""
        writer.body.append(
            writer.starttag(
                node,
                "div",
                CLASS=node.classes(
                    node.CLASSES,
                    node.CLASSES_BG_LIGHT if node.below and node.main_node.dark else [],
                    node.CLASSES_BG_DARK if node.below and not node.main_node.dark else [],
                    node.CLASSES_BELOW if node.below else node.CLASSES_NOT_BELOW,
                ),
            )
        )

        if node.title:
            writer.body.append(writer.starttag(node, "h5", ""))
            writer.body.append(node.title)
            writer.body.append("</h5>\n")

        if node.description:
            writer.body.append(writer.starttag(node, "p"))
            writer.body.append(node.description)
            writer.body.append("\n</p>\n")

    @staticmethod
    def html_depart(writer: HTML5Translator, _):
        """Append closing tags to document body list."""
        writer.body.append("</div>\n")

env_pyrecodes/lib/python3.9/site-packages/sphinx_carousel/nodes.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
"""
Patches Jinja2 v3 to restore compatibility with earlier Sphinx versions.
"""
#
#  Copyright © 2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import os
import sys
from typing import List

# this package
from sphinx_jinja2_compat._installers import install_jinja2, install_markupsafe

__all__: List[str] = []

__author__: str = "Dominic Davis-Foster"
__copyright__: str = "2022 Dominic Davis-Foster"
__license__: str = "MIT License"
__version__: str = "0.2.0"
__email__: str = "dominic@davis-foster.co.uk"

if "NO_SPHINX_JINJA2_COMPAT" not in os.environ:

	if sys.version_info >= (3, 10):
		# stdlib
		import types
		types.Union = types.UnionType

	try:

		# 3rd party
		import markupsafe

		install_markupsafe(markupsafe)

		# 3rd party
		import jinja2
		import jinja2.filters
		import jinja2.utils

		install_jinja2(jinja2, jinja2.filters, jinja2.utils)

	except ImportError:
		# Unable to import one module
		# Perhaps they are in global site-packages and we're not,
		# so they aren't available yet?

		# this package
		from sphinx_jinja2_compat._meta_path import _Finder
		sys.meta_path.insert(0, _Finder())

env_pyrecodes/lib/python3.9/site-packages/sphinx_jinja2_compat/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  _installers.py
"""
Functions to install the various patches.
"""
#
#  Copyright © 2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from types import ModuleType
from typing import Any, Callable, TypeVar

__all__ = ["install_jinja2", "install_markupsafe"]

F = TypeVar('F', bound=Callable[..., Any])


def install_markupsafe(markupsafe: ModuleType) -> None:
	"""
	Install the markupsafe patch.

	:param markupsafe: The ``markupsafe`` module.
	"""

	if not hasattr(markupsafe, "soft_unicode"):

		def soft_unicode(s: Any) -> str:
			return markupsafe.soft_str(s)

		markupsafe.soft_unicode = soft_unicode  # type: ignore[attr-defined]


def install_jinja2(jinja2: ModuleType, filters: ModuleType, utils: ModuleType) -> None:
	"""
	Install the jinja2 patch.

	:param jinja2: The ``jinja2`` module.
	:param filters: The ``jinja2.filters`` module.
	:param utils: The ``jinja2.utils`` module.
	"""

	if not hasattr(filters, "environmentfilter"):

		def environmentfilter(f: F) -> F:
			return utils.pass_environment(f)

		filters.environmentfilter = environmentfilter  # type: ignore[attr-defined]
		jinja2.environmentfilter = environmentfilter  # type: ignore[attr-defined]

	if not hasattr(utils, "contextfunction"):

		def contextfunction(f: F) -> F:
			return utils.pass_context(f)

		utils.contextfunction = contextfunction  # type: ignore[attr-defined]
		jinja2.contextfunction = contextfunction  # type: ignore[attr-defined]

env_pyrecodes/lib/python3.9/site-packages/sphinx_jinja2_compat/_installers.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  _meta_path.py
"""
Meta path finder and loader to install patched at import time.
"""
#
#  Copyright © 2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import importlib.abc
import importlib.util
import sys

# this package
from sphinx_jinja2_compat._installers import install_jinja2, install_markupsafe


class _Finder(importlib.abc.MetaPathFinder):

	def __init__(self):
		# Stack stops us going round in circles
		self._stack = []

	def find_spec(self, fullname, path, target=None):
		if fullname in self._stack:
			return None

		if fullname == "markupsafe":

			if self in sys.meta_path:

				self._stack.append("markupsafe")
				markupsafe = importlib.import_module("markupsafe")
				self._stack.pop()

				# This is necessary to ensure the patched version gets loaded
				del sys.modules["markupsafe"]

				class _MarkupsafeLoader(importlib.abc.Loader):

					def create_module(self, spec):
						install_markupsafe(markupsafe)
						return markupsafe

					def exec_module(self, module):
						pass

				return importlib.util.spec_from_loader(
						"markupsafe",
						_MarkupsafeLoader(),
						origin=markupsafe.__file__,
						)

		elif fullname == "jinja2":

			if self in sys.meta_path:

				self._stack.append("jinja2")
				jinja2 = importlib.import_module("jinja2")
				filters = importlib.import_module("jinja2.filters")
				utils = importlib.import_module("jinja2.utils")
				self._stack.pop()

				# This is necessary to ensure the patched version gets loaded
				del sys.modules["jinja2"]

				class _Jinja2Loader(importlib.abc.Loader):

					def create_module(self, spec):
						install_jinja2(jinja2, filters, utils)
						return jinja2

					def exec_module(self, module):
						pass

				return importlib.util.spec_from_loader("jinja2", _Jinja2Loader(), origin=jinja2.__file__)

env_pyrecodes/lib/python3.9/site-packages/sphinx_jinja2_compat/_meta_path.py
NikolaBlagojevic/pyrecodes
"""A small sphinx extension to add "toggle" buttons to items."""
import os
from docutils.parsers.rst import Directive, directives
from docutils import nodes

__version__ = "0.3.2"


def st_static_path(app):
    static_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "_static"))
    app.config.html_static_path.append(static_path)


def initialize_js_assets(app, config):
    # Update the global context
    app.add_js_file(None, body=f"let toggleHintShow = '{config.togglebutton_hint}';")
    app.add_js_file(None, body=f"let toggleHintHide = '{config.togglebutton_hint_hide}';")
    open_print = str(config.togglebutton_open_on_print).lower()
    app.add_js_file(None, body=f"let toggleOpenOnPrint = '{open_print}';")
    app.add_js_file("togglebutton.js")


# This function reads in a variable and inserts it into JavaScript
def insert_custom_selection_config(app):
    # This is a configuration that you've specified for users in `conf.py`
    selector = app.config["togglebutton_selector"]
    js_text = "var togglebuttonSelector = '%s';" % selector
    app.add_js_file(None, body=js_text)


class Toggle(Directive):
    """Hide a block of markup text by wrapping it in a container."""

    optional_arguments = 1
    final_argument_whitespace = True
    has_content = True

    option_spec = {"id": directives.unchanged, "show": directives.flag}

    def run(self):
        self.assert_has_content()
        classes = ["toggle"]
        if "show" in self.options:
            classes.append("toggle-shown")

        parent = nodes.container(classes=classes)
        self.state.nested_parse(self.content, self.content_offset, parent)
        return [parent]


# We connect this function to the step after the builder is initialized
def setup(app):
    # Add our static path
    app.connect("builder-inited", st_static_path)

    # Add relevant code to headers
    app.add_css_file("togglebutton.css")

    # Add the string we'll use to select items in the JS
    # Tell Sphinx about this configuration variable
    app.add_config_value("togglebutton_selector", ".toggle, .admonition.dropdown", "html")
    app.add_config_value("togglebutton_hint", "Click to show", "html")
    app.add_config_value("togglebutton_hint_hide", "Click to hide", "html")
    app.add_config_value("togglebutton_open_on_print", True, "html")

    # Run the function after the builder is initialized
    app.connect("builder-inited", insert_custom_selection_config)
    app.connect("config-inited", initialize_js_assets)
    app.add_directive("toggle", Toggle)
    return {
        "version": __version__,
        "parallel_read_safe": True,
        "parallel_write_safe": True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinx_togglebutton/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
"""
Box of handy tools for Sphinx.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import sys

# This all has to be up here so it's triggered first.
if sys.version_info >= (3, 10):
	# stdlib
	import types
	types.Union = types.UnionType

# 3rd party
from sphinx.application import Sphinx

# this package
from sphinx_toolbox import (  # noqa: F401
		assets,
		code,
		config,
		confval,
		installation,
		issues,
		rest_example,
		shields,
		source,
		utils,
		wikipedia
		)
from sphinx_toolbox.cache import cache  # noqa: F401

__author__: str = "Dominic Davis-Foster"
__copyright__: str = "2020 Dominic Davis-Foster"
__license__: str = "MIT License"
__version__: str = "3.5.0"
__email__: str = "dominic@davis-foster.co.uk"

__all__ = ("setup", )


def setup(app: Sphinx) -> "utils.SphinxExtMetadata":
	"""
	Setup :mod:`sphinx_toolbox`.

	:param app: The Sphinx application.
	"""

	# Ensure dependencies are set up
	app.setup_extension("sphinx.ext.viewcode")
	app.setup_extension("sphinx_toolbox.github")

	app.connect("config-inited", config.validate_config, priority=850)

	# Setup standalone extensions
	app.setup_extension("sphinx_toolbox.assets")
	app.setup_extension("sphinx_toolbox.changeset")
	app.setup_extension("sphinx_toolbox.code")
	app.setup_extension("sphinx_toolbox.collapse")
	app.setup_extension("sphinx_toolbox.confval")
	app.setup_extension("sphinx_toolbox.decorators")
	app.setup_extension("sphinx_toolbox.formatting")
	app.setup_extension("sphinx_toolbox.installation")
	app.setup_extension("sphinx_toolbox.issues")
	app.setup_extension("sphinx_toolbox.latex")
	app.setup_extension("sphinx_toolbox.rest_example")
	app.setup_extension("sphinx_toolbox.shields")
	app.setup_extension("sphinx_toolbox.sidebar_links")
	app.setup_extension("sphinx_toolbox.source")
	app.setup_extension("sphinx_toolbox.wikipedia")
	app.setup_extension("sphinx_toolbox.more_autodoc.autoprotocol")
	app.setup_extension("sphinx_toolbox.more_autodoc.autotypeddict")
	app.setup_extension("sphinx_toolbox.more_autodoc.autonamedtuple")

	# Hack to get the docutils tab size, as there doesn't appear to be any other way
	app.setup_extension("sphinx_toolbox.tweaks.tabsize")

	return {
			"version": __version__,
			"parallel_read_safe": True,
			}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __main__.py
"""
CLI entry point for sphinx-toolbox.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import sys

# this package
from sphinx_toolbox import cache

__all__ = ("clear_cache", )


def clear_cache() -> int:
	"""
	Clear any cached URLs.
	"""

	if cache.clear():
		print("Cache cleared successfully.")
		return 0
	else:
		return 1


if __name__ == "__main__":
	sys.exit(clear_cache())

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/__main__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  _css.py
"""
Internal Sphinx extension to provide custom CSS.

.. versionadded:: 2.7.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import MutableMapping, Optional

# 3rd party
import dict2css
from domdf_python_tools.paths import PathPlus
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("copy_asset_files", "setup")

installation_styles: MutableMapping[str, dict2css.Style] = {
		'div[id*="installation"] .sphinx-tabs-tab': {"color": "#2980b9"},
		"button.sphinx-tabs-tab,div.sphinx-tabs-panel": {"outline": (None, dict2css.IMPORTANT)},
		}

shields_styles: MutableMapping[str, dict2css.Style] = {
		".table-wrapper td p img.sphinx_toolbox_shield": {"vertical-align": "middle"},
		}

regex_styles: MutableMapping[str, dict2css.Style] = {
		"span.regex_literal": {"color": "dimgrey"},
		"span.regex_at": {"color": "orangered"},
		"span.regex_repeat_brace": {"color": "orangered"},
		"span.regex_branch": {"color": "orangered"},
		"span.regex_subpattern": {"color": "dodgerblue"},
		"span.regex_in": {"color": "darkorange"},
		"span.regex_category": {"color": "darkseagreen"},
		"span.regex_repeat": {"color": "orangered"},
		"span.regex_any": {"color": "orangered"},
		"code.regex": {"font-size": "80%"},
		"span.regex": {"font-weight": "bold"},
		}

tweaks_sphinx_panels_tabs_styles: MutableMapping[str, dict2css.Style] = {
		".docutils.container": {
				"padding-left": (0, dict2css.IMPORTANT),
				"padding-right": (0, dict2css.IMPORTANT),
				},
		"div.ui.top.attached.tabular.menu.sphinx-menu.docutils.container": {
				"margin-left": (0, dict2css.IMPORTANT),
				"margin-right": (0, dict2css.IMPORTANT),
				},
		}


def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None:
	"""
	Copy additional stylesheets into the HTML build directory.

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.format.lower() != "html":  # pragma: no cover
		return

	extensions_selector = ", ".join([
			"p.sphinx-toolbox-extensions",
			"div.sphinx-toolbox-extensions.highlight-python",
			"div.sphinx-toolbox-extensions.highlight-python div.highlight",
			])

	rest_example_style = {
			"padding-left": "5px",
			"border-style": "dotted",
			"border-width": "1px",
			"border-color": "darkgray",
			}

	style: MutableMapping[str, dict2css.Style] = {
			"p.source-link": {"margin-bottom": 0},
			"p.source-link + hr.docutils": {"margin-top": "10px"},
			extensions_selector: {"margin-bottom": "10px"},
			"div.rest-example.docutils.container": rest_example_style,
			**installation_styles,
			**shields_styles,
			**regex_styles,
			}

	css_static_dir = PathPlus(app.outdir) / "_static" / "css"
	css_static_dir.maybe_make(parents=True)
	dict2css.dump(style, css_static_dir / "sphinx-toolbox.css")


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox._css`.

	:param app: The Sphinx application.
	"""

	app.add_css_file("css/sphinx-toolbox.css")
	app.connect("build-finished", copy_asset_files)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/_css.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  _data_documenter.py
"""
Relevant parts of :mod:`sphinx.ext.autodoc` from Sphinx 3.3.1.
"""
#
#  Based on Sphinx
#  Copyright (c) 2007-2020 by the Sphinx team.
#  All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import Any, Optional, get_type_hints

# 3rd party
from docutils.statemachine import StringList
from sphinx.ext.autodoc import (
		SUPPRESS,
		UNINITIALIZED_ATTR,
		ModuleDocumenter,
		ModuleLevelDocumenter,
		annotation_option
		)
from sphinx.util import logging
from sphinx.util.inspect import object_description, safe_getattr

try:
	# 3rd party
	from sphinx.util.typing import stringify_annotation as stringify_typehint  # type: ignore[attr-defined]
except ImportError:
	from sphinx.util.typing import stringify as stringify_typehint

# this package
from sphinx_toolbox.more_autodoc import _documenter_add_content

__all__ = ("DataDocumenter", )

logger = logging.getLogger(__name__)


class DataDocumenter(ModuleLevelDocumenter):
	"""
	Specialized Documenter subclass for data items.
	"""

	objtype = "data"
	member_order = 40
	priority = -10
	option_spec = dict(ModuleLevelDocumenter.option_spec)
	option_spec["annotation"] = annotation_option

	@classmethod
	def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any) -> bool:
		"""
		Called to see if a member can be documented by this documenter.
		"""

		return isinstance(parent, ModuleDocumenter) and isattr

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive header and options to the generated content.
		"""

		super().add_directive_header(sig)
		sourcename = self.get_sourcename()
		if not self.options.annotation:
			# obtain annotation for this data
			try:
				annotations = get_type_hints(self.parent)
			except NameError:
				# Failed to evaluate ForwardRef (maybe TYPE_CHECKING)
				annotations = safe_getattr(self.parent, "__annotations__", {})
			except TypeError:
				annotations = {}
			except KeyError:
				# a broken class found (refs: https://github.com/sphinx-doc/sphinx/issues/8084)
				annotations = {}
			except AttributeError:
				# AttributeError is raised on 3.5.2 (fixed by 3.5.3)
				annotations = {}

			if self.objpath[-1] in annotations:
				objrepr = stringify_typehint(annotations.get(self.objpath[-1]))
				self.add_line("   :type: " + objrepr, sourcename)
			else:
				key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
				if self.analyzer and key in self.analyzer.annotations:
					self.add_line("   :type: " + self.analyzer.annotations[key], sourcename)

			try:
				if self.object is UNINITIALIZED_ATTR:
					pass
				else:
					objrepr = object_description(self.object)
					self.add_line("   :value: " + objrepr, sourcename)
			except ValueError:
				pass
		elif self.options.annotation is SUPPRESS:
			pass
		else:
			self.add_line("   :annotation: %s" % self.options.annotation, sourcename)

	def document_members(self, all_members: bool = False) -> None:  # noqa: D102
		pass

	def get_real_modname(self) -> str:
		"""
		Get the real module name of an object to document.

		It can differ from the name of the module through which the object was imported.
		"""

		return self.get_attr(self.parent or self.object, "__module__", None) or self.modname

	def add_content(self, more_content: Optional[StringList], no_docstring: bool = False) -> None:
		"""
		Add content from docstrings, attribute documentation and user.
		"""

		_documenter_add_content(self, more_content, no_docstring)

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/_data_documenter.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  assets.py
"""
Role to provide a link to open a file within the web browser, rather than downloading it.

.. versionadded:: 0.5.0
.. extensions:: sphinx_toolbox.assets
.. TODO:: Handle non-HTML builders

	Perhaps just put a message to see the online documentation?

Usage
------

.. latex:vspace:: -10px

.. rst:role:: asset

	Adds a link to a local file that can be viewed within the web browser.
	The file will be copied from the directory set in :confval:`assets_dir` to ``/_assets`` in the HTML output.

	This is similar to the :rst:role:`download` role, but that role will download the file to the user's computer instead.
	This role may be useful for PDFs, which most web browsers can display.

	If the file can't be found an error will be shown in the build output:

	.. code-block:: text

		<page where the error occurred>: Asset file '<missing asset file name>' not found.


	**Asset**

	.. rest-example::

		:asset:`hello_world.txt`

		:asset:`hello_world <hello_world.txt>`

	**Download**

	.. rest-example::

		:download:`hello_world.txt <../assets/hello_world.txt>`


Configuration
---------------

.. confval:: assets_dir
	:type: :class:`str`
	:required: False
	:default: ``'./assets'``

	The directory in which to find assets for the :rst:role:`asset` role.


API Reference
---------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import pathlib
import posixpath
import shutil
from typing import Dict, List, Sequence, Tuple

# 3rd party
from docutils import nodes
from docutils.nodes import system_message
from docutils.parsers.rst.states import Inliner
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.utils import stderr_writer
from sphinx.application import Sphinx
from sphinx.util import split_explicit_title
from sphinx.writers.html import HTMLTranslator

__all__ = (
		"asset_role",
		"AssetNode",
		"visit_asset_node",
		"depart_asset_node",
		"setup",
		)

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version


class AssetNode(nodes.reference):
	"""
	Node representing a link to an asset.
	"""


def asset_role(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict = {},
		content: List[str] = []
		) -> Tuple[Sequence[AssetNode], List[system_message]]:
	"""
	Adds a link to an asset.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.
	"""

	has_t, title, target = split_explicit_title(text)
	title = nodes.unescape(title)
	target = nodes.unescape(target)

	if not has_t:
		if target.startswith('~'):
			target = target[1:]
			title = pathlib.PurePosixPath(text[1:]).name

	app = inliner.document.settings.env.app
	base = app.config.assets_dir
	node = AssetNode(rawtext, title, refuri=target, source_file=PathPlus(base) / target, **options)

	return [node], []


def visit_asset_node(translator: HTMLTranslator, node: AssetNode) -> None:
	"""
	Visit an :class:`~.AssetNode`.

	:param translator:
	:param node: The node being visited.
	"""

	if not hasattr(translator, "_asset_node_seen_files"):
		# Files that have already been seen
		translator._asset_node_seen_files = []  # type: ignore[attr-defined]

	assets_out_dir = PathPlus(translator.builder.outdir) / "_assets"
	assets_out_dir.maybe_make(parents=True)

	source_file = PathPlus(translator.builder.confdir) / node["source_file"]

	source_file_exists = source_file.is_file()
	source_file_seen = source_file in translator._asset_node_seen_files  # type: ignore[attr-defined]

	if not source_file_seen and source_file_exists:
		# Avoid unnecessary copies of potentially large files.
		translator._asset_node_seen_files.append(source_file)  # type: ignore[attr-defined]
		shutil.copy2(source_file, assets_out_dir)
	elif not source_file_exists:
		stderr_writer(
				f"\x1b[31m{translator.builder.current_docname}: "
				f"Asset file '{source_file}' not found.\x1b[39m"
				)
		translator.context.append('')
		return

	# Create the HTML
	current_uri = (pathlib.PurePosixPath('/') / translator.builder.current_docname).parent
	refuri = posixpath.relpath(f"/_assets/{node['refuri']}", str(current_uri))
	translator.body.append(f'<a class="reference external" href="{refuri}")/">')
	translator.context.append("</a>")


def depart_asset_node(translator: HTMLTranslator, node: AssetNode) -> None:
	"""
	Depart an :class:`~.AssetNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append(translator.context.pop())


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.assets`.

	.. versionadded:: 1.0.0

	:param app: The Sphinx application.
	"""

	app.add_role("asset", asset_role)
	app.add_config_value("assets_dir", "./assets", "env", [str])
	app.add_node(AssetNode, html=(visit_asset_node, depart_asset_node))

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/assets.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  changeset.py
"""
Customised versions of Sphinx's :rst:dir:`versionadded`, :rst:dir:`versionchanged`
and :rst:dir:`deprecated` directives to correctly handle bullet lists.

.. versionadded:: 2.11.0
.. extensions:: sphinx_toolbox.changeset


Usage
------

.. rst:directive:: .. versionadded:: version

   Documents the version of the project which added the described feature.

   The first argument must be given and is the version in question; you can add
   a second argument consisting of a *brief* explanation of the change.
   Alternatively, a longer description my be given in the body of the directive.

.. rst:directive:: .. versionchanged:: version

   Similar to :rst:dir:`versionadded`, but describes when and what changed in
   the feature in some way (new parameters, changed side effects, etc.).

.. rst:directive:: .. deprecated:: version

   Similar to :rst:dir:`versionchanged`, but describes when the feature was deprecated.
   An explanation can also be given, for example to inform the reader what should be used instead.

.. latex:vspace:: 10px

This extension also adds the following directive:

.. rst:directive:: .. versionremoved:: version [details]

   Similar to :rst:dir:`versionchanged`, but describes when the feature was or will be removed.
   An explanation can also be given, for example to inform the reader what should be used instead.


.. latex:clearpage::

API Reference
----------------

"""  # noqa: D400
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on Sphinx
#  Copyright (c) 2007-2021 by the Sphinx team.
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |    notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |    notice, this list of conditions and the following disclaimer in the
#  |    documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import List, Optional, cast

# 3rd party
import sphinx.domains.changeset
from docutils import nodes
from docutils.nodes import Node
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.locale import _

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("VersionChange", "setup")

versionlabels = {
		"versionremoved": _("Removed in version %s"),
		**sphinx.domains.changeset.versionlabels,
		}

versionlabel_classes = {
		"versionremoved": "removed",
		**sphinx.domains.changeset.versionlabel_classes,
		}


class VersionChange(sphinx.domains.changeset.VersionChange):
	"""
	Directive to describe a addition/change/deprecation/removal in a specific version.
	"""

	def run(self) -> List[Node]:
		"""
		Process the content of the directive.
		"""

		node = addnodes.versionmodified()
		node.document = self.state.document
		self.set_source_info(node)

		node["type"] = self.name
		node["version"] = self.arguments[0]

		text = versionlabels[self.name] % self.arguments[0]

		if len(self.arguments) == 2:
			inodes, messages = self.state.inline_text(self.arguments[1], self.lineno + 1)
			para = nodes.paragraph(self.arguments[1], '', *inodes, translatable=False)
			self.set_source_info(para)
			node.append(para)
		else:
			messages = []  # pylint: disable=W8301

		if self.content:
			self.state.nested_parse(self.content, self.content_offset, node)

		classes = ["versionmodified", versionlabel_classes[self.name]]  # pylint: disable=W8301

		if len(node):
			to_add: Optional[nodes.Node] = None

			if isinstance(node[0], nodes.paragraph) and node[0].rawsource:
				content = nodes.inline(node[0].rawsource, translatable=True)
				content.source = node[0].source
				content.line = node[0].line
				content += node[0].children
				node[0].replace_self(nodes.paragraph('', '', content, translatable=False))

			elif isinstance(node[0], (nodes.bullet_list, nodes.enumerated_list)):
				# Fix for incorrect ordering with bullet lists
				node.insert(0, nodes.compound(''))
				to_add = nodes.paragraph('', '')

			para = cast(nodes.paragraph, node[0])
			para.insert(0, nodes.inline('', f'{text}: ', classes=classes))

			if to_add is not None:
				node.insert(0, to_add)

		else:
			para = nodes.paragraph(
					'',
					'',
					nodes.inline('', f'{text}.', classes=classes),
					translatable=False,
					)
			node.append(para)

		domain = cast(
				sphinx.domains.changeset.ChangeSetDomain,
				self.env.get_domain("changeset"),
				)
		domain.note_changeset(node)

		ret: List[Node] = [node]  # pylint: disable=W8301
		ret += messages
		return ret


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.changeset`.

	:param app: The Sphinx application.
	"""

	app.add_directive("deprecated", VersionChange, override=True)
	app.add_directive("versionadded", VersionChange, override=True)
	app.add_directive("versionchanged", VersionChange, override=True)
	app.add_directive("versionremoved", VersionChange, override=True)

	return {
			"parallel_read_safe": True,
			"parallel_write_safe": True,
			}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/changeset.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  code.py
"""
Customised ``.. code-block::`` directive with an adjustable indent size.

.. extensions:: sphinx_toolbox.code


Usage
------

.. rst:directive:: .. code-block:: [language]
                   .. sourcecode:: [language]

	Customised ``.. code-block::`` directive with an adjustable indent size.

	.. rst:directive:option:: tab-width: width
		:type: integer

		Sets the size of the indentation in spaces.


	All other options from :rst:dir:`sphinx:code-block` are available,
	see the `Sphinx documentation`_ for details.

	.. _Sphinx documentation: https://www.sphinx-doc.org/en/3.x/usage/restructuredtext/directives.html#directive-code-block

	**Examples**

	.. rest-example::

		.. code-block:: python

			def print(text):
				sys.stdout.write(text)


	.. rest-example::

		.. code-block:: python
			:tab-width: 8

			def print(text):
				sys.stdout.write(text)

.. clearpage::

.. rst:directive:: .. code-cell:: [language]
				   .. output-cell:: [language]

	Customised ``.. code-block::`` directives which display an execution count to
	the left of the code block, similar to a Jupyter Notebook cell.

	.. versionadded:: 2.6.0

	.. rst:directive:option:: execution-count: count
		:type: positive integer

		The execution count of the cell.

	All other options from the :rst:dir:`code-block` directive above are available.

	**Examples**

	.. rest-example::

		.. code-cell:: python
			:execution-count: 1

			def print(text):
				sys.stdout.write(text)

			print("hello world")

		.. output-cell::
			:execution-count: 1

			hello world


	.. rest-example::

		.. code-cell:: python
			:execution-count: 2
			:tab-width: 8

			def print(text):
				sys.stdout.write(text)


	.. seealso::

		`nbsphinx <https://nbsphinx.readthedocs.io/en/0.8.3/>`_,
		which inspired these directives and provides additional functionality
		for integrating Jupyter Notebooks with Sphinx.

API Reference
----------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on Sphinx
#  Copyright (c) 2007-2020 by the Sphinx team.
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |    notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |    notice, this list of conditions and the following disclaimer in the
#  |    documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import List, MutableMapping, Optional

# 3rd party
import dict2css
import docutils.nodes
import docutils.statemachine
import sphinx.directives.code
from docutils.nodes import Node
from docutils.parsers.rst import directives
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.stringlist import StringList
from domdf_python_tools.utils import convert_indents
from sphinx.application import Sphinx
from sphinx.writers.html import HTMLTranslator
from sphinx.writers.latex import LaTeXTranslator

# this package
from sphinx_toolbox.utils import Config, OptionSpec, SphinxExtMetadata, metadata_add_version

__all__ = (
		"CodeBlock",
		"CodeCell",
		"OutputCell",
		"Prompt",
		"visit_prompt_html",
		"visit_prompt_latex",
		"copy_asset_files",
		"configure",
		"setup",
		)


class CodeBlock(sphinx.directives.code.CodeBlock):
	"""
	Directive for a code block with special highlighting or line numbering settings.

	The indent_size can be adjusted with the ``:tab-width: <int>`` option.

	.. autoclasssumm:: CodeBlock
		:autosummary-sections: ;;
	"""

	option_spec: OptionSpec = {  # type: ignore[assignment]
		"force": directives.flag,
		"linenos": directives.flag,
		"tab-width": int,
		"dedent": int,
		"lineno-start": int,
		"emphasize-lines": directives.unchanged_required,
		"caption": directives.unchanged_required,
		"class": directives.class_option,
		"name": directives.unchanged,
		}

	def run(self) -> List[Node]:
		"""
		Process the content of the code block.
		"""

		code = '\n'.join(self.content)

		if "tab-width" in self.options:
			tab_width = self.options["tab-width"]
		else:
			tab_width = 4

		code = convert_indents(code, tab_width=tab_width, from_=' ' * self.config.docutils_tab_width)

		self.content = docutils.statemachine.StringList(code.split('\n'))

		return super().run()


class Prompt(docutils.nodes.General, docutils.nodes.FixedTextElement):
	"""
	Represents a cell prompt for a :class:`CodeCell` and :class:`OutputCell`.

	.. versionadded:: 2.6.0
	"""


class CodeCell(CodeBlock):
	"""
	Customised code block which displays an execution count to the left of the code block,
	similar to a Jupyter Notebook cell.

	The indent_size can be adjusted with the ``:tab-width: <int>`` option.

	The execution count can be set using the ``:execution-count: <int>`` option.

	.. autoclasssumm:: CodeCell
		:autosummary-sections: ;;

	.. versionadded:: 2.6.0
	"""  # noqa: D400

	option_spec: OptionSpec = {
			**CodeBlock.option_spec,
			"execution-count": directives.positive_int,
			}

	_prompt: str = "In [%s]:"
	_class: str = "code-cell"

	def run(self) -> List[Node]:
		"""
		Process the content of the code block.
		"""

		self.options.setdefault("class", [])
		self.options["class"].append(f"{self._class}-code")

		prompt = self._prompt % self.options.get("execution-count", ' ')

		outer_node = docutils.nodes.container(classes=[self._class])

		outer_node += Prompt(
				prompt,
				prompt,
				language="none",
				classes=["prompt", f"{self._class}-prompt"],
				)

		outer_node += super().run()[0]

		return [outer_node]


class OutputCell(CodeCell):
	"""
	Variant of :class:`~.CodeCell` for displaying the output of a cell in a Jupyter Notebook.

	The indent_size can be adjusted with the ``:tab-width: <int>`` option.

	The execution count can be set using the ``:execution-count: <int>`` option.

	.. versionadded:: 2.6.0

	.. autoclasssumm:: OutputCell
		:autosummary-sections: ;;
	"""

	_prompt: str = "[%s]:"
	_class: str = "output-cell"


def visit_prompt_html(translator: HTMLTranslator, node: Prompt) -> None:
	"""
	Visit a :class:`~.Prompt` node with the HTML translator.

	.. versionadded:: 2.6.0

	:param translator:
	:param node:
	"""

	starttag = translator.starttag(node, "div", suffix='', CLASS="notranslate")
	translator.body.append(starttag + node.rawsource + '</div>\n')
	raise docutils.nodes.SkipNode


def visit_prompt_latex(translator: LaTeXTranslator, node: Prompt) -> None:
	"""
	Visit a :class:`~.Prompt` node with the LaTeX translator.

	.. versionadded:: 2.6.0

	:param translator:
	:param node:
	"""

	translator.body.append("\n\n")
	translator.body.append(r"\vspace{4mm}")

	if f"code-cell-prompt" in node["classes"]:
		colour = "nbsphinxin"
	elif f"output-cell-prompt" in node["classes"]:
		colour = "nbsphinxout"
	else:  # pragma: no cover
		colour = "black"

	translator.body.append(
			rf"\llap{{\color{{{colour}}}\texttt{{{node.rawsource}}}"
			r"\,\hspace{\fboxrule}\hspace{\fboxrule}\hspace{\fboxsep}}"
			)
	translator.body.append(r"\vspace{-7mm}")

	raise docutils.nodes.SkipNode


def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None:
	"""
	Copy additional stylesheets into the HTML build directory.

	.. versionadded:: 2.6.0

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.format.lower() != "html":  # pragma: no cover
		return

	prompt_style: dict2css.Style = {
			"user-select": None,
			"font-size": "13px",
			"font-family": '"SFMono-Regular", Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace',
			"border": None,
			"padding": "11px 0 0",
			"margin": "0 5px 0 0",
			"box-shadow": None,
			"wrap-option": None,
			"white-space": "nowrap",
			}

	container_style: dict2css.Style = {
			"padding-top": "5px",
			"display": "flex",
			"align-items": "stretch",
			"margin": 0,
			}

	code_style_string = "div.code-cell.container div.code-cell-code, div.output-cell.container div.output-cell-code"
	code_style: dict2css.Style = {
			"width": "100%",
			"padding-top": 0,
			"margin-top": 0,
			}

	style: MutableMapping[str, dict2css.Style] = {
			"div.code-cell.container div.prompt": {"color": "#307FC1"},
			"div.output-cell.container div.prompt": {"color": "#BF5B3D"},
			"div.code-cell.container div.prompt, div.output-cell.container div.prompt": prompt_style,
			"div.code-cell.container, div.output-cell.container": container_style,
			code_style_string: code_style,
			}

	static_dir = PathPlus(app.outdir) / "_static"
	static_dir.maybe_make(parents=True)
	dict2css.dump(style, static_dir / "sphinx-toolbox-code.css")


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.code`.

	.. versionadded:: 2.9.0

	:param app: The Sphinx application.
	:param config:
	"""

	latex_elements = getattr(config, "latex_elements", {})

	latex_preamble = StringList(latex_elements.get("preamble", ''))
	latex_preamble.blankline()
	latex_preamble.append(r"\definecolor{nbsphinxin}{HTML}{307FC1}")
	latex_preamble.append(r"\definecolor{nbsphinxout}{HTML}{BF5B3D}")

	latex_elements["preamble"] = str(latex_preamble)
	config.latex_elements = latex_elements


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.code`.

	.. versionadded:: 1.0.0

	:param app: The Sphinx application.
	"""

	# Code block with customisable indent size.
	app.add_directive("code-block", CodeBlock, override=True)
	app.add_directive("sourcecode", CodeBlock, override=True)

	app.add_directive("code-cell", CodeCell)
	app.add_directive("output-cell", OutputCell)

	# Hack to get the docutils tab size, as there doesn't appear to be any other way
	app.setup_extension("sphinx_toolbox.tweaks.tabsize")

	app.add_node(
			Prompt,
			html=(visit_prompt_html, lambda *args, **kwargs: None),
			latex=(visit_prompt_latex, lambda *args, **kwargs: None)
			)

	app.connect("config-inited", configure)
	app.add_css_file("sphinx-toolbox-code.css")
	app.connect("build-finished", copy_asset_files)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/code.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  collapse.py
r"""
Adds a collapsible section to an HTML page using a details_ element.

.. _details: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/details>

.. versionadded:: 2.5.0
.. extensions:: sphinx_toolbox.collapse


Usage
------

.. rst:directive:: .. collapse:: [label]

	Adds a collapsible section to an HTML page using a details_ element.

	.. _details: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/details>

	With non-HTML builders, the content will be added as-is.

	.. rest-example::

		.. collapse:: Details

			Something small enough to escape casual notice.


		.. collapse:: A Different Label
			:class: custom-summary
			:name: summary0

			Something else that might escape notice.

		.. collapse:: A long code block

			.. code-block:: python

				print("Not really")


	.. rst:directive:option:: open
		:type: flag

		The ``:open:`` option can be used to have the section open by default.

		.. versionadded:: 3.0.0

		.. rest-example::

			.. collapse:: Open
				:open:

				This section is open by default.


API Reference
----------------

"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import Optional, Sequence

# 3rd party
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.parsers.rst.roles import set_classes
from domdf_python_tools.stringlist import DelimitedList
from sphinx.application import Sphinx
from sphinx.util.docutils import SphinxDirective
from sphinx.writers.html import HTMLTranslator

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, flag, metadata_add_version

__all__ = ("CollapseDirective", "CollapseNode", "visit_collapse_node", "depart_collapse_node", "setup")


class CollapseDirective(SphinxDirective):
	r"""
	A Sphinx directive to add a collapsible section to an HTML page using a details_ element.

	.. _details: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/details>
	"""

	final_argument_whitespace: bool = True
	has_content: bool = True

	# The label
	required_arguments: int = 1

	option_spec = {
			"class": directives.class_option,
			"name": directives.unchanged,
			"open": flag,
			}

	def run(self) -> Sequence[nodes.Node]:  # type: ignore[override]
		"""
		Process the content of the directive.
		"""

		set_classes(self.options)
		self.assert_has_content()

		text = '\n'.join(self.content)
		label = self.arguments[0]

		collapse_node = CollapseNode(text, label, **self.options)

		self.add_name(collapse_node)

		collapse_node["classes"].append(f"summary-{nodes.make_id(label)}")

		self.state.nested_parse(self.content, self.content_offset, collapse_node)

		return [collapse_node]


class CollapseNode(nodes.Body, nodes.Element):
	"""
	Node that represents a collapsible section.

	:param rawsource:
	:param label:
	"""

	def __init__(self, rawsource: str = '', label: Optional[str] = None, *children, **attributes):
		super().__init__(rawsource, *children, **attributes)
		self.label = label


def visit_collapse_node(translator: HTMLTranslator, node: CollapseNode) -> None:
	"""
	Visit a :class:`~.CollapseNode`.

	:param translator:
	:param node: The node being visited.
	"""

	tag_parts = DelimitedList(["details"])

	if node.get("names", None):
		names = DelimitedList(node["names"])
		tag_parts.append(f'name="{names: }"')

	if node.get("classes", None):
		classes = DelimitedList(node["classes"])
		tag_parts.append(f'class="{classes: }"')

	if node.attributes.get("open", False):
		tag_parts.append("open")

	translator.body.append(f"<{tag_parts: }>\n<summary>{node.label}</summary>")
	translator.context.append("</details>")


def depart_collapse_node(translator: HTMLTranslator, node: CollapseNode) -> None:
	"""
	Depart a :class:`~.CollapseNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append(translator.context.pop())


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.collapse`.

	:param app: The Sphinx application.
	"""

	app.add_directive("collapse", CollapseDirective)
	app.add_node(
			CollapseNode,
			html=(visit_collapse_node, depart_collapse_node),
			latex=(lambda *args, **kwargs: None, lambda *args, **kwargs: None)
			)

	return {
			"parallel_read_safe": True,
			}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/collapse.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  config.py
"""
Internal configuration for ``sphinx-toolbox``.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import List

# 3rd party
from apeye.requests_url import RequestsURL
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.utils import Config, add_nbsp_substitution

__all__ = ("MissingOptionError", "InvalidOptionError", "validate_config", "ToolboxConfig")


class MissingOptionError(ValueError):
	"""
	Subclass of :exc:`ValueError` to indicate a missing configuration option.
	"""


class InvalidOptionError(ValueError):
	"""
	Subclass of :exc:`ValueError` to indicate an invalid value for a configuration value.
	"""


class ToolboxConfig(Config):
	"""
	Subclass of :class:`sphinx.config.Config` with type annotations for the
	configuration values added by ``sphinx-toolbox``.

	Depending on the extensions enabled not all of these configuration values will be present.

	Functionally compatible with :class:`sphinx.config.Config`.
	"""  # noqa: D400

	source_link_target: str
	"""
	The target of the source link, either ``'github'`` or ``'sphinx'``.
	Will be lowercase after :func:`~.validate_config` has run.
	"""

	#: The username of the GitHub account that owns the repository this documentation corresponds to.
	github_username: str

	github_repository: str
	"""
	The GitHub repository this documentation corresponds to.

	.. clearpage::
	"""

	#: The complete URL of the repository on GitHub.
	github_url: RequestsURL

	#: The base URL for the source code on GitHub.
	github_source_url: RequestsURL

	#: The base URL for the issues on GitHub.
	github_issues_url: RequestsURL

	#: The base URL for the pull requests on GitHub.
	github_pull_url: RequestsURL

	#: List of required Conda channels.
	conda_channels: List[str]

	#: The directory in which to find assets for the :rst:role:`asset` role.
	assets_dir: str

	docutils_tab_width: int
	"""
	The tab size used by docutils.
	This is usually 8 spaces, but can be configured in the ``docutils.conf`` file.
	"""

	#: The Wikipedia language to use for :rst:role:`wikipedia` roles.
	wikipedia_lang: str

	#: A string of reStructuredText that will be included at the beginning of every source file that is read.
	rst_prolog: str

	#: Document all :class:`typing.TypeVar`\s, even if they have no docstring.
	all_typevars: bool

	no_unbound_typevars: bool
	r"""
	Only document :class:`typing.TypeVar`\s that have a constraint of are bound.

	This option has no effect if :confval:`all_typevars` is False.
	"""


def validate_config(app: Sphinx, config: ToolboxConfig) -> None:
	"""
	Validate the provided configuration values.

	See :class:`~sphinx_toolbox.config.ToolboxConfig` for a list of the configuration values.

	:param app: The Sphinx application.
	:param config:
	:type config: :class:`~sphinx.config.Config`
	"""

	# this package
	from sphinx_toolbox import github, source

	source._configure(app, config)
	github.validate_config(app, config)
	add_nbsp_substitution(config)

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/config.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  confval.py
r"""
The confval directive and role for configuration values.

.. extensions:: sphinx_toolbox.confval

Usage
-------

.. rst:directive:: .. confval:: name

	Used to document a configuration value.

	.. raw:: latex

		\begin{multicols}{2}

	.. rst:directive:option:: type
		:type: string

		Indicates the configuration value's type.

	.. rst:directive:option:: required
		:type: flag

		Indicates the whether the configuration value is required.

	.. rst:directive:option:: default
		:type: string

		Indicates the default value.

	.. rst:directive:option:: noindex
		:type: flag

		Disables the index entry and cross-referencing for this configuration value.

		.. versionadded:: 2.11.0

	.. raw:: latex

		\end{multicols}


.. latex:vspace:: -0px


.. rst:role:: confval

	Role which provides a cross-reference to a :rst:dir:`confval` directive.

.. latex:vspace:: 10px

**Examples:**

.. rest-example::

	.. confval:: demo
		:type: string
		:default: ``"Hello World"``
		:required: False

.. latex:vspace:: -20px

.. rest-example::

	To enable this feature set the :confval:`demo` configuration value to "True".


.. latex:clearpage::


API Reference
--------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Based on https://github.com/readthedocs/sphinx_rtd_theme/blob/master/docs/conf.py
#  Copyright (c) 2013-2018 Dave Snider, Read the Docs, Inc. & contributors
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import List

# 3rd party
from docutils.nodes import Node
from docutils.parsers.rst import directives
from docutils.statemachine import StringList
from domdf_python_tools.utils import strtobool
from sphinx.application import Sphinx
from sphinx.domains import ObjType
from sphinx.domains.std import GenericObject, StandardDomain
from sphinx.errors import ExtensionError
from sphinx.roles import XRefRole

# this package
from sphinx_toolbox.utils import OptionSpec, SphinxExtMetadata, flag, metadata_add_version

__all__ = ("ConfigurationValue", "register_confval", "setup")


class ConfigurationValue(GenericObject):
	"""
	The confval directive.

	.. versionchanged:: 1.1.0

		The formatting of the type, required and default options can be customised
		using the ``self.format_*`` methods.

	.. versionchanged:: 2.11.0

		Added the ``:noindex:`` option, which disables the index entry
		and cross-referencing for this configuration value.
	"""

	indextemplate: str = "%s (configuration value)"

	option_spec: OptionSpec = {  # type: ignore[assignment]
		"type": directives.unchanged_required,
		"required": directives.unchanged_required,
		"default": directives.unchanged_required,
		"noindex": flag,
		}

	def run(self) -> List[Node]:
		"""
		Process the content of the directive.
		"""

		content: List[str] = []

		if self.options and set(self.options.keys()) != {"noindex"}:
			content.extend(('', ".. raw:: latex", '', r"    \vspace{-45px}", ''))

		if "type" in self.options:
			content.append(f"| **Type:** {self.format_type(self.options['type'])}")
		if "required" in self.options:
			content.append(f"| **Required:** ``{self.format_required(self.options['required'])}``")
		if "default" in self.options:
			content.append(f"| **Default:** {self.format_default(self.options['default'])}")

		if self.content:

			content.extend((
					'',
					".. raw:: latex",
					'',
					r"    \vspace{-25px}",
					'',
					))
			content.extend(self.content)

		self.content = StringList(content)

		return super().run()

	@staticmethod
	def format_type(the_type: str) -> str:
		"""
		Formats the ``:type:`` option.

		.. versionadded:: 1.1.0

		:param the_type:
		"""

		return the_type

	@staticmethod
	def format_required(required: str) -> bool:
		"""
		Formats the ``:required:`` option.

		.. versionadded:: 1.1.0

		:param required:
		"""

		return strtobool(required)

	@staticmethod
	def format_default(default: str) -> str:
		"""
		Formats the ``:default:`` option.

		.. versionadded:: 1.1.0

		:param default:
		"""

		return default


def register_confval(app: Sphinx, override: bool = False) -> None:
	"""
	Create and register the ``confval`` role and directive.

	:param app: The Sphinx application.
	:param override:
	"""

	if "std" not in app.registry.domains:
		app.add_domain(StandardDomain)  # pragma: no cover

	name = "confval"

	app.registry.add_directive_to_domain("std", name, ConfigurationValue)
	app.registry.add_role_to_domain("std", name, XRefRole())

	object_types = app.registry.domain_object_types.setdefault("std", {})

	if name in object_types and not override:  # pragma: no cover
		raise ExtensionError(f"The {name!r} object_type is already registered")

	object_types[name] = ObjType(name, name)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.confval`.

	.. versionadded:: 0.7.0

	:param app: The Sphinx application.
	"""

	register_confval(app)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/confval.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  decorators.py
"""
reStructuredText XRef role for decorators.

.. versionadded:: 0.9.0
.. extensions:: sphinx_toolbox.decorators


Usage
------

.. rst:role:: deco

	Adds a cross reference to a decorator, prefixed with an ``@``.

	.. rest-example::

		.. decorator:: my_decorator

			A decorator.

		:deco:`my_decorator`

		:deco:`@my_decorator`

		:deco:`Title <my_decorator>`


.. latex:clearpage::


API Reference
----------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on Sphinx
#  Copyright (c) 2007-2020 by the Sphinx team.
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |    notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |    notice, this list of conditions and the following disclaimer in the
#  |    documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import Tuple, Type  # noqa: F401

# 3rd party
from docutils.nodes import Element
from sphinx.application import Sphinx
from sphinx.domains.python import PyXRefRole
from sphinx.environment import BuildEnvironment

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("PyDecoXRefRole", "setup")


class PyDecoXRefRole(PyXRefRole):
	"""
	XRef Role for decorators members.
	"""

	def process_link(
			self,
			env: BuildEnvironment,
			refnode: Element,
			has_explicit_title: bool,
			title: str,
			target: str,
			) -> Tuple[str, str]:
		"""
		Called after parsing title and target text, and creating the reference node (given in ``refnode``).

		This method can alter the reference node and must return a new (or the same) ``(title, target)`` tuple.

		:param env:
		:param refnode:
		:param has_explicit_title:
		:param title:
		:param target:
		"""

		target = target.lstrip('@')

		title, target = super().process_link(
			env=env,
			refnode=refnode,
			has_explicit_title=has_explicit_title,
			title=title,
			target=target,
			)

		if not has_explicit_title and not title.startswith('@'):
			title = f"@{title}"

		# Ensure the reference is correctly found in the broader scope.
		refnode["reftype"] = "obj"

		return title, target


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.decorators`.

	:param app: The Sphinx application.
	"""

	app.add_role_to_domain("py", "deco", PyDecoXRefRole())

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/decorators.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  documentation_summary.py
"""
Allows insertion of a summary line on the title page generated with the LaTeX builder,
and at a custom location throughout the document.

.. versionadded:: 2.2.0
.. extensions:: sphinx_toolbox.documentation_summary


Configuration
--------------

.. latex:vspace:: -10px

.. confval:: documentation_summary
	:type: :class:`str`

	The documentation summary to display on the title page with the LaTeX builder,
	and at the location of :rst:dir:`documentation-summary` directives for other builders.

	If undefined no summary is shown.


Usage
------

.. latex:vspace:: -10px

.. rst:directive:: documentation-summary

	Adds the documentation summary as configured above.

	.. only:: html

		:bold-title:`Example`

		.. rest-example::

			.. documentation-summary::


	.. rst:directive:option:: meta

		Include the summary as a meta_ "description" tag in the HTML output.

		The structure of the description is ``{project} -- {summary}``,
		where ``project`` is configured in ``conf.py``.

		See `the sphinx documentation`_ for more information on the ``project`` option.

		.. versionadded:: 2.10.0

		.. _meta: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta
		.. _the sphinx documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-project


.. latex:clearpage::

API Reference
----------------

"""  # noqa: D400
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import List

# 3rd party
from docutils import nodes
from docutils.statemachine import StringList
from docutils.utils.smartquotes import educateQuotes
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.util.docutils import SphinxDirective

# this package
from sphinx_toolbox.utils import Config, Purger, SphinxExtMetadata, flag, metadata_add_version

__all__ = ("DocumentationSummaryDirective", "configure", "setup")

summary_node_purger = Purger("all_summary_nodes")

RENEW = r"""
\makeatletter
\renewcommand{\py@release}{
	\releasename\space\version
	\par
	\vspace{25pt}
	\textup{\thesummary}
}
\makeatother
"""

RESET = r"\makeatletter\renewcommand{\py@release}{\releasename\space\version}\makeatother"


class DocumentationSummaryDirective(SphinxDirective):
	"""
	A Sphinx directive for creating a summary line.
	"""

	option_spec = {"meta": flag}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the directive.
		"""

		summary = getattr(self.config, "documentation_summary", '').strip()

		if not summary:
			return []  # pragma: no cover

		# if self.env.app.builder.format.lower() == "latex" or not summary:
		# 	return []

		targetid = f'documentation-summary-{self.env.new_serialno("documentation-summary"):d}'

		onlynode = addnodes.only(expr="html")

		content = f'**{summary}**'
		content_node = nodes.paragraph(rawsource=content, ids=[targetid])
		onlynode += content_node
		self.state.nested_parse(StringList([content]), self.content_offset, content_node)
		summary_node_purger.add_node(self.env, content_node, content_node, self.lineno)

		if "meta" in self.options:
			meta_content = f'.. meta::\n    :description: {self.config.project} -- {summary}\n'
			meta_node = nodes.paragraph(rawsource=meta_content, ids=[targetid])
			onlynode += meta_node
			self.state.nested_parse(
					StringList(meta_content.split('\n')),
					self.content_offset,
					meta_node,
					)
			summary_node_purger.add_node(self.env, meta_node, meta_node, self.lineno)

		return [onlynode]


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.documentation_summary`.

	:param app: The Sphinx application.
	:param config:
	"""

	if not hasattr(config, "latex_elements"):  # pragma: no cover
		config.latex_elements = {}

	latex_elements = (config.latex_elements or {})

	latex_preamble = latex_elements.get("preamble", '')
	summary = getattr(config, "documentation_summary", '').strip()

	if not summary:
		return  # pragma: no cover

	# Escape latex special characters
	summary = summary.replace("~ ", r"\textasciitilde\space ")
	summary = summary.replace("^ ", r"\textasciicircum\space ")
	summary = summary.replace("\\ ", r"\textbackslash\space ")

	summary = summary.translate({
			35: r"\#",
			36: r"\$",
			37: r"\%",
			38: r"\&",
			94: r"\textasciicircum",
			95: r"\_",
			123: r"\{",
			125: r"\}",
			126: r"\textasciitilde",
			})

	# TODO: escape backslashes without breaking the LaTeX commands

	summary_command = rf"\newcommand{{\thesummary}}{{{educateQuotes(summary)}}}"

	if summary_command not in latex_preamble:
		config.latex_elements["preamble"] = '\n'.join([
				latex_preamble,
				summary_command,
				RENEW,
				])
		config.latex_elements["maketitle"] = '\n'.join([
				config.latex_elements.get("maketitle", r"\sphinxmaketitle"),
				RESET,
				])


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.documentation_summary`.

	:param app: The Sphinx application.
	"""

	app.connect("config-inited", configure, priority=550)
	app.add_directive("documentation-summary", DocumentationSummaryDirective)
	app.add_config_value("documentation_summary", None, "env", types=[str, None])
	app.connect("env-purge-doc", summary_node_purger.purge_nodes)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/documentation_summary.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  flake8.py
r"""
A Sphinx directive for documenting flake8 codes.

.. versionadded:: 1.6.0
.. extensions:: sphinx_toolbox.flake8


Usage
------

.. rst:directive:: .. flake8-codes:: plugin

	Adds a table documenting a flake8 plugin's codes.

	The directive takes a single argument -- the fully qualified name of the flake8 plugin module.

	Codes to document are given in the body of the directive.


	:bold-title:`Example`

	.. rest-example::

		.. flake8-codes:: flake8_dunder_all

			DALL000


.. latex:clearpage::


API Reference
----------------
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import warnings
from typing import List, Sequence, Tuple

# 3rd party
import docutils
import tabulate
from docutils import nodes
from docutils.statemachine import StringList
from sphinx.application import Sphinx
from sphinx.ext.autodoc.importer import import_module
from sphinx.util.docutils import SphinxDirective

# this package
from sphinx_toolbox.utils import Purger, SphinxExtMetadata, metadata_add_version

__all__ = ("Flake8CodesDirective", "setup")

table_node_purger = Purger("all_flake8_code_table_nodes")


class Flake8CodesDirective(SphinxDirective):
	"""
	A Sphinx directive for documenting flake8 codes.
	"""

	has_content: bool = True

	# the fully qualified name of the flake8 plugin module
	required_arguments: int = 1

	def run(self) -> Sequence[nodes.Node]:  # type: ignore[override]
		"""
		Process the content of the directive.
		"""

		plugin: str = self.arguments[0]

		if not self.content:
			warnings.warn("No codes specified")
			return []

		module = import_module(plugin)
		codes: List[Tuple[str, str]] = []

		for code in self.content:
			if code.strip():
				if hasattr(module, code):
					description = getattr(module, code)
					if description.startswith(code):
						description = description[len(code):]
					codes.append((code, description.strip()))
				else:
					warnings.warn(f"No such code {code!r}")

		if not codes:
			warnings.warn("No codes specified")
			return []

		targetid = f'flake8codes-{self.env.new_serialno("flake8codes"):d}'
		targetnode = nodes.section(ids=[targetid])

		table = tabulate.tabulate(codes, headers=["Code", "Description"], tablefmt="rst")
		content = '\n' + table.replace('\t', "    ") + '\n'

		view = StringList(content.split('\n'))
		table_node = nodes.paragraph(rawsource=content)
		self.state.nested_parse(view, self.content_offset, table_node)

		if docutils.__version_info__ >= (0, 18):
			table_node.children[0]["classes"] += ["colwidths-given"]  # type: ignore[index]

		table_node_purger.add_node(self.env, table_node, targetnode, self.lineno)

		return [table_node]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.flake8`.

	:param app: The Sphinx application.
	"""

	app.add_directive("flake8-codes", Flake8CodesDirective)
	app.connect("env-purge-doc", table_node_purger.purge_nodes)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/flake8.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  formatting.py
"""
Directives, roles and nodes for text formatting.

.. versionadded:: 0.2.0
.. extensions:: sphinx_toolbox.formatting


.. latex:vspace:: -10px

Usage
-------

.. rst:role:: iabbr

	An abbreviation. If the role content contains a parenthesized explanation,
	it will be treated specially: it will be shown in a tool-tip in HTML,
	and output only once in LaTeX.

	Unlike Sphinx's :rst:role:`abbr` role, this one shows the abbreviation in italics.

	.. versionadded:: 0.2.0

	:bold-title:`Example`

	.. rest-example::

		:iabbr:`LIFO (last-in, first-out)`


.. rst:role:: bold-title

	Role for displaying a pseudo title in bold.

	This is useful for breaking up Python docstrings.

	.. versionadded:: 2.12.0

	:bold-title:`Example`

	.. rest-example::

		:bold-title:`Examples:`

		:bold-title:`Other Extensions`


API Reference
---------------
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts of the docstrings based on https://docutils.sourceforge.io/docs/howto/rst-roles.html
#

# stdlib
from typing import List, Tuple

# 3rd party
from docutils import nodes
from docutils.nodes import Node, system_message
from docutils.parsers.rst import roles
from sphinx.application import Sphinx
from sphinx.roles import Abbreviation
from sphinx.util.docutils import SphinxRole
from sphinx.writers.html import HTMLTranslator
from sphinx.writers.latex import LaTeXTranslator

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = (
		"ItalicAbbreviationNode",
		"ItalicAbbreviation",
		"visit_iabbr_node",
		"depart_iabbr_node",
		"latex_visit_iabbr_node",
		"latex_depart_iabbr_node",
		"setup"
		)


class ItalicAbbreviationNode(nodes.abbreviation):
	"""
	Docutils Node to show an abbreviation in italics.
	"""


class ItalicAbbreviation(Abbreviation):
	"""
	Docutils role to show an abbreviation in italics.
	"""

	def run(self) -> Tuple[List[Node], List[system_message]]:
		"""
		Process the content of the italic abbreviation role.
		"""

		options = self.options.copy()
		matched = self.abbr_re.search(self.text)

		if matched:
			text = self.text[:matched.start()].strip()
			options["explanation"] = matched.group(1)
		else:
			text = self.text

		return [ItalicAbbreviationNode(self.rawtext, text, **options)], []


class BoldTitle(SphinxRole):
	"""
	Role for displaying a pseudo title in bold.

	This is useful for breaking up Python docstrings.
	"""

	def run(self) -> Tuple[List[nodes.Node], List[nodes.system_message]]:

		assert self.text is not None

		node_list: List[nodes.Node] = [
				nodes.raw('', r"\vspace{10px}", format="latex"),
				nodes.strong(f"**{self.text}**", self.text),
				]

		return node_list, []


def visit_iabbr_node(translator: HTMLTranslator, node: ItalicAbbreviationNode) -> None:
	"""
	Visit an :class:`~.ItalicAbbreviationNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append('<i class="abbreviation">')
	attrs = {}

	if node.hasattr("explanation"):
		attrs["title"] = node["explanation"]

	translator.body.append(translator.starttag(node, "abbr", '', **attrs))


def depart_iabbr_node(translator: HTMLTranslator, node: ItalicAbbreviationNode) -> None:
	"""
	Depart an :class:`~.ItalicAbbreviationNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append("</i></abbr>")


def latex_visit_iabbr_node(translator: LaTeXTranslator, node: ItalicAbbreviationNode) -> None:
	"""
	Visit an :class:`~.ItalicAbbreviationNode`.

	:param translator:
	:param node: The node being visited.
	"""

	abbr = node.astext()
	translator.body.append(r"\textit{\sphinxstyleabbreviation{")

	# spell out the explanation once
	if node.hasattr("explanation") and abbr not in translator.handled_abbrs:
		translator.context.append(f'}} ({translator.encode(node["explanation"])})')
		translator.handled_abbrs.add(abbr)
	else:
		translator.context.append('}')


def latex_depart_iabbr_node(translator: LaTeXTranslator, node: ItalicAbbreviationNode) -> None:
	"""
	Depart an :class:`~.ItalicAbbreviationNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append(translator.context.pop())
	translator.body.append('}')


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.formatting`.

	:param app: The Sphinx application.
	"""

	roles.register_local_role("iabbr", ItalicAbbreviation())
	app.add_role("bold-title", BoldTitle())

	app.add_node(
			ItalicAbbreviationNode,
			html=(visit_iabbr_node, depart_iabbr_node),
			latex=(latex_visit_iabbr_node, latex_depart_iabbr_node),
			)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/formatting.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
r"""
Sphinx domain for GitHub.com, and related utilities.

.. versionadded:: 2.4.0
.. extensions:: sphinx_toolbox.github


Configuration
--------------

.. latex:vspace:: -10px

.. confval:: github_username
	:type: :class:`str`
	:required: True

	The username of the GitHub account that owns the repository this documentation corresponds to.

.. confval:: github_repository
	:type: :class:`str`
	:required: True

	The GitHub repository this documentation corresponds to.


Usage
------

.. latex:vspace:: -10px

.. rst:role:: github:issue

	Role which shows a link to the given issue on GitHub.

	If the issue exists, the link has a tooltip that shows the title of the issue.

	**Example**

	.. rest-example::

		:github:issue:`1`

	You can also reference an issue in a different repository by adding the repository name inside ``<>``.

	.. rest-example::

		:github:issue:`7680 <pytest-dev/pytest>`


.. rst:role:: github:pull

	Role which shows a link to the given pull request on GitHub.

	If the pull requests exists, the link has a tooltip that shows the title of the pull requests.

	**Example**

	.. rest-example::

		:github:pull:`2`

	You can also reference a pull request in a different repository by adding the repository name inside ``<>``.

	.. rest-example::

		:github:pull:`7671 <pytest-dev/pytest>`


.. rst:role:: github:repo

	Role which shows a link to the given repository on GitHub.

	**Example**

	.. rest-example::

		:github:repo:`sphinx-toolbox/sphinx-toolbox`

	You can also use a different label for the link:.

	.. rest-example::

		See more in the :github:repo:`pytest repository <pytest-dev/pytest>`.


.. rst:role:: github:user

	Role which shows a link to the given user on GitHub.

	**Example**

	.. rest-example::

		:github:user:`domdfcoding`

	You can also use a different label for the link:.

	.. rest-example::

		See more of my :github:user:`repositories <domdfcoding>`.


.. latex:clearpage::

.. rst:role:: github:org

	Role which shows a link to the given organization on GitHub.

	**Example**

	.. rest-example::

		:github:org:`sphinx-toolbox`

	You can also use a different label for the link:.

	.. rest-example::

		See more repositories in the :github:org:`pytest-dev org <pytest-dev>`.


Caching
-----------

HTTP requests to obtain issue/pull request titles are cached for four hours.

To clear the cache manually, run:

.. prompt:: bash

	python3 -m sphinx_toolbox



API Reference
---------------

Enable this extension from your extension's setup function like so:

.. code-block::

	def setup(app: Sphinx) -> Dict[str, Any]:
		app.setup_extension('sphinx_toolbox.github')
		return {}

This will guarantee that the following values will be available via
:attr:`app.config <sphinx.config.Config>`:

* **github_username** (:class:`str`\) -- The username of the GitHub account that owns the repository this documentation corresponds to.
* **github_repository** (:class:`str`\) -- The GitHub repository this documentation corresponds to.
* **github_url** (:class:`apeye.requests_url.RequestsURL`\) -- The complete URL of the repository on GitHub.
* **github_source_url** (:class:`~apeye.requests_url.RequestsURL`\) -- The base URL for the source code on GitHub.
* **github_issues_url** (:class:`~apeye.requests_url.RequestsURL`\) -- The base URL for the issues on GitHub.
* **github_pull_url** (:class:`~apeye.requests_url.RequestsURL`\) -- The base URL for the pull requests on GitHub.

If the user has not provided either ``github_username`` or ``github_repository``
a :exc:`~.MissingOptionError` will be raised.

.. latex:clearpage::
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# 3rd party
from sphinx.application import Sphinx
from sphinx.domains import Domain
from sphinx.environment import BuildEnvironment

# this package
from sphinx_toolbox.config import MissingOptionError, ToolboxConfig
from sphinx_toolbox.github.issues import (
		IssueNode,
		_depart_issue_node_latex,
		_visit_issue_node_latex,
		depart_issue_node,
		issue_role,
		pull_role,
		visit_issue_node
		)
from sphinx_toolbox.github.repos_and_users import (
		GitHubObjectLinkNode,
		_depart_github_object_link_node_latex,
		_visit_github_object_link_node_latex,
		depart_github_object_link_node,
		repository_role,
		user_role,
		visit_github_object_link_node
		)
from sphinx_toolbox.utils import SphinxExtMetadata, make_github_url, metadata_add_version

_ = BuildEnvironment

__all__ = ("GitHubDomain", "validate_config", "setup")


class GitHubDomain(Domain):
	"""
	Sphinx domain for `GitHub.com <https://github.com>`_.
	"""

	name = "github"
	label = "GitHub"
	roles = {
			"issue": issue_role,  # type: ignore[dict-item]
			"pull": pull_role,  # type: ignore[dict-item]
			"user": user_role,  # type: ignore[dict-item]
			"org": user_role,  # type: ignore[dict-item]
			"repo": repository_role,  # type: ignore[dict-item]
			}


def validate_config(app: Sphinx, config: ToolboxConfig) -> None:
	"""
	Validate the provided configuration values.

	See :class:`~sphinx_toolbox.config.ToolboxConfig` for a list of the configuration values.

	:param app: The Sphinx application.
	:param config:
	:type config: :class:`~sphinx.config.Config`
	"""

	if not config.github_username:
		raise MissingOptionError("The 'github_username' option is required.")
	else:
		config.github_username = str(config.github_username)

	if not config.github_repository:
		raise MissingOptionError("The 'github_repository' option is required.")
	else:
		config.github_repository = str(config.github_repository)

	config.github_url = make_github_url(config.github_username, config.github_repository)
	config.github_source_url = config.github_url / "blob" / "master"
	config.github_issues_url = config.github_url / "issues"
	config.github_pull_url = config.github_url / "pull"


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.github`.

	.. versionadded:: 1.0.0

	:param app: The Sphinx application.
	"""

	app.connect("config-inited", validate_config, priority=850)

	app.add_config_value("github_username", None, "env", types=[str])
	app.add_config_value("github_repository", None, "env", types=[str])
	app.add_domain(GitHubDomain)

	# Custom node for issues and PRs
	app.add_node(
			IssueNode,
			html=(visit_issue_node, depart_issue_node),
			latex=(_visit_issue_node_latex, _depart_issue_node_latex)
			)
	app.add_node(
			GitHubObjectLinkNode,
			html=(visit_github_object_link_node, depart_github_object_link_node),
			latex=(_visit_github_object_link_node_latex, _depart_github_object_link_node_latex)
			)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/github/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  issues.py
"""
Roles and nodes for GitHub issues and Pull Requests.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on pyspecific.py from the Python documentation.
#  Copyright 2008-2014 by Georg Brandl.
#  Licensed under the PSF License 2.0
#
#  Parts of the docstrings based on https://docutils.sourceforge.io/docs/howto/rst-roles.html
#

# stdlib
import warnings
import xml.sax.saxutils
from typing import Any, Dict, List, Optional, Tuple, Union

# 3rd party
import requests  # nodep
from apeye.url import URL
from bs4 import BeautifulSoup  # type: ignore[import]
from docutils import nodes
from docutils.nodes import system_message
from docutils.parsers.rst.states import Inliner
from sphinx.util.nodes import split_explicit_title
from sphinx.writers.html import HTMLTranslator
from sphinx.writers.latex import LaTeXTranslator

# this package
from sphinx_toolbox.cache import cache
from sphinx_toolbox.utils import make_github_url

__all__ = (
		"IssueNode",
		"IssueNodeWithName",
		"issue_role",
		"pull_role",
		"visit_issue_node",
		"depart_issue_node",
		"get_issue_title",
		)


class IssueNode(nodes.reference):
	"""
	Docutils Node to represent a link to a GitHub *Issue* or *Pull Request*.

	:param issue_number: The number of the issue or pull request.
	:param refuri: The URL of the issue / pull request on GitHub.
	"""

	has_tooltip: bool
	issue_number: int
	issue_url: str

	def __init__(
			self,
			issue_number: Union[str, int],
			refuri: Union[str, URL],
			**kwargs,
			):
		self.has_tooltip = False
		self.issue_number = int(issue_number)
		self.issue_url = str(refuri)

		source = f"#{issue_number}"
		super().__init__(source, source, refuri=self.issue_url)

	@property
	def _copy_kwargs(self):  # pragma: no cover  # noqa: MAN002
		return {"issue_number": self.issue_number, "refuri": self.issue_url}

	def copy(self) -> "IssueNode":  # pragma: no cover
		"""
		Return a copy of the :class:`sphinx_toolbox.github.issues.IssueNode`.
		"""

		# This was required to stop some breakage, but it doesn't seem to run during the tests.
		obj = self.__class__(**self._copy_kwargs)
		obj.document = self.document
		obj.has_tooltip = self.has_tooltip
		obj.line = self.line
		return obj


class IssueNodeWithName(IssueNode):
	"""
	Docutils Node to represent a link to a GitHub *Issue* or *Pull Request*, with the repository name shown.

	.. versionadded:: 2.4.0

	:param repo_name: The full name of the repository, in the form ``owner/name``.
	:param issue_number: The number of the issue or pull request.
	:param refuri: The URL of the issue / pull request on GitHub.
	"""

	repo_name: str

	def __init__(
			self,
			repo_name: str,
			issue_number: Union[str, int],
			refuri: Union[str, URL],
			**kwargs,
			):
		self.has_tooltip = False
		self.issue_number = int(issue_number)
		self.issue_url = str(refuri)
		self.repo_name = str(repo_name)

		source = f"{repo_name}#{issue_number}"
		nodes.reference.__init__(self, source, source, refuri=self.issue_url)

	@property
	def _copy_kwargs(self) -> Dict[str, Any]:  # pragma: no cover
		return {"repo_name": self.repo_name, "issue_number": self.issue_number, "refuri": self.issue_url}


def issue_role(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict[str, Any] = {},
		content: List[str] = []
		) -> Tuple[List[IssueNode], List[system_message]]:
	"""
	Adds a link to the given issue on GitHub.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.issue_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.

	.. latex:clearpage::
	"""

	has_t, issue_number, repository = split_explicit_title(text)
	issue_number = nodes.unescape(issue_number)

	messages: List[system_message] = []
	refnode: IssueNode

	if has_t:
		repository_parts = nodes.unescape(repository).split('/')
		if len(repository_parts) != 2:
			warning_message = inliner.document.reporter.warning(
					f"Invalid repository '{repository}' for issue #{issue_number}.",
					)
			messages.append(warning_message)

		else:
			refnode = IssueNodeWithName(
					repo_name=repository,
					issue_number=issue_number,
					refuri=make_github_url(*repository_parts) / "issues" / str(int(issue_number)),
					)
			return [refnode], messages

	issues_url = inliner.document.settings.env.app.config.github_issues_url
	refnode = IssueNode(issue_number=issue_number, refuri=issues_url / str(int(issue_number)))

	return [refnode], messages


def pull_role(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict[str, Any] = {},
		content: List[str] = []
		) -> Tuple[List[IssueNode], List[system_message]]:
	"""
	Adds a link to the given pulll request on GitHub.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.pull_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.
	"""

	has_t, issue_number, repository = split_explicit_title(text)
	issue_number = nodes.unescape(issue_number)

	messages: List[system_message] = []
	refnode: IssueNode

	if has_t:
		repository_parts = nodes.unescape(repository).split('/')
		if len(repository_parts) != 2:
			warning_message = inliner.document.reporter.warning(
					f"Invalid repository '{repository}' for pull request #{issue_number}."
					)
			messages.append(warning_message)

		else:
			refnode = IssueNodeWithName(
					repo_name=repository,
					issue_number=issue_number,
					refuri=make_github_url(*repository_parts) / "pull" / str(int(issue_number)),
					)
			return [refnode], messages

	pull_url = inliner.document.settings.env.app.config.github_pull_url
	refnode = IssueNode(issue_number=issue_number, refuri=pull_url / str(int(issue_number)))

	return [refnode], messages


def visit_issue_node(translator: HTMLTranslator, node: IssueNode) -> None:
	"""
	Visit an :class:`~.IssueNode`.

	If the node points to a valid issue / pull request,
	add a tooltip giving the title of the issue / pull request and a hyperlink to the page on GitHub.

	:param translator:
	:param node: The node being visited.
	"""

	issue_title = get_issue_title(node.issue_url)

	if issue_title:
		node.has_tooltip = True
		translator.body.append(f'<abbr title="{issue_title}">')
		translator.visit_reference(node)
	else:
		warnings.warn(f"Issue/Pull Request #{node.issue_number} not found.")


def depart_issue_node(translator: HTMLTranslator, node: IssueNode) -> None:
	"""
	Depart an :class:`~.IssueNode`.

	:param translator:
	:param node: The node being visited.
	"""

	if node.has_tooltip:
		translator.depart_reference(node)
		translator.body.append("</abbr>")


def _visit_issue_node_latex(translator: LaTeXTranslator, node: IssueNode) -> None:
	"""
	Visit an :class:`~.IssueNode`.

	If the node points to a valid issue / pull request,
	add a tooltip giving the title of the issue / pull request and a hyperlink to the page on GitHub.

	:param translator:
	:param node: The node being visited.
	"""

	node.children = node.children[:1]
	translator.visit_reference(node)


def _depart_issue_node_latex(translator: LaTeXTranslator, node: IssueNode) -> None:
	"""
	Depart an :class:`~.IssueNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.depart_reference(node)


def get_issue_title(issue_url: str) -> Optional[str]:
	"""
	Returns the title of the issue with the given url,
	or :py:obj:`None` if the issue isn't found.

	:param issue_url:
	"""  # noqa: D400

	try:
		r = cache.session.get(issue_url, timeout=30)
	except requests.exceptions.RequestException:
		return None

	if r.status_code == 200:
		soup = BeautifulSoup(r.content, "html5lib")
		try:
			content = soup.find_all("span", attrs={"class": "js-issue-title"})[0].text
		except IndexError:
			# As of 13 Jan 2023 GitHub seems to use a bidirectional text tag instead
			content = soup.find_all("bdi", attrs={"class": "js-issue-title"})[0].text

		content = xml.sax.saxutils.escape(content).replace('"', "&quot;")
		return content.strip()

	return None

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/github/issues.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  repos_and_users.py
"""
Roles and nodes for referencing GitHub repositories and organizations.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts of the docstrings based on https://docutils.sourceforge.io/docs/howto/rst-roles.html
#

# stdlib
from typing import Any, Dict, List, Tuple, Union

# 3rd party
from apeye.url import URL
from docutils import nodes
from docutils.nodes import system_message
from docutils.parsers.rst.states import Inliner
from sphinx.util.nodes import split_explicit_title
from sphinx.writers.html import HTMLTranslator
from sphinx.writers.latex import LaTeXTranslator

# this package
from sphinx_toolbox.utils import GITHUB_COM, make_github_url

__all__ = (
		"GitHubObjectLinkNode",
		"repository_role",
		"user_role",
		"visit_github_object_link_node",
		"depart_github_object_link_node",
		)


class GitHubObjectLinkNode(nodes.reference):
	"""
	Docutils Node to represent a link to a GitHub repository.

	:param repo_name: The full name of the repository, in the form ``owner/name``.
	:param refuri: The URL of the issue / pull request on GitHub.

	.. clearpage::
	"""

	name: str
	url: str

	def __init__(
			self,
			name: str,
			refuri: Union[str, URL],
			**kwargs,
			):
		self.name = str(name)
		self.url = str(refuri)

		super().__init__(self.name, self.name, refuri=self.url)

	def copy(self) -> "GitHubObjectLinkNode":  # pragma: no cover
		"""
		Return a copy of the :class:`sphinx_toolbox.github.repos.GitHubObjectLinkNode`.
		"""

		# This was required to stop some breakage, but it doesn't seem to run during the tests.
		obj = self.__class__(self.name, self.url)
		obj.document = self.document
		obj.source = self.source
		obj.line = self.line
		return obj


def repository_role(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict[str, Any] = {},
		content: List[str] = []
		) -> Tuple[List[nodes.reference], List[system_message]]:
	"""
	Adds a link to the given repository on GitHub.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.repository_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.
	"""

	has_t, text, repo_name = split_explicit_title(text)
	repo_name = nodes.unescape(repo_name)
	repository_parts = nodes.unescape(repo_name).split('/')

	if len(repository_parts) != 2:
		return [], [inliner.document.reporter.warning(f"Invalid repository '{repo_name}'.")]

	# refnode: nodes.reference

	if has_t:
		refnode = nodes.reference(
				text,
				text,
				refuri=str(make_github_url(*repository_parts)),
				)

	else:
		refnode = GitHubObjectLinkNode(
				name=repo_name,
				refuri=make_github_url(*repository_parts),
				)

	return [refnode], []


def user_role(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict[str, Any] = {},
		content: List[str] = []
		) -> Tuple[List[nodes.reference], List[system_message]]:
	"""
	Adds a link to the given user / organization on GitHub.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.user_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.

	.. clearpage::
	"""

	has_t, text, username = split_explicit_title(text)
	username = nodes.unescape(username)

	messages: List[system_message] = []

	if has_t:
		refnode = nodes.reference(
				text,
				text,
				refuri=str(GITHUB_COM / username),
				)

	else:
		refnode = GitHubObjectLinkNode(
				name=f"@{username}",
				refuri=GITHUB_COM / username,
				)

	return [refnode], messages


def visit_github_object_link_node(translator: HTMLTranslator, node: GitHubObjectLinkNode) -> None:
	"""
	Visit a :class:`~.GitHubObjectLinkNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append(f'<b class="github-object">')
	translator.visit_reference(node)


def depart_github_object_link_node(translator: HTMLTranslator, node: GitHubObjectLinkNode) -> None:
	"""
	Depart an :class:`~.GitHubObjectLinkNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.depart_reference(node)
	translator.body.append("</b>")


def _visit_github_object_link_node_latex(translator: LaTeXTranslator, node: GitHubObjectLinkNode) -> None:
	"""
	Visit a :class:`~.GitHubObjectLinkNode`.

	:param translator:
	:param node: The node being visited.
	"""

	node.children = node.children[:1]
	translator.visit_reference(node)


def _depart_github_object_link_node_latex(translator: LaTeXTranslator, node: GitHubObjectLinkNode) -> None:
	"""
	Depart an :class:`~.GitHubObjectLinkNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.depart_reference(node)

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/github/repos_and_users.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  installation.py
r"""
.. extensions:: sphinx_toolbox.installation


Configuration
--------------

.. confval:: conda_channels
	:type: :class:`~typing.List`\[:class:`str`\]
	:required: False
	:default: ``[]``

	The conda channels required to install the library from Anaconda.

	An alternative to setting it within the :rst:dir:`installation` directive.


Usage
-------

.. rst:directive:: .. installation:: name

	Adds a series of tabs providing installation instructions for the project from a number of sources.

	The directive takes a single required argument -- the name of the project.
	If the project uses a different name on PyPI and/or Anaconda,
	the ``:pypi-name:`` and ``:conda-name:`` options can be used to set the name
	for those repositories.

	.. rst:directive:option:: pypi
		:type: flag

		Flag to indicate the project can be installed from PyPI.

	.. rst:directive:option:: pypi-name: name
		:type: string

		The name of the project on PyPI.

	.. rst:directive:option:: conda
		:type: flag

		Flag to indicate the project can be installed with Conda.

	.. rst:directive:option:: conda-name: name
		:type: string

		The name of the project on Conda.

	.. rst:directive:option:: conda-channels: channels
		:type: comma separated strings

		Comma-separated list of required Conda channels.

		This can also be set via the :confval:`conda_channels` option.

	.. rst:directive:option:: github
		:type: flag

		Flag to indicate the project can be installed from GitHub.

		To use this option add the following to your ``conf.py``:

		.. code-block:: python

			extensions = [
					...
					'sphinx_toolbox.github',
					]

			github_username = '<your username>'
			github_repository = '<your repository>'


		See :mod:`sphinx_toolbox.github` for more information.


	:bold-title:`Example`

	.. rest-example::

		.. installation:: sphinx-toolbox
			:pypi:
			:anaconda:
			:conda-channels: domdfcoding,conda-forge
			:github:



.. rst:directive:: extensions

	Shows instructions on how to enable a Sphinx extension.

	The directive takes a single argument -- the name of the extension.

	.. rst:directive:option:: import-name
		:type: string

		The name used to import the extension, if different from the name of the extension.

	.. rst:directive:option:: no-preamble
		:type: flag

		Disables the preamble text.

	.. rst:directive:option:: no-postamble
		:type: flag

		Disables the postamble text.

	.. rst:directive:option:: first
		:type: flag

		Puts the entry for extension before its dependencies.
		By default is is placed at the end.

		.. versionadded:: 0.4.0


	:bold-title:`Example`

	.. rest-example::

		.. extensions:: sphinx-toolbox
			:import-name: sphinx_toolbox

			sphinx.ext.viewcode
			sphinx_tabs.tabs
			sphinx-prompt


API Reference
--------------
"""  # noqa: D400
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import inspect
import re
import textwrap
import warnings
from typing import Any, Callable, Dict, List, Optional, Tuple

# 3rd party
import dict2css
import sphinx.environment
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.statemachine import ViewList
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.stringlist import StringList
from domdf_python_tools.words import word_join
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util.docutils import SphinxDirective

# this package
from sphinx_toolbox import _css
from sphinx_toolbox.utils import Config, OptionSpec, Purger, SphinxExtMetadata, flag, metadata_add_version

__all__ = (
		"InstallationDirective",
		"ExtensionsDirective",
		"make_installation_instructions",
		"Sources",
		"sources",
		"pypi_installation",
		"conda_installation",
		"github_installation",
		"installation_node_purger",
		"extensions_node_purger",
		"copy_asset_files",
		"setup",
		)


class _Purger(Purger):

	def purge_nodes(self, app: Sphinx, env: BuildEnvironment, docname: str) -> None:  # pragma: no cover
		"""
		Remove all redundant nodes.

		:param app: The Sphinx application.
		:param env: The Sphinx build environment.
		:param docname: The name of the document to remove nodes for.
		"""

		if not hasattr(env, self.attr_name):
			return

		setattr(env, self.attr_name, [])


installation_node_purger = _Purger("all_installation_node_nodes")
extensions_node_purger = Purger("all_extensions_node_nodes")


class Sources(List[Tuple[str, str, Callable, Callable, Optional[Dict[str, Callable]]]]):
	"""
	Class to store functions that provide installation instructions for different sources.

	The syntax of each entry is:

	.. code-block:: python

		(option_name, source_name, getter_function, validator_function, extra_options)

	* ``option_name`` -- a string to use in the directive to specify the source to use,
	* ``source_name`` -- a string to use in the tabs to indicate the installation source,
	* ``getter_function`` -- the function that returns the installation instructions,
	* ``validator_function`` -- a function to validate the option value provided by the user,
	* ``extra_options`` -- a mapping of additional options for the directive that are used by the getter_function.

	.. autoclasssumm:: Sources
		:autosummary-sections: ;;
	"""

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)

	_args = ["options", "env"]  # pylint: disable=W8301
	_directive_name = "installation"

	def register(
			self,
			option_name: str,
			source_name: str,
			validator: Callable = directives.unchanged,
			extra_options: Optional[Dict[str, Callable]] = None,
			) -> Callable:
		"""
		Decorator to register a function.

		The function must have the following signature:

		.. code-block:: python

			def function(
				options: Dict[str, Any],  # Mapping of option names to values.
				env: sphinx.environment.BuildEnvironment,  # The Sphinx build environment.
				) -> List[str]: ...

		:param option_name: A string to use in the directive to specify the source to use.
		:param source_name: A string to use in tabbed installation instructions to represent this source.
		:param validator: A function to validate the option value provided by the user.
		:default validator: :func:`docutils.parsers.rst.directives.unchanged`
		:param extra_options: An optional mapping of extra option names to validator functions.
		:default extra_options: ``{}``

		:return: The registered function.

		:raises: :exc:`SyntaxError` if the decorated function does not take the correct arguments.
		"""

		def _decorator(function: Callable) -> Callable:
			signature = inspect.signature(function)

			if list(signature.parameters.keys()) != self._args:
				raise SyntaxError(  # pragma: no cover
					"The decorated function must take only the following arguments: "
					f"{word_join(self._args, use_repr=True, oxford=True)}"
					)

			self.append((option_name, source_name, function, validator, extra_options or {}))

			setattr(function, f"_{self._directive_name}_registered", True)

			return function

		return _decorator


#: Instance of :class:`~.Sources`.
sources: Sources = Sources()


# pypi_name: The name of the project on PyPI.
@sources.register("pypi", "PyPI", flag, {"pypi-name": directives.unchanged})
def pypi_installation(
		options: Dict[str, Any],
		env: sphinx.environment.BuildEnvironment,
		) -> List[str]:
	"""
	Source to provide instructions for installing from PyPI.

	:param options: Mapping of option names to values.
	:param env: The Sphinx build environment.
	"""

	if "pypi-name" in options:
		pypi_name = options["pypi-name"]
	elif "project_name" in options:
		pypi_name = options["project_name"]
	else:
		raise ValueError("No PyPI project name supplied for the PyPI installation instructions.")

	return [".. prompt:: bash", '', f"    python3 -m pip install {pypi_name} --user"]


# conda_name: The name of the project on PyPI.
@sources.register(
		"anaconda",
		"Anaconda",
		flag,
		{"conda-name": directives.unchanged, "conda-channels": directives.unchanged},
		)
def conda_installation(
		options: Dict[str, Any],
		env: sphinx.environment.BuildEnvironment,
		) -> List[str]:
	"""
	Source to provide instructions for installing from Anaconda.

	:param options: Mapping of option names to values.
	:param env: The Sphinx build environment.
	"""

	if "conda-name" in options:
		conda_name = options["conda-name"]
	elif "pypi-name" in options:
		conda_name = options["pypi-name"]
	elif "project_name" in options:
		conda_name = options["project_name"]
	else:
		raise ValueError("No username supplied for the Anaconda installation instructions.")

	lines: StringList = StringList()
	lines.indent_type = "    "

	if "conda-channels" in options:
		channels = str(options["conda-channels"]).split(',')
	else:
		channels = env.config.conda_channels

	if channels:
		lines.append("First add the required channels\n\n.. prompt:: bash\n")

		with lines.with_indent_size(lines.indent_size + 1):
			for channel in channels:  # pylint: disable=use-list-copy
				lines.append(f"conda config --add channels https://conda.anaconda.org/{channel.strip()}")

		lines.append("\nThen install")

	if lines:
		lines.blankline(ensure_single=True)

	lines.append(f".. prompt:: bash")
	lines.blankline(ensure_single=True)

	with lines.with_indent_size(lines.indent_size + 1):
		lines.append(f"conda install {conda_name}")
		lines.blankline(ensure_single=True)

	return list(lines)


@sources.register("github", "GitHub", flag)
def github_installation(
		options: Dict[str, Any],
		env: sphinx.environment.BuildEnvironment,
		) -> List[str]:
	"""
	Source to provide instructions for installing from GitHub.

	:param options: Mapping of option names to values.
	:param env: The Sphinx build environment.
	"""

	if "sphinx_toolbox.github" not in env.app.extensions:
		raise ValueError(
				"The 'sphinx_toolbox.github' extension is required for the "
				":github: option but it is not enabled!"
				)

	username = getattr(env.config, "github_username", None)
	if username is None:
		raise ValueError("'github_username' has not been set in 'conf.py'!")

	repository = getattr(env.config, "github_repository", None)
	if repository is None:
		raise ValueError("'github_repository' has not been set in 'conf.py'!")

	return [
			".. prompt:: bash",
			'',
			f"    python3 -m pip install git+https://github.com/{username}/{repository}@master --user"
			]


class InstallationDirective(SphinxDirective):
	"""
	Directive to show installation instructions.
	"""

	has_content: bool = True
	optional_arguments: int = 1  # The name of the project; can be overridden for each source

	# Registered sources
	option_spec: OptionSpec = {  # type: ignore[assignment]
		source[0].lower(): source[3]
		for source in sources  # pylint: disable=not-an-iterable
		}

	# Extra options for registered sources
	for source in sources:  # pylint: disable=not-an-iterable,loop-global-usage
		if source[4] is not None:
			option_spec.update(source[4])  # type: ignore[attr-defined]

	options: Dict[str, Any]
	"""
	Mapping of option names to values.

	The options are as follows:

	* **pypi**: Flag to indicate the project can be installed from PyPI.
	* **pypi-name**: The name of the project on PyPI.
	* **conda**: Flag to indicate the project can be installed with Conda.
	* **conda-name**: The name of the project on Conda.
	* **conda-channels**: Comma-separated list of required Conda channels.
	* **github**:  Flag to indicate the project can be installed from GitHub.

	The GitHub username and repository are configured in ``conf.py`` and are available in ``env.config``.
	"""

	def run_html(self) -> List[nodes.Node]:
		"""
		Generate output for ``HTML`` builders.

		:rtype:

		.. latex:clearpage::
		"""

		targetid = f'installation-{self.env.new_serialno("sphinx-toolbox installation"):d}'
		targetnode = nodes.target('', '', ids=[targetid])

		content = make_installation_instructions(self.options, self.env)
		view = ViewList(content)

		installation_node = nodes.paragraph(rawsource=content)  # type: ignore[arg-type]
		self.state.nested_parse(view, self.content_offset, installation_node)  # type: ignore[arg-type]
		installation_node_purger.add_node(self.env, installation_node, targetnode, self.lineno)

		return [targetnode, installation_node]

	def run_generic(self) -> List[nodes.Node]:
		"""
		Generate generic reStructuredText output.
		"""

		targetid = f'installation-{self.env.new_serialno("sphinx-toolbox installation"):d}'
		targetnode = nodes.target('', '', ids=[targetid])

		tabs: Dict[str, List[str]] = _get_installation_instructions(self.options, self.env)

		if not tabs:
			warnings.warn("No installation source specified. No installation instructions will be shown.")
			return []

		nodes_to_return: List[nodes.Node] = [targetnode]

		non_word_sub = re.compile(r'\W+')

		for tab_name, tab_content in tabs.items():
			# pylint: disable=loop-global-usage
			section_id = non_word_sub.sub('_', tab_name)
			section = nodes.section(ids=[f"{targetid}-{section_id}"])
			section += nodes.title(tab_name, tab_name)
			nodes_to_return.append(section)
			installation_node_purger.add_node(self.env, section, targetnode, self.lineno)

			view = ViewList(tab_content)
			paragraph_node = nodes.paragraph(rawsource=tab_content)  # type: ignore[arg-type]
			self.state.nested_parse(view, self.content_offset, paragraph_node)  # type: ignore[arg-type]
			nodes_to_return.append(paragraph_node)
			installation_node_purger.add_node(self.env, paragraph_node, targetnode, self.lineno)
			# pylint: enable=loop-global-usage

		return nodes_to_return

	def run(self) -> List[nodes.Node]:
		"""
		Create the installation node.
		"""

		assert self.env.app.builder is not None

		if self.arguments:
			self.options["project_name"] = self.arguments[0]

		if self.env.app.builder.format.lower() == "html":
			return self.run_html()
		else:
			return self.run_generic()


def make_installation_instructions(options: Dict[str, Any], env: BuildEnvironment) -> List[str]:
	"""
	Make the content of an installation node.

	:param options:
	:param env: The Sphinx build environment.
	"""

	tabs: Dict[str, List[str]] = _get_installation_instructions(options, env)

	if not tabs:
		warnings.warn("No installation source specified. No installation instructions will be shown.")
		return []

	content = StringList([".. tabs::", ''])
	content.set_indent_type("    ")

	for tab_name, tab_content in tabs.items():
		with content.with_indent_size(1):
			content.append(f".. tab:: {tab_name}")
			content.blankline(ensure_single=True)

		with content.with_indent_size(2):
			content.extend([f"{line}" if line else '' for line in tab_content])  # pylint: disable=loop-invariant-statement

	return list(content)


def _get_installation_instructions(options: Dict[str, Any], env: BuildEnvironment) -> Dict[str, List[str]]:
	"""
	Returns a mapping of tab/section names to their content.

	:param options:
	:param env: The Sphinx build environment.
	"""

	tabs: Dict[str, List[str]] = {}

	# pylint: disable=loop-global-usage,use-dict-comprehension
	for option_name, source_name, getter_function, validator_function, extra_options in sources:
		# pylint: enable=loop-global-usage,use-dict-comprehension
		if option_name in options:
			tabs[f"from {source_name}"] = getter_function(options, env)

	return tabs


class ExtensionsDirective(SphinxDirective):
	"""
	Directive to show instructions for enabling the extension.
	"""

	has_content: bool = True  # Other required extensions, one per line
	optional_arguments: int = 1  # The name of the project
	option_spec: OptionSpec = {  # type: ignore[assignment]
		"import-name": directives.unchanged_required,  # If different to project name
		"no-preamble": flag,
		"no-postamble": flag,
		"first": flag,
		}

	def run(self) -> List[nodes.Node]:
		"""
		Create the extensions node.

		:rtype:

		.. latex:clearpage::
		"""

		extensions = list(self.content)
		first = self.options.get("first", False)

		if "import-name" in self.options and first:
			extensions.insert(0, self.options["import-name"])
		elif "import-name" in self.options:
			extensions.append(self.options["import-name"])
		elif first:
			extensions.insert(0, self.arguments[0])
		else:
			extensions.append(self.arguments[0])

		targetid = f'extensions-{self.env.new_serialno("sphinx-toolbox extensions"):d}'
		targetnode = nodes.target('', '', ids=[targetid])

		top_text = (
				".. latex:vspace:: 10px",
				".. rst-class:: sphinx-toolbox-extensions",
				'',
				f"    Enable ``{self.arguments[0]}`` by adding the following",
				f"    to the ``extensions`` variable in your ``conf.py``:",
				)
		bottom_text = textwrap.dedent(
				r"""
		.. raw:: latex

			\begin{flushleft}

		For more information see https://www.sphinx-doc.org/en/master/usage/extensions#third-party-extensions .

		.. raw:: latex

			\end{flushleft}
		"""
				).expandtabs(4).splitlines()

		if "no-preamble" in self.options:
			content = []
		else:
			content = [*top_text, '']

		content.append(".. code-block:: python", )

		if "no-postamble" not in self.options:
			content.append("    :class: sphinx-toolbox-extensions")

		content.extend([
				'',
				"    extensions = [",
				"        ...",
				])

		for extension in extensions:  # pylint: disable=use-list-copy
			content.append(f"        {extension!r},")

		content.extend(["        ]", ''])

		if "no-postamble" not in self.options:
			content.extend(bottom_text)

		extensions_node = nodes.paragraph(rawsource=content)  # type: ignore[arg-type]
		self.state.nested_parse(ViewList(content), self.content_offset, extensions_node)  # type: ignore[arg-type]
		extensions_node_purger.add_node(self.env, extensions_node, targetnode, self.lineno)

		return [targetnode, extensions_node]


def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None:
	"""
	Copy additional stylesheets into the HTML build directory.

	.. versionadded:: 1.2.0

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.format.lower() != "html":  # pragma: no cover
		return

	static_dir = PathPlus(app.outdir) / "_static"
	static_dir.maybe_make(parents=True)
	dict2css.dump(_css.installation_styles, static_dir / "sphinx_toolbox_installation.css", minify=True)

	(static_dir / "sphinx_toolbox_installation.js").write_lines([
			"// Based on https://github.com/executablebooks/sphinx-tabs/blob/master/sphinx_tabs/static/tabs.js",
			"// Copyright (c) 2017 djungelorm",
			"// MIT Licensed",
			'',
			"function deselectTabset(target) {",
			"  const parent = target.parentNode;",
			"  const grandparent = parent.parentNode;",
			'',
			'  if (parent.parentNode.parentNode.getAttribute("id").startsWith("installation")) {',
			'',
			"    // Hide all tabs in current tablist, but not nested",
			"    Array.from(parent.children).forEach(t => {",
			'      if (t.getAttribute("name") !== target.getAttribute("name")) {',
			'        t.setAttribute("aria-selected", "false");',
			"      }",
			"    });",
			'',
			"    // Hide all associated panels",
			"    Array.from(grandparent.children).slice(1).forEach(p => {  // Skip tablist",
			'      if (p.getAttribute("name") !== target.getAttribute("name")) {',
			'        p.setAttribute("hidden", "false")',
			"      }",
			"    });",
			"  }",
			'',
			"  else {",
			"    // Hide all tabs in current tablist, but not nested",
			"    Array.from(parent.children).forEach(t => {",
			'      t.setAttribute("aria-selected", "false");',
			"    });",
			'',
			"    // Hide all associated panels",
			"    Array.from(grandparent.children).slice(1).forEach(p => {  // Skip tablist",
			'      p.setAttribute("hidden", "true")',
			"    });",
			"  }",
			'',
			'}',
			'',
			"// Compatibility with sphinx-tabs 2.1.0 and later",
			"function deselectTabList(tab) {deselectTabset(tab)}",
			'',
			])


def _on_config_inited(app: Sphinx, config: Config) -> None:
	app.add_css_file("sphinx_toolbox_installation.css")
	app.add_js_file("sphinx_toolbox_installation.js")


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.installation`.

	.. versionadded:: 0.7.0

	:param app: The Sphinx application.
	"""

	if "sphinx_inline_tabs" not in getattr(app, "extensions", ()):
		app.setup_extension("sphinx_tabs.tabs")

	app.setup_extension("sphinx-prompt")
	app.setup_extension("sphinx_toolbox._css")
	app.setup_extension("sphinx_toolbox.latex")

	app.add_config_value("conda_channels", [], "env", types=[list])

	# Instructions for installing a python package
	app.add_directive("installation", InstallationDirective)
	app.connect("env-get-outdated", installation_node_purger.get_outdated_docnames)
	# app.connect("env-purge-doc", installation_node_purger.purge_nodes)

	# Instructions for enabling a sphinx extension
	app.add_directive("extensions", ExtensionsDirective)
	app.connect("env-purge-doc", extensions_node_purger.purge_nodes)

	# Ensure this happens after tabs.js has been added
	app.connect("config-inited", _on_config_inited, priority=510)
	app.connect("build-finished", copy_asset_files)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/installation.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  issues.py
r"""
Add links to GitHub issues and Pull Requests.

.. extensions:: sphinx_toolbox.issues


Usage
------

.. latex:vspace:: -10px

.. rst:role:: issue

	Role which shows a link to the given issue on GitHub.

	If the issue exists, the link has a tooltip that shows the title of the issue.

	**Example**

	.. rest-example::

		:issue:`1`

	You can also reference an issue in a different repository by adding the repository name inside ``<>``.

	.. rest-example::

		:issue:`7680 <pytest-dev/pytest>`


.. rst:role:: pull

	Role which shows a link to the given pull request on GitHub.

	If the pull requests exists, the link has a tooltip that shows the title of the pull requests.

	**Example**

	.. rest-example::

		:pull:`2`

	You can also reference a pull request in a different repository by adding the repository name inside ``<>``.

	.. rest-example::

		:pull:`7671 <pytest-dev/pytest>`


.. versionchanged:: 2.4.0

	:rst:role:`issue` and :rst:role:`pull` now show the repository name
	when the name differs from that configured in ``conf.py``.

.. versionchanged:: 2.4.0

	These directives are also available in the :mod:`~.sphinx_toolbox.github` domain.

The only difference between the :rst:role:`issue` and :rst:role:`pull` roles
is in the URL. GitHub uses the same numbering scheme for issues and
pull requests, and automatically redirects to the pull request if
the user tries to navigate to an issue with that same number.


Caching
-----------

HTTP requests to obtain issue/pull request titles are cached for four hours.

To clear the cache manually, run:

.. prompt:: bash

	python3 -m sphinx_toolbox


API Reference
---------------

.. versionchanged:: 2.4.0

	The following moved to :mod:`sphinx_toolbox.github.issues`:

	* :class:`~.IssueNode`
	* :class:`~.IssueNodeWithName`
	* :func:`~.issue_role`
	* :func:`~.pull_role`
	* :func:`~.visit_issue_node`
	* :func:`~.depart_issue_node`
	* :func:`~.get_issue_title`

.. latex:vspace:: 10px

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on pyspecific.py from the Python documentation.
#  Copyright 2008-2014 by Georg Brandl.
#  Licensed under the PSF License 2.0
#
#  Parts of the docstrings based on https://docutils.sourceforge.io/docs/howto/rst-roles.html
#

# 3rd party
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.github.issues import issue_role, pull_role
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = (
		"issue_role",
		"pull_role",
		"setup",
		)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.issues`.

	.. versionadded:: 1.0.0

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx_toolbox.github")

	# Link to GH issue
	app.add_role("issue", issue_role)

	# Link to GH pull request
	app.add_role("pr", pull_role)
	app.add_role("pull", pull_role)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/issues.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  latex.py
r"""
Sphinx utilities for LaTeX builders.

.. versionadded:: 2.8.0

In addition to the developer API (see below), :mod:`sphinx_toolbox.latex`
configures Sphinx and LaTeX to correctly handle symbol footnotes.

.. versionchanged:: 2.12.0

	Sphinx is also configured to respect ``.. only:: html`` etc. directives surrounding
	toctree directives when determining the overall toctree depth.

.. extensions:: sphinx_toolbox.latex


.. latex:vspace:: -5px

Example Footnotes
--------------------

.. latex:vspace:: -10px

| Hello [1]_
| Goodbye [2]_
| Symbol [*]_
| Another Symbol [*]_
| Number Again [3]_
| Symbol 3 [*]_
| Symbol 4 [*]_
| Symbol 5 [*]_
| Symbol 6 [*]_
| Symbol 7 [*]_
| Symbol 8 [*]_
| Symbol 9 [*]_

.. latex:vspace:: 20px

.. [1] One
.. [2] Two
.. [*] Buckle my shoe
.. [*] The second symbol
.. [3] The number after the symbol
.. [*] Symbol 3
.. [*] Symbol 4
.. [*] Symbol 5
.. [*] Symbol 6
.. [*] Symbol 7
.. [*] Symbol 8
.. [*] Symbol 9


Usage
-------

.. latex:vspace:: -10px


.. rst:directive:: latex:samepage
                   samepage

	Configures LaTeX to make all content within this directive appear on the same page.

	This can be useful to avoid awkward page breaks.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.9.0


.. rst:directive:: latex:clearpage
                   clearpage

	Configures LaTeX to start a new page.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.10.0
	.. seealso:: :rst:dir:`latex:cleardoublepage`


.. rst:directive:: latex:cleardoublepage
                   cleardoublepage

	Configures LaTeX to start a new page.

	In a two-sided printing it also makes the next page a right-hand (odd-numbered) page,
	inserting a blank page if necessary.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.10.0
	.. seealso:: :rst:dir:`latex:clearpage`


.. rst:directive:: .. latex:vspace:: space

	Configures LaTeX to add or remove vertical space.

	The value for ``space`` is passed verbatim to the ``\vspace{}`` command.
	Ensure you pass a valid value.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.11.0


.. latex:clearpage::

API Reference
----------------

.. latex:vspace:: -20px
"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  PatchedLaTeXBuilder based on Sphinx
#  Copyright (c) 2007-2020 by the Sphinx team.
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |    notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |    notice, this list of conditions and the following disclaimer in the
#  |    documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import re
from os.path import join as joinpath
from textwrap import dedent
from typing import Any, List, Optional, cast

# 3rd party
import sphinx
from docutils import nodes
from docutils.frontend import OptionParser
from docutils.transforms.references import Footnotes
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.stringlist import DelimitedList
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.builders.latex import LaTeXBuilder
from sphinx.domains import Domain
from sphinx.environment import BuildEnvironment
from sphinx.locale import __
from sphinx.util.docutils import SphinxDirective, SphinxFileOutput
from sphinx.util.nodes import process_only_nodes
from sphinx.writers.latex import LaTeXTranslator, LaTeXWriter

# this package
from sphinx_toolbox.utils import Config, SphinxExtMetadata, metadata_add_version

try:
	# 3rd party
	from sphinx.util.display import progress_message  # type: ignore[import]
except ImportError:
	# 3rd party
	from sphinx.util import progress_message

_ = BuildEnvironment

__all__ = (
		"use_package",
		"visit_footnote",
		"depart_footnote",
		"SamepageDirective",
		"ClearPageDirective",
		"ClearDoublePageDirective",
		"VSpaceDirective",
		"LaTeXDomain",
		"replace_unknown_unicode",
		"better_header_layout",
		"configure",
		"setup",
		)

footmisc_symbols = ['0', *Footnotes.symbols]
# footmisc_symbols = ['0', '*', '†', '‡', '§', '¶', '‖', "**", "††", "‡‡"]


def visit_footnote(translator: LaTeXTranslator, node: nodes.footnote) -> None:
	"""
	Visit a :class:`docutils.nodes.footnote` node with the LaTeX translator.

	Unlike the default ``visit_footnote`` function, this one handles footnotes using symbols.

	.. versionadded:: 2.8.0

	:param translator:
	:param node:
	"""

	translator.in_footnote += 1
	footnote_id = str(cast(nodes.label, node[0]).astext())

	if not translator.in_parsed_literal:
		translator.body.append("%\n")

	if not footnote_id.isnumeric() and footnote_id in footmisc_symbols:
		footnote_id = str(footmisc_symbols.index(footnote_id))
		translator.body.append(r"\renewcommand\thefootnote{\thesymbolfootnote}")

	translator.body.append(rf"\begin{{footnote}}[{footnote_id}]")
	translator.body.append("\\sphinxAtStartFootnote\n")


def depart_footnote(translator: LaTeXTranslator, node: nodes.footnote) -> None:
	"""
	Depart a :class:`docutils.nodes.footnote` node with the LaTeX translator.

	.. versionadded:: 2.8.0

	:param translator:
	:param node:
	"""

	if not translator.in_parsed_literal:
		translator.body.append("%\n")

	translator.body.append(r"\end{footnote}")
	translator.body.append(r"\renewcommand\thefootnote{\thenumberfootnote}")
	translator.in_footnote -= 1


def use_package(package: str, config: Config, *args: str, **kwargs: str) -> None:
	r"""
	Configure LaTeX to use the given package.

	The ``\usepackage`` entry is added to the
	:py:obj:`sphinx.config.Config.latex_elements` ``["preamble"]`` attribute.

	:param package:
	:param config:
	:param \*args: Passed as options to the LaTeX package.
	:param \*\*kwargs: Passed as named options to the LaTeX package.
	"""

	options: DelimitedList[str] = DelimitedList()
	options.extend(args)
	options.extend(map("{}={}".format, kwargs.items()))

	use_string = rf"\usepackage[{options:,}]{{{package}}}"

	if not hasattr(config, "latex_elements") or not config.latex_elements:  # pragma: no cover
		config.latex_elements = {}

	latex_preamble = config.latex_elements.get("preamble", '')

	if use_string not in latex_preamble:
		config.latex_elements["preamble"] = f"{latex_preamble}\n{use_string}"


class SamepageDirective(SphinxDirective):
	"""
	Directive which configures LaTeX to make all content within this directive appear on the same page.

	This can be useful to avoid awkward page breaks.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.9.0
	"""

	has_content = True

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the directive.
		"""

		content_node = nodes.container(rawsource='\n'.join(self.content))
		self.state.nested_parse(self.content, self.content_offset, content_node)

		return [
				nodes.raw('', r"\par\begin{samepage}", format="latex"),
				content_node,
				nodes.raw('', r"\end{samepage}\par", format="latex"),
				]


class ClearPageDirective(SphinxDirective):
	"""
	Directive which configures LaTeX to start a new page.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.10.0
	.. seealso:: :class:`~.ClearDoublePageDirective`
	"""

	has_content = True

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the directive.
		"""

		return [nodes.raw('', r"\clearpage", format="latex")]


class ClearDoublePageDirective(SphinxDirective):
	"""
	Directive which configures LaTeX to start a new page.

	In a two-sided printing it also makes the next page a right-hand (odd-numbered) page,
	inserting a blank page if necessary.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.10.0
	.. seealso:: :class:`~.ClearPageDirective`
	"""

	has_content = True

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the directive.
		"""

		return [nodes.raw('', r"\cleardoublepage", format="latex")]


class VSpaceDirective(SphinxDirective):
	"""
	Directive which configures LaTeX to add or remove vertical space.

	This directive has no effect with non-LaTeX builders.

	.. versionadded:: 2.11.0
	"""

	required_arguments = 1  # the space

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the directive.
		"""

		return [nodes.raw('', f"\n\\vspace{{{self.arguments[0]}}}\n", format="latex")]


class LaTeXDomain(Domain):
	"""
	Domain containing various LaTeX-specific directives.

	.. versionadded:: 2.11.0
	"""

	name = "latex"
	label = "LaTeX"
	directives = {
			"samepage": SamepageDirective,
			"clearpage": ClearPageDirective,
			"cleardoublepage": ClearDoublePageDirective,
			"vspace": VSpaceDirective,
			}


def replace_unknown_unicode(app: Sphinx, exception: Optional[Exception] = None) -> None:
	r"""
	Replaces certain unknown unicode characters in the Sphinx LaTeX output with the best equivalents.

	.. only:: html

		The mapping is as follows:

		* ♠ -- \spadesuit
		* ♥ -- \heartsuit
		* ♦ -- \diamondsuit
		* ♣ -- \clubsuit
		* Zero width space -- \hspace{0pt}
		* μ -- \textmu
		* ≡ -- \equiv (new in version 2.11.0)
		* ≈ -- \approx (new in version 2.12.0)
		* ≥ -- \geq (new in version 2.13.0)
		* ≤ -- \leq (new in version 2.13.0)

	This function can be hooked into the :event:`build-finished` event as follows:

	.. code-block:: python

		app.connect("build-finished", replace_unknown_unicode)

	.. versionadded:: 2.9.0

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.name.lower() != "latex":
		return

	builder = cast(LaTeXBuilder, app.builder)
	output_file = PathPlus(builder.outdir) / f"{builder.titles[0][1].lower()}.tex"

	output_content = output_file.read_text()

	output_content = output_content.replace('♠', r' $\spadesuit$ ')
	output_content = output_content.replace('♥', r' $\heartsuit$ ')
	output_content = output_content.replace('♦', r' $\diamondsuit$ ')
	output_content = output_content.replace('♣', r' $\clubsuit$ ')
	output_content = output_content.replace('\u200b', r'\hspace{0pt}')  # Zero width space
	output_content = output_content.replace('μ', r"\textmu{}")
	output_content = output_content.replace('≡', r" $\equiv$ ")
	output_content = output_content.replace('≈', r" $\approx$ ")
	output_content = output_content.replace('≥', r" $\geq$ ")
	output_content = output_content.replace('≤', r" $\leq$ ")

	output_file.write_clean(output_content)


def better_header_layout(
		config: Config,
		space_before: int = 10,
		space_after: int = 20,
		) -> None:
	"""
	Makes LaTeX chapter names lowercase, and adjusts the spacing above and below the chapter name.

	.. versionadded:: 2.10.0

	:param config: The Sphinx configuration object.
	:param space_before: The space, in pixels, before the chapter name.
	:param space_after: The space, in pixels, after the chapter name.
	"""

	begin = "% begin st better header layout"
	end = "% end st better header layout"

	commands = rf"""
	{begin}
	\makeatletter
		\renewcommand{{\DOCH}}{{%
			\mghrulefill{{\RW}}\par\nobreak
			\CNV\FmN{{\@chapapp}}\par\nobreak
			\CNoV\TheAlphaChapter\par\nobreak
			\vskip -1\baselineskip\vskip 5pt\mghrulefill{{\RW}}\par\nobreak
			\vskip 10\p@
			}}
		\renewcommand{{\DOTI}}[1]{{%
			\CTV\FmTi{{#1}}\par\nobreak
			\vskip {space_before}\p@
			}}
		\renewcommand{{\DOTIS}}[1]{{%
			\CTV\FmTi{{#1}}\par\nobreak
			\vskip {space_after}\p@
			}}
	\makeatother
	{end}
	"""

	if not hasattr(config, "latex_elements") or not config.latex_elements:  # pragma: no cover
		config.latex_elements = {}

	latex_preamble = config.latex_elements.get("preamble", '')

	if begin in latex_preamble:
		config.latex_elements["preamble"] = re.sub(
				f"{begin}.*{end}",
				dedent(commands),
				latex_preamble,
				count=1,
				flags=re.DOTALL,
				)
	else:
		config.latex_elements["preamble"] = f"{latex_preamble}\n{dedent(commands)}"

	config.latex_elements["fncychap"] = "\\usepackage[Bjarne]{fncychap}\n\\ChNameAsIs\n\\ChTitleAsIs\n"


class PatchedLaTeXBuilder(LaTeXBuilder):
	"""
	Patched version of Sphinx's LaTeX builder which skips toctrees under ``.. only:: html`` etc. directives
	when determining the max toctree depth.

	The default behaviour uses the ``:maxdepth:`` option of the first toctree,
	irrespective of whether the toctree will exist with the LaTeX builder.
	"""  # noqa: D400

	# TODO: respect toctree caption for LaTeX table of contents too
	# \addto\captionsenglish{\renewcommand{\contentsname}{Documentation}}

	def write(self, *ignored: Any) -> None:
		assert self.env is not None

		docwriter = LaTeXWriter(self)
		docsettings: Any = OptionParser(
				defaults=self.env.settings,
				components=(docwriter, ),
				read_config_files=True,
				).get_default_values()

		if sphinx.version_info <= (4, 0):
			# 3rd party
			from sphinx.builders.latex import patch_settings  # type: ignore[attr-defined]
			patch_settings(docsettings)

		self.init_document_data()
		self.write_stylesheet()

		for entry in self.document_data:
			docname, targetname, title, author, themename = entry[:5]
			theme = self.themes.get(themename)
			toctree_only = False
			if len(entry) > 5:
				toctree_only = entry[5]
			destination = SphinxFileOutput(
					destination_path=joinpath(self.outdir, targetname),
					encoding="utf-8",
					overwrite_if_changed=True
					)
			with progress_message(__("processing %s") % targetname):
				doctree = self.env.get_doctree(docname)
				process_only_nodes(doctree, self.tags)

				if hasattr(doctree, "findall"):
					toctree = next(doctree.findall(addnodes.toctree), None)  # type: ignore[attr-defined]
				else:
					toctree = next(iter(doctree.traverse(addnodes.toctree)), None)

				if toctree and toctree.get("maxdepth") > 0:
					tocdepth = toctree.get("maxdepth")
				else:
					tocdepth = None

				doctree = self.assemble_doctree(
						docname,
						toctree_only,
						appendices=(self.config.latex_appendices if theme.name != "howto" else [])
						)
				doctree["docclass"] = theme.docclass
				doctree["contentsname"] = self.get_contentsname(docname)
				doctree["tocdepth"] = tocdepth
				self.post_process_images(doctree)
				self.update_doc_context(title, author, theme)

				if hasattr(self, "update_context"):  # pragma: no cover
					# Only present in newer Sphinx versions
					self.update_context()

			with progress_message(__("writing")):
				# pylint: disable=loop-invariant-statement
				docsettings._author = author
				docsettings._title = title
				docsettings._contentsname = doctree["contentsname"]
				docsettings._docname = docname
				docsettings._docclass = theme.name

				doctree.settings = docsettings
				docwriter.theme = theme
				docwriter.write(doctree, destination)
				# pylint: enable=loop-invariant-statement


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.latex`.

	:param app: The Sphinx application.
	:param config:
	"""

	if not hasattr(config, "latex_elements") or not config.latex_elements:  # pragma: no cover
		config.latex_elements = {}

	latex_preamble = config.latex_elements.get("preamble", '')

	command_string = r"\newcommand\thesymbolfootnote{\fnsymbol{footnote}}\let\thenumberfootnote\thefootnote"

	if command_string not in latex_preamble:
		config.latex_elements["preamble"] = f"{latex_preamble}\n{command_string}"


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.latex`.

	.. versionadded:: 2.8.0

	:param app: The Sphinx application.
	"""

	app.add_node(nodes.footnote, latex=(visit_footnote, depart_footnote), override=True)

	app.add_directive("samepage", SamepageDirective)
	app.add_directive("clearpage", ClearPageDirective)
	app.add_directive("cleardoublepage", ClearDoublePageDirective)

	app.add_domain(LaTeXDomain)

	app.add_builder(PatchedLaTeXBuilder, override=True)

	app.connect("config-inited", configure)

	return {}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/latex/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  layout.py
r"""
Makes minor adjustments to the LaTeX layout.

* Increases the whitespace above function signatures by 5px,
  to prevent the function visually merging with the previous one.
* Remove unnecessary indentation and allow "raggedright" for the fields in
  the body of functions, which prevents ugly whitespace and line breaks.
* Disables justification for function signatures.
  This is a backport of changes from Sphinx 4 added in :github:pull:`8997 <sphinx-doc/sphinx>`.

  .. versionadded:: 2.12.0

* With Sphinx 3.5, doesn't add ``\sphinxAtStartPar`` before every paragraph.
  The change in :github:issue:`8781 <sphinx-doc/sphinx>` was to solve an issue with *tables*,
  but it isn't clear why it then gets added for *every* paragraph so this extension removes it.

  .. versionadded:: 2.13.0

* Configures hyperref to apply correct page numbering to the frontmatter.

  .. versionadded:: 2.14.0

* Optionally, configures the ``needspace`` package.

  The :confval:`needspace_amount` option can be set in ``conf.py`` to add the ``\needspace{}`` command
  before each ``addnodes.desc`` node (i.e. a function or class description).
  The amount of space is set by the ``needspace_amount`` option, e.g.:

  .. code-block:: python

      needspace_amount = r"4\baselineskip"

  .. versionadded:: 3.0.0


.. confval:: needspace_amount
	:type: :class:`str`
	:default: :py:obj:`None`

	Sets the value for the ``\needspace{}`` command.

	Values should be in the form ``r"4\baselineskip"``.

	.. versionadded:: 3.0.0


.. versionadded:: 2.10.0
.. versionchanged:: 3.0.0  Moved from :mod:`sphinx_toolbox.tweaks.latex_layout`.
.. latex:clearpage::
.. extensions:: sphinx_toolbox.latex.layout

-----

"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# 3rd party
from docutils import nodes
from domdf_python_tools.stringlist import StringList
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.builders.latex.nodes import footnotetext
from sphinx.config import Config
from sphinx.writers.latex import LaTeXTranslator

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("setup", )


def visit_desc(translator: LaTeXTranslator, node: addnodes.desc) -> None:
	translator.body.append('\n\n\\vspace{5px}')

	needspace_amount = getattr(translator.config, "needspace_amount")
	if needspace_amount:
		translator.body.append(f"\\needspace{{{needspace_amount}}}")

	if "sphinxcontrib.toctree_plus" in translator.config.extensions:
		# 3rd party
		from sphinxcontrib import toctree_plus  # type: ignore[import]  # nodep

		toctree_plus.visit_desc(translator, node)
	else:
		LaTeXTranslator.visit_desc(translator, node)


def visit_field_list(translator: LaTeXTranslator, node: nodes.field_list) -> None:
	translator.body.append('\\vspace{10px}\\begin{flushleft}\\begin{description}\n')
	if translator.table:  # pragma: no cover
		translator.table.has_problematic = True


def depart_field_list(translator: LaTeXTranslator, node: nodes.field_list) -> None:
	translator.body.append('\\end{description}\\end{flushleft}\\vspace{10px}\n')


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure Sphinx Extension.

	:param app: The Sphinx application.
	:param config:
	"""

	if not hasattr(config, "latex_elements"):  # pragma: no cover
		config.latex_elements = {}  # type: ignore[attr-defined]

	latex_elements = (config.latex_elements or {})

	latex_preamble = latex_elements.get("preamble", '')

	# Backported from Sphinx 4
	# See https://github.com/sphinx-doc/sphinx/pull/8997
	config.latex_elements["preamble"] = '\n'.join([
			latex_preamble,
			r"\makeatletter",
			'',
			r"\renewcommand{\py@sigparams}[2]{%",
			r"  \parbox[t]{\py@argswidth}{\raggedright #1\sphinxcode{)}#2\strut}%",
			"  % final strut is to help get correct vertical separation in case of multi-line",
			"  % box with the item contents.",
			'}',
			r"\makeatother",
			])

	config.latex_elements["hyperref"] = '\n'.join([
			r"% Include hyperref last.",
			r"\usepackage[pdfpagelabels,hyperindex,hyperfigures]{hyperref}",
			r"% Fix anchor placement for figures with captions.",
			r"\usepackage{hypcap}% it must be loaded after hyperref.",
			])

	config.latex_elements["maketitle"] = '\n'.join([
			r"\begingroup",
			r"\let\oldthepage\thepage",
			r"\renewcommand{\thepage}{T\oldthepage}",
			config.latex_elements.get("maketitle", r"\sphinxmaketitle"),
			r"\endgroup"
			])

	needspace_amount = getattr(config, "needspace_amount")
	if needspace_amount:
		latex_extrapackages = StringList(latex_elements.get("extrapackages", ''))
		latex_extrapackages.append(r"\usepackage{needspace}")
		latex_elements["extrapackages"] = str(latex_extrapackages)


def visit_paragraph(translator: LaTeXTranslator, node: nodes.paragraph) -> None:
	index = node.parent.index(node)
	if (
			index > 0 and isinstance(node.parent, nodes.compound)
			and not isinstance(node.parent[index - 1], nodes.paragraph)
			and not isinstance(node.parent[index - 1], nodes.compound)
			):
		# insert blank line, if the paragraph follows a non-paragraph node in a compound
		translator.body.append("\\noindent\n")
	elif index == 1 and isinstance(node.parent, (nodes.footnote, footnotetext)):
		# don't insert blank line, if the paragraph is second child of a footnote
		# (first one is label node)
		pass
	else:
		# Sphinx 3.5 adds \sphinxAtStartPar here, but I don't see what it gains.
		translator.body.append('\n')


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.latex.layout`.

	:param app: The Sphinx application.
	"""

	app.connect("config-inited", configure, priority=500)

	app.add_node(addnodes.desc, latex=(visit_desc, LaTeXTranslator.depart_desc), override=True)
	app.add_node(nodes.field_list, latex=(visit_field_list, depart_field_list), override=True)
	app.add_node(nodes.paragraph, latex=(visit_paragraph, LaTeXTranslator.depart_paragraph), override=True)
	app.add_config_value("needspace_amount", default=None, rebuild="latex", types=[str])

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/latex/layout.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  succinct_seealso.py
"""
Sphinx extension which customises :rst:dir:`seealso` directives to be on one line with the LaTeX builder.

.. extensions:: sphinx_toolbox.latex.succinct_seealso

.. versionadded:: 3.0.0
"""
#
#  Copyright © 2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# 3rd party
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.locale import admonitionlabels
from sphinx.writers.latex import LaTeXTranslator

__all__ = ("setup", )

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version


def visit_seealso(translator: LaTeXTranslator, node: addnodes.seealso) -> None:
	"""
	Visit an :class:`addnodes.seealso`` node.

	:param translator:
	:param node:
	"""

	# translator.body.append('\n\n\\begin{description}\\item[{%s:}] \\leavevmode' % admonitionlabels['seealso'])
	# translator.body.append('\n\n\\sphinxstrong{%s:} ' % admonitionlabels["seealso"])
	if len(node) > 1:
		LaTeXTranslator.visit_seealso(translator, node)
	else:
		translator.body.append('\n\n\\sphinxstrong{%s:} ' % admonitionlabels["seealso"])


def depart_seealso(translator: LaTeXTranslator, node: addnodes.seealso) -> None:
	"""
	Depart an :class:`addnodes.seealso`` node.

	:param translator:
	:param node:
	"""

	if len(node) > 1:
		LaTeXTranslator.depart_seealso(translator, node)
	else:
		# translator.body.append("\\end{description}\n\n")
		translator.body.append("\n\n")


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.latex.succinct_seealso`.

	:param app: The Sphinx application.
	"""

	app.add_node(addnodes.seealso, latex=(visit_seealso, depart_seealso), override=True)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/latex/succinct_seealso.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  toc.py
"""
Adjusts the default LaTeX output as follows:

* The captions from ``toctree`` directives are converted into document parts.
* The PDF outline has the correct hierarchy, including having the indices as top-level elements.

.. versionadded:: 2.1.0
.. versionchanged:: 3.0.0  Moved from :mod:`sphinx_toolbox.tweaks.latex_toc`.
.. latex:clearpage::
.. extensions:: sphinx_toolbox.latex.toc

-----

"""  # noqa: D400
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import List

# 3rd party
import sphinx.directives.other
import sphinx.writers.latex
from docutils import nodes
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.latex import use_package
from sphinx_toolbox.utils import Config, SphinxExtMetadata, metadata_add_version

__all__ = ("setup", )

nest_bookmark_level_part = "\\bookmarksetupnext{{level=part}}\n"


class latex_toc(nodes.raw):
	pass


class LaTeXTranslator(sphinx.writers.latex.LaTeXTranslator):

	def generate_indices(self) -> str:

		super_output = super().generate_indices()

		if not super_output:
			return nest_bookmark_level_part

		return '\n'.join([
				nest_bookmark_level_part,
				*super_output.splitlines(),
				'',
				nest_bookmark_level_part,
				])

	def visit_latex_toc(self, node: latex_toc) -> None:
		if not self.is_inline(node):
			self.body.append('\n')
		if "latex" in node.get("format", '').split():
			self.body.append(f"\\{self.sectionnames[self.sectionlevel]}{{{node.astext()}}}")
		if not self.is_inline(node):
			self.body.append('\n')
		raise nodes.SkipNode

	def depart_latex_toc(self, node: latex_toc) -> None:  # pragma: no cover
		pass


class LatexTocTreeDirective(sphinx.directives.other.TocTree):

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the directive.
		"""

		assert self.env.app.builder is not None

		output: List[nodes.Node] = []
		caption = self.options.get("caption")

		if (
				caption is not None and "hidden" not in self.options
				and self.env.app.builder.format.lower() == "latex"
				and self.env.docname == self.env.config.master_doc
				):

			output.append(latex_toc(text=caption, format="latex"))

		output.extend(super().run())

		return output


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.latex.toc`.

	:param app: The Sphinx application.
	:param config:
	"""

	use_package("bookmark", config)


def purge_outdated(app: Sphinx, env, added, changed, removed) -> List[str]:  # noqa: MAN001
	return [env.config.master_doc]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.latex_toc`.

	:param app: The Sphinx application.
	"""

	app.connect("env-get-outdated", purge_outdated)
	app.connect("config-inited", configure)
	app.add_directive("toctree", LatexTocTreeDirective, override=True)
	app.set_translator("latex", LaTeXTranslator, override=True)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/latex/toc.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
"""
Extensions to :mod:`sphinx.ext.autodoc`.

.. versionadded:: 0.6.0
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import TYPE_CHECKING, Any, List, Optional, Tuple

# 3rd party
from docutils.statemachine import StringList
from sphinx.application import Sphinx
from sphinx.ext.autodoc import Documenter

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

if TYPE_CHECKING:
	# 3rd party
	from sphinx.ext.autodoc import ObjectMembers

	ObjectMembers = ObjectMembers
else:
	ObjectMembers = List[Tuple[str, Any]]

__all__ = ("setup", )


def _documenter_add_content(
		self: Documenter,
		more_content: Optional[StringList],
		no_docstring: bool = False,
		) -> None:
	"""
	Add content from docstrings, attribute documentation and user.
	"""

	# set sourcename and add content from attribute documentation
	sourcename = self.get_sourcename()
	if self.analyzer:
		attr_docs = self.analyzer.find_attr_docs()
		if self.objpath:
			key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
			if key in attr_docs:
				no_docstring = True
				# make a copy of docstring for attributes to avoid cache
				# the change of autodoc-process-docstring event.
				docstrings = [list(attr_docs[key])]

				for i, line in enumerate(self.process_doc(docstrings)):
					self.add_line(line, sourcename, i)

	# add content from docstrings
	if not no_docstring:
		docstrings = self.get_doc() or []
		if docstrings is None:
			# Do not call autodoc-process-docstring on get_doc() returns None.
			pass
		else:
			if not docstrings:
				# append at least a dummy docstring, so that the event
				# autodoc-process-docstring is fired and can add some
				# content if desired
				docstrings.append([])
			for i, line in enumerate(self.process_doc(docstrings)):
				self.add_line(line, sourcename, i)

	# add additional content (e.g. from document), if present
	if more_content:
		for line, src in zip(more_content.data, more_content.items):
			self.add_line(line, src[0], src[1])


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc`.

	:param app: The Sphinx application.
	"""

	# Setup sub-extensions
	app.setup_extension("sphinx_toolbox.more_autodoc.augment_defaults")
	app.setup_extension("sphinx_toolbox.more_autodoc.autoprotocol")
	app.setup_extension("sphinx_toolbox.more_autodoc.autotypeddict")
	app.setup_extension("sphinx_toolbox.more_autodoc.autonamedtuple")
	app.setup_extension("sphinx_toolbox.more_autodoc.genericalias")
	app.setup_extension("sphinx_toolbox.more_autodoc.typehints")
	app.setup_extension("sphinx_toolbox.more_autodoc.variables")
	app.setup_extension("sphinx_toolbox.more_autodoc.sourcelink")
	app.setup_extension("sphinx_toolbox.more_autodoc.no_docstring")
	app.setup_extension("sphinx_toolbox.more_autodoc.regex")
	app.setup_extension("sphinx_toolbox.more_autodoc.typevars")
	app.setup_extension("sphinx_toolbox.more_autodoc.overloads")
	app.setup_extension("sphinx_toolbox.more_autodoc.generic_bases")

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  augment_defaults.py
"""
Sphinx's autodoc module allows for default options to be set,
and allows for those defaults to be disabled for an ``:auto*:`` directive
and different values given instead.

However, it does not appear to be possible to augment the defaults,
such as to globally exclude certain members and then exclude additional members of a single class.
This module monkeypatches in that behaviour.

.. extensions:: sphinx_toolbox.more_autodoc.augment_defaults

.. versionchanged:: 0.6.0

	Moved from :mod:`sphinx_toolbox.autodoc_augment_defaults`.
"""  # noqa: D400
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on Sphinx
#  Copyright (c) 2007-2020 by the Sphinx team.
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |    notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |    notice, this list of conditions and the following disclaimer in the
#  |    documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import Dict, List, Type

# 3rd party
import autodocsumm  # type: ignore[import]
import sphinx.ext.autodoc.directive
from docutils.utils import assemble_option_dict
from sphinx.application import Sphinx
from sphinx.errors import ExtensionError
from sphinx.ext.autodoc import Documenter, Options

# this package
import sphinx_toolbox
import sphinx_toolbox.more_autosummary
from sphinx_toolbox.utils import Config, SphinxExtMetadata

__all__ = ("process_documenter_options", "setup")


def process_documenter_options(
		documenter: Type[Documenter],
		config: Config,
		options: Dict,
		) -> Options:
	"""
	Recognize options of Documenter from user input.

	:param documenter:
	:param config:
	:param options:
	"""

	for name in sphinx.ext.autodoc.directive.AUTODOC_DEFAULT_OPTIONS:
		if name not in documenter.option_spec:  # pragma: no cover
			continue
		else:
			negated = options.pop("no-" + name, True) is None

			if name in config.autodoc_default_options and not negated:
				# pylint: disable=loop-invariant-statement
				default_value = config.autodoc_default_options[name]
				existing_value = options.get(name, None)
				values: List[str] = [v for v in [default_value, existing_value] if v not in {None, True, False}]

				if values:
					options[name] = ','.join(values)
				else:
					options[name] = None  # pragma: no cover
				# pylint: enable=loop-invariant-statement

	return Options(assemble_option_dict(options.items(), documenter.option_spec))


def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.augment_defaults`.

	:param app: The Sphinx application.
	"""

	if "sphinx.ext.autodoc" in app.extensions:
		raise ExtensionError(
				"'sphinx_toolbox.more_autodoc.augment_defaults' "
				"must be loaded before 'sphinx.ext.autodoc'."
				)

	sphinx.ext.autodoc.directive.process_documenter_options = process_documenter_options  # type: ignore[assignment]
	autodocsumm.process_documenter_options = process_documenter_options
	sphinx_toolbox.more_autosummary.process_documenter_options = process_documenter_options  # type: ignore[assignment]

	app.setup_extension("sphinx.ext.autodoc")

	return {"version": sphinx_toolbox.__version__, "parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/augment_defaults.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  autonamedtuple.py
r"""
A Sphinx directive for documenting :class:`NamedTuples <typing.NamedTuple>` in Python.

.. versionadded:: 0.8.0
.. extensions:: sphinx_toolbox.more_autodoc.autonamedtuple
.. versionchanged:: 1.5.0

	``__new__`` methods are documented regardless of other exclusion settings
	if the annotations differ from the namedtuple itself.


Usage
---------

.. rst:directive:: autonamedtuple

	Directive to automatically document a :class:`typing.NamedTuple` or :func:`collections.namedtuple`.

	The output is based on the :rst:dir:`autoclass` directive.
	The list of parameters and the attributes are replaced by a list of Fields,
	combining the types and docstrings from the class docstring individual attributes.
	These will always be shown regardless of the state of the ``:members:`` option.

	Otherwise the directive behaves the same as :rst:dir:`autoclass`, and takes all of its arguments.
	See https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html for further information.

	.. versionadded:: 0.8.0


.. rst:role:: namedtuple

	Role which provides a cross-reference to the documentation generated by :rst:dir:`autonamedtuple`.

.. rst:role:: namedtuple-field

	Role which provides a cross-reference to a field in the namedtuple documentation.

	.. versionadded:: 3.0.0

	.. note::

		Due to limitations in docutils cross-references for all fields in a namedtuple
		will point to the top of the fields list, rather than the individial fields.


.. seealso:: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html


:bold-title:`Examples`

.. literalinclude:: ../../../autonamedtuple_demo.py
	:language: python
	:tab-width: 4
	:linenos:
	:lines: 1-59

.. rest-example::

	.. automodule:: autonamedtuple_demo
		:no-autosummary:
		:exclude-members: Movie

	.. autonamedtuple:: autonamedtuple_demo.Movie

	This function takes a single argument, the :namedtuple:`~.Movie` to watch.
	The name of the movie can be accessed with the :namedtuple-field:`~.Movie.name` attribute.


API Reference
---------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import inspect
import re
import sys
import textwrap
from textwrap import dedent
from typing import Any, Dict, List, Optional, Tuple, Type, cast, get_type_hints

# 3rd party
from docutils import nodes
from docutils.statemachine import StringList
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.domains import ObjType
from sphinx.domains.python import PyAttribute, PyClasslike, PythonDomain, PyXRefRole
from sphinx.ext.autodoc import ClassDocumenter, Documenter, Options
from sphinx.ext.autodoc.directive import DocumenterBridge
from sphinx.locale import _
from sphinx.pycode import ModuleAnalyzer
from sphinx.util.nodes import make_id

# this package
from sphinx_toolbox.more_autodoc import ObjectMembers, _documenter_add_content
from sphinx_toolbox.more_autodoc.typehints import format_annotation
from sphinx_toolbox.utils import (
		Param,
		SphinxExtMetadata,
		add_fallback_css_class,
		add_nbsp_substitution,
		allow_subclass_add,
		baseclass_is_private,
		is_namedtuple,
		metadata_add_version,
		parse_parameters
		)

__all__ = ("NamedTupleDocumenter", "setup")


class NamedTupleDocumenter(ClassDocumenter):
	r"""
	Sphinx autodoc :class:`~sphinx.ext.autodoc.Documenter`
	for documenting :class:`typing.NamedTuple`\s.

	.. versionadded:: 0.8.0

	.. versionchanged:: 0.1.0

		Will no longer attempt to find attribute docstrings from other namedtuple classes.
	"""  # noqa: D400

	objtype = "namedtuple"
	directivetype = "namedtuple"
	priority = 20
	object: Type  # noqa: A003  # pylint: disable=redefined-builtin

	field_alias_re = re.compile("Alias for field number [0-9]+")

	def __init__(self, directive: DocumenterBridge, name: str, indent: str = '') -> None:
		super().__init__(directive, name, indent)
		self.options = Options(self.options.copy())

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		return is_namedtuple(member)

	def add_content(self, more_content: Optional[StringList], no_docstring: bool = True) -> None:
		r"""
		Add extra content (from docstrings, attribute docs etc.),
		but not the :class:`typing.NamedTuple`\'s docstring.

		:param more_content:
		:param no_docstring:
		"""  # noqa: D400

		_documenter_add_content(self, more_content, True)

		# set sourcename and add content from attribute documentation
		sourcename = self.get_sourcename()

		params, pre_output, post_output = self._get_docstring()

		for line in pre_output:
			self.add_line(line, sourcename)

	def _get_docstring(self) -> Tuple[Dict[str, Param], List[str], List[str]]:
		"""
		Returns params, pre_output, post_output.
		"""

		# Size varies depending on docutils config
		tab_size = self.env.app.config.docutils_tab_width

		if self.object.__doc__:
			docstring = dedent(self.object.__doc__).expandtabs(tab_size).split('\n')
		elif "show-inheritance" not in self.options:
			docstring = [":class:`typing.NamedTuple`."]
		else:
			docstring = ['']  # pylint: disable=W8301

		docstring = list(self.process_doc([docstring]))

		return parse_parameters(docstring, tab_size=tab_size)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header, and the inheritance information if the ``:show-inheritance:`` flag set.

		:param sig: The NamedTuple's signature.
		"""

		super().add_directive_header(sig)

		if "show-inheritance" not in self.options:
			return

		acceptable_bases = {
				"   Bases: :class:`tuple`",
				"   Bases: :py:class:`tuple`",
				"   Bases: :class:`tuple`, :class:`typing.Generic`",
				"   Bases: :py:class:`tuple`, :py:class:`typing.Generic`",
				"   Bases: :class:`NamedTuple`",
				"   Bases: :py:class:`NamedTuple`",
				}

		# TODO: support generic bases for multiple inheritance

		if self.directive.result[-1] in acceptable_bases or baseclass_is_private(self.object):
			# TODO: multiple inheritance

			if getattr(self.env.config, "generic_bases_fully_qualified", False):
				# Might not be present; extension might not be enabled
				prefix = ''
			else:
				prefix = '~'

			if hasattr(self.object, "__annotations__"):
				self.directive.result[-1] = f"   Bases: :class:`{prefix}typing.NamedTuple`"
			else:
				self.directive.result[-1] = f"   Bases: :func:`{prefix}collections.namedtuple`"

	def filter_members(
			self,
			members: ObjectMembers,
			want_all: bool,
			) -> List[Tuple[str, Any, bool]]:
		"""
		Filter the list of members to always include ``__new__`` if it has a different signature to the tuple.

		:param members:
		:param want_all:
		"""

		all_hints = get_type_hints(self.object)
		class_hints = {k: all_hints[k] for k in self.object._fields if k in all_hints}

		# TODO: need a better way to partially resolve type hints, and warn about failures
		new_hints = get_type_hints(
				self.object.__new__,
				globalns=sys.modules[self.object.__module__].__dict__,
				localns=self.object.__dict__,  # type: ignore[arg-type]
				)

		# Stock NamedTuples don't have these, but customised collections.namedtuple or hand-rolled classes may
		if "cls" in new_hints:
			new_hints.pop("cls")
		if "return" in new_hints:
			new_hints.pop("return")

		if class_hints and new_hints and class_hints != new_hints:
			#: __new__ has a different signature or different annotations

			def unskip_new(app, what, name, obj, skip, options) -> Optional[bool]:  # noqa: MAN001
				if name == "__new__":
					return False
				return None

			listener_id = self.env.app.connect("autodoc-skip-member", unskip_new)
			members_ = super().filter_members(members, want_all)
			self.env.app.disconnect(listener_id)
			return members_

		else:
			return super().filter_members(members, want_all)

	def sort_members(
			self,
			documenters: List[Tuple[Documenter, bool]],
			order: str,
			) -> List[Tuple[Documenter, bool]]:
		r"""
		Sort the :class:`typing.NamedTuple`\'s members.

		:param documenters:
		:param order:
		"""

		# The documenters for the fields and methods, in the desired order
		# The fields will be in bysource order regardless of the order option
		documenters = super().sort_members(documenters, order)

		# Size varies depending on docutils config
		a_tab = ' ' * self.env.app.config.docutils_tab_width

		# Mapping of member names to docstrings (as list of strings)
		member_docstrings: Dict[str, List[str]]

		try:
			namedtuple_source = textwrap.dedent(inspect.getsource(self.object))

			# Mapping of member names to docstrings (as list of strings)
			member_docstrings = {
					k[1]: v
					for k,
					v in ModuleAnalyzer.for_string(namedtuple_source, self.object.__module__
													).find_attr_docs().items()
					}

		except (TypeError, OSError):
			member_docstrings = {}

		# set sourcename and add content from attribute documentation
		sourcename = self.get_sourcename()

		params, pre_output, post_output = self._get_docstring()

		self.add_line('', sourcename)

		self.add_line(":Fields:", sourcename)
		# TODO: support for default_values
		self.add_line('', sourcename)

		fields = self.object._fields

		fields_body = []

		for pos, field in enumerate(fields):
			doc: List[str] = ['']  # pylint: disable=W8201
			arg_type: str = ''

			# Prefer doc from class docstring
			if field in params:
				doc, arg_type = params.pop(field).values()  # type: ignore[assignment]

			# Otherwise use attribute docstring
			if not ''.join(doc).strip() and field in member_docstrings:
				doc = member_docstrings[field]

			# Fallback to namedtuple's default docstring
			if not ''.join(doc).strip():
				doc = [getattr(self.object, field).__doc__]  # pylint: disable=W8301

			# Prefer annotations over docstring types
			type_hints = get_type_hints(self.object)
			if type_hints:
				if field in type_hints:
					arg_type = format_annotation(type_hints[field])

			self.add_line(f"{a_tab}.. namedtuple-field:: {field}", sourcename)
			self.add_line('', sourcename)
			# field_entry = [f"{a_tab}{pos})", "|nbsp|", f"**{field}**"]
			field_entry = [f"{a_tab}{pos}) \u00A0**{field}**"]

			if arg_type:
				field_entry.append(f"({arg_type}\\)")
			field_entry.append("--")
			field_entry.extend(doc)

			if self.field_alias_re.match(getattr(self.object, field).__doc__ or ''):
				getattr(self.object, field).__doc__ = ' '.join(doc)

			fields_body.append(' '.join(field_entry))
			fields_body.append('')

		for line in fields_body:
			self.add_line(line, sourcename)
		self.add_line('', sourcename)

		for line in post_output:
			self.add_line(line, sourcename)

		self.add_line('', sourcename)

		# Stop a duplicate set of parameters being added with `autodoc_typehints = 'both'`
		self.env.temp_data["annotations"] = {}

		# Remove documenters corresponding to fields and return the rest
		return [d for d in documenters if d[0].name.split('.')[-1] not in fields]


class _PyNamedTuplelike(PyClasslike):
	"""
	Description of a namedtuple-like object.
	"""

	def get_index_text(self, modname: str, name_cls: Tuple[str, str]) -> str:
		if self.objtype == "namedtuple":
			return _("%s (namedtuple in %s)") % (name_cls[0], modname)
		else:
			return super().get_index_text(modname, name_cls)


class _PyNamedTupleField(PyAttribute):
	"""
	Index entry and target for a namedtuple field.
	"""

	def run(self) -> List[nodes.Node]:

		if ':' in self.name:
			self.domain, self.objtype = self.name.split(':', 1)
		else:
			self.domain, self.objtype = '', self.name

		self.indexnode = addnodes.index(entries=[])

		modname = self.options.get("module", self.env.ref_context.get("py:module"))
		classname = self.env.ref_context.get("py:class")
		name = self.arguments[0]
		full_name = f"{classname}.{name}"

		fullname = (modname + '.' if modname else '') + full_name
		node_id = make_id(self.env, self.state.document, '', fullname)
		node = nodes.target('', '', ids=[node_id])
		node.document = self.state.document

		if "noindex" not in self.options:
			# only add target and index entry if this is the first
			# description of the object with this name in this desc block

			domain = cast(PythonDomain, self.env.get_domain("py"))
			domain.note_object(fullname, self.objtype, node_id, location=node)

			if "noindexentry" not in self.options:
				key = name[0].upper()
				pair = [  # pylint: disable=W8301
					_("%s (namedtuple in %s)") % (classname, modname),
					_("%s (namedtuple field)") % name,
					]
				self.indexnode["entries"].append(("pair", "; ".join(pair), node_id, '', key))

		return [self.indexnode, node]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.autonamedtuple`.

	.. versionadded:: 0.8.0

	:param app: The Sphinx application.
	"""

	# Hack to get the docutils tab size, as there doesn't appear to be any other way
	app.setup_extension("sphinx_toolbox.tweaks.tabsize")

	app.registry.domains["py"].object_types["namedtuple"] = ObjType(_("namedtuple"), "namedtuple", "class", "obj")
	app.add_directive_to_domain("py", "namedtuple", _PyNamedTuplelike)
	app.add_role_to_domain("py", "namedtuple", PyXRefRole())

	app.add_directive_to_domain("py", "namedtuple-field", _PyNamedTupleField)
	app.add_role_to_domain("py", "namedtuple-field", PyXRefRole())
	app.registry.domains["py"].object_types["namedtuple-field"] = ObjType(
			_("namedtuple-field"),
			"namedtuple-field",
			"attr",
			"obj",
			)

	app.connect("object-description-transform", add_fallback_css_class({"namedtuple": "class"}))

	allow_subclass_add(app, NamedTupleDocumenter)

	app.connect("config-inited", lambda _, config: add_nbsp_substitution(config))

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/autonamedtuple.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  autoprotocol.py
r"""
A Sphinx directive for documenting :class:`Protocols <typing.Protocol>` in Python.

.. versionadded:: 0.2.0
.. extensions:: sphinx_toolbox.more_autodoc.autoprotocol
.. versionchanged:: 0.6.0  Moved from :mod:`sphinx_toolbox.autoprotocol`.
.. versionchanged:: 2.13.0  Added support for generic bases, such as ``class SupportsAbs(Protocol[T_co]): ...``.


Usage
-------

.. latex:vspace:: -20px

.. rst:directive:: autoprotocol

	Directive to automatically document a :class:`typing.Protocol`.

	The output is based on the :rst:dir:`autoclass` directive, but with a few differences:

	* Private members are always excluded.
	* Special members (dunder methods) are always included.
	* Undocumented members are always included.

	The following options from :rst:dir:`autoclass` are available:

	.. rst:directive:option:: noindex
		:type: flag

		Do not generate index entries for the documented object (and all autodocumented members).

	.. rst:directive:option:: member-order
		:type: string

		Override the global value of :any:`sphinx:autodoc_member_order` for one directive.

	.. rst:directive:option:: show-inheritance
		:type: flag

		Inserts a list of base classes just below the protocol's signature.


.. rst:role:: protocol

	Role which provides a cross-reference to the documentation generated by :rst:dir:`autoprotocol`.

.. latex:vspace:: 5px
.. seealso:: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
.. latex:clearpage::

:bold-title:`Examples:`

.. literalinclude:: ../../../autoprotocol_demo.py
	:language: python
	:tab-width: 4
	:lines: 1-31
	:linenos:

.. rest-example::

	.. automodule:: autoprotocol_demo
		:members:
		:no-autosummary:
		:exclude-members: HasGreaterThan

	.. autoprotocol:: autoprotocol_demo.HasGreaterThan

	The objects being sorted must implement the :protocol:`~.HasGreaterThan` protocol.


.. latex:vspace:: 30px


API Reference
--------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import sys
from typing import Any, Callable, Dict, List, Optional, Tuple

# 3rd party
from docutils.statemachine import StringList
from sphinx.application import Sphinx
from sphinx.domains import ObjType
from sphinx.domains.python import PyClasslike, PyXRefRole
from sphinx.ext.autodoc import (
		INSTANCEATTR,
		ClassDocumenter,
		Documenter,
		Options,
		exclude_members_option,
		member_order_option
		)
from sphinx.ext.autodoc.directive import DocumenterBridge
from sphinx.locale import _
from sphinx.util.inspect import getdoc, safe_getattr

# this package
from sphinx_toolbox.more_autodoc import ObjectMembers, _documenter_add_content
from sphinx_toolbox.more_autodoc.generic_bases import _add_generic_bases
from sphinx_toolbox.utils import (
		SphinxExtMetadata,
		add_fallback_css_class,
		allow_subclass_add,
		filter_members_warning,
		flag,
		metadata_add_version
		)

if sys.version_info < (3, 8):  # pragma: no cover (>=py38)
	# 3rd party
	from typing_extensions import _ProtocolMeta  # type: ignore[attr-defined]
else:  # pragma: no cover (<py38)
	# stdlib
	from typing import _ProtocolMeta

__all__ = ("ProtocolDocumenter", "setup")

runtime_message = (
		"This protocol is `runtime checkable "
		"<https://www.python.org/dev/peps/pep-0544/#runtime-checkable-decorator-and-narrowing-types-by-isinstance>`_."
		)


class ProtocolDocumenter(ClassDocumenter):
	r"""
	Sphinx autodoc :class:`~sphinx.ext.autodoc.Documenter`
	for documenting :class:`typing.Protocol`\s.
	"""  # noqa: D400

	objtype = "protocol"
	directivetype = "protocol"
	priority = 20
	option_spec: Dict[str, Callable] = {
			"noindex": flag,
			"member-order": member_order_option,
			"show-inheritance": flag,
			"exclude-protocol-members": exclude_members_option,
			}

	globally_excluded_methods = {
			"__module__",
			"__new__",
			"__init__",
			"__subclasshook__",
			"__doc__",
			"__tree_hash__",
			"__extra__",
			"__orig_bases__",
			"__origin__",
			"__parameters__",
			"__next_in_mro__",
			"__slots__",
			"__args__",
			"__dict__",
			"__weakref__",
			"__annotations__",
			"__abstractmethods__",
			"__class_getitem__",
			"__init_subclass__",
			"__protocol_attrs__",
			}

	def __init__(self, directive: DocumenterBridge, name: str, indent: str = '') -> None:
		super().__init__(directive, name, indent)
		self.options = Options(self.options.copy())

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		# _is_protocol = True
		return isinstance(member, _ProtocolMeta)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive header.

		:param sig:
		"""

		sourcename = self.get_sourcename()

		if self.doc_as_attr:
			self.directivetype = "attribute"

		Documenter.add_directive_header(self, sig)

		if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
			self.add_line("   :final:", sourcename)

		# add inheritance info, if wanted
		if not self.doc_as_attr and self.options.show_inheritance:
			_add_generic_bases(self)

	def format_signature(self, **kwargs: Any) -> str:
		"""
		Protocols do not have a signature.
		"""

		return ''  # pragma: no cover

	def add_content(self, more_content: Optional[StringList], no_docstring: bool = False) -> None:
		"""
		Add the autodocumenter content.

		:param more_content:
		:param no_docstring:
		"""

		_documenter_add_content(self, more_content, no_docstring)

		sourcename = self.get_sourcename()

		if not getdoc(self.object) and "show-inheritance" not in self.options:
			self.add_line(":class:`typing.Protocol`.", sourcename)
			self.add_line('', sourcename)

		if hasattr(self.object, "_is_runtime_protocol") and self.object._is_runtime_protocol:
			self.add_line(runtime_message, sourcename)
			self.add_line('', sourcename)

		self.add_line(
				"Classes that implement this protocol must have the following methods / attributes:", sourcename
				)
		self.add_line('', sourcename)

	def document_members(self, all_members: bool = False) -> None:
		"""
		Generate reST for member documentation.

		All members are always documented.
		"""

		super().document_members(True)

	def filter_members(
			self,
			members: ObjectMembers,
			want_all: bool,
			) -> List[Tuple[str, Any, bool]]:
		"""
		Filter the given member list.

		:param members:
		:param want_all:
		"""

		ret = []

		# process members and determine which to skip
		for (membername, member) in members:
			# if isattr is True, the member is documented as an attribute

			if safe_getattr(member, "__sphinx_mock__", False):
				# mocked module or object
				keep = False  # pragma: no cover

			elif (
					self.options.get("exclude-protocol-members", [])
					and membername in self.options["exclude-protocol-members"]
					):
				# remove members given by exclude-protocol-members
				keep = False  # pragma: no cover

			elif membername.startswith('_') and not (membername.startswith("__") and membername.endswith("__")):
				keep = False

			elif membername not in self.globally_excluded_methods:
				# Magic method you wouldn't overload, or private method.
				if membername in dir(self.object.__base__):
					keep = member is not getattr(self.object.__base__, membername)
				else:
					keep = True

			else:
				keep = False

			# give the user a chance to decide whether this member
			# should be skipped
			if self.env.app:
				# let extensions preprocess docstrings
				try:  # pylint: disable=R8203
					skip_user = self.env.app.emit_firstresult(
							"autodoc-skip-member",
							self.objtype,
							membername,
							member,
							not keep,
							self.options,
							)

					if skip_user is not None:
						keep = not skip_user

				except Exception as exc:
					filter_members_warning(member, exc)
					keep = False

			if keep:
				ret.append((membername, member, member is INSTANCEATTR))

		return ret


class _PyProtocollike(PyClasslike):
	"""
	Description of a Protocol-like object.
	"""

	def get_index_text(self, modname: str, name_cls: Tuple[str, str]) -> str:
		if self.objtype == "protocol":
			return _("%s (protocol in %s)") % (name_cls[0], modname)
		else:
			return super().get_index_text(modname, name_cls)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.autoprotocol`.

	:param app: The Sphinx application.
	"""

	app.registry.domains["py"].object_types["protocol"] = ObjType(_("protocol"), "protocol", "class", "obj")
	app.add_directive_to_domain("py", "protocol", _PyProtocollike)
	app.add_role_to_domain("py", "protocol", PyXRefRole())
	app.connect("object-description-transform", add_fallback_css_class({"protocol": "class"}))

	allow_subclass_add(app, ProtocolDocumenter)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/autoprotocol.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  autotypeddict.py
r"""
A Sphinx directive for documenting :class:`TypedDicts <typing.TypedDict>` in Python.

Only supports :mod:`typing_extensions`\'s TypedDict until :pull:`700 <python/typing>` is implemented in CPython.

.. versionadded:: 0.5.0
.. extensions:: sphinx_toolbox.more_autodoc.autotypeddict
.. versionchanged:: 0.6.0  Moved from :mod:`sphinx_toolbox.autotypeddict`.


Usage
---------

.. latex:vspace:: -20px

.. rst:directive:: autotypeddict

	Directive to automatically document a :class:`typing.TypedDict`.

	The output is based on the :rst:dir:`autoclass` directive, but with a few differences:

	* Private and Special members are always excluded.
	* Undocumented members are always included.
	* The default sort order is ``bysource``.

	The following options are available:

	.. rst:directive:option:: noindex
		:type: flag

		Do not generate index entries for the documented object (and all autodocumented members).

	.. rst:directive:option:: alphabetical
		:type: flag

		Sort the keys alphabetically. By default the keys are listed in the order they were defined.

	.. rst:directive:option:: show-inheritance
		:type: flag

		Inserts a list of base classes just below the TypedDict's signature.


.. rst:role:: typeddict

	Role which provides a cross-reference to the documentation generated by :rst:dir:`autotypeddict`.

.. latex:vspace:: 5px
.. seealso:: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
.. latex:clearpage::

:bold-title:`Examples:`

.. literalinclude:: ../../../autotypeddict_demo.py
	:language: python
	:tab-width: 4
	:linenos:

.. rest-example::

	.. automodule:: autotypeddict_demo
		:no-autosummary:
		:exclude-members: Movie,AquaticBird,OldStyleAnimal

	.. autotypeddict:: autotypeddict_demo.Movie

	This function takes a single argument, the :typeddict:`~.Movie` to watch.

.. latex:vspace:: -5px

API Reference
---------------

.. latex:vspace:: -20px
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import Any, Callable, Dict, List, Tuple, Type, get_type_hints

# 3rd party
import docutils.statemachine
from domdf_python_tools.stringlist import StringList
from sphinx.application import Sphinx
from sphinx.domains import ObjType
from sphinx.domains.python import PyClasslike, PyXRefRole
from sphinx.ext.autodoc import INSTANCEATTR, ClassDocumenter, Documenter, Options
from sphinx.locale import _
from sphinx.pycode import ModuleAnalyzer
from sphinx.util.inspect import safe_getattr

# this package
from sphinx_toolbox.more_autodoc import ObjectMembers
from sphinx_toolbox.more_autodoc.typehints import format_annotation
from sphinx_toolbox.utils import (
		SphinxExtMetadata,
		add_fallback_css_class,
		allow_subclass_add,
		filter_members_warning,
		flag,
		metadata_add_version
		)

__all__ = ("TypedDictDocumenter", "setup")


class TypedDictDocumenter(ClassDocumenter):
	r"""
	Sphinx autodoc :class:`~sphinx.ext.autodoc.Documenter`
	for documenting :class:`typing.TypedDict`\s.
	"""  # noqa: D400

	objtype = "typeddict"
	directivetype = "typeddict"
	priority = 20
	option_spec: Dict[str, Callable] = {
			"noindex": flag,
			"alphabetical": flag,
			"show-inheritance": flag,
			}

	def __init__(self, *args: Any) -> None:
		super().__init__(*args)

		self.options = Options(self.options.copy())

		alphabetical = self.options.get("alphabetical", False)
		if alphabetical:
			self.options["member-order"] = "alphabetical"
		else:
			self.options["member-order"] = "bysource"

		for key in {"inherited-members", "special-members"}:  # pragma: no cover
			if key in self.options:
				del self.options[key]

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		for attr in {"__optional_keys__", "__required_keys__", "__total__"}:
			if not hasattr(member, attr):
				return False

		return True

	def format_signature(self, **kwargs: Any) -> str:
		"""
		Typed Dicts do not have a signature.

		:rtype:

		.. latex:clearpage::
		"""

		return ''

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive header.

		:param sig:
		"""

		sourcename = self.get_sourcename()

		if self.doc_as_attr:
			self.directivetype = "attribute"

		Documenter.add_directive_header(self, sig)

		if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
			self.add_line("   :final:", sourcename)

		# add inheritance info, if wanted
		if not self.doc_as_attr and self.options.show_inheritance:
			self.add_line('', sourcename)

			def get_bases(obj: Type) -> List[Type]:
				bases: Tuple[Type, ...] = ()
				if hasattr(obj, "__orig_bases__") and len(obj.__orig_bases__):
					# A subclass of generic types
					# refs: PEP-560 <https://www.python.org/dev/peps/pep-0560/>
					bases = obj.__orig_bases__
				elif hasattr(obj, "__bases__") and len(obj.__bases__):
					# A normal class
					bases = obj.__bases__

				output = []
				for base in bases:
					if base.__name__.startswith('_'):
						output.extend(get_bases(base))
					else:
						output.append(base)

				return output

			bases = [format_annotation(cls) for cls in get_bases(self.object)]
			self.add_line("   " + _("Bases: %s") % ", ".join(bases), sourcename)
			self.add_line('', sourcename)

	def add_content(self, more_content: Any, no_docstring: bool = False) -> None:
		"""
		Add the autodocumenter content.

		:param more_content:
		:param no_docstring:
		"""

		if self.doc_as_attr:  # pragma: no cover (verbatim from Sphinx)
			classname = safe_getattr(self.object, "__qualname__", None)
			if not classname:
				classname = safe_getattr(self.object, "__name__", None)
			if classname:
				module = safe_getattr(self.object, "__module__", None)
				parentmodule = safe_getattr(self.parent, "__module__", None)
				if module and module != parentmodule:
					classname = str(module) + '.' + str(classname)
				more_content = docutils.statemachine.StringList([_("alias of :class:`%s`") % classname], source='')
				no_docstring = True

		# set sourcename and add content from attribute documentation
		sourcename = self.get_sourcename()
		if self.analyzer:
			attr_docs = self.analyzer.find_attr_docs()
			if self.objpath:
				key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
				if key in attr_docs:
					no_docstring = True
					# make a copy of docstring for attributes to avoid cache
					# the change of autodoc-process-docstring event.
					docstrings = [list(attr_docs[key])]

					for i, line in enumerate(self.process_doc(docstrings)):
						self.add_line(line, sourcename, i)

		# add content from docstrings
		if not no_docstring:
			docstrings = self.get_doc() or []
			if not docstrings:
				# append at least a dummy docstring, so that the event
				# autodoc-process-docstring is fired and can add some
				# content if desired
				docstrings.append([":class:`typing.TypedDict`.", ''])
			for i, line in enumerate(self.process_doc(docstrings)):
				self.add_line(line, sourcename, i)

		# add additional content (e.g. from document), if present
		if more_content:  # pragma: no cover (verbatim from Sphinx)
			for line, src in zip(more_content.data, more_content.items):
				self.add_line(line, src[0], src[1])

	def document_members(self, all_members: bool = False) -> None:
		"""
		Generate reST for member documentation. All members are always documented.
		"""

		super().document_members(True)

	def sort_members(
			self,
			documenters: List[Tuple[Documenter, bool]],
			order: str,
			) -> List[Tuple[Documenter, bool]]:
		"""
		Sort the TypedDict's members.

		:param documenters:
		:param order:
		"""

		# The documenters for the keys, in the desired order
		documenters = super().sort_members(documenters, order)

		# Mapping of key names to docstrings (as list of strings)
		docstrings = {
				k[1]: v
				for k, v in ModuleAnalyzer.for_module(self.object.__module__).find_attr_docs().items()
				}

		required_keys = []
		optional_keys = []
		types = get_type_hints(self.object)

		for d in documenters:
			name = d[0].name.split('.')[-1]
			if name in self.object.__required_keys__:
				required_keys.append(name)
			elif name in self.object.__optional_keys__:
				optional_keys.append(name)
			# else: warn user. This shouldn't ever happen, though.

		sourcename = self.get_sourcename()

		if required_keys:
			self.add_line('', sourcename)
			self.add_line(":Required Keys:", sourcename)
			self.document_keys(required_keys, types, docstrings)
			self.add_line('', sourcename)

		if optional_keys:
			self.add_line('', sourcename)
			self.add_line(":Optional Keys:", sourcename)
			self.document_keys(optional_keys, types, docstrings)
			self.add_line('', sourcename)

		return []

	def document_keys(
			self,
			keys: List[str],
			types: Dict[str, Type],
			docstrings: Dict[str, List[str]],
			) -> None:
		"""
		Document keys in a :class:`typing.TypedDict`.

		:param keys: List of key names to document.
		:param types: Mapping of key names to types.
		:param docstrings: Mapping of key names to docstrings.
		"""

		content = StringList()

		for key in keys:
			if key in types:
				key_type = f"({format_annotation(types[key])}) "
			else:
				key_type = ''

			if key in docstrings:
				content.append(f"    * **{key}** {key_type}-- {' '.join(docstrings.get(key, ''))}")
			else:
				content.append(f"    * **{key}** {key_type}")

		for line in content:
			self.add_line(line, self.get_sourcename())

	def filter_members(
			self,
			members: ObjectMembers,
			want_all: bool,
			) -> List[Tuple[str, Any, bool]]:
		"""
		Filter the given member list.

		:param members:
		:param want_all:
		"""

		ret = []

		# process members and determine which to skip
		for (membername, member) in members:
			# if isattr is True, the member is documented as an attribute

			if safe_getattr(member, "__sphinx_mock__", False):
				# mocked module or object
				keep = False
			elif membername.startswith('_'):
				keep = False
			else:
				keep = True

			# give the user a chance to decide whether this member
			# should be skipped
			if self.env.app:
				# let extensions preprocess docstrings
				try:  # pylint: disable=R8203
					skip_user = self.env.app.emit_firstresult(
							"autodoc-skip-member",
							self.objtype,
							membername,
							member,
							not keep,
							self.options,
							)

					if skip_user is not None:
						keep = not skip_user

				except Exception as exc:
					filter_members_warning(member, exc)
					keep = False

			if keep:
				ret.append((membername, member, member is INSTANCEATTR))

		return ret


class _PyTypedDictlike(PyClasslike):
	"""
	Description of a typeddict-like object.
	"""

	def get_index_text(self, modname: str, name_cls: Tuple[str, str]) -> str:
		if self.objtype == "typeddict":
			return _("%s (typeddict in %s)") % (name_cls[0], modname)
		else:
			return super().get_index_text(modname, name_cls)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.autotypeddict`.

	:param app: The Sphinx application.
	"""

	app.registry.domains["py"].object_types["typeddict"] = ObjType("typeddict", "typeddict", "class", "obj")
	app.add_directive_to_domain("py", "typeddict", _PyTypedDictlike)
	app.add_role_to_domain("py", "typeddict", PyXRefRole())
	app.connect("object-description-transform", add_fallback_css_class({"typeddict": "class"}))

	allow_subclass_add(app, TypedDictDocumenter)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/autotypeddict.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  generic_bases.py
r"""
Modifies :class:`sphinx.ext.autodoc.ClassDocumenter`\'s ``:show-inheritence:`` option
to show generic base classes.

This requires a relatively new version of the :mod:`typing` module that implements ``__orig_bases__``.

.. versionadded:: 1.5.0
.. extensions:: sphinx_toolbox.more_autodoc.generic_bases


Configuration
----------------

.. confval:: generic_bases_fully_qualified
	:type: :class:`bool`
	:required: False
	:default: :py:obj:`False`

	Determines whether the fully qualified name should be shown for bases.

	If :py:obj:`False` (the default):

		.. class:: Foo
			:noindex:

			**Bases**: :class:`~typing.List`\[:class:`str`\]

	If :py:obj:`True`:

		.. class:: Foo
			:noindex:

			**Bases**: :class:`typing.List`\[:class:`str`\]

	Corresponds to the ``fully_qualified`` argument to
	:func:`sphinx_toolbox.more_autodoc.typehints.format_annotation`.

	.. versionadded:: 2.13.0

Example
--------

.. autoclass:: sphinx_toolbox.more_autodoc.generic_bases.Example


API Reference
-----------------
"""  # noqa: D400
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import sys
from typing import List, Tuple

# 3rd party
from sphinx.application import Sphinx
from sphinx.ext.autodoc import Documenter
from sphinx.locale import _
from typing_extensions import final

# this package
from sphinx_toolbox.more_autodoc.typehints import format_annotation
from sphinx_toolbox.more_autosummary import PatchedAutoSummClassDocumenter
from sphinx_toolbox.utils import SphinxExtMetadata, allow_subclass_add, metadata_add_version

if sys.version_info >= (3, 8):  # pragma: no cover (<py38)
	# stdlib
	from typing import get_origin
else:  # pragma: no cover (py38+)
	# 3rd party
	from typing_inspect import get_origin  # type: ignore[import]

__all__ = ("GenericBasesClassDocumenter", "setup")


class GenericBasesClassDocumenter(PatchedAutoSummClassDocumenter):
	"""
	Class documenter that adds inheritance info, with support for generics.
	"""

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive header.

		:param sig:
		"""

		sourcename = self.get_sourcename()

		if self.doc_as_attr:
			self.directivetype = "attribute"

		Documenter.add_directive_header(self, sig)

		if self.analyzer and '.'.join(self.objpath) in self.analyzer.finals:
			self.add_line("   :final:", sourcename)

		# add inheritance info, if wanted
		if not self.doc_as_attr and self.options.show_inheritance:
			_add_generic_bases(self)


def _add_generic_bases(documenter: Documenter) -> None:
	"""
	Add the generic bases to the output of the given Documenter.

	.. versionadded:: 2.13.0  (undocumented)

	:param documenter:
	"""

	sourcename = documenter.get_sourcename()

	# add inheritance info, if wanted
	fully_qualified = getattr(documenter.env.config, "generic_bases_fully_qualified", False)

	documenter.add_line('', sourcename)
	bases = []  # pylint: disable=W8301

	if (
			hasattr(documenter.object, "__orig_bases__") and len(documenter.object.__orig_bases__)
			and get_origin(documenter.object.__orig_bases__[0]) is documenter.object.__bases__[0]
			):
		# Last condition guards against classes that don't directly subclass a Generic.
		bases = [format_annotation(b, fully_qualified) for b in documenter.object.__orig_bases__]

	elif hasattr(documenter.object, "__bases__") and len(documenter.object.__bases__):
		bases = [format_annotation(b, fully_qualified) for b in documenter.object.__bases__]

	if bases:
		bases_string = ", ".join(bases).replace("typing_extensions.", "typing.")
		documenter.add_line("   " + _("Bases: %s") % bases_string, sourcename)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.generic_bases`.

	.. versionadded:: 1.5.0

	:param app: The Sphinx application.
	"""

	allow_subclass_add(app, GenericBasesClassDocumenter)
	app.add_config_value(
			"generic_bases_fully_qualified",
			default=False,
			rebuild="env",
			types=[bool],
			)

	return {"parallel_read_safe": True}


class Example(List[Tuple[str, float, List[str]]]):
	"""
	An example of :mod:`sphinx_toolbox.more_autodoc.generic_bases`.
	"""

	def __init__(self, iterable=()):  # pragma: no cover  # noqa: MAN001
		pass


class Example2(Example):
	"""
	An example of :mod:`sphinx_toolbox.more_autodoc.generic_bases`.

	This one does not directly subclass a Generic.
	"""


@final
class FinalExample(List[Tuple[str, float, List[str]]]):
	"""
	An example of :mod:`sphinx_toolbox.more_autodoc.generic_bases` decorated with ``@final``.
	"""

	def __init__(self, iterable=()):  # pragma: no cover  # noqa: MAN001
		pass

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/generic_bases.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  genericalias.py
"""
Documenter for alias, which usually manifest as `type aliases`_.

.. _type aliases: https://docs.python.org/3/library/typing.html#type-aliases

.. versionadded:: 0.6.0
.. extensions:: sphinx_toolbox.more_autodoc.genericalias

.. note::

	:mod:`sphinx_toolbox.more_autodoc.genericalias` is only supported on Python 3.7 and above.

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import Any

# 3rd party
from docutils.statemachine import StringList
from sphinx.application import Sphinx
from sphinx.ext.autodoc import SUPPRESS, Options
from sphinx.ext.autodoc.directive import DocumenterBridge  # noqa: F401
from sphinx.locale import _
from sphinx.util import inspect

# this package
from sphinx_toolbox._data_documenter import DataDocumenter
from sphinx_toolbox.more_autodoc.typehints import format_annotation
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("PrettyGenericAliasDocumenter", "setup")


class PrettyGenericAliasDocumenter(DataDocumenter):
	"""
	Specialized Documenter subclass for GenericAliases, with prettier output than Sphinx's one.
	"""

	objtype = "genericalias"
	directivetype = "data"
	priority = DataDocumenter.priority + 2

	@classmethod
	def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any) -> bool:
		"""
		Called to see if a member can be documented by this documenter.
		"""

		return inspect.isgenericalias(member)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive header and options to the generated content.
		"""

		self.options = Options(self.options)
		self.options["annotation"] = SUPPRESS
		super().add_directive_header(sig)

	def add_content(self, more_content: Any, no_docstring: bool = False) -> None:
		"""
		Add the autodocumenter content.

		:param more_content:
		:param no_docstring:
		"""

		name = format_annotation(self.object)
		content = StringList([_("Alias of %s") % name], source='')
		DataDocumenter.add_content(self, content)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.genericalias`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx.ext.autodoc")
	app.add_autodocumenter(PrettyGenericAliasDocumenter, override=True)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/genericalias.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  no_docstring.py
"""
Adds the ``:no-docstring:`` option to automodule directives to exclude the docstring from the output.

.. versionadded:: 1.0.0

	Functions in this module moved from ``sphinx_toolbox.more_autodoc.__init__.py``

.. extensions:: sphinx_toolbox.more_autodoc.no_docstring
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from types import ModuleType
from typing import List

# 3rd party
import autodocsumm  # type: ignore[import]
import sphinx.ext.autodoc
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.more_autosummary import PatchedAutoSummModuleDocumenter
from sphinx_toolbox.utils import SphinxExtMetadata, flag, metadata_add_version

__all__ = ("automodule_add_nodocstring", "no_docstring_process_docstring", "setup")


def automodule_add_nodocstring(app: Sphinx) -> None:
	"""
	Add the ``:no-docstring:`` option to automodule directives.

	The option is used to exclude the docstring from the output

	:param app: The Sphinx application.
	"""

	sphinx.ext.autodoc.ModuleDocumenter.option_spec["no-docstring"] = flag
	autodocsumm.AutoSummModuleDocumenter.option_spec["no-docstring"] = flag
	PatchedAutoSummModuleDocumenter.option_spec["no-docstring"] = flag

	app.setup_extension("sphinx.ext.autodoc")
	app.connect("autodoc-process-docstring", no_docstring_process_docstring, priority=1000)


def no_docstring_process_docstring(  # noqa: MAN001
		app: Sphinx,
		what,
		name: str,
		obj,
		options,
		lines: List[str],
		) -> None:
	"""
	Remove module docstrings if the ``:no-docstring:`` flag was set.

	:param app: The Sphinx application.
	:param what:
	:param name: The name of the object being documented.
	:param obj: The object being documented.
	:param options: Mapping of autodoc options to values.
	:param lines: List of strings representing the current contents of the docstring.
	"""

	if isinstance(obj, ModuleType):
		no_docstring = options.get("no-docstring", False)
		if no_docstring:
			lines.clear()


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.no_docstring`.

	:param app: The Sphinx application.
	"""

	automodule_add_nodocstring(app)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/no_docstring.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  overloads.py
r"""
Documenters for functions and methods which display overloads differently.

.. versionadded:: 1.4.0
.. extensions:: sphinx_toolbox.more_autodoc.overloads


Configuration
----------------

.. latex:vspace:: -20px


.. confval:: overloads_location
	:type: :class:`str`
	:default: ``'signature'``

	The location to display overloads at:

	* ``'signature'`` -- Display overloads above the function signature.
	* ``'top'`` -- Display overloads at the top of the docstring, immediately below the signature.
	* ``'bottom'`` -- Display overloads at the bottom of the docstring, or immediately below the return type.


API Reference
----------------

.. latex:vspace:: -20px

.. automodulesumm:: sphinx_toolbox.more_autodoc.overloads
	:autosummary-sections: Classes

.. latex:vspace:: -15px

.. automodulesumm:: sphinx_toolbox.more_autodoc.overloads
	:autosummary-sections: Functions

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import re
from contextlib import suppress
from inspect import Parameter, Signature
from types import ModuleType
from typing import TYPE_CHECKING, Any, Dict, ForwardRef, List

# 3rd party
from domdf_python_tools.stringlist import StringList
from sphinx.application import Sphinx
from sphinx.config import ENUM
from sphinx.ext import autodoc
from sphinx.ext.autodoc.directive import DocumenterBridge
from sphinx.util import inspect
from sphinx.util.inspect import evaluate_signature, safe_getattr, stringify_signature

# this package
from sphinx_toolbox.more_autodoc import _documenter_add_content
from sphinx_toolbox.more_autodoc.typehints import _resolve_forwardref, default_preprocessors, format_annotation
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

if TYPE_CHECKING:
	_OverloadMixinBase = autodoc.ModuleLevelDocumenter
else:
	_OverloadMixinBase = object

_ = DocumenterBridge

__all__ = (
		"OverloadMixin",
		"FunctionDocumenter",
		"MethodDocumenter",
		"setup",
		)

_return_type_re = re.compile("^:(rtype|return(s)?):")


class OverloadMixin(_OverloadMixinBase):
	"""
	Mixin class for function and class documenters that changes the appearance of overloaded functions.
	"""

	def create_body_overloads(self) -> StringList:
		"""
		Create the overloaded implementations for insertion into to the body of the documenter's output.
		"""

		output = StringList()
		formatted_overloads = []

		output.blankline()
		# output.append(":Overloaded Implementations:")
		output.append(":Overloads:")
		output.blankline()

		# Size varies depending on docutils config
		output.indent_type = ' '
		output.indent_size = self.env.app.config.docutils_tab_width

		if self.analyzer and '.'.join(self.objpath) in self.analyzer.overloads:

			the_overloads = self.analyzer.overloads.get('.'.join(self.objpath))
			assert the_overloads is not None
			for overload in the_overloads:
				overload = self.process_overload_signature(overload)

				buf = [format_annotation(self.object), r"\("]

				for name, param in overload.parameters.items():
					buf.append(f"**{name}**")

					if param.annotation is not Parameter.empty:
						buf.append(r"\: ")
						buf.append(format_annotation(param.annotation))

					if param.default is not Parameter.empty:
						default = param.default

						# pylint: disable=loop-global-usage
						if hasattr(inspect, "DefaultValue") and isinstance(default, inspect.DefaultValue):
							default = default.value
						# pylint: enable=loop-global-usage

						buf.append(" = ")
						buf.append(default)

					buf.append(r"\, ")

				if buf[-2][-1] != '`':
					buf[-1] = r" )"
				else:
					buf[-1] = r")"

				if overload.return_annotation is not Parameter.empty:
					return_annotation = overload.return_annotation

					if isinstance(return_annotation, ForwardRef):
						with suppress(Exception):
							if isinstance(self.parent, ModuleType):
								module_name = self.parent.__name__
							else:
								module_name = self.parent.__module__

							return_annotation = _resolve_forwardref(return_annotation, module_name)

					buf.append(" -> ")
					buf.append(format_annotation(return_annotation))

				formatted_overloads.append(''.join(buf))

			if len(formatted_overloads) == 1:
				output.append(formatted_overloads[0])
			else:
				for line in formatted_overloads:
					output.append(f"* {line}")
					output.blankline(ensure_single=True)

			return output

		return StringList()

	def process_overload_signature(self, overload: Signature) -> Signature:
		"""
		Processes the signature of the given overloaded implementation.

		:param overload:
		"""

		parameters = []
		non_overload_sig = inspect.signature(self.object)

		for param, non_overload_param in zip(overload.parameters.values(), non_overload_sig.parameters.values()):
			default = param.default

			if default is not Parameter.empty:
				for check, preprocessor in default_preprocessors:
					if check(default):
						default = preprocessor(default)
						break

			if param.annotation is non_overload_param.annotation:
				annotation = Parameter.empty
			else:
				annotation = param.annotation
			parameters.append(param.replace(default=default, annotation=annotation))

		if non_overload_sig.return_annotation is overload.return_annotation:
			overload = overload.replace(parameters=parameters, return_annotation=Parameter.empty)
		else:
			overload = overload.replace(parameters=parameters)

		return overload

	def add_content(self, more_content: Any, no_docstring: bool = False) -> None:
		"""
		Add content from docstrings, attribute documentation and the user.

		:param more_content:
		:param no_docstring:
		"""

		if self.env.config.overloads_location == "bottom":

			def process_docstring(
					app: Sphinx,
					what: str,
					name: str,
					obj: Any,
					options: Dict[str, Any],
					lines: List[str],
					) -> None:

				if callable(obj):
					seen_return = False

					n_lines = len(lines)

					for i, line in enumerate(lines):
						if not line:
							continue

						if line[0].isspace():
							continue

						if _return_type_re.match(line):  # pylint: disable=loop-global-usage
							seen_return = True
							continue

						if seen_return and i != n_lines:
							lines.insert(i, '')
							for inner_line in reversed(self.create_body_overloads()):  # pylint: disable=W8402
								lines.insert(i, inner_line)
							lines.insert(i, '')

							break
					else:
						lines.append('')
						lines.extend(self.create_body_overloads())
						lines.append('')

			listener_id = self.env.app.connect("autodoc-process-docstring", process_docstring, priority=600)
			try:
				_documenter_add_content(self, more_content, no_docstring)
			finally:
				self.env.app.disconnect(listener_id)

		else:
			_documenter_add_content(self, more_content, no_docstring)


class FunctionDocumenter(OverloadMixin, autodoc.FunctionDocumenter):
	"""
	Custom :class:`autodoc.FunctionDocumenter <sphinx.ext.autodoc.FunctionDocumenter>`
	which renders overloads differently.
	"""  # noqa: D400

	def format_signature(self, **kwargs: Any) -> str:
		"""
		Format the function's signature, including those for any overloaded implementations.

		:return: The signature(s), as a multi-line string.
		"""

		sigs = []

		if self.analyzer and '.'.join(self.objpath) in self.analyzer.overloads:
			overloaded = True
		else:
			overloaded = False
			sig = super(autodoc.FunctionDocumenter, self).format_signature(**kwargs)
			sigs.append(sig)

		if inspect.is_singledispatch_function(self.object):
			# append signature of singledispatch'ed functions
			for typ, func in self.object.registry.items():
				if typ is object:
					pass  # default implementation. skipped.
				else:
					self.annotate_to_first_argument(func, typ)

					documenter = FunctionDocumenter(self.directive, '')
					documenter.object = func
					documenter.objpath = [None]  # type: ignore[list-item]
					sigs.append(documenter.format_signature())

		if overloaded:
			if self.env.config.overloads_location == "signature":
				the_overloads = self.analyzer.overloads.get('.'.join(self.objpath))
				assert the_overloads is not None
				for overload in the_overloads:
					sig = stringify_signature(self.process_overload_signature(overload), **kwargs)
					sigs.append(sig)

			sig = super(autodoc.FunctionDocumenter, self).format_signature(**kwargs)
			sigs.append(sig)

		return '\n'.join(sigs)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header.

		:param sig:
		"""

		super().add_directive_header(sig)

		if self.env.config.overloads_location == "top":
			for line in self.create_body_overloads():
				self.add_line(f"{self.content_indent}{line}", self.get_sourcename())

	def process_overload_signature(self, overload: Signature) -> Signature:
		"""
		Processes the signature of the given overloaded implementation.

		:param overload:
		"""

		__globals__ = safe_getattr(self.object, "__globals__", {})
		overload = evaluate_signature(overload, __globals__)
		return super().process_overload_signature(overload)


class MethodDocumenter(OverloadMixin, autodoc.MethodDocumenter):
	"""
	Custom :class:`autodoc.MethodDocumenter <sphinx.ext.autodoc.MethodDocumenter>`
	which renders overloads differently.
	"""  # noqa: D400

	def format_signature(self, **kwargs: Any) -> str:
		"""
		Format the method's signature, including those for any overloaded implementations.

		:param kwargs:

		:return: The signature(s), as a multi-line string.
		"""

		sigs = []

		if self.analyzer and '.'.join(self.objpath) in self.analyzer.overloads:
			overloaded = True
		else:
			overloaded = False
			sig = super(autodoc.MethodDocumenter, self).format_signature(**kwargs)
			sigs.append(sig)

		meth = self.parent.__dict__.get(self.objpath[-1])
		if inspect.is_singledispatch_method(meth):
			# append signature of singledispatch'ed functions
			for typ, func in meth.dispatcher.registry.items():
				if typ is object:
					pass  # default implementation. skipped.
				else:
					self.annotate_to_first_argument(func, typ)

					documenter = MethodDocumenter(self.directive, '')
					documenter.parent = self.parent
					documenter.object = func
					documenter.objpath = [None]  # type: ignore[list-item]
					sigs.append(documenter.format_signature())

		if overloaded:
			if self.env.config.overloads_location == "signature":
				the_overloads = self.analyzer.overloads.get('.'.join(self.objpath))
				assert the_overloads is not None
				for overload in the_overloads:
					sig = stringify_signature(self.process_overload_signature(overload), **kwargs)
					sigs.append(sig)

			sig = super(autodoc.MethodDocumenter, self).format_signature(**kwargs)
			sigs.append(sig)

		return '\n'.join(sigs)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header.

		:param sig:
		"""

		super().add_directive_header(sig)

		if self.env.config.overloads_location == "top":
			for line in self.create_body_overloads():
				self.add_line(f"{self.content_indent}{line}", self.get_sourcename())

	def process_overload_signature(self, overload: Signature) -> Signature:
		"""
		Processes the signature of the given overloaded implementation.

		:param overload:
		"""

		__globals__ = safe_getattr(self.object, "__globals__", {})
		overload = evaluate_signature(overload, __globals__)

		if not inspect.isstaticmethod(self.object, cls=self.parent, name=self.object_name):
			overload = overload.replace(parameters=list(overload.parameters.values())[1:])

		return super().process_overload_signature(overload)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.overloads`.

	:param app: The Sphinx application.
	"""

	app.add_autodocumenter(FunctionDocumenter, override=True)
	app.add_autodocumenter(MethodDocumenter, override=True)
	app.add_config_value(
			"overloads_location",
			"signature",
			"env",
			ENUM("top", "bottom", "signature"),  # top (of body), bottom (of body)
			)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/overloads.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  regex.py
r"""
Specialized Documenter for regular expression variables, similar to :rst:dir:`autodata`.

.. versionadded:: 1.2.0
.. extensions:: sphinx_toolbox.more_autodoc.regex

Usage
-------

.. rst:directive:: autoregex

	Directive to automatically document a regular expression variable.

	The output is based on the :rst:dir:`autodata` directive, and takes all of its options except ``:annotation:``.

	.. rst:directive:option:: no-value

		Don't show the value of the variable.

	.. rst:directive:option:: value: value
		:type: string

		Show this instead of the value taken from the Python source code.

	.. rst:directive:option:: no-type

		Don't show the type of the variable.

	.. rst:directive:option:: no-flags

		Don't show the flags of the :class:`~typing.Pattern` object.

	.. rst:directive:option:: flags: flags
		:type: string

		Show this instead of the flags taken from the :class:`~typing.Pattern` object.
		This should be correctly formatted for insertion into reStructuredText, such as ``:py:data:`re.ASCII```.


	.. versionchanged:: 2.7.0

		The flags :py:data:`re.DEBUG` and :py:data:`re.VERBOSE` are now hidden
		as they don't affect the regex itself.

.. latex:clearpage::

.. rst:role:: regex

	Formats a regular expression with coloured output.

	.. rest-example::

		:regex:`^Hello\s+[Ww]orld[.,](Lovely|Horrible) weather, isn't it (.*)?`

	.. versionchanged:: 2.11.0  Now generates coloured output with the LaTeX builder.

"""
#
#  Copyright © 2020-2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import itertools
import re
import sre_parse
from sre_constants import (
		ANY,
		AT,
		AT_BEGINNING,
		AT_BEGINNING_STRING,
		AT_BOUNDARY,
		AT_END,
		AT_END_STRING,
		AT_NON_BOUNDARY,
		BRANCH,
		CATEGORY,
		CATEGORY_DIGIT,
		CATEGORY_NOT_DIGIT,
		CATEGORY_NOT_SPACE,
		CATEGORY_NOT_WORD,
		CATEGORY_SPACE,
		CATEGORY_WORD,
		IN,
		LITERAL,
		MAX_REPEAT,
		MAXREPEAT,
		MIN_REPEAT,
		RANGE,
		SUBPATTERN
		)
from textwrap import dedent
from typing import Any, Callable, List, Optional, Pattern, Tuple

# 3rd party
import dict2css
from docutils import nodes
from docutils.nodes import Node, system_message
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.stringlist import StringList
from sphinx.application import Sphinx
from sphinx.ext.autodoc import UNINITIALIZED_ATTR, ModuleDocumenter
from sphinx.util import texescape
from sphinx.util.docutils import SphinxRole
from sphinx.writers.html import HTMLTranslator

# this package
from sphinx_toolbox import _css
from sphinx_toolbox.more_autodoc.variables import VariableDocumenter
from sphinx_toolbox.utils import Config, SphinxExtMetadata, add_nbsp_substitution, flag, metadata_add_version

__all__ = (
		"RegexDocumenter",
		"RegexParser",
		"TerminalRegexParser",
		"HTMLRegexParser",
		"LaTeXRegexParser",
		"parse_regex_flags",
		"no_formatting",
		"span",
		"latex_textcolor",
		"copy_asset_files",
		"setup",
		)


class RegexDocumenter(VariableDocumenter):
	"""
	Specialized Documenter subclass for regex patterns.
	"""

	directivetype = "data"
	objtype = "regex"
	priority = VariableDocumenter.priority + 1
	option_spec = {
			**VariableDocumenter.option_spec,
			"no-flag": flag,
			"flag": str,
			}
	del option_spec["type"]
	del option_spec["annotation"]

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		return isinstance(parent, ModuleDocumenter) and isattr and isinstance(member, Pattern)

	def add_content(self, more_content: Any, no_docstring: bool = False) -> None:
		"""
		Add content from docstrings, attribute documentation and the user.

		:param more_content:
		:param no_docstring:
		"""

		# set sourcename and add content from attribute documentation
		sourcename = self.get_sourcename()

		if self.analyzer:
			attr_docs = self.analyzer.find_attr_docs()
			if self.objpath:
				key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
				if key in attr_docs:
					no_docstring = True
					# make a copy of docstring for attributes to avoid cache
					# the change of autodoc-process-docstring event.
					docstrings = [list(attr_docs[key])]

					for i, line in enumerate(self.process_doc(docstrings)):
						self.add_line(line, sourcename, i)

		# add content from docstrings
		if not no_docstring:
			docstrings = self.get_doc() or []
			if not docstrings:
				# append at least a dummy docstring, so that the event
				# autodoc-process-docstring is fired and can add some
				# content if desired
				docstrings.append([])

			if docstrings == [["Compiled regular expression objects", '']] or docstrings == [[]]:
				docstrings = [["Compiled regular expression object.", '']]  # pylint: disable=W8301

			for i, line in enumerate(self.process_doc(docstrings)):
				self.add_line(line, sourcename, i)

		# add additional content (e.g. from document), if present
		if more_content:
			for line, src in zip(more_content.data, more_content.items):
				self.add_line(line, src[0], src[1])

		no_value = self.options.get("no-value", False)
		no_flag = self.options.get("no-flag", False)

		if self.object is not UNINITIALIZED_ATTR and (not no_value or not no_flag):

			self.add_line('', sourcename)
			self.add_line('', sourcename)

			the_flag: Optional[str] = None

			if not no_flag:
				if "flag" in self.options:
					the_flag = self.options["flag"]
				else:
					raw_flags = self.object.flags
					raw_flags = (raw_flags & ~re.DEBUG) & ~re.VERBOSE
					the_flag = parse_regex_flags(raw_flags)

			if no_value and not the_flag:
				return

			self.add_line(".. csv-table::", sourcename)
			self.add_line("     :widths: auto", sourcename)
			self.add_line("     :stub-columns: 1", sourcename)
			self.add_line('', sourcename)

			if not no_value:

				if "value" in self.options:
					the_pattern = self.options["value"]
				else:
					the_pattern = self.object.pattern

				the_pattern = the_pattern.replace('`', r"\`")

				leading_spaces = len(tuple(itertools.takewhile(str.isspace, the_pattern)))
				trailing_spaces = len(tuple(itertools.takewhile(str.isspace, the_pattern[::-1])))
				the_pattern = the_pattern.strip(' ')

				if leading_spaces > 1:
					the_pattern = f"[ ]{leading_spaces}{the_pattern}"
				elif leading_spaces == 1:
					the_pattern = f"[ ]{the_pattern}"

				if trailing_spaces > 1:
					the_pattern += f" {trailing_spaces}"
				elif trailing_spaces == 1:
					the_pattern += "[ ]"

				self.add_line(f'     **Pattern**, ":regex:`{the_pattern}`"', sourcename)

			if the_flag:
				self.add_line(f"     **Flags**, {the_flag}", sourcename)

			self.add_line('', sourcename)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header.

		:param sig:
		"""

		user_no_value = self.options.get("no-value", False)
		self.options["no-value"] = True
		super().add_directive_header(sig)
		self.options["no-value"] = user_no_value


def parse_regex_flags(flags: int) -> str:
	"""
	Convert regex flags into "bitwise-or'd" Sphinx xrefs.

	:param flags:
	"""

	buf = []

	if flags & re.ASCII:
		buf.append("ASCII")
	if flags & re.DEBUG:
		buf.append("DEBUG")
	if flags & re.IGNORECASE:
		buf.append("IGNORECASE")
	if flags & re.LOCALE:
		buf.append("LOCALE")
	if flags & re.MULTILINE:
		buf.append("MULTILINE")
	if flags & re.DOTALL:
		buf.append("DOTALL")
	if flags & re.VERBOSE:
		buf.append("VERBOSE")

	return " ``|`` ".join(f":py:data:`re.{x}`" for x in buf)


def no_formatting(value: Any) -> str:
	"""
	No-op that returns the value as a string.

	Used for unformatted output.
	"""

	return str(value)


class RegexParser:
	r"""
	Parser for regular expressions that outputs coloured output.

	The formatting is controlled by the following callable attributes:

	* ``AT_COLOUR`` -- Used for e.g. :regex:`^\A\b\B\Z$`
	* ``SUBPATTERN_COLOUR`` -- Used for the parentheses around subpatterns, e.g. :regex:`(Hello) World`
	* ``IN_COLOUR`` -- Used for the square brackets around character sets, e.g. :regex:`[Hh]ello`
	* ``REPEAT_COLOUR`` -- Used for repeats, e.g. :regex:`A?B+C*D{2,4}E{5}`
	* ``REPEAT_BRACE_COLOUR`` -- Used for the braces around numerical repeats.
	* ``CATEGORY_COLOUR`` -- Used for categories, e.g. :regex:`\d\D\s\D\w\W`
	* ``BRANCH_COLOUR`` -- Used for branches, e.g. :regex:`(Lovely|Horrible) Weather`
	* ``LITERAL_COLOUR`` -- Used for literal characters.
	* ``ANY_COLOUR`` -- Used for the "any" dot.

	These are all :class:`~typing.Callable`\[[:class:`~typing.Any`], :class:`str`\].

	By default no formatting is performed.
	Subclasses should set these attributes to appropriate functions.
	"""

	# Colours
	AT_COLOUR: Callable[[Any], str] = no_formatting
	SUBPATTERN_COLOUR: Callable[[Any], str] = no_formatting
	IN_COLOUR: Callable[[Any], str] = no_formatting
	REPEAT_COLOUR: Callable[[Any], str] = no_formatting
	REPEAT_BRACE_COLOUR: Callable[[Any], str] = no_formatting
	CATEGORY_COLOUR: Callable[[Any], str] = no_formatting
	BRANCH_COLOUR: Callable[[Any], str] = no_formatting
	LITERAL_COLOUR: Callable[[Any], str] = no_formatting
	ANY_COLOUR: Callable[[Any], str] = no_formatting

	def parse_pattern(self, regex: Pattern) -> str:
		"""
		Parse the given regular expression and return the formatted pattern.

		:param regex:
		"""

		buf = []

		def _parse_pattern(pattern) -> None:  # noqa: MAN001

			for what, content in pattern:
				# print(what, content)

				if what is AT:
					if content is AT_BEGINNING:
						buf.append(type(self).AT_COLOUR('^'))
						continue
					elif content is AT_END:
						buf.append(type(self).AT_COLOUR('$'))
						continue
					elif content is AT_BEGINNING_STRING:
						buf.append(type(self).AT_COLOUR(r"\A"))
						continue
					elif content is AT_BOUNDARY:
						buf.append(type(self).AT_COLOUR(r"\b"))
						continue
					elif content is AT_NON_BOUNDARY:
						buf.append(type(self).AT_COLOUR(r"\B"))
						continue
					elif content is AT_END_STRING:
						buf.append(type(self).AT_COLOUR(r"\Z"))
						continue

				if what is SUBPATTERN:
					buf.append(type(self).SUBPATTERN_COLOUR('('))
					group, add_flags, del_flags, subpattern = content
					# print(group, add_flags, del_flags)
					_parse_pattern(subpattern)
					buf.append(type(self).SUBPATTERN_COLOUR(')'))
					continue

				if what is LITERAL:
					# TODO: escape characters that have meaning to avoid ambiguity
					buf.append(type(self).LITERAL_COLOUR(chr(content)))
					continue

				if what is IN:
					if len(content) > 1 or content[0][0] is RANGE:
						buf.append(type(self).IN_COLOUR('['))

					_parse_pattern(content)

					if len(content) > 1 or content[0][0] is RANGE:
						buf.append(type(self).IN_COLOUR(']'))
					continue

				if what is MAX_REPEAT or what is MIN_REPEAT:
					min_, max_, item = content
					_parse_pattern(item)

					if min_ == 0 and max_ is MAXREPEAT:
						buf.append(type(self).REPEAT_COLOUR('*'))
					elif min_ == 1 and max_ is MAXREPEAT:
						buf.append(type(self).REPEAT_COLOUR('+'))
					elif min_ == 0 and max_ == 1:
						buf.append(type(self).REPEAT_COLOUR('?'))
					elif min_ == max_:
						buf.append(type(self).REPEAT_BRACE_COLOUR('{'))
						buf.append(type(self).REPEAT_COLOUR(str(min_)))
						buf.append(type(self).REPEAT_BRACE_COLOUR('}'))
					else:
						buf.append(type(self).REPEAT_BRACE_COLOUR('{'))
						buf.append(type(self).REPEAT_COLOUR(str(min_)))
						buf.append(type(self).LITERAL_COLOUR(','))
						buf.append(type(self).REPEAT_COLOUR(str(max_)))
						buf.append(type(self).REPEAT_BRACE_COLOUR('}'))
					if what is MIN_REPEAT:
						buf.append(type(self).REPEAT_COLOUR('?'))

					continue
				#
				# if what is MIN_REPEAT:
				# 	min_, max_, item = content
				# 	_parse_pattern(item)
				# 	print(min_, max_, item)
				# 	input(">>>")

				if what is CATEGORY:
					if content is CATEGORY_DIGIT:
						buf.append(type(self).CATEGORY_COLOUR(r"\d"))
						continue
					elif content is CATEGORY_NOT_DIGIT:
						buf.append(type(self).CATEGORY_COLOUR(r"\D"))
						continue
					elif content is CATEGORY_SPACE:
						buf.append(type(self).CATEGORY_COLOUR(r"\s"))
						continue
					elif content is CATEGORY_NOT_SPACE:
						buf.append(type(self).CATEGORY_COLOUR(r"\S"))
						continue
					elif content is CATEGORY_WORD:
						buf.append(type(self).CATEGORY_COLOUR(r"\w"))
						continue
					elif content is CATEGORY_NOT_WORD:
						buf.append(type(self).CATEGORY_COLOUR(r"\W"))
						continue

				if what is BRANCH:
					for branch in content[1]:
						_parse_pattern(branch)
						buf.append(type(self).BRANCH_COLOUR('|'))
					buf.pop(-1)
					continue

				if what is RANGE:
					buf.append(type(self).LITERAL_COLOUR(chr(content[0])))
					buf.append(type(self).AT_COLOUR('-'))
					buf.append(type(self).LITERAL_COLOUR(chr(content[1])))
					continue

				if what is ANY:
					buf.append(type(self).ANY_COLOUR('.'))
					continue

				print(what, content)  # pragma: no cover

		pattern = regex.pattern.replace('\t', r"\t")

		# Remove leading and trailing spaces from the pattern. They will be added back at the end.
		leading_spaces = len(tuple(itertools.takewhile(str.isspace, pattern)))
		trailing_spaces = len(tuple(itertools.takewhile(str.isspace, pattern[::-1])))
		pattern = pattern.strip(' ')

		tokens: List = list(sre_parse.parse(pattern, regex.flags))  # type: ignore[call-overload]

		if not leading_spaces:
			while tokens[0] == (LITERAL, ord(' ')):
				leading_spaces += 1
				tokens.pop(0)

		if not trailing_spaces:
			while tokens[-1] == (LITERAL, ord(' ')):
				trailing_spaces += 1
				tokens.pop(-1)

		if leading_spaces:
			buf.append(type(self).IN_COLOUR('['))
			buf.append(type(self).LITERAL_COLOUR(' '))
			buf.append(type(self).IN_COLOUR(']'))
			if leading_spaces > 1:
				buf.append(type(self).REPEAT_BRACE_COLOUR('{'))
				buf.append(type(self).REPEAT_COLOUR(str(leading_spaces)))
				buf.append(type(self).REPEAT_BRACE_COLOUR('}'))

		_parse_pattern(tokens)

		if trailing_spaces == 1:
			buf.append(type(self).IN_COLOUR('['))
			buf.append(type(self).LITERAL_COLOUR(' '))
			buf.append(type(self).IN_COLOUR(']'))
		elif trailing_spaces > 1:
			buf.append(type(self).LITERAL_COLOUR(' '))
			buf.append(type(self).REPEAT_BRACE_COLOUR('{'))
			buf.append(type(self).REPEAT_COLOUR(str(trailing_spaces)))
			buf.append(type(self).REPEAT_BRACE_COLOUR('}'))

		return ''.join(buf)


def span(css_class: str) -> Callable[[Any], str]:
	"""
	Returns a function that wraps a value in a ``span`` tag with the given class.

	:param css_class:
	"""

	def f(value: Any) -> str:
		return f'<span class="{css_class}">{value}</span>'

	return f


def latex_textcolor(colour_name: str) -> Callable[[Any], str]:
	"""
	Returns a function that wraps a value in a LaTeX ``textcolor`` command for the given colour.

	.. versionadded:: 2.11.0

	:param colour_name:
	"""

	def f(value: Any) -> str:
		if value == ' ':
			return "\\enspace"
		return f'\\textcolor{{{colour_name}}}{{{texescape.escape(value)}}}'

	return f


class HTMLRegexParser(RegexParser):
	r"""
	:class:`~.RegexParser` that outputs styled HTML.

	The formatting is controlled by the following functions, which
	wrap the character in a ``span`` tag with an appropriate CSS class:

	* ``AT_COLOUR`` -> ``regex_at`` -- Used for e.g. :regex:`^\A\b\B\Z$`
	* ``SUBPATTERN_COLOUR`` -> ``regex_subpattern`` -- Used for the parentheses around subpatterns, e.g. :regex:`(Hello) World`
	* ``IN_COLOUR`` -> ``regex_in`` -- Used for the square brackets around character sets, e.g. :regex:`[Hh]ello`
	* ``REPEAT_COLOUR`` -> ``regex_repeat`` -- Used for repeats, e.g. :regex:`A?B+C*D{2,4}E{5}`
	* ``REPEAT_BRACE_COLOUR`` -> ``regex_repeat_brace`` -- Used for the braces around numerical repeats.
	* ``CATEGORY_COLOUR`` -> ``regex_category`` -- Used for categories, e.g. :regex:`\d\D\s\D\w\W`
	* ``BRANCH_COLOUR`` -> ``regex_branch`` -- Used for branches, e.g. :regex:`(Lovely|Horrible) Weather`
	* ``LITERAL_COLOUR`` -> ``regex_literal`` -- Used for literal characters.
	* ``ANY_COLOUR`` -> ``regex_any`` -- Used for the "any" dot.

	Additionally, all ``span`` tags the ``regex`` class,
	and the surrounding ``code`` tag has the following classes:
	``docutils literal notranslate regex``.
	"""

	# Colours
	AT_COLOUR = span("regex regex_at")
	SUBPATTERN_COLOUR = span("regex regex_subpattern")
	IN_COLOUR = span("regex regex_in")
	REPEAT_COLOUR = span("regex regex_repeat")
	REPEAT_BRACE_COLOUR = span("regex regex_repeat_brace")
	CATEGORY_COLOUR = span("regex regex_category")
	BRANCH_COLOUR = span("regex regex_branch")
	LITERAL_COLOUR = span("regex regex_literal")
	ANY_COLOUR = span("regex regex_any")

	def parse_pattern(self, regex: Pattern) -> str:
		"""
		Parse the given regular expression and return the formatted pattern.

		:param regex:
		"""

		return dedent(
				f"""
		<code class="docutils literal notranslate regex">
		{super().parse_pattern(regex)}
		</code>
		"""
				)


class LaTeXRegexParser(RegexParser):
	r"""
	:class:`~.RegexParser` that outputs styled LaTeX.

	The formatting is controlled by the following functions, which
	wrap the character in a LaTeX ``textcolor`` command for an appropriate colour:

	* ``AT_COLOUR`` -> ``regex_at`` -- Used for e.g. :regex:`^\A\b\B\Z$`
	* ``SUBPATTERN_COLOUR`` -> ``regex_subpattern`` -- Used for the parentheses around subpatterns, e.g. :regex:`(Hello) World`
	* ``IN_COLOUR`` -> ``regex_in`` -- Used for the square brackets around character sets, e.g. :regex:`[Hh]ello`
	* ``REPEAT_COLOUR`` -> ``regex_repeat`` -- Used for repeats, e.g. :regex:`A?B+C*D{2,4}E{5}`
	* ``REPEAT_BRACE_COLOUR`` -> ``regex_repeat_brace`` -- Used for the braces around numerical repeats.
	* ``CATEGORY_COLOUR`` -> ``regex_category`` -- Used for categories, e.g. :regex:`\d\D\s\D\w\W`
	* ``BRANCH_COLOUR`` -> ``regex_branch`` -- Used for branches, e.g. :regex:`(Lovely|Horrible) Weather`
	* ``LITERAL_COLOUR`` -> ``regex_literal`` -- Used for literal characters.
	* ``ANY_COLOUR`` -> ``regex_any`` -- Used for the "any" dot.

	.. versionadded:: 2.11.0
	"""

	# Colours
	AT_COLOUR = latex_textcolor("regex_at")
	SUBPATTERN_COLOUR = latex_textcolor("regex_subpattern")
	IN_COLOUR = latex_textcolor("regex_in")
	REPEAT_COLOUR = latex_textcolor("regex_repeat")
	REPEAT_BRACE_COLOUR = latex_textcolor("regex_repeat_brace")
	CATEGORY_COLOUR = latex_textcolor("regex_category")
	BRANCH_COLOUR = latex_textcolor("regex_branch")
	LITERAL_COLOUR = latex_textcolor("regex_literal")
	ANY_COLOUR = latex_textcolor("regex_any")

	def parse_pattern(self, regex: Pattern) -> str:
		"""
		Parse the given regular expression and return the formatted pattern.

		:param regex:
		"""

		return f"\\sphinxcode{{\\sphinxupquote{{{super().parse_pattern(regex)}}}}}"


class TerminalRegexParser(RegexParser):
	r"""
	:class:`~.RegexParser` that outputs ANSI coloured output for the terminal.


	The formatting is controlled by the following callable attributes,
	which set ANSI escape codes for the appropriate colour:

	* ``AT_COLOUR`` -> YELLOW, Used for e.g. :regex:`^\A\b\B\Z$`
	* ``SUBPATTERN_COLOUR`` -> LIGHTYELLOW_EX, Used for the parentheses around subpatterns, e.g. :regex:`(Hello) World`
	* ``IN_COLOUR`` -> LIGHTRED_EX, Used for the square brackets around character sets, e.g. :regex:`[Hh]ello`
	* ``REPEAT_COLOUR`` -> LIGHTBLUE_EX, Used for repeats, e.g. :regex:`A?B+C*D{2,4}E{5}`
	* ``REPEAT_BRACE_COLOUR`` -> YELLOW, Used for the braces around numerical repeats.
	* ``CATEGORY_COLOUR`` -> LIGHTYELLOW_EX, Used for categories, e.g. :regex:`\d\D\s\D\w\W`
	* ``BRANCH_COLOUR`` -> YELLOW, Used for branches, e.g. :regex:`(Lovely|Horrible) Weather`
	* ``LITERAL_COLOUR`` -> GREEN, Used for literal characters.
	* ``ANY_COLOUR`` -> YELLOW, Used for the "any" dot.
	"""

	# Colours

	@staticmethod
	def AT_COLOUR(s: str) -> str:  # noqa: D102
		return f"\x1b[33m{s}\x1b[39m"

	@staticmethod
	def SUBPATTERN_COLOUR(s: str) -> str:  # noqa: D102
		return f"\x1b[93m{s}\x1b[39m"

	@staticmethod
	def IN_COLOUR(s: str) -> str:  # noqa: D102
		return f"\x1b[91m{s}\x1b[39m"

	@staticmethod
	def REPEAT_COLOUR(s: str) -> str:  # noqa: D102
		return f"\x1b[94m{s}\x1b[39m"

	@staticmethod
	def LITERAL_COLOUR(s: str) -> str:  # noqa: D102
		return f"\x1b[32m{s}\x1b[39m"

	REPEAT_BRACE_COLOUR = BRANCH_COLOUR = ANY_COLOUR = AT_COLOUR
	CATEGORY_COLOUR = SUBPATTERN_COLOUR


class RegexNode(nodes.literal):
	"""
	Docutils Node to show a highlighted regular expression.
	"""

	def __init__(self, rawsource: str = '', text: str = '', *children, **attributes) -> None:
		super().__init__(rawsource, text, *children, **attributes)
		self.pattern = re.compile(':'.join(rawsource.split(':')[2:])[1:-1])


class Regex(SphinxRole):
	"""
	Docutils role to show a highlighted regular expression.
	"""

	def run(self) -> Tuple[List[Node], List[system_message]]:
		"""
		Process the content of the regex role.
		"""

		options = self.options.copy()

		return [RegexNode(self.rawtext, self.text, **options)], []


def visit_regex_node(translator: HTMLTranslator, node: RegexNode) -> None:
	"""
	Visit an :class:`~.RegexNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append(regex_parser.parse_pattern(node.pattern))


def depart_regex_node(translator: HTMLTranslator, node: RegexNode) -> None:
	"""
	Depart an :class:`~.RegexNode`.

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.pop(-1)


def visit_regex_node_latex(translator: HTMLTranslator, node: RegexNode) -> None:
	"""
	Visit an :class:`~.RegexNode` with the LaTeX builder.

	.. versionadded:: 2.11.0

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.append(latex_regex_parser.parse_pattern(node.pattern))


def depart_regex_node_latex(translator: HTMLTranslator, node: RegexNode) -> None:
	"""
	Depart an :class:`~.RegexNode` with the LaTeX builder.

	.. versionadded:: 2.11.0

	:param translator:
	:param node: The node being visited.
	"""

	translator.body.pop(-1)


def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None:
	"""
	Copy additional stylesheets into the HTML build directory.

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.format.lower() != "html":  # pragma: no cover
		return

	static_dir = PathPlus(app.outdir) / "_static"
	static_dir.maybe_make(parents=True)
	dict2css.dump(_css.regex_styles, static_dir / "regex.css", minify=True)


regex_parser = HTMLRegexParser()
latex_regex_parser = LaTeXRegexParser()


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.code`.

	.. versionadded:: 2.11.0

	:param app: The Sphinx application.
	:param config:
	"""

	latex_elements = getattr(app.config, "latex_elements", {})

	latex_preamble = StringList(latex_elements.get("preamble", ''))
	latex_preamble.blankline()
	latex_preamble.append(r"\definecolor{regex_literal}{HTML}{696969}")
	latex_preamble.append(r"\definecolor{regex_at}{HTML}{FF4500}")
	latex_preamble.append(r"\definecolor{regex_repeat_brace}{HTML}{FF4500}")
	latex_preamble.append(r"\definecolor{regex_branch}{HTML}{FF4500}")
	latex_preamble.append(r"\definecolor{regex_subpattern}{HTML}{1e90ff}")
	latex_preamble.append(r"\definecolor{regex_in}{HTML}{ff8c00}")
	latex_preamble.append(r"\definecolor{regex_category}{HTML}{8fbc8f}")
	latex_preamble.append(r"\definecolor{regex_repeat}{HTML}{FF4500}")
	latex_preamble.append(r"\definecolor{regex_any}{HTML}{FF4500}")

	latex_elements["preamble"] = str(latex_preamble)
	app.config.latex_elements = latex_elements  # type: ignore[attr-defined]
	add_nbsp_substitution(config)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.regex`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx.ext.autodoc")
	app.setup_extension("sphinx_toolbox._css")

	app.connect("config-inited", configure)

	app.add_autodocumenter(RegexDocumenter)

	app.add_role("regex", Regex())
	app.add_node(
			RegexNode,
			html=(visit_regex_node, depart_regex_node),
			latex=(visit_regex_node_latex, depart_regex_node_latex)
			)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/regex.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  sourcelink.py
r"""
Show a link to the corresponding source code at the top of :rst:dir:`automodule` output.

.. versionadded:: 0.6.0
.. extensions:: sphinx_toolbox.more_autodoc.sourcelink


Configuration
----------------

.. raw:: latex

	\begin{flushleft}

:mod:`sphinx_toolbox.more_autodoc.sourcelink` can be configured using the :confval:`autodoc_default_options`
option in ``conf.py``, or with the :rst:dir:`:sourcelink: <sourcelink>` option flag to :rst:dir:`automodule`.

.. raw:: latex

	\end{flushleft}


.. confval:: autodoc_show_sourcelink
	:type: :class:`bool`
	:default: :py:obj:`False`

	If :py:obj:`True`, shows a link to the corresponding source code
	at the top of each :rst:dir:`automodule` directive.

.. rst:directive:option:: sourcelink

	When passed as an option flag to an :rst:dir:`automodule` directive,
	show a link to the corresponding source code at the top of the output *for that module only*.

.. latex:vspace:: 5px

.. versionchanged:: 1.1.0

	Added support for the :rst:dir:`:sourcelink: <sourcelink>` option flag to :rst:dir:`automodule`.


.. latex:clearpage::

API Reference
--------------


"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from types import ModuleType
from typing import Any, List, Mapping

# 3rd party
import autodocsumm  # type: ignore[import]
import sphinx.ext.autodoc
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.more_autosummary import PatchedAutoSummModuleDocumenter
from sphinx_toolbox.utils import SphinxExtMetadata, flag, metadata_add_version

__all__ = ("sourcelinks_process_docstring", "setup")


def sourcelinks_process_docstring(  # noqa: MAN001
		app: Sphinx,
		what,
		name: str,
		obj,
		options: Mapping[str, Any],
		lines: List[str],
		) -> None:
	"""
	Process the docstring of a module and add a link to the source code if given in the configuration.

	:param app: The Sphinx application.
	:param what:
	:param name: The name of the object being documented.
	:param obj: The object being documented.
	:param options: Mapping of autodoc options to values.
	:param lines: List of strings representing the current contents of the docstring.
	"""

	show_sourcelink = options.get("sourcelink", app.config.autodoc_show_sourcelink)

	if isinstance(obj, ModuleType) and what == "module" and show_sourcelink:
		if not obj.__file__:
			return
		elif obj.__file__.endswith("/__init__.py"):
			source_target = f"{name.replace('.', '/')}/__init__.py"
		elif obj.__file__.endswith("\\__init__.py"):
			source_target = f"{name.replace('.', '/')}/__init__.py"
		elif obj.__file__.endswith(".py"):
			source_target = f"{name.replace('.', '/')}.py"
		else:
			return

		lines_to_insert = (
				".. rst-class:: source-link",
				'',
				f"    **Source code:** :source:`{source_target}`",
				'',
				"--------------------",
				'',
				)

		for line in reversed(lines_to_insert):  # pylint: disable=W8402
			lines.insert(0, line)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.sourcelink`.

	:param app: The Sphinx application.
	"""

	sphinx.ext.autodoc.ModuleDocumenter.option_spec["sourcelink"] = flag
	autodocsumm.AutoSummModuleDocumenter.option_spec["sourcelink"] = flag
	PatchedAutoSummModuleDocumenter.option_spec["sourcelink"] = flag

	app.setup_extension("sphinx_toolbox.source")
	app.setup_extension("sphinx_toolbox._css")
	app.setup_extension("sphinx.ext.autodoc")

	app.connect("autodoc-process-docstring", sourcelinks_process_docstring)
	app.add_config_value("autodoc_show_sourcelink", False, "env", [bool])

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/sourcelink.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  autodoc_typehints.py
r"""
| Enhanced version of `sphinx-autodoc-typehints <https://pypi.org/project/sphinx-autodoc-typehints/>`_.
| Copyright (c) Alex Grönholm

The changes are:

* *None* is formatted as :py:obj:`None` and not ``None``.
  If `intersphinx <https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html>`_
  is used this will now be a link to the Python documentation.

  Since :github:pull:`154 <agronholm/sphinx-autodoc-typehints>` this feature is now available upstream.

* If the signature of the object cannot be read, the signature provided by Sphinx will be used
  rather than raising an error.

  This usually occurs for methods of builtin types.

* :class:`typing.TypeVar`\s are linked to if they have been included in the documentation.

* If a function/method argument has a :class:`module <types.ModuleType>`, :class:`class <typing.Type>`
  or :py:obj:`function <types.FunctionType>` object as its default value a better
  representation will be shown in the signature.

  For example:

	.. autofunction:: sphinx_toolbox.more_autodoc.typehints.serialise
		:noindex:

  Previously this would have shown the full path to the source file. Now it displays ``<module 'json'>``.

* The ability to hook into the :func:`~.process_docstring` function to edit the object's properties before the
  annotations are added to the docstring.

  This is used by `attr-utils <https://attr-utils.readthedocs.io>`_
  to add annotations based on converter functions in `attrs <https://www.attrs.org>`_ classes.

  To use this, in your extension's ``setup`` function:

  .. code-block:: python

      def setup(app: Sphinx) -> Dict[str, Any]:
          from sphinx_toolbox.more_autodoc.typehints import docstring_hooks
          docstring_hooks.append((my_hook, 75))
          return {}

  .. latex:clearpage::

  ``my_hook`` is a function that takes the object being documented as its only argument
  and returns that object after modification. The ``75`` is the priority of the hook:

   * ``< 20`` runs before ``fget`` functions are extracted from properties
   * ``< 90`` runs before ``__new__`` functions are extracted from :class:`~typing.NamedTuple`\s.
   * ``< 100`` runs before ``__init__`` functions are extracted from classes.

* Unresolved forward references are handled better.

* Many of the built in types from the :mod:`types` module are now formatted and linked to correctly.

-----

.. versionadded:: 0.4.0
.. versionchanged:: 0.6.0  Moved from :mod:`sphinx_toolbox.autodoc_typehints`.
.. versionchanged:: 0.8.0  Added support for :func:`collections.namedtuple`.

-----

.. extensions:: sphinx_toolbox.more_autodoc.typehints

.. latex:vspace:: -20px

| For configuration information see https://github.com/agronholm/sphinx-autodoc-typehints
| In addition, the following configuration value is added by this extension:

.. latex:vspace:: -20px

.. confval:: hide_none_rtype
	:type: :class:`bool`
	:default: :py:obj:`False`

	Hides return types of :py:obj:`None`.


API Reference
-----------------

"""  # noqa: SXL001
#
#  Copyright (c) Alex Grönholm
#  Changes copyright © 2020-2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import inspect
import itertools
import json
import re
import sys
import types
from contextlib import suppress
from operator import itemgetter
from tempfile import TemporaryDirectory
from types import FunctionType, ModuleType
from typing import (
		Any,
		AnyStr,
		Callable,
		Dict,
		ForwardRef,
		List,
		Mapping,
		NewType,
		Optional,
		Tuple,
		Type,
		TypeVar,
		Union,
		get_type_hints
		)

# 3rd party
import sphinx.util.typing
import sphinx_autodoc_typehints
from domdf_python_tools.stringlist import DelimitedList
from domdf_python_tools.typing import (
		ClassMethodDescriptorType,
		MethodDescriptorType,
		MethodWrapperType,
		WrapperDescriptorType
		)
from domdf_python_tools.utils import etc
from sphinx.application import Sphinx
from sphinx.errors import ExtensionError
from sphinx.util.inspect import signature as Signature
from sphinx.util.inspect import stringify_signature

# this package
from sphinx_toolbox.utils import (
		SphinxExtMetadata,
		code_repr,
		escape_trailing__,
		is_namedtuple,
		metadata_add_version
		)

try:
	# 3rd party
	import attr
except ImportError:  # pragma: no cover
	# attrs is used in a way that it is only required in situations
	# where it is available to import, so it is fine to do this.
	pass

try:
	# 3rd party
	from sphinx_autodoc_typehints import logger as sat_logger  # type: ignore[attr-defined]
	from sphinx_autodoc_typehints import pydata_annotations  # type: ignore[attr-defined]
except ImportError:
	# Privatised in 1.14.1
	# 3rd party
	from sphinx_autodoc_typehints import _LOGGER as sat_logger
	from sphinx_autodoc_typehints import _PYDATA_ANNOTATIONS as pydata_annotations

try:
	# 3rd party
	from sphinx_autodoc_typehints import _future_annotations_imported
except ImportError:

	def _future_annotations_imported(obj: Any) -> bool:
		if sys.version_info < (3, 7):  # pragma: no cover (py37+)
			# Only Python ≥ 3.7 supports PEP563.
			return False

		_annotations = getattr(inspect.getmodule(obj), "annotations", None)
		if _annotations is None:
			return False

		# Make sure that annotations is imported from __future__ - defined in cpython/Lib/__future__.py
		# annotations become strings at runtime
		CO_FUTURE_ANNOTATIONS = 0x100000 if sys.version_info[0:2] == (3, 7) else 0x1000000
		return _annotations.compiler_flag == CO_FUTURE_ANNOTATIONS


__all__ = (
		"ObjectAlias",
		"Module",
		"Function",
		"Class",
		"process_signature",
		"process_docstring",
		"format_annotation",
		"get_all_type_hints",
		"docstring_hooks",
		"setup",
		"get_annotation_module",
		"get_annotation_class_name",
		"get_annotation_args",
		"backfill_type_hints",
		"load_args",
		"split_type_comment_args",
		"default_preprocessors",
		"Preprocessor",
		)

get_annotation_module = sphinx_autodoc_typehints.get_annotation_module
get_annotation_class_name = sphinx_autodoc_typehints.get_annotation_class_name
get_annotation_args = sphinx_autodoc_typehints.get_annotation_args
backfill_type_hints = sphinx_autodoc_typehints.backfill_type_hints
load_args = sphinx_autodoc_typehints.load_args
split_type_comment_args = sphinx_autodoc_typehints.split_type_comment_args


# Demonstration of module default argument in signature
def serialise(obj: Any, library=json) -> str:  # noqa: MAN001
	"""
	Serialise an object into a JSON string.

	:param obj: The object to serialise.
	:param library: The JSON library to use.
	:no-default library:

	:return: The JSON string.
	"""


class ObjectAlias:
	"""
	Used to represent a module, class, function etc in a Sphinx function/class signature.

	.. versionadded:: 0.9.0

	:param name: The name of the object being aliased.
	"""

	_alias_type: str

	def __init__(self, name: str):
		self.name: str = name

	def __repr__(self) -> str:
		"""
		Returns a string representation of the :class:`~.ObjectAlias`.
		"""

		return f"<{self._alias_type} {self.name!r}>"


class Module(ObjectAlias):
	"""
	Used to represent a module in a Sphinx function/class signature.

	:param name: The name of the module.
	"""

	_alias_type = "module"


class Function(ObjectAlias):
	"""
	Used to represent a function in a Sphinx function/class signature.

	.. versionadded:: 0.9.0

	:param name: The name of the function.
	"""

	_alias_type = "function"


class Class(ObjectAlias):
	"""
	Used to represent a class in a Sphinx function/class signature.

	.. versionadded:: 0.9.0

	:param name: The name of the class.
	"""

	_alias_type = "class"


def format_annotation(annotation: Any, fully_qualified: bool = False) -> str:
	"""
	Format a type annotation.

	:param annotation:
	:param fully_qualified: Whether the fully qualified name should be shown (e.g. ``typing.List``)
		or only the object name (e.g. ``List``).

	:rtype:

	.. versionchanged:: 2.13.0  Added support for :py:obj:`True` and :py:obj:`False`

	.. latex:clearpage::
	"""

	prefix = '' if fully_qualified else '~'

	# Special cases
	if annotation is None or annotation is type(None):  # noqa: E721
		return ":py:obj:`None`"
	elif isinstance(annotation, bool):
		return f":py:obj:`{annotation}`"
	elif annotation is Ellipsis:
		return "..."
	elif annotation is itertools.cycle:
		return f":func:`{prefix}itertools.cycle`"
	elif annotation is types.GetSetDescriptorType:  # noqa: E721
		return f":py:data:`{prefix}types.GetSetDescriptorType`"
	elif annotation is types.MemberDescriptorType:  # noqa: E721
		return f":py:data:`{prefix}types.MemberDescriptorType`"
	elif annotation is types.MappingProxyType:  # noqa: E721
		return f":py:class:`{prefix}types.MappingProxyType`"
	elif annotation is types.ModuleType:  # noqa: E721
		return f":py:class:`{prefix}types.ModuleType`"
	elif annotation is types.FunctionType:  # noqa: E721
		return f":py:data:`{prefix}types.FunctionType`"
	elif annotation is types.BuiltinFunctionType:  # noqa: E721
		return f":py:data:`{prefix}types.BuiltinFunctionType`"
	elif annotation is types.MethodType:  # noqa: E721
		return f":py:data:`{prefix}types.MethodType`"
	elif annotation is MethodDescriptorType:
		return f":py:data:`{prefix}types.MethodDescriptorType`"
	elif annotation is ClassMethodDescriptorType:
		return f":py:data:`{prefix}types.ClassMethodDescriptorType`"
	elif annotation is MethodWrapperType:
		return f":py:data:`{prefix}types.MethodWrapperType`"
	elif annotation is WrapperDescriptorType:
		return f":py:data:`{prefix}types.WrapperDescriptorType`"
	elif isinstance(annotation, ForwardRef):
		# Unresolved forward ref
		return f":py:obj:`{prefix}.{annotation.__forward_arg__}`"
	elif annotation is type(re.compile('')):  # noqa: E721
		return f":py:class:`{prefix}typing.Pattern`"
	elif annotation is TemporaryDirectory:
		return f":py:obj:`{prefix}tempfile.TemporaryDirectory`"
	elif sys.version_info >= (3, 10):  # pragma: no cover (<py310)
		if annotation is types.UnionType:  # noqa: E721
			return f":py:data:`{prefix}types.UnionType`"

	try:
		module = get_annotation_module(annotation)
		class_name = get_annotation_class_name(annotation, module)

		# Special case for typing.NewType being a class in 3.10
		# Fixed upstream in 1.13.0
		if sys.version_info >= (3, 10) and isinstance(annotation, NewType):  # pragma: no cover (<py310)
			module, class_name = "typing", "NewType"

		args = get_annotation_args(annotation, module, class_name)
	except ValueError:
		return f":py:obj:`~.{annotation}`"

	if module == "_io":
		module = "io"
	elif module == "_ast":
		module = "ast"
	# Redirect all typing_extensions types to the stdlib typing module
	elif module == "typing_extensions":
		module = "typing"
	elif module == "typing" and class_name == "AbstractContextManager":
		module = "contextlib"

	full_name = (f"{module}.{class_name}") if module != "builtins" else class_name
	prefix = '' if fully_qualified or full_name == class_name else '~'
	role = "data" if class_name in pydata_annotations else "class"
	args_format = "\\[{}]"
	formatted_args = ''

	# Type variables are also handled specially
	with suppress(TypeError):
		if isinstance(annotation, TypeVar) and annotation is not AnyStr:
			typevar_name = (annotation.__module__ + '.' + annotation.__name__)
			return f":py:data:`{repr(annotation)} <{typevar_name}>`"

	# Some types require special handling
	if full_name == "typing.NewType":
		args_format = f"\\(:py:data:`~{annotation.__name__}`, {{}})"
		role = "class" if sys.version_info > (3, 10) else "func"
	elif full_name in {"typing.Union", "types.UnionType"} and len(args) == 2 and type(None) in args:
		full_name = "typing.Optional"
	elif full_name == "types.UnionType":
		full_name = "typing.Union"
		role = "data"
	elif full_name == "typing.Callable" and args and args[0] is not ...:
		formatted_args = "\\[\\[" + ", ".join(format_annotation(arg) for arg in args[:-1]) + ']'
		formatted_args += ", " + format_annotation(args[-1]) + ']'
	elif full_name == "typing.Literal":
		# TODO: Enums?
		formatted_arg_list: DelimitedList[str] = DelimitedList()
		for arg in args:
			if isinstance(arg, bool):
				formatted_arg_list.append(format_annotation(arg))
			else:
				formatted_arg_list.append(code_repr(arg))

		formatted_args = f"\\[{formatted_arg_list:, }]"

	if full_name == "typing.Optional":
		args = tuple(x for x in args if x is not type(None))  # noqa: E721

	# TODO: unions with one or more forward refs

	if args and not formatted_args:
		formatted_args = args_format.format(", ".join(format_annotation(arg, fully_qualified) for arg in args))

	return f":py:{role}:`{prefix}{full_name}`{formatted_args}"


#: Type hint for default preprocessor functions.
Preprocessor = Callable[[Type], Any]

default_preprocessors: List[Tuple[Callable[[Type], bool], Preprocessor]] = [
		(lambda d: isinstance(d, ModuleType), lambda d: Module(d.__name__)),
		(lambda d: isinstance(d, FunctionType), lambda d: Function(d.__name__)),
		(inspect.isclass, lambda d: Class(d.__name__)),
		(lambda d: d is Ellipsis, lambda d: etc),
		(lambda d: d == "...", lambda d: etc),
		]
"""
A list of 2-element tuples, comprising a function to check the default value against and a preprocessor to
pass the function to if True.
"""


def preprocess_function_defaults(obj: Callable) -> Tuple[Optional[inspect.Signature], List[inspect.Parameter]]:
	"""
	Pre-processes the default values for the arguments of a function.

	.. versionadded:: 0.8.0

	:param obj: The function.

	:return: The function signature and a list of arguments/parameters.
	"""

	try:
		signature = Signature(inspect.unwrap(obj))
	except ValueError:  # pragma: no cover
		return None, []

	parameters = []
	preprocessor_list = default_preprocessors

	for param in signature.parameters.values():
		default = param.default

		# pylint: disable=dotted-import-in-loop
		if default is not inspect.Parameter.empty:
			for check, preprocessor in preprocessor_list:
				if check(default):
					default = preprocessor(default)
					break

		parameters.append(param.replace(annotation=inspect.Parameter.empty, default=default))
		# pylint: enable=dotted-import-in-loop

	return signature, parameters


def preprocess_class_defaults(
		obj: Callable
		) -> Tuple[Optional[Callable], Optional[inspect.Signature], List[inspect.Parameter]]:
	"""
	Pre-processes the default values for the arguments of a class.

	.. versionadded:: 0.8.0

	:param obj: The class.

	:return: The class signature and a list of arguments/parameters.
	"""

	init: Optional[Callable[..., Any]] = getattr(obj, "__init__", getattr(obj, "__new__", None))

	if is_namedtuple(obj):
		init = getattr(obj, "__new__")

	try:
		signature = Signature(inspect.unwrap(init))  # type: ignore[arg-type]
	except ValueError:  # pragma: no cover
		return init, None, []

	parameters = []
	preprocessor_list = default_preprocessors

	for argname, param in signature.parameters.items():
		default = param.default

		if default is not inspect.Parameter.empty:
			for check, preprocessor in preprocessor_list:
				if check(default):
					default = preprocessor(default)
					break

			else:
				if hasattr(obj, "__attrs_attrs__") and default is attr.NOTHING:
					# Special casing for attrs classes
					for value in obj.__attrs_attrs__:  # type: ignore[attr-defined]
						if value.name == argname:
							if isinstance(value.default, attr.Factory):  # type: ignore[arg-type]
								default = value.default.factory()

		parameters.append(param.replace(annotation=inspect.Parameter.empty, default=default))

	return init, signature, parameters


def process_signature(  # noqa: MAN001
		app: Sphinx,
		what: str,
		name: str,
		obj,
		options,
		signature,
		return_annotation: Any,
		) -> Optional[Tuple[str, None]]:
	"""
	Process the signature for a function/method.

	:param app: The Sphinx application.
	:param what:
	:param name: The name of the object being documented.
	:param obj: The object being documented.
	:param options: Mapping of autodoc options to values.
	:param signature:
	:param return_annotation:

	:rtype:

	.. versionchanged:: 0.8.0

		Added support for factory function default values in attrs classes.
	"""

	if not callable(obj):
		return None

	original_obj = obj

	if inspect.isclass(obj):
		obj, signature, parameters = preprocess_class_defaults(obj)
	else:
		signature, parameters = preprocess_function_defaults(obj)

	obj = inspect.unwrap(obj)

	if not getattr(obj, "__annotations__", None):
		return None

	# The generated dataclass __init__() and class are weird and need extra checks
	# This helper function operates on the generated class and methods
	# of a dataclass, not an instantiated dataclass object. As such,
	# it cannot be replaced by a call to `dataclasses.is_dataclass()`.
	def _is_dataclass(name: str, what: str, qualname: str) -> bool:
		if what == "method" and name.endswith(".__init__"):
			# generated __init__()
			return True
		if what == "class" and qualname.endswith(".__init__"):
			# generated class
			return True
		return False

	# The generated dataclass __init__() is weird and needs the second condition
	if (
			hasattr(obj, "__qualname__") and "<locals>" in obj.__qualname__
			and not _is_dataclass(name, what, obj.__qualname__)
			):
		sat_logger.warning(
				"Cannot treat a function defined as a local function: '%s'  (use @functools.wraps)", name
				)
		return None

	if parameters:
		if inspect.isclass(original_obj) or (what == "method" and name.endswith(".__init__")):
			del parameters[0]
		elif what == "method":

			try:
				outer = inspect.getmodule(obj)
				if outer is not None:
					for clsname in obj.__qualname__.split('.')[:-1]:
						outer = getattr(outer, clsname)
			except AttributeError:
				outer = None

			method_name = obj.__name__
			if method_name.startswith("__") and not method_name.endswith("__"):
				# If the method starts with double underscore (dunder)
				# Python applies mangling so we need to prepend the class name.
				# This doesn't happen if it always ends with double underscore.
				class_name = obj.__qualname__.split('.')[-2]
				method_name = f"_{class_name}{method_name}"

			if outer is not None:
				method_object = outer.__dict__[method_name] if outer else obj
				if not isinstance(method_object, (classmethod, staticmethod)):
					del parameters[0]

			else:
				if not inspect.ismethod(obj) and parameters[0].name in {"self", "cls", "_cls"}:
					del parameters[0]

	signature = signature.replace(parameters=parameters, return_annotation=inspect.Signature.empty)

	return stringify_signature(signature), None  # .replace('\\', '\\\\')


def _docstring_property_hook(obj: Any) -> Callable:
	if isinstance(obj, property):
		obj = obj.fget

	return obj


def _docstring_class_hook(obj: Any) -> Callable:
	if callable(obj):
		if inspect.isclass(obj):
			obj = getattr(obj, "__init__")

	return obj


def _docstring_namedtuple_hook(obj: Any) -> Callable:
	if is_namedtuple(obj):
		obj = getattr(obj, "__new__")

	return obj


docstring_hooks: List[Tuple[Callable[[Any], Callable], int]] = [
		(_docstring_property_hook, 20),
		(_docstring_namedtuple_hook, 90),
		(_docstring_class_hook, 100),
		]
"""
List of additional hooks to run in :func:`~sphinx_toolbox.more_autodoc.typehints.process_docstring`.

Each entry in the list consists of:

* a function that takes the object being documented as its only argument
  and returns that object after modification.

* a number giving the priority of the hook, in ascending order.

   * ``< 20`` runs before ``fget`` functions are extracted from properties
   * ``< 90`` runs before ``__new__`` functions are extracted from :class:`NamedTuples <typing.NamedTuple>`.
   * ``< 100`` runs before ``__init__`` functions are extracted from classes.
"""


def process_docstring(
		app: Sphinx,
		what: str,
		name: str,
		obj: Any,
		options: Dict[str, Any],
		lines: List[str],
		) -> None:
	"""
	Process the docstring of a class, function, method etc.

	:param app: The Sphinx application.
	:param what:
	:param name: The name of the object being documented.
	:param obj: The object being documented.
	:param options: Mapping of autodoc options to values.
	:param lines: List of strings representing the current contents of the docstring.

	.. versionchanged:: 1.1.0

		An empty ``:rtype:`` flag can be used to control the position of the return type annotation in the docstring.
	"""

	if what in {"variable", "regex"}:
		# Doesn't have parameters or return type
		return

	original_obj = obj

	for hook, priority in sorted(docstring_hooks, key=itemgetter(1)):
		obj = hook(obj)

	if callable(obj):
		obj = inspect.unwrap(obj)
		type_hints = get_all_type_hints(obj, name, original_obj)

		signature_params: Mapping[str, Any]
		try:
			signature_params = inspect.signature(obj).parameters
		except Exception:
			# Ignore errors. Can be a multitude of things including builtin functions or extension modules.
			signature_params = {}

		for argname, annotation in type_hints.items():
			if argname == "return":
				continue  # this is handled separately later

			if argname in signature_params:
				# Ensure *args and **kwargs have *s
				if signature_params[argname].kind == 2:  # *args
					argname = f"\\*{argname}"
				elif signature_params[argname].kind == 4:  # **kwargs
					argname = f"\\*\\*{argname}"

			argname = escape_trailing__(argname)

			formatted_annotation = format_annotation(
					annotation,
					fully_qualified=app.config.typehints_fully_qualified,
					)

			searchfor = [f":{field} {argname}:" for field in ("param", "parameter", "arg", "argument")]
			insert_index = None

			for i, line in enumerate(lines):
				if any(line.startswith(search_string) for search_string in searchfor):
					insert_index = i
					break

			if insert_index is None and app.config.always_document_param_types:
				lines.append(f":param {argname}:")
				insert_index = len(lines)

			if insert_index is not None:
				lines.insert(insert_index, f":type {argname}: {formatted_annotation}")

		if "return" in type_hints and not inspect.isclass(original_obj):
			# This avoids adding a return type for data class __init__ methods
			if what == "method" and name.endswith(".__init__"):
				return

			formatted_annotation = format_annotation(
					type_hints["return"],
					fully_qualified=app.config.typehints_fully_qualified,
					)

			insert_index = len(lines)
			for i, line in enumerate(lines):
				if line.startswith(":rtype:"):
					if line[7:].strip():
						insert_index = None
					else:
						insert_index = i
						lines.pop(i)

					break

				elif line.startswith(":return:") or line.startswith(":returns:"):
					insert_index = i

			if insert_index is not None and app.config.typehints_document_rtype:

				if insert_index == len(lines):
					# Ensure that :rtype: doesn't get joined with a paragraph of text, which
					# prevents it being interpreted.
					lines.append('')
					insert_index += 1

				if not (formatted_annotation == ":py:obj:`None`" and app.config.hide_none_rtype):
					lines.insert(insert_index, f":rtype: {formatted_annotation}")


def _class_get_type_hints(obj, globalns=None, localns=None):  # noqa: MAN001,MAN002
	"""
	Return type hints for an object.

	For classes, unlike :func:`typing.get_type_hints` this will attempt to
	use the global namespace of the modules where the class and its parents
	were defined until it can resolve all forward references.
	"""

	if not inspect.isclass(obj):
		return get_type_hints(obj, localns=localns, globalns=globalns)

	mro_stack = list(obj.__mro__)
	if localns is None:
		localns = {}

	while True:

		try:  # pylint: disable=R8203
			return get_type_hints(obj.__init__, localns=localns, globalns=globalns)
		except NameError:
			if not mro_stack:
				raise
			klasse = mro_stack.pop(0)
			if klasse is object or klasse.__module__ == "builtins":
				raise
			localns = {**sys.modules[klasse.__module__].__dict__, **localns}


def get_all_type_hints(obj: Any, name: str, original_obj: Any) -> Dict[str, Any]:
	"""
	Returns the resolved type hints for the given objects.

	:param obj:
	:param name:
	:param original_obj: The original object, before the class if ``obj`` is its ``__init__`` method.
	"""

	def log(exc: Exception) -> None:
		sat_logger.warning(
				'Cannot resolve forward reference in type annotations of "%s": %s',
				name,
				exc,
				)

	rv = {}

	try:
		if inspect.isclass(original_obj):
			rv = _class_get_type_hints(original_obj)
		else:
			rv = get_type_hints(obj)

	except (AttributeError, TypeError, RecursionError) as exc:
		# Introspecting a slot wrapper will raise TypeError, and some recursive type
		# definitions will cause a RecursionError (https://github.com/python/typing/issues/574).

		# If one is using PEP563 annotations, Python will raise a (e.g.,)
		# TypeError("TypeError: unsupported operand type(s) for |: 'type' and 'NoneType'")
		# on 'str | None', therefore we accept TypeErrors with that error message
		# if 'annotations' is imported from '__future__'.
		if isinstance(exc, TypeError):
			if _future_annotations_imported(obj) and "unsupported operand type" in str(exc):
				rv = obj.__annotations__

	except NameError as exc:
		log(exc)
		rv = obj.__annotations__

	if rv:
		return rv

	rv = backfill_type_hints(obj, name)

	try:
		if rv != {}:
			obj.__annotations__ = rv
	except (AttributeError, TypeError):
		return rv

	try:
		if inspect.isclass(original_obj):
			rv = _class_get_type_hints(original_obj)
		else:
			rv = get_type_hints(obj)
	except (AttributeError, TypeError):
		pass
	except NameError as exc:
		log(exc)
		rv = obj.__annotations__

	return rv


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.typehints`.

	:param app: The Sphinx application.
	"""

	if "sphinx_autodoc_typehints" in app.extensions:
		raise ExtensionError(
				"'sphinx_toolbox.more_autodoc.typehints' must be loaded before 'sphinx_autodoc_typehints'."
				)

	sphinx_autodoc_typehints.format_annotation = format_annotation  # type: ignore[assignment]
	sphinx_autodoc_typehints.process_signature = process_signature
	sphinx_autodoc_typehints.process_docstring = process_docstring  # type: ignore[assignment]

	app.setup_extension("sphinx.ext.autodoc")
	app.setup_extension("sphinx_autodoc_typehints")

	app.add_config_value("hide_none_rtype", False, "env", [bool])

	return {"parallel_read_safe": True}


def _resolve_forwardref(
		fr: Union[ForwardRef, sphinx.util.typing.ForwardRef],
		module: str,
		) -> object:
	"""
	Resolve a forward reference.

	This is not part of the public API.

	:param fr:
	:param module: The module the forward reference was defined in.

	:return: The evaluated object.
	"""

	module_dict = sys.modules[module].__dict__
	if sys.version_info < (3, 9):
		return fr._evaluate(module_dict, module_dict)
	else:
		return fr._evaluate(module_dict, module_dict, set())

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/typehints.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  typevars.py
r"""
Documenter for module level :class:`typing.TypeVar`\'s, similar to Sphinx's
``autotypevar`` but with a different appearance.

.. versionadded:: 1.3.0
.. extensions:: sphinx_toolbox.more_autodoc.typevars

.. latex:vspace:: -15px

Configuration
-------------

.. confval:: all_typevars
	:type: :class:`bool`
	:default: False

	Document all :class:`typing.TypeVar`\s, even if they have no docstring.


.. confval:: no_unbound_typevars
	:type: :class:`bool`
	:default: True

	Only document :class:`typing.TypeVar`\s that have a constraint of are bound.

	This option has no effect if :confval:`all_typevars` is False.


.. latex:clearpage::

Usage
----------

.. rst:directive:: autotypevar

	Directive to automatically document a :class:`typing.TypeVar`.

	The output is based on the :rst:dir:`autodata` directive, and takes all of its options
	plus these additional ones:

	.. rst:directive:option:: no-value

		Don't show the value of the variable.

	.. rst:directive:option:: value: value
		:type: string

		Show this instead of the value taken from the Python source code.

	.. rst:directive:option:: no-type

		Don't show the type of the variable.


API Reference
----------------
"""  # noqa: D400
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import sys
from typing import Any, Dict, ForwardRef, List, Optional, Tuple, Type, TypeVar, Union

# 3rd party
from docutils.statemachine import StringList
from domdf_python_tools.words import word_join
from sphinx.application import Sphinx
from typing_extensions import Protocol

# this package
from sphinx_toolbox._data_documenter import DataDocumenter
from sphinx_toolbox.config import ToolboxConfig
from sphinx_toolbox.more_autodoc import _documenter_add_content
from sphinx_toolbox.more_autodoc.typehints import format_annotation
from sphinx_toolbox.more_autodoc.variables import VariableDocumenter
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = (
		"TypeVarDocumenter",
		"unskip_typevars",
		"setup",
		)


class TypeVarDocumenter(VariableDocumenter):
	r"""
	Alternative version of :class:`sphinx.ext.autodoc.TypeVarDocumenter`
	with better type hint rendering.

	Specialized Documenter subclass for :class:`typing.TypeVar`\s.
	"""  # noqa: D400

	objtype = "typevar"
	directivetype = "data"
	priority = DataDocumenter.priority + 1

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		return isinstance(member, TypeVar)

	def resolve_type(self, forward_ref: ForwardRef) -> Type:
		"""
		Resolve a :class:`typing.ForwardRef` using the module the :class:`~typing.TypeVar` belongs to.

		:param forward_ref:
		"""

		if forward_ref.__forward_evaluated__:
			return forward_ref.__forward_value__  # type: ignore[return-value]
		else:
			globanls = sys.modules[self.object.__module__].__dict__

			eval_ = eval
			return eval_(forward_ref.__forward_code__, globanls, globanls)

	def add_content(self, more_content: Optional[StringList], no_docstring: bool = False) -> None:
		"""
		Add content from docstrings, attribute documentation and user.

		:param more_content:
		:param no_docstring:
		"""

		obj: _TypeVar = self.object
		sourcename = self.get_sourcename()
		constraints = [self.resolve_type(c) if isinstance(c, ForwardRef) else c for c in obj.__constraints__]
		description = []

		bound_to: Optional[Type]

		if isinstance(obj.__bound__, ForwardRef):
			bound_to = self.resolve_type(obj.__bound__)
		else:
			bound_to = obj.__bound__

		if obj.__covariant__:
			description.append("Covariant")
		elif obj.__contravariant__:
			description.append("Contravariant")
		else:
			description.append("Invariant")

		description.append(":class:`~typing.TypeVar`")

		if constraints:
			description.append("constrained to")
			description.append(word_join(format_annotation(c, fully_qualified=True) for c in constraints))
		elif bound_to:
			description.append("bound to")
			description.append(format_annotation(bound_to, fully_qualified=True))

		# if self.analyzer:
		# 	attr_docs = self.analyzer.find_attr_docs()
		# 	if self.objpath:
		# 		key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
		# 		if key in attr_docs:
		# 			return

		self.add_line('', sourcename)
		self.add_line(' '.join(description).rstrip() + '.', sourcename)  # "   " +
		self.add_line('', sourcename)

		_documenter_add_content(self, more_content, no_docstring)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header.

		:param sig:
		"""

		obj: _TypeVar = self.object
		sourcename = self.get_sourcename()
		constraints = [self.resolve_type(c) if isinstance(c, ForwardRef) else c for c in obj.__constraints__]
		sig_elements = [obj.__name__, *(c.__name__ for c in constraints)]

		bound_to: Optional[Type]

		if isinstance(obj.__bound__, ForwardRef):
			bound_to = self.resolve_type(obj.__bound__)
		else:
			bound_to = obj.__bound__

		if bound_to is not None:
			try:
				sig_elements.append(f"bound={bound_to.__name__}")
			except AttributeError:
				sig_elements.append(f"bound={repr(bound_to)}")

		if obj.__covariant__:
			sig_elements.append(f"covariant=True")
		elif obj.__contravariant__:
			sig_elements.append(f"contravariant=True")

		self.options["value"] = f"TypeVar({', '.join(sig_elements)})"
		self.add_line('', sourcename)

		super().add_directive_header(sig)

	def get_doc(
			self,
			encoding: Optional[str] = None,
			ignore: Optional[int] = None,
			) -> List[List[str]]:
		"""
		Decode and return lines of the docstring(s) for the object.

		:param encoding:
		:param ignore:

		:rtype:

		.. latex:clearpage::
		"""

		if self.object.__doc__ != TypeVar.__doc__:
			return super().get_doc() or []
		else:
			return []


def validate_config(app: Sphinx, config: ToolboxConfig) -> None:
	r"""
	Validate the provided configuration values.

	See :class:`~sphinx_toolbox.config.ToolboxConfig` for a list of the configuration values.

	:param app: The Sphinx application.
	:param config:
	:type config: :class:`~sphinx.config.Config`
	"""

	if config.all_typevars:
		app.connect("autodoc-skip-member", unskip_typevars)


def unskip_typevars(
		app: Sphinx,
		what: str,
		name: str,
		obj: Any,
		skip: bool,
		options: Dict[str, Any],
		) -> Optional[bool]:
	r"""
	Unskip undocumented :class:`typing.TypeVar`\s if :confval:`all_typevars` is :py:obj:`True`.

	:param app: The Sphinx application.
	:param what: The type of the object which the docstring belongs to (one of
		``'module'``, ``'class'``, ``'exception'``, ``'function'``, ``'method'``,
		``'attribute'``).
	:param name: The fully qualified name of the object.
	:param obj: The object itself.
	:param skip: A boolean indicating if autodoc will skip this member if the
		user handler does not override the decision.
	:param options: The options given to the directive: an object with attributes
		``inherited_members``, ``undoc_members``, ``show_inheritance`` and
		``noindex`` that are true if the flag option of same name was given to the
		auto directive.
	"""

	assert app.env is not None

	if isinstance(obj, TypeVar):
		if app.env.config.no_unbound_typevars:
			if obj.__bound__ or obj.__constraints__:
				return False
			else:
				return True
		else:
			return False

	return None


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.typevars`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx.ext.autodoc")
	app.add_autodocumenter(TypeVarDocumenter, override=True)
	app.add_config_value("all_typevars", False, "env", types=[bool])
	app.add_config_value("no_unbound_typevars", True, "env", types=[bool])

	app.connect("config-inited", validate_config, priority=850)

	return {"parallel_read_safe": True}


class _TypeVar(Protocol):
	__constraints__: Tuple[Union[Type, ForwardRef], ...]
	__bound__: Union[Type, ForwardRef, None]
	__covariant__: bool
	__contravariant__: bool
	__name__: str

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/typevars.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  variables.py
r"""
Documenter for module level variables, similar to :rst:dir:`autodata` but
with a different appearance and more customisation options.

.. versionadded:: 0.6.0
.. extensions:: sphinx_toolbox.more_autodoc.variables
.. versionchanged:: 0.7.0  Added ``*AttributeDocumenter``\s
.. versionchanged:: 1.1.0  Added :class:`~.SlotsAttributeDocumenter`


Usage
----------

.. rst:directive:: autovariable

	Directive to automatically document a variable.

	The output is based on the :rst:dir:`autodata` directive, and takes all of its options,
	plus these additional ones:

	.. rst:directive:option:: no-value

		Don't show the value of the variable.

	.. rst:directive:option:: value: value
		:type: string

		Show this instead of the value taken from the Python source code.

	.. rst:directive:option:: no-type

		Don't show the type of the variable.

	.. rst:directive:option:: type: type
		:type: string

		Show this instead of the type taken from the Python source code.

	An example of the output cen be seen below for :py:obj:`~.type_template`.

API Reference
----------------
"""  # noqa: D400
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import importlib
import sys
import warnings
from contextlib import suppress
from typing import Any, List, Optional, cast, get_type_hints

# 3rd party
import sphinx
from docutils.statemachine import StringList
from sphinx.application import Sphinx
from sphinx.errors import PycodeError
from sphinx.ext.autodoc import (
		INSTANCEATTR,
		SLOTSATTR,
		UNINITIALIZED_ATTR,
		ClassLevelDocumenter,
		DocstringStripSignatureMixin,
		Documenter,
		ModuleDocumenter,
		ModuleLevelDocumenter,
		Options,
		annotation_option,
		import_object,
		logger,
		mock
		)
from sphinx.ext.autodoc.directive import DocumenterBridge
from sphinx.pycode import ModuleAnalyzer
from sphinx.util import inspect
from sphinx.util.inspect import ForwardRef, getdoc, object_description, safe_getattr

# this package
from sphinx_toolbox._data_documenter import DataDocumenter
from sphinx_toolbox.more_autodoc import _documenter_add_content
from sphinx_toolbox.more_autodoc.typehints import _resolve_forwardref, format_annotation
from sphinx_toolbox.utils import (
		RemovedInSphinx50Warning,
		SphinxExtMetadata,
		add_nbsp_substitution,
		flag,
		metadata_add_version,
		prepare_docstring
		)

__all__ = (
		"VariableDocumenter",
		"TypedAttributeDocumenter",
		"InstanceAttributeDocumenter",
		"SlotsAttributeDocumenter",
		"type_template",
		"old_type_template",
		"get_variable_type",
		"setup",
		)


def get_variable_type(documenter: Documenter) -> str:
	"""
	Returns the formatted type annotation for a variable.

	:param documenter:
	"""

	try:
		annotations = get_type_hints(documenter.parent)
	except NameError:
		# Failed to evaluate ForwardRef (maybe TYPE_CHECKING)
		annotations = safe_getattr(documenter.parent, "__annotations__", {})
	except (TypeError, KeyError, AttributeError):
		# KeyError: a broken class found (refs: https://github.com/sphinx-doc/sphinx/issues/8084)
		# AttributeError is raised on 3.5.2 (fixed by 3.5.3)
		annotations = {}

	if documenter.objpath[-1] in annotations:
		ann = annotations.get(documenter.objpath[-1])

		if isinstance(ann, str):
			return format_annotation(ann.strip("'\""))

		return format_annotation(ann)
	else:
		# Instance attribute
		key = ('.'.join(documenter.objpath[:-1]), documenter.objpath[-1])

		if documenter.analyzer and key in documenter.analyzer.annotations:
			# Forward references will have quotes
			annotation: str = documenter.analyzer.annotations[key].strip("'\"")

			try:
				module_dict = sys.modules[documenter.parent.__module__].__dict__

				if annotation.isidentifier() and annotation in module_dict:
					return format_annotation(module_dict[annotation])
				else:
					ref = ForwardRef(annotation)
					evaled = _resolve_forwardref(ref, documenter.parent.__module__)
					return format_annotation(evaled)

			except (NameError, TypeError, ValueError, AttributeError):
				return annotation
		else:
			return ''


old_type_template = "   **Type:** |nbsp| |nbsp| |nbsp| |nbsp| %s"
"""
Old template for rendering type annotations in :class:`~.VariableDocumenter`,
:class:`~.TypedAttributeDocumenter` and :class:`~.InstanceAttributeDocumenter`.

Renders like:

	**Type:** |nbsp| |nbsp| |nbsp| |nbsp| :class:`str`

.. note::

	Be sure to call :func:~.add_nbsp_substitution` in the ``setup`` function
	of any extensions using this template.
"""

type_template = "   **Type:**    %s"
"""
Template for rendering type annotations in :class:`~.VariableDocumenter`,
:class:`~.TypedAttributeDocumenter` and :class:`~.InstanceAttributeDocumenter`.

Renders like:

	**Type:** \u00A0\u00A0\u00A0\u00A0 :class:`str`

.. versionchanged:: 3.4.0

	This template now uses the unicode codepoint for a non-breaking space,
	rather than the ReStructuredText substitution used in 3.3.0 and earlier.
	The old template is available as ``old_type_template``.
"""


class VariableDocumenter(DataDocumenter):
	"""
	Specialized Documenter subclass for data items.
	"""

	directivetype = "data"
	objtype = "variable"

	# keeps it below TypeVarDocumenter
	priority: float = DataDocumenter.priority + 0.5  # type: ignore[assignment]

	option_spec = {
			"no-value": flag,
			"no-type": flag,
			"type": str,
			"value": str,
			**DataDocumenter.option_spec,
			}

	def __init__(self, directive: DocumenterBridge, name: str, indent: str = '') -> None:
		super().__init__(directive=directive, name=name, indent=indent)
		self.options = Options(self.options.copy())
		add_nbsp_substitution(self.env.app.config)

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header.

		:param sig:
		"""

		sourcename = self.get_sourcename()

		no_value = self.options.get("no-value", False)
		no_type = self.options.get("no-type", False)

		if not self.options.get("annotation", ''):
			ModuleLevelDocumenter.add_directive_header(self, sig)

			if not no_value:
				if "value" in self.options:
					self.add_line(f"   :value: {self.options['value']}", sourcename)
				else:
					with suppress(ValueError):
						if self.object is not UNINITIALIZED_ATTR:
							objrepr = object_description(self.object)
							self.add_line(f"   :value: {objrepr}", sourcename)

			self.add_line('', sourcename)

			if not no_type:
				if "type" in self.options:
					the_type = self.options["type"]
				else:
					# obtain type annotation for this data
					the_type = get_variable_type(self)
					if not the_type.strip():
						obj_type = type(self.object)

						if obj_type is object:
							return

						try:
							the_type = format_annotation(obj_type)
						except Exception:
							return

				line = type_template % the_type
				self.add_line(line, sourcename)

		else:
			super().add_directive_header(sig)


class TypedAttributeDocumenter(DocstringStripSignatureMixin, ClassLevelDocumenter):
	"""
	Alternative version of
	:class:`autodoc.AttributeDocumenter <sphinx.ext.autodoc.AttributeDocumenter>`
	with better type hint rendering.

	Specialized Documenter subclass for attributes.

	.. versionadded:: 0.7.0
	.. versionchanged:: 1.0.0  Now uses the type of the variable if it is not explicitly annotated.
	"""  # noqa: D400

	objtype = "attribute"
	member_order = 60
	option_spec = dict(ModuleLevelDocumenter.option_spec)
	option_spec["annotation"] = annotation_option

	# must be higher than the MethodDocumenter, else it will recognize
	# some non-data descriptors as methods
	priority = 10

	def __init__(self, directive: DocumenterBridge, name: str, indent: str = '') -> None:
		super().__init__(directive=directive, name=name, indent=indent)
		self.options = Options(self.options.copy())
		self._datadescriptor = True

	@staticmethod
	def is_function_or_method(obj: Any) -> bool:  # noqa: D102
		return inspect.isfunction(obj) or inspect.isbuiltin(obj) or inspect.ismethod(obj)

	@classmethod
	def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any) -> bool:
		"""
		Called to see if a member can be documented by this documenter.
		"""

		if inspect.isattributedescriptor(member):
			return True
		elif (
				not isinstance(parent, ModuleDocumenter) and not inspect.isroutine(member)
				and not isinstance(member, type)
				):
			return True
		else:
			return False

	def document_members(self, all_members: bool = False) -> None:  # noqa: D102
		pass

	def isinstanceattribute(self) -> bool:
		"""
		Check the subject is an instance attribute.
		"""

		try:
			analyzer = ModuleAnalyzer.for_module(self.modname)
			attr_docs = analyzer.find_attr_docs()
			if self.objpath:
				key = ('.'.join(self.objpath[:-1]), self.objpath[-1])
				if key in attr_docs:
					return True

			return False
		except PycodeError:
			return False

	def import_object(self, raiseerror: bool = False) -> bool:
		"""
		Import the object given by *self.modname* and *self.objpath* and set it as ``self.object``.

		:returns: :py:obj:`True` if successful, :py:obj:`False` if an error occurred.
		"""

		try:
			ret = super().import_object(raiseerror=True)
			if inspect.isenumattribute(self.object):
				self.object = self.object.value
			if inspect.isattributedescriptor(self.object):
				self._datadescriptor = True
			else:
				# if it's not a data descriptor
				self._datadescriptor = False
		except ImportError as exc:
			if self.isinstanceattribute():
				self.object = INSTANCEATTR
				self._datadescriptor = False
				ret = True
			elif raiseerror:
				raise
			else:
				logger.warning(exc.args[0], type="autodoc", subtype="import_object")
				self.env.note_reread()
				ret = False

		return ret

	def get_real_modname(self) -> str:
		"""
		Get the real module name of an object to document.

		It can differ from the name of the module through which the object was imported.
		"""

		return self.get_attr(self.parent or self.object, "__module__", None) or self.modname

	def add_directive_header(self, sig: str) -> None:
		"""
		Add the directive's header.

		:param sig:
		"""

		sourcename = self.get_sourcename()

		no_value = self.options.get("no-value", False)
		no_type = self.options.get("no-type", False)

		if not self.options.get("annotation", ''):
			ClassLevelDocumenter.add_directive_header(self, sig)

			# data descriptors do not have useful values
			if not no_value and not self._datadescriptor:
				if "value" in self.options:
					self.add_line("   :value: " + self.options["value"], sourcename)
				else:
					with suppress(ValueError):
						if self.object is not INSTANCEATTR:
							objrepr = object_description(self.object)
							self.add_line("   :value: " + objrepr, sourcename)

			self.add_line('', sourcename)

			if not no_type:
				if "type" in self.options:
					self.add_line(type_template % self.options["type"], sourcename)
				else:
					# obtain type annotation for this attribute
					the_type = get_variable_type(self)
					if not the_type.strip():
						obj_type = type(self.object)

						if obj_type is object:
							return

						try:
							the_type = format_annotation(obj_type)
						except Exception:
							return

					line = type_template % the_type
					self.add_line(line, sourcename)

		else:
			super().add_directive_header(sig)

	def get_doc(
			self,
			encoding: Optional[str] = None,
			ignore: Optional[int] = None,
			) -> List[List[str]]:
		"""
		Decode and return lines of the docstring(s) for the object.

		:param encoding:
		:param ignore:
		"""

		# Disable `autodoc_inherit_docstring` temporarily to avoid to obtain
		# a docstring from the value which descriptor returns unexpectedly.
		# ref: https://github.com/sphinx-doc/sphinx/issues/7805
		orig = self.env.config.autodoc_inherit_docstrings

		try:
			self.env.config.autodoc_inherit_docstrings = False  # type: ignore[attr-defined]

			# Sphinx's signature is wrong wrt Optional
			if sphinx.version_info >= (4, 0):
				if encoding is not None:
					raise TypeError("The 'encoding' argument to get_doc was removed in Sphinx 4")

				if self._new_docstrings is not None:
					return self._new_docstrings or []

				docstring = getdoc(
						self.object,
						self.get_attr,
						self.config.autodoc_inherit_docstrings,
						self.parent,
						self.object_name,
						)

				if docstring:
					tab_width = self.directive.state.document.settings.tab_width
					return [prepare_docstring(docstring, ignore, tab_width)]
				return []

			else:
				enc = cast(str, encoding)
				ign = cast(int, ignore)
				super_get_doc = super().get_doc(enc, ign)  # type: ignore[call-arg]
				return super_get_doc or []
		finally:
			self.env.config.autodoc_inherit_docstrings = orig  # type: ignore[attr-defined]

	def add_content(self, more_content: Optional[StringList], no_docstring: bool = False) -> None:
		"""
		Add content from docstrings, attribute documentation and user.
		"""

		if not self._datadescriptor:
			# if it's not a data descriptor, its docstring is very probably the wrong thing to display
			no_docstring = True

		_documenter_add_content(self, more_content, no_docstring)


class InstanceAttributeDocumenter(TypedAttributeDocumenter):
	"""
	Alternative version of
	:class:`autodoc.InstanceAttributeDocumenter <sphinx.ext.autodoc.InstanceAttributeDocumenter>`
	with better type hint rendering.

	Specialized Documenter subclass for attributes that cannot be imported
	because they are instance attributes (e.g. assigned in ``__init__``).

	.. versionadded:: 0.7.0
	.. versionchanged:: 1.0.0  Now uses the type of the variable if it is not explicitly annotated.
	"""  # noqa: D400

	objtype = "instanceattribute"
	directivetype = "attribute"
	member_order = 60

	# must be higher than TypedAttributeDocumenter
	priority = 11

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		This documenter only documents INSTANCEATTR members.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		return not isinstance(parent, ModuleDocumenter) and isattr and member is INSTANCEATTR

	def import_parent(self) -> Any:
		"""
		Import and return the attribute's parent.
		"""

		try:
			parent = importlib.import_module(self.modname)
			for name in self.objpath[:-1]:
				parent = self.get_attr(parent, name)

			return parent
		except (ImportError, AttributeError):
			return None

	def import_object(self, raiseerror: bool = False) -> bool:
		"""
		Never import anything.

		:param raiseerror:
		"""

		# disguise as an attribute
		self.objtype = "attribute"
		self.object = INSTANCEATTR
		self.parent = self.import_parent()
		self._datadescriptor = False
		return True

	def add_content(self, more_content: Any, no_docstring: bool = False) -> None:
		"""
		Never try to get a docstring from the object.
		"""

		super().add_content(more_content, no_docstring=True)


class SlotsAttributeDocumenter(TypedAttributeDocumenter):
	r"""
	Alternative version of
	:class:`autodoc.InstanceAttributeDocumenter <sphinx.ext.autodoc.InstanceAttributeDocumenter>`
	with better type hint rendering.

	Specialized Documenter subclass for attributes that cannot be imported
	because they are attributes in __slots__.

	.. versionadded:: 1.1.0

	.. latex:vspace:: 10px

	"""  # noqa: D400

	objtype = "slotsattribute"
	directivetype = "attribute"
	member_order = 60

	# must be higher than AttributeDocumenter
	priority = 11

	@classmethod
	def can_document_member(
			cls,
			member: Any,
			membername: str,
			isattr: bool,
			parent: Any,
			) -> bool:
		"""
		Called to see if a member can be documented by this documenter.

		This documenter only documents SLOTSATTR members.

		:param member: The member being checked.
		:param membername: The name of the member.
		:param isattr:
		:param parent: The parent of the member.
		"""

		return member is SLOTSATTR

	def import_object(self, raiseerror: bool = False) -> bool:
		"""
		Never import anything.

		:param raiseerror:
		"""

		# disguise as an attribute
		self.objtype = "attribute"
		self._datadescriptor = True

		with mock(self.env.config.autodoc_mock_imports):
			try:
				ret = import_object(
						self.modname,
						self.objpath[:-1],
						"class",
						attrgetter=self.get_attr,
						warningiserror=self.env.config.autodoc_warningiserror
						)
				self.module, _, _, self.parent = ret
				return True
			except ImportError as exc:
				if raiseerror:
					raise
				else:
					logger.warning(exc.args[0], type="autodoc", subtype="import_object")
					self.env.note_reread()
					return False

	def get_doc(
			self,
			encoding: Optional[str] = None,
			ignore: Optional[int] = None,
			) -> List[List[str]]:
		"""
		Decode and return lines of the docstring(s) for the object.

		:param encoding:
		:param ignore:
		"""

		if sphinx.version_info >= (4, 0):
			if encoding is not None:
				raise TypeError("The 'encoding' argument to get_doc was removed in Sphinx 4")

		if ignore is not None:
			if sphinx.version_info >= (5, 0):
				raise TypeError("The 'ignore' argument to get_doc was removed in Sphinx 5")
			else:
				warnings.warn(
						"The 'ignore' argument to get_doc() is deprecated.",
						RemovedInSphinx50Warning,  # type: ignore[arg-type]
						stacklevel=2
						)

		name = self.objpath[-1]
		__slots__ = safe_getattr(self.parent, "__slots__", [])

		if isinstance(__slots__, dict) and isinstance(__slots__.get(name), str):
			return [prepare_docstring(__slots__[name])]
		else:
			return []


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autodoc.variables`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx.ext.autodoc")
	app.setup_extension("sphinx_toolbox.more_autosummary")

	app.add_autodocumenter(VariableDocumenter)
	app.add_autodocumenter(TypedAttributeDocumenter, override=True)
	app.add_autodocumenter(InstanceAttributeDocumenter, override=True)
	app.add_autodocumenter(SlotsAttributeDocumenter, override=True)

	app.connect("config-inited", lambda _, config: add_nbsp_substitution(config))

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autodoc/variables.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  __init__.py
r"""
Extensions to :mod:`sphinx.ext.autosummary`.

Provides an enhanced version of https://autodocsumm.readthedocs.io/
which respects the autodoc ``member-order`` option.
This can be given for an individual directive, in the
`autodoc_member_order <https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#confval-autodoc_member_order>`_
configuration value, or via :confval:`autodocsumm_member_order`.

Also patches :class:`sphinx.ext.autosummary.Autosummary` to fix an issue where
the module name is sometimes duplicated.
I.e. ``foo.bar.baz()`` became ``foo.bar.foo.bar.baz()``, which of course doesn't exist
and created a broken link.


.. versionadded:: 0.7.0

.. versionchanged:: 1.3.0

	Autosummary now selects the appropriate documenter for attributes rather than
	falling back to :class:`~sphinx.ext.autodoc.DataDocumenter`.

.. versionchanged:: 2.13.0

	Also patches :class:`sphinx.ext.autodoc.ModuleDocumenter` to fix an issue where
	``__all__`` is not respected for autosummary tables.



Configuration
--------------

.. latex:vspace:: -20px

.. confval:: autodocsumm_member_order
	:type: :py:obj:`str`
	:default: ``'alphabetical'``

	Determines the sort order of members in ``autodocsumm`` summary tables.
	Valid values are ``'alphabetical'`` and ``'bysource'``.

	Note that for ``'bysource'`` the module must be a Python module with the source code available.

	The member order can also be set on a per-directive basis using the ``:member-order: [order]`` option.
	This applies not only to :rst:dir:`automodule` etc. directives,
	but also to :rst:dir:`automodulesumm` etc. directives.

.. confval:: autosummary_col_type
	:type: :py:obj:`str`
	:default: ``'\X'``

	The LaTeX column type to use for autosummary tables.

	Custom columns can be defined in the LaTeX preamble for use with this option.

	For example:

	.. code-block:: python

		latex_elements["preamble"] = r'''
			\makeatletter
			\newcolumntype{\Xx}[2]{>{\raggedright\arraybackslash}p{\dimexpr
				(\linewidth-\arrayrulewidth)*#1/#2-\tw@\tabcolsep-\arrayrulewidth\relax}}
			\makeatother
			'''

		autosummary_col_type = "\\Xx"


	.. versionadded:: 2.13.0



API Reference
----------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#  Builds on top of, and PatchedAutoDocSummDirective based on, https://github.com/Chilipp/autodocsumm
#  | Copyright 2016-2019, Philipp S. Sommer
#  | Copyright 2020-2021, Helmholtz-Zentrum Hereon
#  |
#  | Licensed under the Apache License, Version 2.0 (the "License");
#  | you may not use this file except in compliance with the License.
#  | You may obtain a copy of the License at
#  |
#  |     http://www.apache.org/licenses/LICENSE-2.0
#  |
#  | Unless required by applicable law or agreed to in writing,
#  | software distributed under the License is distributed on an "AS IS" BASIS,
#  | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  | See the License for the specific language governing permissions and limitations under the License.
#

# stdlib
import inspect
import operator
import re
from typing import Any, Dict, List, Optional, Tuple, Type

# 3rd party
import autodocsumm  # type: ignore[import]
import docutils
import sphinx
from docutils import nodes
from domdf_python_tools.stringlist import StringList
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.config import ENUM
from sphinx.ext.autodoc import (
		ALL,
		INSTANCEATTR,
		ClassDocumenter,
		Documenter,
		ModuleDocumenter,
		logger,
		special_member_re
		)
from sphinx.ext.autodoc.directive import DocumenterBridge, process_documenter_options
from sphinx.ext.autosummary import Autosummary, FakeDirective, autosummary_table
from sphinx.locale import __
from sphinx.util.inspect import getdoc, safe_getattr

# this package
from sphinx_toolbox._data_documenter import DataDocumenter
from sphinx_toolbox.more_autodoc import ObjectMembers
from sphinx_toolbox.utils import SphinxExtMetadata, allow_subclass_add, get_first_matching, metadata_add_version

if sphinx.version_info > (4, 1):
	# 3rd party
	from sphinx.util.docstrings import separate_metadata
else:
	# 3rd party
	from sphinx.util.docstrings import extract_metadata

__all__ = (
		"PatchedAutosummary",
		"PatchedAutoSummModuleDocumenter",
		"PatchedAutoSummClassDocumenter",
		"get_documenter",
		"setup",
		)

try:
	# 3rd party
	from sphinx.ext.autodoc.importer import get_module_members as _get_module_members  # type: ignore[attr-defined]
except ImportError:
	# 3rd party
	from sphinx.util.inspect import getannotations

	def _get_module_members(module: Any) -> List[Tuple[str, Any]]:
		"""Get members of target module."""
		# 3rd party
		from sphinx.ext.autodoc import INSTANCEATTR

		members: Dict[str, Tuple[str, Any]] = {}
		for name in dir(module):
			try:
				value = safe_getattr(module, name, None)
				members[name] = (name, value)
			except AttributeError:
				continue

		# annotation only member (ex. attr: int)
		for name in getannotations(module):
			if name not in members:
				members[name] = (name, INSTANCEATTR)

		return sorted(list(members.values()))


def add_autosummary(self, relative_ref_paths: bool = False) -> None:
	"""
	Add the :rst:dir:`autosummary` table of this documenter.

	:param relative_ref_paths: Use paths relative to the current module
		instead of absolute import paths for each object.
	"""

	if not self.options.get("autosummary", False):
		return

	content = StringList()
	content.indent_type = ' ' * 4
	sourcename = self.get_sourcename()
	grouped_documenters = self.get_grouped_documenters()

	if not self.options.get("autosummary-no-titles", False) and grouped_documenters:
		content.blankline()
		content.append(".. latex:vspace:: 10px")
		content.blankline()

	member_order = get_first_matching(
			lambda x: x != "groupwise",
			(
					self.options.get("member-order", ''),
					self.env.config.autodocsumm_member_order,
					self.env.config.autodoc_member_order,
					),
			default="alphabetical",
			)

	for section, documenters in grouped_documenters.items():
		if not self.options.get("autosummary-no-titles", False):
			content.append(f"**{section}:**")
			content.blankline()
			content.append(".. latex:vspace:: -5px")

		content.blankline(ensure_single=True)

		# TODO transform to make caption associated with table in LaTeX

		content.append(".. autosummary::")

		if self.options.autosummary_nosignatures:
			content.append("    :nosignatures:")

		content.blankline(ensure_single=True)

		with content.with_indent_size(content.indent_size + 1):
			for documenter, _ in self.sort_members(documenters, member_order):
				obj_ref_path = documenter.fullname

				# if relative_ref_paths:
				# 	modname = self.modname + '.'
				# 	if documenter.fullname.startswith(modname):
				# 		obj_ref_path = documenter.fullname[len(modname):]

				content.append(f"~{obj_ref_path}")

		content.blankline()

	for line in content:
		self.add_line(line, sourcename)


class PatchedAutosummary(Autosummary):
	"""
	Pretty table containing short signatures and summaries of functions etc.

	Patched version of :class:`sphinx.ext.autosummary.Autosummary` to fix an issue where
	the module name is sometimes duplicated.

	I.e. ``foo.bar.baz()`` became ``foo.bar.foo.bar.baz()``, which of course doesn't exist
	and created a broken link.

	.. versionadded:: 0.5.1
	.. versionchanged:: 0.7.0  Moved from :mod:`sphinx_toolbox.patched_autosummary`.

	.. versionchanged:: 2.13.0

		Added support for customising the column type with the :confval:`autosummary_col_type` option.
	"""

	def import_by_name(self, name: str, prefixes: List[Optional[str]]) -> Tuple[str, Any, Any, str]:
		"""
		Import the object with the give name.

		:param name:
		:param prefixes:

		:return: The real name of the object, the object, the parent of the object, and the name of the module.
		"""

		real_name, obj, parent, modname = super().import_by_name(name=name, prefixes=prefixes)
		real_name = re.sub(rf"((?:{modname}\.)+)", f"{modname}.", real_name)
		return real_name, obj, parent, modname

	def create_documenter(
			self,
			app: Sphinx,
			obj: Any,
			parent: Any,
			full_name: str,
			) -> Documenter:
		"""
		Get an :class:`autodoc.Documenter` class suitable for documenting the given object.

		:param app: The Sphinx application.
		:param obj: The object being documented.
		:param parent: The parent of the object (e.g. a module or a class).
		:param full_name: The full name of the object.

		.. versionchanged:: 1.3.0

			Now selects the appropriate documenter for attributes rather than
			falling back to :class:`~sphinx.ext.autodoc.DataDocumenter`.
		"""

		doccls = get_documenter(app, obj, parent)
		return doccls(self.bridge, full_name)

	def get_table(self, items: List[Tuple[str, str, str, str]]) -> List[nodes.Node]:
		"""
		Generate a list of table nodes for the :rst:dir:`autosummary` directive.

		:param items: A list produced by ``self.get_items``.

		:rtype:

		.. latex:clearpage::
		"""

		table_spec, table, *other_nodes = super().get_table(items)
		assert isinstance(table_spec, addnodes.tabular_col_spec)
		assert isinstance(table, autosummary_table)

		if docutils.__version_info__ >= (0, 18):
			table.children[0]["classes"] += ["colwidths-given"]  # type: ignore[index]

		column_type = getattr(self.env.config, "autosummary_col_type", r"\X")
		table_spec["spec"] = f'{column_type}{{1}}{{2}}{column_type}{{1}}{{2}}'

		return [table_spec, table, *other_nodes]


def get_documenter(app: Sphinx, obj: Any, parent: Any) -> Type[Documenter]:
	"""
	Returns an :class:`autodoc.Documenter` class suitable for documenting the given object.

	.. versionadded:: 1.3.0

	:param app: The Sphinx application.
	:param obj: The object being documented.
	:param parent: The parent of the object (e.g. a module or a class).
	"""

	if inspect.ismodule(obj):
		# ModuleDocumenter.can_document_member always returns False
		return ModuleDocumenter

	# Construct a fake documenter for *parent*
	if parent is not None:
		parent_doc_cls = get_documenter(app, parent, None)
	else:
		parent_doc_cls = ModuleDocumenter

	if hasattr(parent, "__name__"):
		parent_doc = parent_doc_cls(FakeDirective(), parent.__name__)
	else:
		parent_doc = parent_doc_cls(FakeDirective(), '')

	# Get the correct documenter class for *obj*
	classes = [
			cls for cls in app.registry.documenters.values()
			if cls.can_document_member(obj, '', False, parent_doc)
			]

	data_doc_classes = [
			cls for cls in app.registry.documenters.values() if cls.can_document_member(obj, '', True, parent_doc)
			]

	if classes:
		classes.sort(key=lambda cls: cls.priority)
		return classes[-1]
	elif data_doc_classes:
		data_doc_classes.sort(key=lambda cls: cls.priority)
		return data_doc_classes[-1]
	else:
		return DataDocumenter


class PatchedAutoSummModuleDocumenter(autodocsumm.AutoSummModuleDocumenter):
	"""
	Patched version of :class:`autodocsumm.AutoSummClassDocumenter`
	which works around a bug in Sphinx 3.4 and above where ``__all__`` is not respected.

	.. versionadded:: 2.13.0
	"""  # noqa: D400

	def filter_members(self, members: ObjectMembers, want_all: bool) -> List[Tuple[str, Any, bool]]:
		"""
		Filter the given member list.

		Members are skipped if:

		* they are private (except if given explicitly or the ``private-members`` option is set)
		* they are special methods (except if given explicitly or the ``special-members`` option is set)
		* they are undocumented (except if the ``undoc-members`` option is set)

		The user can override the skipping decision by connecting to the :event:`autodoc-skip-member` event.
		"""

		def is_filtered_inherited_member(name: str) -> bool:
			if inspect.isclass(self.object):
				for cls in self.object.__mro__:
					if cls.__name__ == self.options.inherited_members and cls != self.object:
						# given member is a member of specified *super class*
						return True
					elif name in cls.__dict__:
						return False
					elif name in self.get_attr(cls, "__annotations__", {}):
						return False

			return False

		ret = []

		# search for members in source code too
		namespace = '.'.join(self.objpath)  # will be empty for modules

		if self.analyzer:
			attr_docs = self.analyzer.find_attr_docs()
		else:
			attr_docs = {}

		doc: Optional[str]
		sphinx_gt_41 = sphinx.version_info > (4, 1)

		# process members and determine which to skip
		for (membername, member) in members:
			# if isattr is True, the member is documented as an attribute
			isattr = (member is INSTANCEATTR or (namespace, membername) in attr_docs)

			doc = getdoc(
					member,
					self.get_attr,
					self.env.config.autodoc_inherit_docstrings,
					self.parent,
					self.object_name
					)
			if not isinstance(doc, str):
				# Ignore non-string __doc__
				doc = None

			# if the member __doc__ is the same as self's __doc__, it's just
			# inherited and therefore not the member's doc
			cls = self.get_attr(member, "__class__", None)
			if cls:
				cls_doc = self.get_attr(cls, "__doc__", None)
				if cls_doc == doc:
					doc = None

			if sphinx_gt_41:
				doc, metadata = separate_metadata(doc)  # type: ignore[arg-type]
			else:
				metadata = extract_metadata(doc)  # type: ignore[arg-type]

			has_doc = bool(doc)

			if "private" in metadata:
				# consider a member private if docstring has "private" metadata
				isprivate = True
			elif "public" in metadata:
				# consider a member public if docstring has "public" metadata
				isprivate = False
			else:
				isprivate = membername.startswith('_')

			keep = False
			if safe_getattr(member, "__sphinx_mock__", False):
				# mocked module or object
				pass
			elif self.options.exclude_members and membername in self.options.exclude_members:
				# remove members given by exclude-members
				keep = False
			elif want_all and special_member_re.match(membername):
				# special __methods__
				if self.options.special_members and membername in self.options.special_members:
					if membername == "__doc__":
						keep = False
					elif is_filtered_inherited_member(membername):
						keep = False
					else:
						keep = has_doc or self.options.undoc_members
				else:
					keep = False
			elif (namespace, membername) in attr_docs:
				if want_all and isprivate:
					if self.options.private_members is None:
						keep = False
					else:
						keep = membername in self.options.private_members
				else:
					# keep documented attributes
					keep = True
				isattr = True
			elif want_all and isprivate:
				if has_doc or self.options.undoc_members:
					if self.options.private_members is None:
						keep = False
					elif is_filtered_inherited_member(membername):
						keep = False
					else:
						keep = membername in self.options.private_members
				else:
					keep = False
			else:
				if self.options.members is ALL and is_filtered_inherited_member(membername):
					keep = False
				else:
					# ignore undocumented members if :undoc-members: is not given
					keep = has_doc or self.options.undoc_members

			# give the user a chance to decide whether this member
			# should be skipped
			if self.env.app:
				# let extensions preprocess docstrings
				try:  # pylint: disable=R8203
					skip_user = self.env.app.emit_firstresult(
							"autodoc-skip-member",
							self.objtype,
							membername,
							member,
							not keep,
							self.options,
							)
					if skip_user is not None:
						keep = not skip_user
				except Exception as exc:
					msg = 'autodoc: failed to determine %r to be documented, the following exception was raised:\n%s'
					logger.warning(__(msg), member, exc, type="autodoc")
					keep = False

			if keep:
				ret.append((membername, member, isattr))

		return ret

	def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]:
		"""
		Return a tuple of  ``(members_check_module, members)``,
		where ``members`` is a list of ``(membername, member)`` pairs of the members of ``self.object``.

		If ``want_all`` is :py:obj:`True`, return all members.
		Otherwise, only return those members given by ``self.options.members`` (which may also be none).
		"""  # noqa: D400

		if want_all:
			if self.__all__:
				memberlist = self.__all__
			else:
				# for implicit module members, check __module__ to avoid
				# documenting imported objects
				return True, _get_module_members(self.object)
		else:
			memberlist = self.options.members or []
		ret = []
		for mname in memberlist:
			try:  # pylint: disable=R8203
				ret.append((mname, safe_getattr(self.object, mname)))
			except AttributeError:
				# pylint: disable=dotted-import-in-loop)
				logger.warning(
						operator.mod(
								__("missing attribute mentioned in :members: or __all__: module %s, attribute %s"),
								(safe_getattr(self.object, "__name__", "???"), mname),
								),
						type="autodoc"
						)
				# pylint: enable=dotted-import-in-loop)
		return False, ret


class PatchedAutoSummClassDocumenter(autodocsumm.AutoSummClassDocumenter):
	"""
	Patched version of :class:`autodocsumm.AutoSummClassDocumenter`
	which doesn't show summary tables for aliased objects.

	.. versionadded:: 0.9.0
	"""  # noqa: D400

	def add_content(self, *args, **kwargs) -> None:
		"""
		Add content from docstrings, attribute documentation and user.
		"""

		ClassDocumenter.add_content(self, *args, **kwargs)

		if not self.doc_as_attr:
			self.add_autosummary()


class PatchedAutoDocSummDirective(autodocsumm.AutoDocSummDirective):
	"""
	Patched ``AutoDocSummDirective`` which uses :py:obj:`None` for the members option rather than an empty string.

	.. attention:: This class is not part of the public API.
	"""

	def run(self) -> List[nodes.Node]:
		reporter = self.state.document.reporter

		if hasattr(reporter, "get_source_and_line"):
			_, lineno = reporter.get_source_and_line(self.lineno)
		else:
			_, lineno = (None, None)

		# look up target Documenter
		objtype = self.name[4:-4]  # strip prefix (auto-) and suffix (-summ).
		doccls = self.env.app.registry.documenters[objtype]

		self.options["autosummary-force-inline"] = "True"
		self.options["autosummary"] = "True"
		if "no-members" not in self.options:
			self.options["members"] = None

		# process the options with the selected documenter's option_spec
		try:
			documenter_options = process_documenter_options(doccls, self.config, self.options)
		except (KeyError, ValueError, TypeError) as exc:
			# an option is either unknown or has a wrong type
			logger.error(
					"An option to %s is either unknown or has an invalid value: %s",
					self.name,
					exc,
					location=(self.env.docname, lineno),
					)
			return []

		# generate the output
		params = DocumenterBridge(self.env, reporter, documenter_options, lineno, self.state)
		documenter = doccls(params, self.arguments[0])
		documenter.add_autosummary()

		node = nodes.paragraph()
		node.document = self.state.document
		self.state.nested_parse(params.result, 0, node)

		return node.children


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autosummary`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx.ext.autosummary")
	app.setup_extension("autodocsumm")
	app.setup_extension("sphinx_toolbox.latex")

	app.add_directive("autosummary", PatchedAutosummary, override=True)
	app.add_directive("autoclasssumm", PatchedAutoDocSummDirective, override=True)
	app.add_directive("automodulesumm", PatchedAutoDocSummDirective, override=True)

	autodocsumm.AutosummaryDocumenter.add_autosummary = add_autosummary
	allow_subclass_add(app, PatchedAutoSummModuleDocumenter)
	allow_subclass_add(app, PatchedAutoSummClassDocumenter)

	app.add_config_value(
			"autodocsumm_member_order",
			default="alphabetical",
			rebuild=True,
			types=ENUM("alphabetic", "alphabetical", "bysource"),
			)

	app.add_config_value(
			"autosummary_col_type",
			default=r"\X",
			rebuild="latex",
			types=[str],
			)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autosummary/__init__.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  column_widths.py
"""
Sphinx extension to allow customisation of column widths in autosummary tables with the LaTeX builder.

.. versionadded:: 3.0.0


Usage
--------------

This extension provides the :rst:dir:`autosummary-widths` directive.
This sets the autosummary table's column widths with the LaTeX builder
until the end of the current reStructuredText document,
or until the next :rst:dir:`autosummary-widths` directive.

.. rst:directive:: autosummary-widths

	Set the width of the autosummary table's columns with the LaTeX builder.

	The directive takes up to two arguments -- the column widths as vulgar fractions (e.g. ``5/10``).
	If only one argument is provided, this sets the width of the first column,
	and the width of the second column is calculated from it.
	If both arguments are provided, they set the width of the first and second columns respectively.

	.. latex:clearpage::

	:bold-title:`Examples:`

	.. code-block:: rst

		.. autosummary-widths:: 5/10
		.. autosummary-widths:: 3/10, 7/10
		.. autosummary-widths:: 35/100

	.. attention:: This directive ignores the :confval:`autosummary_col_type` configuration option.


API Reference
----------------


"""
#  Copyright © 2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx/blob/3.x/sphinx/ext/autosummary/__init__.py
#  |  Copyright (c) 2007-2022 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from contextlib import suppress
from fractions import Fraction
from itertools import chain
from typing import Iterable, List, Tuple, cast

# 3rd party
import docutils
from docutils import nodes
from docutils.statemachine import StringList
from domdf_python_tools import stringlist
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.config import Config
from sphinx.ext.autosummary import autosummary_table
from sphinx.util import rst
from sphinx.util.docutils import SphinxDirective, switch_source_input

# this package
from sphinx_toolbox import latex
from sphinx_toolbox.more_autosummary import PatchedAutosummary
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("AutosummaryWidths", "WidthsDirective", "configure", "setup")


class AutosummaryWidths(PatchedAutosummary):
	"""
	Customised :rst:dir:`autosummary` directive with customisable width with the LaTeX builder.

	.. attention:: This directive ignores the :confval:`autosummary_col_type` configuration option.
	"""

	def get_table(self, items: List[Tuple[str, str, str, str]]) -> List[nodes.Node]:
		"""
		Generate a proper list of table nodes for autosummary:: directive.

		:param items: A list produced by ``self.get_items``.
		"""

		table_spec = addnodes.tabular_col_spec()

		widths = tuple(chain.from_iterable(getattr(self.state.document, "autosummary_widths", ((1, 2), (1, 2)))))
		assert len(widths) == 4

		table_spec["spec"] = r'\Xx{%d}{%d}\Xx{%d}{%d}' % widths

		table = autosummary_table('')
		classes = ["autosummary", "longtable"]

		if docutils.__version_info__ >= (0, 18):
			classes.append("colwidths-given")

		real_table = nodes.table('', classes=classes)
		table.append(real_table)

		group = nodes.tgroup('', cols=2)
		real_table.append(group)

		group.append(nodes.colspec('', colwidth=10))
		group.append(nodes.colspec('', colwidth=90))

		body = nodes.tbody('')
		group.append(body)

		def append_row(*column_texts: str) -> None:
			row = nodes.row('')
			source, line = self.state_machine.get_source_and_line()

			for text in column_texts:
				node = nodes.paragraph('')
				vl = StringList()
				vl.append(text, f"{source}:{line:d}:<autosummary>")  # pylint: disable=loop-invariant-statement

				with switch_source_input(self.state, vl):
					self.state.nested_parse(vl, 0, node)

					with suppress(IndexError):
						if isinstance(node[0], nodes.paragraph):
							node = node[0]

					row.append(nodes.entry('', node))

			body.append(row)

		add_signature = "nosignatures" not in self.options
		for name, sig, summary, real_name in items:
			col1 = f":obj:`{name} <{real_name}>`"

			if add_signature:
				col1 += f"\\ {rst.escape(sig)}"

			append_row(col1, summary)

		return [table_spec, table]


class WidthsDirective(SphinxDirective):
	"""
	Sphinx directive which configures the column widths of an :rst:dir:`autosummary` table
	for the remainder of the document, or until the next `autosummary-widths` directive.
	"""  # noqa: D400

	required_arguments = 1
	optional_arguments = 1

	@staticmethod
	def parse_widths(raw_widths: Iterable[str]) -> List[Tuple[int, int]]:
		"""
		Parse a width string (as a vulgar fraction) into a list of 2-element ``(numerator, denominator)`` tuples.

		For example, ``'5/10'`` becomes ``(5, 10)``.

		:param raw_widths:
		"""

		widths = [cast(Tuple[int, int], tuple(map(int, arg.split('/')))) for arg in raw_widths]

		if len(widths) == 1:
			left_width = Fraction(*widths[0])
			right_width = 1 - left_width
			widths.append((right_width.numerator, right_width.denominator))

		return widths

	def run(self) -> List:
		"""
		Process the directive's arguments.
		"""

		self.state.document.autosummary_widths = self.parse_widths(self.arguments)  # type: ignore[attr-defined]

		return []


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.more_autosummary.column_widths`.

	:param app: The Sphinx application.
	:param config:
	"""

	latex_elements = getattr(config, "latex_elements", {})

	latex_preamble = stringlist.StringList(latex_elements.get("preamble", ''))
	latex_preamble.blankline()
	latex_preamble.append(r"\makeatletter")
	latex_preamble.append(r"\newcolumntype{\Xx}[2]{>{\raggedright\arraybackslash}p{\dimexpr")
	latex_preamble.append(r"    (\linewidth-\arrayrulewidth)*#1/#2-\tw@\tabcolsep-\arrayrulewidth\relax}}")
	latex_preamble.append(r"\makeatother")
	latex_preamble.blankline()

	latex_elements["preamble"] = str(latex_preamble)
	config.latex_elements = latex_elements  # type: ignore[attr-defined]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.more_autosummary.column_widths`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx_toolbox.more_autosummary")

	app.add_directive("autosummary", AutosummaryWidths, override=True)
	app.add_directive("autosummary-widths", WidthsDirective)

	app.connect("config-inited", configure)
	app.connect("build-finished", latex.replace_unknown_unicode)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/more_autosummary/column_widths.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  pre_commit.py
"""
Sphinx extension to show examples of ``.pre-commit-config.yaml`` configuration.

.. versionadded:: 1.6.0
.. extensions:: sphinx_toolbox.pre_commit


Usage
------

.. rst:directive:: pre-commit

	Directive which shows an example snippet of ``.pre-commit-config.yaml``.

	.. rst:directive:option:: rev
		:type: string

		The revision or tag to clone at.

	.. rst:directive:option:: hooks
		:type: comma separated list

		A list of hooks IDs to document.

		If not given the hooks will be parsed from ``.pre-commit-hooks.yaml``.

	.. rst:directive:option:: args
		:type: comma separated list

		A list arguments that should or can be provided to the first hook ID.

		.. versionadded:: 1.7.2


	:bold-title:`Example`

	.. rest-example::

		.. pre-commit::
			:rev: v0.0.4
			:hooks: some-hook,some-other-hook

.. clearpage::

.. rst:directive:: .. pre-commit:flake8:: version

	Directive which shows an example snippet of ``.pre-commit-config.yaml`` for a flake8 plugin.

	The directive takes a single argument -- the version of the flake8 plugin to install from PyPI.

	.. rst:directive:option:: flake8-version
		:type: string

		The version of flake8 to use. Default ``3.8.4``.

	.. rst:directive:option:: plugin-name
		:type: string

		The name of the plugin to install from PyPI. Defaults to the repository name.

	:bold-title:`Example`

	.. rest-example::

		.. pre-commit:flake8:: 0.0.4

	.. versionchanged:: 2.8.0  The repository URL now points to GitHub.

API Reference
----------------
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import re
import warnings
from io import StringIO
from textwrap import indent
from typing import Any, List, Sequence

# 3rd party
import sphinx.util.docutils
from docutils import nodes
from docutils.statemachine import StringList
from domdf_python_tools.paths import PathPlus
from ruamel.yaml import YAML
from sphinx.application import Sphinx
from sphinx.util.docutils import SphinxDirective
from typing_extensions import TypedDict

# this package
from sphinx_toolbox.utils import Purger, SphinxExtMetadata, make_github_url, metadata_add_version

__all__ = (
		"pre_commit_node_purger",
		"pre_commit_f8_node_purger",
		"parse_hooks",
		"PreCommitDirective",
		"Flake8PreCommitDirective",
		"setup",
		)

pre_commit_node_purger = Purger("all_pre_commit_nodes")
pre_commit_f8_node_purger = Purger("all_pre_commit_f8_nodes")


def parse_hooks(hooks: str) -> List[str]:
	"""
	Parses the comma, semicolon and/or space delimited list of hook IDs.

	:param hooks:
	"""

	return list(filter(None, re.split("[,; ]", hooks)))


class _BaseHook(TypedDict):
	id: str  # noqa: A003  # pylint: disable=redefined-builtin


class _Hook(_BaseHook, total=False):
	args: List[str]


class _BaseConfig(TypedDict):
	repo: str


class _Config(_BaseConfig, total=False):
	rev: str
	hooks: List[_Hook]


class PreCommitDirective(SphinxDirective):
	"""
	A Sphinx directive for documenting pre-commit hooks.

	.. clearpage::
	"""

	has_content: bool = False
	option_spec = {
			"rev": str,  # the revision or tag to clone at.
			"hooks": parse_hooks,
			"args": parse_hooks,
			}

	def run(self) -> Sequence[nodes.Node]:  # type: ignore[override]
		"""
		Process the content of the directive.
		"""

		if "hooks" in self.options:
			hooks = self.options["hooks"]
		else:
			cwd = PathPlus.cwd()

			for directory in (cwd, *cwd.parents):
				hook_file = directory / ".pre-commit-hooks.yaml"
				if hook_file.is_file():
					hooks_dict = YAML(typ="safe", pure=True).load(hook_file.read_text())
					hooks = [h["id"] for h in hooks_dict]  # pylint: disable=loop-invariant-statement
					break
			else:
				warnings.warn("No hooks specified and no .pre-commit-hooks.yaml file found.")
				return []

		repo = make_github_url(self.env.config.github_username, self.env.config.github_repository)
		config: _Config = {"repo": str(repo)}

		if "rev" in self.options:
			config["rev"] = self.options["rev"]

		config["hooks"] = [{"id": hook_name} for hook_name in hooks]

		if "args" in self.options:
			config["hooks"][0]["args"] = self.options["args"]

		targetid = f'pre-commit-{self.env.new_serialno("pre-commit"):d}'
		targetnode = nodes.section(ids=[targetid])

		yaml_dumper = YAML()
		yaml_dumper.default_flow_style = False

		yaml_output_stream = StringIO()
		yaml_dumper.dump([config], stream=yaml_output_stream)

		yaml_output = yaml_output_stream.getvalue()

		if not yaml_output:
			return []

		content = f".. code-block:: yaml\n\n{indent(yaml_output, '    ')}\n\n"
		view = StringList(content.split('\n'))
		pre_commit_node = nodes.paragraph(rawsource=content)
		self.state.nested_parse(view, self.content_offset, pre_commit_node)

		pre_commit_node_purger.add_node(self.env, pre_commit_node, targetnode, self.lineno)

		return [pre_commit_node]


class Flake8PreCommitDirective(SphinxDirective):
	"""
	A Sphinx directive for documenting flake8 plugins' pre-commit hooks.
	"""

	has_content: bool = False
	option_spec = {
			"flake8-version": str,
			"plugin-name": str,  # defaults to repository name
			}
	required_arguments = 1  # the plugin version

	def run(self) -> Sequence[nodes.Node]:  # type: ignore[override]
		"""
		Process the content of the directive.
		"""

		plugin_name = self.options.get("plugin-name", self.env.config.github_repository)
		flake8_version = self.options.get("flake8-version", "3.8.4")

		config = {
				"repo": "https://github.com/pycqa/flake8",
				"rev": flake8_version,
				"hooks": [{"id": "flake8", "additional_dependencies": [f"{plugin_name}=={self.arguments[0]}"]}]
				}

		targetid = f'pre-commit-{self.env.new_serialno("pre-commit"):d}'
		targetnode = nodes.section(ids=[targetid])

		yaml_dumper = YAML()
		yaml_dumper.default_flow_style = False

		yaml_output_stream = StringIO()
		yaml_dumper.dump([config], stream=yaml_output_stream)

		yaml_output = yaml_output_stream.getvalue()

		if not yaml_output:
			return []

		content = f".. code-block:: yaml\n\n{indent(yaml_output, '    ')}\n\n"
		view = StringList(content.split('\n'))
		pre_commit_node = nodes.paragraph(rawsource=content)
		self.state.nested_parse(view, self.content_offset, pre_commit_node)

		pre_commit_f8_node_purger.add_node(self.env, pre_commit_node, targetnode, self.lineno)

		return [pre_commit_node]


def revert_8345() -> None:  # pragma: no cover  # Only for Sphinx 4+
	"""
	Remove the incorrect warning emitted since https://github.com/sphinx-doc/sphinx/pull/8345.
	"""

	#  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
	#  BSD Licensed
	#  All rights reserved.
	#
	#  Redistribution and use in source and binary forms, with or without
	#  modification, are permitted provided that the following conditions are
	#  met:
	#
	#  * Redistributions of source code must retain the above copyright
	#   notice, this list of conditions and the following disclaimer.
	#
	#  * Redistributions in binary form must reproduce the above copyright
	#   notice, this list of conditions and the following disclaimer in the
	#   documentation and/or other materials provided with the distribution.
	#
	#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	#  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	def lookup_domain_element(self, type: str, name: str) -> Any:  # noqa: A002  # pylint: disable=redefined-builtin
		"""
		Lookup a markup element (directive or role), given its name which can be a full name (with domain).
		"""

		name = name.lower()
		# explicit domain given?
		if ':' in name:
			domain_name, name = name.split(':', 1)
			if domain_name in self.env.domains:
				domain = self.env.get_domain(domain_name)
				element = getattr(domain, type)(name)
				if element is not None:
					return element, []
		# else look in the default domain
		else:
			def_domain = self.env.temp_data.get("default_domain")
			if def_domain is not None:
				element = getattr(def_domain, type)(name)
				if element is not None:
					return element, []

		# always look in the std domain
		element = getattr(self.env.get_domain("std"), type)(name)
		if element is not None:
			return element, []

		raise sphinx.util.docutils.ElementLookupError

	sphinx.util.docutils.sphinx_domains.lookup_domain_element = lookup_domain_element  # type: ignore[assignment]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.pre_commit`.

	:param app: The Sphinx application.
	"""

	app.add_directive("pre-commit", PreCommitDirective)
	app.add_directive("pre-commit:flake8", Flake8PreCommitDirective)
	app.connect("env-purge-doc", pre_commit_node_purger.purge_nodes)
	app.connect("env-purge-doc", pre_commit_f8_node_purger.purge_nodes)

	if sphinx.version_info >= (4, 0):
		revert_8345()

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/pre_commit.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  rest_example.py
"""
Directive to show example reStructuredText and the rendered output.

.. extensions:: sphinx_toolbox.rest_example


Usage
---------

.. rst:directive:: rest-example

	Directive to show example reStructuredText and the rendered output.

	.. rst:directive:option:: force
		:type: flag

		If given, minor errors on highlighting are ignored.

	.. rst:directive:option:: emphasize-lines: line numbers
		:type: comma separated numbers

		Emphasize particular lines of the code block:

	.. rst:directive:option:: tab-width: number
		:type: number

		Sets the size of the indentation in spaces.

	.. rst:directive:option:: dedent: number
		:type: number

		Strip indentation characters from the code block,

	.. latex:clearpage::

	:bold-title:`Example`

	.. rest-example::

		.. rest-example::

			:source:`sphinx_toolbox/config.py`

			Here is the :source:`source code <sphinx_toolbox/config.py>`


API Reference
---------------

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import Any, Dict, List, Sequence

# 3rd party
import sphinx.environment
from docutils import nodes
from docutils.parsers.rst import directives
from docutils.statemachine import ViewList
from domdf_python_tools.stringlist import StringList
from sphinx.application import Sphinx
from sphinx.util.docutils import SphinxDirective

# this package
from sphinx_toolbox.utils import OptionSpec, Purger, SphinxExtMetadata, metadata_add_version

__all__ = ("reSTExampleDirective", "make_rest_example", "rest_example_purger", "setup")


class reSTExampleDirective(SphinxDirective):
	"""
	Directive to show some reStructuredText source, and the rendered output.
	"""

	has_content: bool = True

	# Options to pass through to .. code-block::
	option_spec: OptionSpec = {  # type: ignore[assignment]
		"force": directives.flag,
		"emphasize-lines": directives.unchanged,
		"tab-width": int,
		"dedent": int,
		}

	def run(self) -> List[nodes.Node]:
		"""
		Create the rest_example node.
		"""

		targetid = f'example-{self.env.new_serialno("sphinx-toolbox rest_example"):d}'
		targetnode = nodes.target('', '', ids=[targetid])

		content = make_rest_example(
				self.options,
				self.env,
				self.content,  # type: ignore[arg-type]
				)
		view = ViewList(content)

		example_node = nodes.paragraph(rawsource=content)  # type: ignore[arg-type]
		self.state.nested_parse(view, self.content_offset, example_node)  # type: ignore[arg-type]

		rest_example_purger.add_node(self.env, example_node, targetnode, self.lineno)

		return [targetnode, example_node]


def make_rest_example(
		options: Dict[str, Any],
		env: sphinx.environment.BuildEnvironment,
		content: Sequence[str],
		) -> List[str]:
	"""
	Make the content of a reST Example node.

	:param options:
	:param content: The user-provided content of the directive.
	"""

	output = StringList(".. container:: rest-example")
	output.indent_type = ' ' * env.config.docutils_tab_width

	output.blankline()

	with output.with_indent_size(1):

		output.append(".. code-block:: rest")

		with output.with_indent_size(2):
			for option, value in options.items():
				if value is None:
					output.append(f":{option}:")
				else:
					output.append(f":{option}: {value}")

			output.blankline()
			output.extend(content)

		output.blankline(ensure_single=True)
		output.extend(content)
		output.blankline(ensure_single=True)

	return list(output)


#: Purger to track rest-example nodes, and remove redundant ones.
rest_example_purger = Purger("all_rest_example_nodes")


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.rest_example`.

	.. versionadded:: 0.7.0

	:param app: The Sphinx application.
	"""

	# Hack to get the docutils tab size, as there doesn't appear to be any other way
	app.setup_extension("sphinx_toolbox.tweaks.tabsize")
	app.setup_extension("sphinx_toolbox._css")

	app.add_directive("rest-example", reSTExampleDirective)
	app.connect("env-purge-doc", rest_example_purger.purge_nodes)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/rest_example.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  shields.py
r"""
Directives for shield/badge images.

.. extensions:: sphinx_toolbox.shields

Usage
---------

Several shield/badge directives are available.

They function similarly to the ``.. image::`` directives, although not all options are available.
As with the image directive, shields can be used as part of substitutions, e.g.

.. code-block:: rest

	This repository uses pre-commit |pre-commit|

	.. |pre-commit| pre-commit::


All shields have the following options:

.. rst:directive:option:: alt

	Alternative text for the shield, used when the image cannot be displayed or the user uses a screen reader.

.. rst:directive:option:: height
						  width
						  scale

	The height/width/scale of the shield.

.. rst:directive:option:: name

.. rst:directive:option:: class
	:type: string

	Additional CSS class for the shield.
	All shields have the ``sphinx_toolbox_shield`` class by default.


.. latex:clearpage::

Shields
^^^^^^^^^^^^

.. rst:directive:: rtfd-shield

	Shield to show the `ReadTheDocs <https://readthedocs.org/>`_ documentation build status.

	.. rst:directive:option:: project

		The name of the project on *ReadTheDocs*.

	.. rst:directive:option:: version

		The documentation version. Default ``latest``.

	.. rst:directive:option:: target

		The hyperlink target of the shield. Useful if the documentation uses a custom domain.

		.. versionadded:: 1.8.0


	.. only:: html

		**Examples**

		.. rest-example::

			.. rtfd-shield::
				:project: sphinx-toolbox

		.. rest-example::

			.. rtfd-shield::
				:project: attrs
				:target: https://www.attrs.org/


.. rst:directive:: pypi-shield

	Shield to show information about the project on `PyPI <https://pypi.org/>`_.

	.. rst:directive:option:: project

		The name of the project on *PyPI*.

	Only one of the following options is permitted:

	.. rst:directive:option:: version

		Show the package version.

	.. rst:directive:option:: py-versions

		Show the supported python versions.

	.. rst:directive:option:: implementations

		Show the supported python implementations.

	.. rst:directive:option:: wheel

		Show whether the package has a wheel.

	.. rst:directive:option:: license

		Show the license listed on PyPI.

	.. rst:directive:option:: downloads

		Show the downloads for the given period (day / week / month)

		.. versionchanged:: 2.5.0  Shields created with this option now link to pypistats_.

		.. _pypistats: https://pypistats.org


	.. only:: html

		**Examples**

		.. rest-example::

			.. pypi-shield::
				:version:

			\

			.. pypi-shield::
				:project: sphinx
				:downloads: month


.. rst:directive:: maintained-shield

	Shield to indicate whether the project is maintained.

	Takes a single argument: the current year.


	.. only:: html

		**Example**

		.. rest-example::

			.. maintained-shield:: 2020


.. rst:directive:: github-shield

	Shield to show information about a GitHub repository.

	.. rst:directive:option:: username

		The GitHub username. Defaults to :confval:`github_username`.

	.. rst:directive:option:: repository

		The GitHub repository. Defaults to :confval:`github_repository`.

	.. rst:directive:option:: branch

		The branch to show information about. Default ``master``.

		Required for ``commits-since`` and ``last-commit``.

	Only one of the following options is permitted:

	.. rst:directive:option:: contributors
		:type: flag

		Show the number of contributors.

	.. rst:directive:option:: commits-since: tag
		:type: string

		Show the number of commits since the given tag.

	.. rst:directive:option:: last-commit
		:type: flag

		Show the date of the last commit.

	.. rst:directive:option:: top-language
		:type: flag

		Show the top language and percentage.

	.. rst:directive:option:: license
		:type: flag

		Show the license detected by GitHub.


	.. only:: html

		**Examples**

		.. rest-example::

			.. github-shield::
				:last-commit:

			\

			.. github-shield::
				:commits-since: v0.1.0


.. rst:directive:: actions-shield

	Shield to show the *GitHub Actions* build status.

	.. rst:directive:option:: username

		The GitHub username. Defaults to :confval:`github_username`.

	.. rst:directive:option:: repository

		The GitHub repository. Defaults to :confval:`github_repository`.

	.. rst:directive:option:: workflow

		The workflow to show the status for.

	.. only:: html

		**Example**

		.. rest-example::

			.. actions-shield::
				:workflow: Windows Tests


.. rst:directive:: requires-io-shield

	Shield to show the *Requires.io* status.

	.. rst:directive:option:: username

		The GitHub username. Defaults to :confval:`github_username`.

	.. rst:directive:option:: repository

		The GitHub repository. Defaults to :confval:`github_repository`.

	.. rst:directive:option:: branch

		The branch to show the build status for. Default ``master``.


	.. only:: html

		**Example**

		.. rest-example::

			.. requires-io-shield::


.. rst:directive:: coveralls-shield

	Shield to show the code coverage from `Coveralls.io <https://coveralls.io/>`_.

	.. rst:directive:option:: username

		The GitHub username. Defaults to :confval:`github_username`.

	.. rst:directive:option:: repository

		The GitHub repository. Defaults to :confval:`github_repository`.

	.. rst:directive:option:: branch

		The branch to show the build status for. Default ``master``.


	.. only:: html

		**Example**

		.. rest-example::

			.. coveralls-shield::


.. rst:directive:: codefactor-shield

	Shield to show the code quality score from `Codefactor <https://www.codefactor.io>`_.

	.. rst:directive:option:: username

		The GitHub username. Defaults to :confval:`github_username`.

	.. rst:directive:option:: repository

		The GitHub repository. Defaults to :confval:`github_repository`.


	.. only:: html

		**Example**

		.. rest-example::

			.. codefactor-shield::


.. rst:directive:: pre-commit-shield

	Shield to indicate that the project uses `pre-commit <https://pre-commit.com/>`_.


	.. only:: html

		**Example**

		.. rest-example::

			.. pre-commit-shield::


.. rst:directive:: pre-commit-ci-shield

	.. versionadded:: 1.7.0

	Shield to show the `pre-commit.ci <https://pre-commit.ci/>`_ status.

	.. rst:directive:option:: username

		The GitHub username. Defaults to :confval:`github_username`.

	.. rst:directive:option:: repository

		The GitHub repository. Defaults to :confval:`github_repository`.

	.. rst:directive:option:: branch

		The branch to show the status for. Default ``master``.


	.. only:: html

		**Example**

		.. rest-example::

			.. pre-commit-ci-shield::

.. latex:vspace:: 5mm

"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on public domain code from Docutils
#

# stdlib
from typing import List, Optional, Tuple
from urllib.parse import quote

# 3rd party
import dict2css
import docutils
from apeye.url import URL
from docutils import nodes
from docutils.nodes import fully_normalize_name, whitespace_normalize_name
from docutils.parsers.rst import directives
from docutils.parsers.rst.roles import set_classes
from domdf_python_tools.paths import PathPlus
from sphinx.application import Sphinx
from sphinx.util.docutils import SphinxDirective

# this package
from sphinx_toolbox import _css
from sphinx_toolbox.utils import OptionSpec, SphinxExtMetadata, flag, make_github_url, metadata_add_version

__all__ = (
		"SHIELDS_IO",
		"shield_default_option_spec",
		"Shield",
		"RTFDShield",
		"PyPIShield",
		"MaintainedShield",
		"GitHubBackedShield",
		"GitHubShield",
		"GitHubActionsShield",
		"RequiresIOShield",
		"CoverallsShield",
		"CodefactorShield",
		"PreCommitShield",
		"PreCommitCIShield",
		"copy_asset_files",
		"setup",
		)

#: Base URL for shields.io
SHIELDS_IO = URL("https://img.shields.io")

#: Options common to all shields.
shield_default_option_spec: OptionSpec = {
		"alt": directives.unchanged,
		"height": directives.length_or_unitless,
		"width": directives.length_or_percentage_or_unitless,
		"scale": directives.percentage,
		"name": directives.unchanged,
		"class": directives.class_option,
		}


class Shield(SphinxDirective):
	"""
	Directive for `shields.io <https://shields.io>`_ shields/badges.
	"""

	required_arguments = 1
	optional_arguments = 0
	final_argument_whitespace = True
	option_spec: OptionSpec = {  # type: ignore[assignment]
		"target": directives.unchanged_required,
		**shield_default_option_spec,
		}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		if "class" in self.options:
			self.options["class"].append("sphinx_toolbox_shield")
		else:
			self.options["class"] = ["sphinx_toolbox_shield"]

		self.arguments = [str(x) for x in self.arguments]

		messages = []
		reference = directives.uri(self.arguments[0])
		self.options["uri"] = reference
		reference_node = None

		if "target" in self.options:
			block = docutils.utils.escape2null(self.options["target"]).splitlines()
			block = [line for line in block]

			target_type, data = self.state.parse_target(block, self.block_text, self.lineno)  # type: ignore[attr-defined]

			if target_type == "refuri":
				reference_node = nodes.reference(refuri=data)
			elif target_type == "refname":  # pragma: no cover
				reference_node = nodes.reference(
						refname=fully_normalize_name(data),
						name=whitespace_normalize_name(data),
						)
				reference_node.indirect_reference_name = data  # type: ignore[attr-defined]
				self.state.document.note_refname(reference_node)
			else:  # pragma: no cover
				# malformed target
				# data is a system message
				messages.append(data)

			del self.options["target"]

		set_classes(self.options)
		image_node = nodes.image(self.block_text, **self.options)
		self.add_name(image_node)

		if reference_node:
			reference_node += image_node
			return messages + [reference_node]
		else:
			return messages + [image_node]


class GitHubBackedShield(Shield):
	"""
	Base class for badges that are based around GitHub.
	"""

	required_arguments = 0
	option_spec: OptionSpec = {
			"username": str,  # Defaults to "github_username" if undefined
			"repository": str,  # Defaults to "github_repository" if undefined
			**shield_default_option_spec,
			}

	def get_repo_details(self) -> Tuple[str, str]:
		"""
		Returns the username and repository name, either parsed from the directive's options or from ``conf.py``.
		"""

		username = self.options.pop("username", self.env.config.github_username)
		if username is None:
			raise ValueError("'github_username' has not been set in 'conf.py'!")

		repository = self.options.pop("repository", self.env.config.github_repository)
		if repository is None:
			raise ValueError("'github_repository' has not been set in 'conf.py'!")

		return username, repository


class RTFDShield(Shield):
	"""
	Shield to show the `ReadTheDocs <https://readthedocs.org/>`_ documentation build status.

	.. versionchanged:: 1.8.0

		Added the ``:target:`` option, to allow a custom target to be specified.
		Useful if the documentation uses a custom domain.
	"""

	required_arguments = 0
	option_spec: OptionSpec = {
			"project": directives.unchanged_required,
			"version": str,
			"target": str,
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		project = self.options.pop("project")
		version = self.options.pop("version", "latest")

		image = SHIELDS_IO / "readthedocs" / project / f"{version}?logo=read-the-docs"
		self.arguments = [image]

		if "target" not in self.options:
			self.options["target"] = f"https://{project}.readthedocs.io/en/{version}/"

		return super().run()


class GitHubActionsShield(GitHubBackedShield):
	"""
	Shield to show the *GitHub Actions* build status.
	"""

	option_spec: OptionSpec = {
			"username": str,  # Defaults to "github_username" if undefined
			"repository": str,  # Defaults to "github_repository" if undefined
			"workflow": directives.unchanged_required,  # The name of the workflow
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		username, repository = self.get_repo_details()
		workflow = quote(self.options["workflow"])

		self.arguments = [str(make_github_url(username, repository) / "workflows" / workflow / "badge.svg")]
		self.options["target"] = str(
				make_github_url(username, repository) / f"actions?query=workflow%3A%22{workflow}%22"
				)

		return super().run()


class RequiresIOShield(GitHubBackedShield):
	"""
	Shield to show the `Requires.io <https://requires.io>`_ status.
	"""

	option_spec: OptionSpec = {
			"username": str,  # Defaults to "github_username" if undefined
			"repository": str,  # Defaults to "github_repository" if undefined
			"branch": str,
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		username, repository = self.get_repo_details()
		branch = self.options.pop("branch", "master")
		base_url = URL("https://requires.io/github/") / username / repository

		self.arguments = [str(base_url / f"requirements.svg?branch={branch}")]
		self.options["target"] = str(base_url / f"requirements/?branch={branch}")

		return super().run()


class CoverallsShield(GitHubBackedShield):
	"""
	Shield to show the code coverage from `Coveralls.io <https://coveralls.io/>`_.
	"""

	option_spec: OptionSpec = {
			"username": str,  # Defaults to "github_username" if undefined
			"repository": str,  # Defaults to "github_repository" if undefined
			"branch": str,
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		username, repository = self.get_repo_details()
		branch = self.options.pop("branch", "master")

		url = SHIELDS_IO / "coveralls" / "github" / username / repository / f"{branch}?logo=coveralls"
		self.arguments = [str(url)]
		self.options["target"] = f"https://coveralls.io/github/{username}/{repository}?branch={branch}"

		return super().run()


class CodefactorShield(GitHubBackedShield):
	"""
	Shield to show the code quality score from `Codefactor <https://www.codefactor.io>`_.
	"""

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		username, repository = self.get_repo_details()

		url = SHIELDS_IO / "codefactor" / "grade" / "github" / username / f"{repository}?logo=codefactor"
		self.arguments = [str(url)]
		self.options["target"] = f"https://codefactor.io/repository/github/{username}/{repository}"

		return super().run()


class PyPIShield(Shield):
	"""
	Shield to show information about the project on `PyPI <https://pypi.org/>`_.
	"""

	required_arguments = 0
	option_spec: OptionSpec = {
			"project": directives.unchanged_required,
			"version": flag,  # Show the package version.
			"py-versions": flag,  # Show the supported python versions.
			"implementations": flag,  # Show the supported python implementations.
			"wheel": flag,  # Show whether the package has a wheel on PyPI.
			"license": flag,  # Show the license listed on PyPI.
			"downloads": str,  # Show the downloads for the given period (day / week / month).
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		base_url = SHIELDS_IO / "pypi"

		project = self.options.pop("project", self.env.config.github_repository)

		self.options["target"] = f"https://pypi.org/project/{project}"

		info = {
				'v': self.options.pop("version", False),
				"py-versions": self.options.pop("py-versions", False),
				"implementation": self.options.pop("implementations", False),
				"wheel": self.options.pop("wheel", False),
				'l': self.options.pop("license", False),
				"downloads": self.options.pop("downloads", False),
				}

		n_info_options: int = len([k for k, v in info.items() if v])

		if n_info_options > 1:
			raise ValueError("Only one information option is allowed for the 'pypi-badge' directive.")
		elif n_info_options == 0:
			raise ValueError("An information option is required for the 'pypi-badge' directive.")

		for option in {'v', "implementation", "wheel", 'l'}:
			if info[option]:
				self.arguments = [base_url / option / project]
				break

		if info["py-versions"]:
			self.arguments = [str(base_url / "pyversions" / f"{project}?logo=python&logoColor=white")]

		elif info["downloads"]:
			if info["downloads"] in {"week", "dw"}:
				self.arguments = [base_url / "dw" / project]
			elif info["downloads"] in {"month", "dm"}:
				self.arguments = [base_url / "dm" / project]
			elif info["downloads"] in {"day", "dd"}:
				self.arguments = [base_url / "dd" / project]
			else:
				raise ValueError("Unknown time period for the PyPI download statistics.")

			self.options["target"] = f"https://pypistats.org/packages/{project.lower()}"

		return super().run()


class GitHubShield(GitHubBackedShield):
	"""
	Shield to show information about a GitHub repository.
	"""

	option_spec: OptionSpec = {
			"username": str,  # Defaults to "github_username" if undefined
			"repository": str,  # Defaults to "github_repository" if undefined
			"branch": str,
			"contributors": flag,  # Show the number of contributors.
			"commits-since": str,  # Show the number of commits since the given tag.
			"last-commit": flag,  # Show the date of the last commit.
			"top-language": flag,  # Show the top language and %
			"license": flag,
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		base_url = "https://img.shields.io/github"
		username, repository = self.get_repo_details()
		branch = self.options.pop("branch", "master")

		info = {
				"contributors": self.options.pop("contributors", False),
				"commits-since": self.options.pop("commits-since", False),
				"last-commit": self.options.pop("last-commit", False),
				"top-language": self.options.pop("top-language", False),
				"license": self.options.pop("license", False),
				}

		n_info_options: int = len([k for k, v in info.items() if v])

		if n_info_options > 1:
			raise ValueError("Only one information option is allowed for the 'github-badge' directive.")
		elif n_info_options == 0:
			raise ValueError("An information option is required for the 'github-badge' directive.")

		if info["contributors"]:
			self.arguments = [f"{base_url}/contributors/{username}/{repository}"]
			self.options["target"] = f"https://github.com/{username}/{repository}/graphs/contributors"

		elif info["commits-since"]:
			self.arguments = [f"{base_url}/commits-since/{username}/{repository}/{info['commits-since']}/{branch}"]
			self.options["target"] = f"https://github.com/{username}/{repository}/pulse"

		elif info["last-commit"]:
			self.arguments = [f"{base_url}/last-commit/{username}/{repository}/{branch}"]
			self.options["target"] = f"https://github.com/{username}/{repository}/commit/{branch}"

		elif info["top-language"]:
			self.arguments = [f"{base_url}/languages/top/{username}/{repository}"]

		elif info["license"]:
			self.arguments = [f"{base_url}/license/{username}/{repository}"]
			self.options["target"] = f"https://github.com/{username}/{repository}/blob/master/LICENSE"

		return super().run()


class MaintainedShield(Shield):
	"""
	Shield to indicate whether the project is maintained.
	"""

	required_arguments = 1  # The year
	option_spec = dict(shield_default_option_spec)

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		self.arguments = [f"https://img.shields.io/maintenance/yes/{self.arguments[0]}"]
		return super().run()


class PreCommitShield(Shield):
	"""
	Shield to indicate that the project uses `pre-commit <https://pre-commit.com/>`_.
	"""

	required_arguments = 0
	option_spec = dict(shield_default_option_spec)

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		self.arguments = [
				"https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white"
				]
		self.options["target"] = "https://github.com/pre-commit/pre-commit"
		return super().run()


RESULTS_PRE_COMMIT_CI = URL("https://results.pre-commit.ci")


class PreCommitCIShield(GitHubBackedShield):
	"""
	Shield to show the `pre-commit.ci <https://pre-commit.ci/>`_ status.

	.. versionadded:: 1.7.0
	"""

	option_spec: OptionSpec = {
			"username": str,  # Defaults to "github_username" if undefined
			"repository": str,  # Defaults to "github_repository" if undefined
			"branch": str,
			**shield_default_option_spec,
			}

	def run(self) -> List[nodes.Node]:
		"""
		Process the content of the shield directive.
		"""

		username, repository = self.get_repo_details()
		branch = self.options.pop("branch", "master")

		url = RESULTS_PRE_COMMIT_CI / "badge" / "github" / username / repository / f"{branch}.svg"
		self.arguments = [str(url)]
		self.options["target"] = str(RESULTS_PRE_COMMIT_CI / "latest" / "github" / username / repository / branch)

		return super().run()


def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None:
	"""
	Copy additional stylesheets into the HTML build directory.

	.. versionadded:: 2.3.1

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.format.lower() != "html":  # pragma: no cover
		return

	static_dir = PathPlus(app.outdir) / "_static"
	static_dir.maybe_make(parents=True)
	dict2css.dump(_css.shields_styles, static_dir / "toolbox-shields.css", minify=True)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.shields`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx_toolbox.github")
	app.setup_extension("sphinx_toolbox._css")

	# Shields/badges
	app.add_directive("rtfd-shield", RTFDShield)
	app.add_directive("actions-shield", GitHubActionsShield)
	app.add_directive("requires-io-shield", RequiresIOShield)
	app.add_directive("coveralls-shield", CoverallsShield)
	app.add_directive("codefactor-shield", CodefactorShield)
	app.add_directive("pypi-shield", PyPIShield)
	app.add_directive("github-shield", GitHubShield)
	app.add_directive("maintained-shield", MaintainedShield)
	app.add_directive("pre-commit-shield", PreCommitShield)
	app.add_directive("pre-commit-ci-shield", PreCommitCIShield)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/shields.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  sidebar_links.py
r"""
Directive which adds a toctree to the sidebar containing links to the GitHub repository, PyPI project page etc.

.. versionadded:: 2.9.0

.. extensions:: sphinx_toolbox.sidebar_links


.. latex:vspace:: -5px

Usage
-------

.. latex:vspace:: -10px

.. rst:directive:: sidebar-links

	Adds a toctree to the sidebar containing links to the GitHub repository, PyPI project page etc.
	The toctree is only shown in the sidebar and is hidden with non-HTML builders.

	.. only:: html

		You can see an example of this in the sidebar of this documentation.

	.. note:: This directive can only be used on the root document (i.e. index.rst).

	.. rst:directive:option:: github
		:type: flag

		Flag to add a link to the project's GitHub repository.

		To use this option add the following to your ``conf.py``:

		.. code-block:: python

			extensions = [
					...
					'sphinx_toolbox.github',
					]

			github_username = '<your username>'
			github_repository = '<your repository>'


		See :mod:`sphinx_toolbox.github` for more information.

	.. rst:directive:option:: pypi
		:type: string

		Flag to add a link to the project page on PyPI.

		The name of the project on PyPI must be passed as the option's value.

	.. rst:directive:option:: caption
		:type: string

		The caption of the toctree. Defaults to ``Links``

	Additional toctree entries may be added as the content of the directive, in the same manner as normal toctrees.


API Reference
--------------
"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
import warnings
from typing import List

# 3rd party
import docutils
from docutils import nodes
from docutils.parsers.rst import directives
from domdf_python_tools.stringlist import StringList
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.util.docutils import SphinxDirective

# this package
from sphinx_toolbox.utils import OptionSpec, SphinxExtMetadata, flag, metadata_add_version

__all__ = ("SidebarLinksDirective", "setup")


class SidebarLinksDirective(SphinxDirective):
	"""
	Directive which adds a toctree to the sidebar containing links to the GitHub repository, PyPI project page etc.
	"""

	has_content: bool = True

	option_spec: OptionSpec = {  # type: ignore[assignment]
		"pypi": directives.unchanged_required,
		"github": flag,
		"caption": directives.unchanged_required,
		}

	def process_github_option(self) -> str:
		"""
		Process the ``:github:`` flag.
		"""

		if "sphinx_toolbox.github" not in self.env.app.extensions:
			raise ValueError(
					"The 'sphinx_toolbox.github' extension is required for the "
					":github: option but it is not enabled!"
					)

		username = getattr(self.env.config, "github_username", None)
		if username is None:
			raise ValueError("'github_username' has not been set in 'conf.py'!")

		repository = getattr(self.env.config, "github_repository", None)
		if repository is None:
			raise ValueError("'github_repository' has not been set in 'conf.py'!")

		return f"GitHub <https://github.com/{username}/{repository}>"

	def run(self) -> List[nodes.Node]:
		"""
		Create the installation node.
		"""

		if self.env.docname != self.env.config.master_doc:  # pragma: no cover
			warnings.warn(
					"The 'sidebar-links' directive can only be used on the Sphinx master doc. "
					"No links will be shown.",
					UserWarning,
					)
			return []

		body = StringList([
				".. toctree::",
				"    :hidden:",
				])

		with body.with_indent("    ", 1):
			if "caption" in self.options:
				body.append(f":caption: {self.options['caption']}")
			else:  # pragma: no cover
				body.append(":caption: Links")

			body.blankline()

			if "github" in self.options:
				body.append(self.process_github_option())
			if "pypi" in self.options:
				body.append(f"PyPI <https://pypi.org/project/{self.options['pypi']}>")

			body.extend(self.content)

		body.blankline()
		body.blankline()

		only_node = addnodes.only(expr="html")
		content_node = nodes.paragraph(rawsource=str(body))
		only_node += content_node
		self.state.nested_parse(docutils.statemachine.StringList(body), self.content_offset, content_node)

		return [only_node]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.sidebar_links`.

	:param app: The Sphinx application.
	"""

	app.add_directive("sidebar-links", SidebarLinksDirective)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/sidebar_links.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  source.py
"""
Add hyperlinks to source files, either on GitHub or in the documentation itself.

.. extensions:: sphinx_toolbox.source

If you're looking for a ``[source]`` button to go at the end of your class and
function signatures, checkout :mod:`sphinx.ext.linkcode` and :mod:`sphinx.ext.viewcode`.


Usage
-------

.. confval:: source_link_target
	:type: :class:`str`
	:required: False
	:default: ``'Sphinx'``

	The target of the source link, either ``'GitHub'`` or ``'Sphinx'``.
	Case insensitive.


.. rst:role:: source

	Role which shows a link to the given source file, either on GitHub or within the Sphinx documentation.

	By default, the link points to the code within the documentation,
	but can be configured to point to GitHub by setting :confval:`source_link_target` to ``'GitHub'``.


	:bold-title:`Example`

	.. rest-example::

		:source:`sphinx_toolbox/config.py`

		Here is the :source:`source code <sphinx_toolbox/config.py>`


API Reference
--------------
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on pyspecific.py from the Python documentation.
#  Copyright 2008-2014 by Georg Brandl.
#  Licensed under the PSF License 2.0
#
#  Parts of the docstrings based on https://docutils.sourceforge.io/docs/howto/rst-roles.html
#

# stdlib
from typing import TYPE_CHECKING, Dict, List, Sequence, Tuple

# 3rd party
import sphinx
from docutils import nodes
from docutils.nodes import system_message
from docutils.parsers.rst.states import Inliner
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.errors import NoUri
from sphinx.util import split_explicit_title

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

if TYPE_CHECKING:
	# this package
	from sphinx_toolbox.utils import Config

__all__ = ("source_role", "setup")

# TODO: rawstring: Return it as a problematic node linked to a system message if a problem is encountered.

_sphinx_version = sphinx.version_info[:3]


def _make_viewcode_node(
		title: str,
		pagename: str,
		env: BuildEnvironment,
		) -> nodes.Node:
	"""
	Construct a node for the :mod:`sphinx.ext.viewcode` link.

	Handles Sphinx 3.5+ compatibility.
	"""

	if _sphinx_version < (3, 5, 0):
		return addnodes.pending_xref(
				title,
				nodes.inline(title, title),
				reftype="viewcode",
				refdomain="std",
				refexplicit=False,
				reftarget=pagename,
				refid=title,
				refdoc=env.docname,
				)
	else:
		# 3rd party
		from sphinx.util.nodes import make_refnode

		assert env.app.builder is not None

		try:
			return make_refnode(
					env.app.builder,
					fromdocname=env.docname,
					todocname=pagename,
					targetid=title,
					child=nodes.inline(title, title),
					)
		except NoUri:
			return nodes.inline(title, title)


def source_role(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict = {},
		content: List[str] = []
		) -> Tuple[Sequence[nodes.Node], List[system_message]]:
	"""
	Adds a link to the given Python source file in the documentation or on GitHub.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.

	.. versionchanged:: 2.8.0

		Now returns a sequence of :class:`nodes.reference <docutils.nodes.reference>` and
		:class:`addnodes.pending_xref <sphinx.addnodes.pending_xref>` as the first tuple element,
		rather than :class:`nodes.reference <docutils.nodes.reference>` and
		:class:`addnodes.pending_xref <sphinx.addnodes.only>` as in previous versions.
	"""

	has_t, title, target = split_explicit_title(text)
	title = nodes.unescape(title)
	target = nodes.unescape(target)

	env = inliner.document.settings.env
	config = env.app.config

	nodes_: List[nodes.Node] = []
	messages: List[system_message] = []
	refnode: nodes.Node

	if config.source_link_target == "sphinx":
		if target.endswith("/__init__.py"):
			pagename = "_modules/" + target.rsplit('/', 1)[0]
		else:
			pagename = "_modules/" + target.replace(".py", '')

		# refnode = addnodes.only(expr="html")
		# refnode += addnodes.pending_xref(

		refnode = _make_viewcode_node(
				title,
				pagename,
				env,
				)

		# refnode = addnodes.pending_xref(
		# 		title,
		# 		nodes.inline(title, title),
		# 		reftype="viewcode",
		# 		refdomain="std",
		# 		refexplicit=False,
		# 		reftarget=pagename,
		# 		refid=title,
		# 		refdoc=env.docname,
		# 		)

		nodes_.append(refnode)

	elif config.source_link_target == "github":
		refnode = nodes.reference(
				title,
				title,
				refuri=str(config.github_source_url / target),
				)

		nodes_.append(refnode)

	else:
		message = inliner.document.reporter.error(f"Unsupported source link target '{config.source_link_target}'.")
		messages.append(message)

	return nodes_, messages


def _configure(app: Sphinx, config: "Config") -> None:
	"""
	Validate the provided configuration values.

	:param app: The Sphinx application.
	:param config:
	"""

	config.source_link_target = str(config.source_link_target).lower().strip()  # type: ignore[attr-defined]

	if config.source_link_target not in {"sphinx", "github"}:
		# this package
		from sphinx_toolbox.config import InvalidOptionError

		raise InvalidOptionError("Invalid value for 'source_link_target'.")


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.source`.

	:param app: The Sphinx application.
	"""

	# Link to source code
	app.add_role("source", source_role)

	# The target for the source link. One of GitHub or Sphinx (GitLab coming soon™)
	app.add_config_value("source_link_target", "Sphinx", "env", types=[str])
	app.connect("config-inited", _configure)

	app.setup_extension("sphinx_toolbox.github")

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/source.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  testing.py
r"""
Functions for testing Sphinx extensions.

.. extras-require:: testing
	:pyproject:

.. seealso:: Sphinx's own ``testing`` library: https://github.com/sphinx-doc/sphinx/tree/3.x/sphinx/testing

.. latex:vspace:: 10px

.. _pytest-regressions: https://pypi.org/project/pytest-regressions/
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on Sphinx
#  Copyright (c) 2007-2020 by the Sphinx team.
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |    notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |    notice, this list of conditions and the following disclaimer in the
#  |    documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
import copy
import re
import sys
import tempfile
from functools import partial
from types import SimpleNamespace
from typing import Any, Callable, Dict, List, NamedTuple, Optional, Set, Tuple, Type, Union, cast

# 3rd party
import pytest  # nodep
import sphinx.application
from bs4 import BeautifulSoup  # type: ignore[import]
from coincidence.regressions import (  # nodep
		AdvancedFileRegressionFixture,
		check_file_output,
		check_file_regression
		)
from docutils import __version_info__ as docutils_version
from docutils import nodes
from docutils.parsers.rst import Directive, roles
from docutils.transforms import Transform
from domdf_python_tools.doctools import prettify_docstrings
from domdf_python_tools.paths import PathPlus
from domdf_python_tools.stringlist import StringList
from domdf_python_tools.typing import PathLike
from jinja2 import Template  # nodep
from pygments.lexer import Lexer  # type: ignore[import]  # nodep
from pytest_regressions.common import check_text_files  # nodep
from pytest_regressions.file_regression import FileRegressionFixture  # nodep
from sphinx.builders import Builder
from sphinx.domains import Domain, Index
from sphinx.domains.python import PythonDomain
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.events import EventListener
from sphinx.events import EventManager as BaseEventManager
from sphinx.ext.autodoc.directive import AutodocDirective
from sphinx.highlighting import lexer_classes
from sphinx.registry import SphinxComponentRegistry
from sphinx.roles import XRefRole
from sphinx.util import docutils
from sphinx.util.typing import RoleFunction, TitleGetter

# this package
from sphinx_toolbox.utils import Config, SphinxExtMetadata

# _ModuleWrapper unwrapping BS due to
# https://github.com/sphinx-doc/sphinx/commit/8866adeacfb045c97302cc9c7e3b60dec5ca38fd

try:
	# 3rd party
	from sphinx.deprecation import _ModuleWrapper
	if isinstance(docutils, _ModuleWrapper):
		docutils = docutils._module
except ImportError:
	# Unnecessary if unimportable
	pass

__all__ = (
		"Sphinx",
		"run_setup",
		"RunSetupOutput",
		"remove_html_footer",
		"check_html_regression",
		"remove_html_link_tags",
		"check_asset_copy",
		"HTMLRegressionFixture",
		"html_regression",
		"LaTeXRegressionFixture",
		"latex_regression",
		)


class FakeBuilder(Builder):
	pass


class EventManager(BaseEventManager):

	def connect(self, name: str, callback: Callable, priority: int) -> int:
		"""
		Connect a handler to specific event.
		"""

		listener_id = self.next_listener_id
		self.next_listener_id += 1
		self.listeners[name].append(EventListener(listener_id, callback, priority))
		return listener_id


class Sphinx:
	"""
	A class that pretends to be :class:`sphinx.application.Sphinx` but that is stripped
	back to allow the internals to be inspected. This can be used in tests to ensure the
	nodes, roles etc. being registered in an extension's ``setup()`` function are actually
	being registered.
	"""  # noqa: D400

	registry: SphinxComponentRegistry  #: Instance of :class:`sphinx.registry.SphinxComponentRegistry`
	config: Config  #: Instance of :class:`sphinx.config.Config`
	events: EventManager  #: Instance of :class:`sphinx.events.EventManager`
	html_themes: Dict[str, str]  #: Mapping of HTML theme names to filesystem paths.

	# builder: Builder  #: Instance of :class:`sphinx.builder.Builder`

	def __init__(self):  # , buildername: str = "html"
		self.registry = SphinxComponentRegistry()
		self.config = Config({}, {})
		self.events = EventManager(self)  # type: ignore[arg-type]
		self.html_themes: Dict[str, str] = {}
		# self.builder = self.registry.create_builder(self, buildername)

	def add_builder(self, builder: Type[Builder], override: bool = False) -> None:
		r"""
		Register a new builder.

		The registered values are stored in the ``app.registry.builders`` dictionary
		(:class:`typing.Dict`\[:class:`str`\, :class:`typing.Type`\[:class:`sphinx.builders.Builder`\]]).
		"""

		self.registry.add_builder(builder, override=override)

	def add_config_value(
			self,
			name: str,
			default: Any,
			rebuild: Union[bool, str],
			types: Any = (),
			) -> None:
		r"""
		Register a configuration value.

		The registered values are stored in the ``app.config.values`` dictionary
		(:class:`typing.Dict`\[:class:`str`\, :class:`typing.Tuple`]).
		"""

		if rebuild in {False, True}:
			rebuild = "env" if rebuild else ''

		self.config.add(name, default, rebuild, types)

	def add_event(self, name: str) -> None:
		r"""
		Register an event called ``name``.

		The registered values are stored in the ``app.events.events`` dictionary
		(:class:`typing.Dict`\[:class:`str`\, :class:`str`\]).
		"""

		self.events.add(name)

	def set_translator(
			self,
			name: str,
			translator_class: Type[nodes.NodeVisitor],
			override: bool = False,
			) -> None:
		r"""
		Register or override a Docutils translator class.

		The registered values are stored in the ``app.registry.translators`` dictionary.
		(:class:`typing.Dict`\[:class:`str`\, :class:`typing.Type`\[:class:`docutils.nodes.NodeVisitor`\]]).

		.. clearpage::
		"""

		self.registry.add_translator(name, translator_class, override=override)

	def add_node(
			self,
			node: Type[nodes.Element],
			override: bool = False,
			**kwargs: Tuple[Callable, Callable],
			) -> None:
		r"""
		Register a Docutils node class.

		The registered values are stored in the ``additional_nodes`` set returned by
		:func:`~sphinx_toolbox.testing.run_setup`
		(:class:`typing.Set`\[:class:`typing.Type`\[:class:`docutils.nodes.Node`\]]).
		"""

		if not override and docutils.is_node_registered(node):
			raise ValueError(
					f"node class {node.__name__!r} is already registered, its visitors will be overridden"
					)

		docutils.register_node(node)
		self.registry.add_translation_handlers(node, **kwargs)

	def add_enumerable_node(
			self,
			node: Type[nodes.Element],
			figtype: str,
			title_getter: Optional[TitleGetter] = None,
			override: bool = False,
			**kwargs: Tuple[Callable, Callable],
			) -> None:
		"""
		Register a Docutils node class as a numfig target.
		"""

		# Sphinx's signature is wrong WRT Optional
		self.registry.add_enumerable_node(
				node,
				figtype,
				title_getter,
				override=override,
				)
		self.add_node(node, override=override, **kwargs)

	def add_directive(self, name: str, cls: Type[Directive], override: bool = False) -> None:
		"""
		Register a Docutils directive.
		"""

		if not override and docutils.is_directive_registered(name):
			raise ValueError(f"directive {name!r} is already registered, it will be overridden")

		docutils.register_directive(name, cls)

	def add_role(self, name: str, role: Any, override: bool = False) -> None:
		r"""
		Register a Docutils role.

		The registered values are stored in the ``roles`` dictionary returned by
		:func:`~sphinx_toolbox.testing.run_setup`.
		(:class:`typing.Dict`\[:class:`str`\, :class:`typing.Callable`\]).
		"""

		if not override and docutils.is_role_registered(name):
			raise ValueError(f"role {name!r} is already registered, it will be overridden")

		docutils.register_role(name, role)

	def add_generic_role(self, name: str, nodeclass: Any, override: bool = False) -> None:
		"""
		Register a generic Docutils role.
		"""

		if not override and docutils.is_role_registered(name):
			raise ValueError(f"role {name!r} is already registered, it will be overridden")

		role = roles.GenericRole(name, nodeclass)

		docutils.register_role(name, role)

	def add_domain(
			self,
			domain: Type[Domain],
			override: bool = False,
			) -> None:
		"""
		Register a domain.
		"""

		self.registry.add_domain(domain, override=override)

	def add_directive_to_domain(
			self,
			domain: str,
			name: str,
			cls: Type[Directive],
			override: bool = False,
			) -> None:
		"""
		Register a Docutils directive in a domain.
		"""

		self.registry.add_directive_to_domain(domain, name, cls, override=override)

	def add_role_to_domain(
			self,
			domain: str,
			name: str,
			role: Union[RoleFunction, XRefRole],
			override: bool = False,
			) -> None:
		"""
		Register a Docutils role in a domain.
		"""

		self.registry.add_role_to_domain(domain, name, role, override=override)

	def add_index_to_domain(
			self,
			domain: str,
			index: Type[Index],
			override: bool = False,
			) -> None:
		"""
		Register a custom index for a domain.
		"""

		self.registry.add_index_to_domain(domain, index)

	def add_object_type(
			self,
			directivename: str,
			rolename: str,
			indextemplate: str = '',
			parse_node: Optional[Callable] = None,
			ref_nodeclass: Optional[Type[nodes.TextElement]] = None,
			objname: str = '',
			doc_field_types: List = [],
			override: bool = False,
			) -> None:
		"""
		Register a new object type.
		"""

		# Sphinx's signature is wrong WRT Optional
		self.registry.add_object_type(
				directivename,
				rolename,
				indextemplate,
				parse_node,
				ref_nodeclass,
				objname,
				doc_field_types,
				override=override,
				)

	def add_crossref_type(
			self,
			directivename: str,
			rolename: str,
			indextemplate: str = '',
			ref_nodeclass: Optional[Type[nodes.TextElement]] = None,
			objname: str = '',
			override: bool = False,
			) -> None:
		"""
		Register a new crossref object type.
		"""

		# Sphinx's signature is wrong WRT Optional
		self.registry.add_crossref_type(
				directivename,
				rolename,
				indextemplate,
				ref_nodeclass,
				objname,
				override=override,
				)

	def add_transform(self, transform: Type[Transform]) -> None:
		"""
		Register a Docutils transform to be applied after parsing.
		"""

		self.registry.add_transform(transform)

	def add_post_transform(self, transform: Type[Transform]) -> None:
		"""
		Register a Docutils transform to be applied before writing.
		"""

		self.registry.add_post_transform(transform)

	def add_js_file(self, filename: str, **kwargs: str) -> None:
		"""
		Register a JavaScript file to include in the HTML output.

		.. versionadded:: 2.8.0
		"""

		self.registry.add_js_file(filename, **kwargs)

	# 	if hasattr(self.builder, 'add_js_file'):
	# 		self.builder.add_js_file(filename, **kwargs)
	#

	def add_css_file(self, filename: str, **kwargs: str) -> None:
		"""
		Register a stylesheet to include in the HTML output.

		.. versionadded:: 2.7.0
		"""

		self.registry.add_css_files(filename, **kwargs)

	# 	if hasattr(self.builder, 'add_css_file'):
	# 		self.builder.add_css_file(filename, **kwargs)

	def add_latex_package(
			self,
			packagename: str,
			options: Optional[str] = None,
			after_hyperref: bool = False,
			) -> None:
		"""
		Register a package to include in the LaTeX source code.
		"""

		# Sphinx's signature is wrong WRT Optional
		self.registry.add_latex_package(packagename, cast(str, options), after_hyperref)

	def add_lexer(self, alias: str, lexer: Type[Lexer]) -> None:
		"""
		Register a new lexer for source code.
		"""

		if isinstance(lexer, Lexer):
			raise TypeError("app.add_lexer() API changed; Please give lexer class instead instance")
		else:
			lexer_classes[alias] = lexer

	def add_autodocumenter(self, cls: Any, override: bool = False) -> None:
		"""
		Register a new documenter class for the autodoc extension.
		"""

		self.registry.add_documenter(cls.objtype, cls)
		self.add_directive("auto" + cls.objtype, AutodocDirective, override=override)

	def add_autodoc_attrgetter(
			self,
			typ: Type,
			getter: Callable[[Any, str, Any], Any],
			) -> None:
		"""
		Register a new ``getattr``-like function for the autodoc extension.
		"""

		self.registry.add_autodoc_attrgetter(typ, getter)

	def add_source_suffix(self, suffix: str, filetype: str, override: bool = False) -> None:
		"""
		Register a suffix of source files.
		"""

		self.registry.add_source_suffix(suffix, filetype, override=override)

	def add_source_parser(self, *args: Any, **kwargs: Any) -> None:
		"""
		Register a parser class.
		"""

		self.registry.add_source_parser(*args, **kwargs)

	def add_env_collector(self, collector: Type[EnvironmentCollector]) -> None:
		"""
		No-op for now.

		.. TODO:: Make this do something
		"""

	# def add_env_collector(self, collector: Type[EnvironmentCollector]) -> None:
	# 	"""
	# 	Register an environment collector class.
	# 	"""
	#
	# 	collector().enable(self)

	def add_html_theme(self, name: str, theme_path: str) -> None:
		"""
		Register an HTML Theme.
		"""

		self.html_themes[name] = theme_path

	def add_html_math_renderer(
			self,
			name: str,
			inline_renderers: Optional[Tuple[Callable, Callable]] = None,
			block_renderers: Optional[Tuple[Callable, Callable]] = None,
			) -> None:
		"""
		Register a math renderer for HTML.
		"""

		self.registry.add_html_math_renderer(name, inline_renderers, block_renderers)  # type: ignore[arg-type]

	def setup_extension(self, extname: str) -> None:
		"""
		Import and setup a Sphinx extension module.

		.. TODO:: implement this
		"""

		# self.registry.load_extension(self, extname)

	def require_sphinx(self, version: str) -> None:
		"""
		Check the Sphinx version if requested.

		No-op when testing
		"""

	# event interface
	def connect(self, event: str, callback: Callable, priority: int = 500) -> int:
		"""
		Register *callback* to be called when *event* is emitted.
		"""

		listener_id = self.events.connect(event, callback, priority)
		return listener_id


@prettify_docstrings
class RunSetupOutput(NamedTuple):
	"""
	:class:`~typing.NamedTuple` representing the output from :func:`~sphinx_toolbox.testing.run_setup`.
	"""

	setup_ret: Union[None, Dict[str, Any], "SphinxExtMetadata"]  #: The output from the ``setup()`` function.
	directives: Dict[str, Callable]  #: Mapping of directive names to directive functions.
	roles: Dict[str, Callable]  #: Mapping of role names to role functions.
	additional_nodes: Set[Type[Any]]  #: Set of custom docutils nodes registered in ``setup()``.
	app: Sphinx  #: Instance of :class:`sphinx_toolbox.testing.Sphinx`.


_sphinx_dict_setup = Callable[[sphinx.application.Sphinx], Optional[Dict[str, Any]]]
_sphinx_metadata_setup = Callable[[sphinx.application.Sphinx], Optional["SphinxExtMetadata"]]
_fake_dict_setup = Callable[[Sphinx], Optional[Dict[str, Any]]]
_fake_metadata_setup = Callable[[Sphinx], Optional["SphinxExtMetadata"]]
_setup_func_type = Union[_sphinx_dict_setup, _sphinx_metadata_setup, _fake_dict_setup, _fake_metadata_setup]


def run_setup(setup_func: _setup_func_type) -> RunSetupOutput:  # , buildername: str = "html"
	"""
	Function for running an extension's ``setup()`` function for testing.

	:param setup_func: The ``setup()`` function under test.

	:returns: 5-element namedtuple
	"""

	app = Sphinx()  # buildername

	app.add_domain(PythonDomain)

	_additional_nodes = copy.copy(docutils.additional_nodes)
	try:
		docutils.additional_nodes = set()

		with docutils.docutils_namespace():
			setup_ret = setup_func(app)  # type: ignore[arg-type]
			directives = copy.copy(docutils.directives._directives)  # type: ignore[attr-defined]
			roles = copy.copy(docutils.roles._roles)  # type: ignore[attr-defined]
			additional_nodes = copy.copy(docutils.additional_nodes)
	finally:
		docutils.additional_nodes = _additional_nodes

	return RunSetupOutput(setup_ret, directives, roles, additional_nodes, app)


def remove_html_footer(page: BeautifulSoup) -> BeautifulSoup:
	"""
	Remove the Sphinx footer from HTML pages.

	The footer contains the Sphinx and theme versions and therefore changes between versions.
	This can cause unwanted, false positive test failures.

	:param page: The page to remove the footer from.

	:return: The page without the footer.
	"""

	for div in page.select("div.footer"):
		div.extract()

	return page


def remove_html_link_tags(page: BeautifulSoup) -> BeautifulSoup:
	"""
	Remove link tags from HTML pages.

	These may vary between different versions of Sphinx and its extensions.
	This can cause unwanted, false positive test failures.

	:param page: The page to remove the link tags from.

	:return: The page without the link tags.
	"""

	for div in page.select("head link"):
		div.extract()

	return page


def check_html_regression(page: BeautifulSoup, file_regression: FileRegressionFixture) -> None:
	"""
	Check an HTML page generated by Sphinx for regressions, using `pytest-regressions`_.

	:param page: The page to test.
	:param file_regression: The file regression fixture.

	**Example usage**

	.. code-block:: python

		@pytest.mark.parametrize("page", ["index.html"], indirect=True)
		def test_page(page: BeautifulSoup, file_regression: FileRegressionFixture):
			check_html_regression(page, file_regression)
	"""  # noqa: RST306

	__tracebackhide__ = True

	page = remove_html_footer(page)
	page = remove_html_link_tags(page)

	for div in page.select("script"):
		if "_static/language_data.js" in str(div):
			div.extract()

	for div in page.select("div.sphinxsidebar"):
		div.extract()

	check_file_regression(
			StringList(page.prettify()),
			file_regression,
			extension=".html",
			)


class HTMLRegressionFixture(FileRegressionFixture):
	"""
	Subclass of :class:`pytest_regressions.file_regression.FileRegressionFixture` for checking HTML files.

	.. versionadded:: 2.0.0
	"""

	def check(  # type: ignore[override]
		self,
		page: BeautifulSoup,
		*,
		extension: str = ".html",
		jinja2: bool = False,
		jinja2_namespace: Optional[Dict[str, Any]] = None,
		**kwargs
		) -> None:
		r"""
		Check an HTML page generated by Sphinx for regressions, using `pytest-regressions`_.

		:param page: The page to test.
		:param jinja2: Whether to render the reference file as a jinja2 template.
		:param jinja2_namespace: If ``jinja2`` is :py:obj:`True`,
			a mapping of variable names to values to make available in the jinja2 template.
		:param \*\*kwargs: Additional keyword arguments passed to
			:meth:`pytest_regressions.file_regression.FileRegressionFixture.check`.

		.. versionchanged:: 2.14.0  Added the ``jinja2`` keyword argument.
		.. versionchanged:: 2.17.0  Added the ``jinja2_namespace`` keyword argument.

		.. latex:clearpage::

		When ``jinja2`` is :py:obj:`True`, the reference file will be rendered as a jinja2 template.
		The template is passed the following variables:

		* ``sphinx_version`` -- the Sphinx version number, as a tuple of integers.
		* ``python_version`` -- the Python version number, in the form returned by :data:`sys.version_info`.
		* ``docutils_version`` -- the docutils version number, as a tuple of integers (*New in version 2.16.0*).

		**Example usage**

		.. code-block:: python

			@pytest.mark.parametrize("page", ["index.html"], indirect=True)
			def test_page(page: BeautifulSoup, html_regression: HTMLRegressionFixture):
				html_regression.check(page, file_regression)
		"""  # noqa: RST306

		__tracebackhide__ = True

		page = remove_html_footer(page)
		page = remove_html_link_tags(page)

		for div in page.select("script"):
			if "_static/language_data.js" in str(div):
				div.extract()

		for div in page.select("div.sphinxsidebar"):
			div.extract()

		if sphinx.version_info >= (4, 3):  # pragma: no cover
			for div in page.select("span.w"):
				div.extract()
			for div in page.select("span.p"):
				if div.string == '=':
					sibling = div.next_sibling
					div.replace_with('')
					sibling.replace_with(f"= {sibling.text}")

		kwargs.pop("encoding", None)
		kwargs.pop("extension", None)

		if jinja2:

			def check_fn(obtained_filename: PathPlus, expected_filename: PathPlus):  # noqa: MAN002
				__tracebackhide__ = True

				expected_filename = PathPlus(expected_filename)
				template = Template(expected_filename.read_text())

				expected_filename.write_text(
						template.render(
								sphinx_version=sphinx.version_info,
								python_version=sys.version_info,
								docutils_version=docutils_version,
								**jinja2_namespace or {},
								)
						)

				return check_text_files(obtained_filename, expected_filename, encoding="UTF-8")

		else:
			check_fn = partial(check_text_files, encoding="UTF-8")

		super().check(
				str(StringList(page.prettify())),
				encoding="UTF-8",
				extension=extension,
				check_fn=check_fn,
				)


@pytest.fixture()
def html_regression(datadir, original_datadir, request) -> HTMLRegressionFixture:  # noqa: MAN001
	"""
	Returns an :class:`~.HTMLRegressionFixture` scoped to the test function.

	.. versionadded:: 2.0.0
	"""

	return HTMLRegressionFixture(datadir, original_datadir, request)


def check_asset_copy(
		func: Callable[[sphinx.application.Sphinx, Exception], Any],
		*asset_files: PathLike,
		file_regression: FileRegressionFixture,
		) -> None:
	r"""
	Helper to test functions which respond to Sphinx ``build-finished`` events and copy asset files.

	.. versionadded:: 2.0.0

	:param func: The function to test.
	:param \*asset_files: The paths of asset files copied by the function, relative to the Sphinx output directory.
	:param file_regression:
	"""

	__tracebackhide__ = True

	with tempfile.TemporaryDirectory() as tmpdir:
		tmp_pathplus = PathPlus(tmpdir)

		fake_app = SimpleNamespace()
		fake_app.builder = SimpleNamespace()
		fake_app.builder.format = "html"
		fake_app.outdir = fake_app.builder.outdir = tmp_pathplus

		func(fake_app, None)  # type: ignore[arg-type]

		for filename in asset_files:
			filename = tmp_pathplus / filename

			check_file_output(filename, file_regression, extension=f"_{filename.stem}{filename.suffix}")


_latex_date_re = re.compile(r"\\date{.*}")


class LaTeXRegressionFixture(AdvancedFileRegressionFixture):
	"""
	Subclass of :class:`coincidence.regressions.AdvancedFileRegressionFixture` for checking LaTeX files.

	.. versionadded:: 2.17.0
	"""

	def check(  # type: ignore[override]  # noqa: MAN002
		self,
		contents: Union[str, StringList],
		*,
		extension: str = ".html",
		jinja2: bool = False,
		jinja2_namespace: Optional[Dict[str, Any]] = None,
		**kwargs
		):
		r"""
		Check a LaTeX file generated by Sphinx for regressions,
		using `pytest-regressions <https://pypi.org/project/pytest-regressions/>`__

		:param contents:
		:param jinja2: Whether to render the reference file as a jinja2 template.
		:param jinja2_namespace: If ``jinja2`` is :py:obj:`True`,
			a mapping of variable names to values to make available in the jinja2 template.
		:param \*\*kwargs: Additional keyword arguments passed to
			:meth:`pytest_regressions.file_regression.FileRegressionFixture.check`.

		When ``jinja2`` is :py:obj:`True`, the reference file will be rendered as a jinja2 template.
		The template is passed the following variables:

		* ``sphinx_version`` -- the Sphinx version number, as a tuple of integers.
		* ``python_version`` -- the Python version number, in the form returned by :data:`sys.version_info`.
		* ``docutils_version`` -- the docutils version number, as a tuple of integers (*New in version 2.16.0*).

		.. note::

			Unlike standard HTML jinja2 templates,
			this class expects the use of ``<`` and ``>`` rather than ``{`` and ``}``.

			For example::

				<% if foo %>
				<# This should only happen on Tuesdays #>
				<< foo.upper() >>
				<% endif %>

		**Example usage**

		.. code-block:: python

			@pytest.mark.sphinx("latex")
			def test_latex_output(app: Sphinx, latex_regression: LaTeXRegressionFixture):
				app.build()
				output_file = app.outdir / "python.tex"
				latex_regression.check(output_file.read_text())

		"""  # noqa: D400

		__tracebackhide__ = True

		if jinja2:

			def check_fn(obtained_filename: PathPlus, expected_filename: PathLike):  # noqa: MAN002
				__tracebackhide__ = True

				expected_filename = PathPlus(expected_filename)

				template = Template(
						expected_filename.read_text(),
						block_start_string="<%",
						block_end_string="%>",
						variable_start_string="<<",
						variable_end_string=">>",
						comment_start_string="<#",
						comment_end_string="#>",
						)

				expected_filename.write_text(
						template.render(
								sphinx_version=sphinx.version_info,
								python_version=sys.version_info,
								docutils_version=docutils_version,
								**jinja2_namespace or {},
								)
						)

				return check_text_files(obtained_filename, expected_filename, encoding="UTF-8")

		else:
			check_fn = partial(check_text_files, encoding="UTF-8")

		new_contents = _latex_date_re.sub(
				r"\\date{Mar 11, 2021}",
				str(contents).replace("\\sphinxAtStartPar\n", ''),
				)
		new_contents = new_contents.replace("%% let collapsible ", "%% let collapsable ")  # changed in Sphinx 4.2

		return super().check(
				new_contents,
				extension=".tex",
				check_fn=check_fn,
				)


@pytest.fixture()
def latex_regression(datadir, original_datadir, request) -> LaTeXRegressionFixture:  # noqa: MAN001
	"""
	Returns a :class:`~.LaTeXRegressionFixture` scoped to the test function.

	.. versionadded:: 2.17.0
	"""

	return LaTeXRegressionFixture(datadir, original_datadir, request)

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/testing.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  footnote_symbols.py
r"""
Tweak which monkeypatches docutils to use the following symbols for footnotes:

.. rst-class:: bullet-hidden

* † -- dagger
* ‡ -- double dagger
* § -- section mark
* ¶ -- paragraph mark (pilcrow)
* # -- number sign
* ♠ -- spade suit
* ♥ -- heart suit
* ♦ -- diamond suit
* ♣ -- club suit

With some themes the superscript asterisk becomes very hard to see.

.. versionadded:: 2.7.0
.. extensions:: sphinx_toolbox.tweaks.footnote_symbols

-----

"""  # noqa: D400
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import List

# 3rd party
from docutils.transforms.references import Footnotes
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("setup", )

#: The list of symbols
symbols: List[str] = [
		# Entries 1-3 and 5 below are from section 12.51 of
		# The Chicago Manual of Style, 14th edition.
		'†',  # dagger &dagger;
		'‡',  # double dagger &Dagger;
		'§',  # section mark &sect;
		'¶',  # paragraph mark (pilcrow) &para;
		# (parallels ['||'] in CMoS)
		'#',  # number sign
		# The entries below were chosen arbitrarily.
		'♠',  # spade suit &spades;
		'♥',  # heart suit &hearts;
		'♦',  # diamond suit &diams;
		'♣',  # club suit &clubs;
		]


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.footnote_symbols`.

	:param app: The Sphinx application.
	"""

	Footnotes.symbols = symbols

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/footnote_symbols.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  latex_layout.py
r"""
Makes minor adjustments to the LaTeX layout.

* Increases the whitespace above function signatures by 5px,
  to prevent the function visually merging with the previous one.
* Remove unnecessary indentation and allow "raggedright" for the fields in
  the body of functions, which prevents ugly whitespace and line breaks.
* Disables justification for function signatures.
  This is a backport of changes from Sphinx 4 added in :github:pull:`8997 <sphinx-doc/sphinx>`.

  .. versionadded:: 2.12.0

* With Sphinx 3.5, doesn't add ``\sphinxAtStartPar`` before every paragraph.
  The change in :github:issue:`8781 <sphinx-doc/sphinx>` was to solve an issue with *tables*,
  but it isn't clear why it then gets added for *every* paragraph so this extension removes it.

  .. versionadded:: 2.13.0

* Configures hyperref to apply correct page numbering to the frontmatter.

  .. versionadded:: 2.14.0

* Optionally, configures the ``needspace`` package.

  The :confval:`needspace_amount` option can be set in ``conf.py`` to add the ``\needspace{}`` command
  before each ``addnodes.desc`` node (i.e. a function or class description).
  The amount of space is set by the ``needspace_amount`` option, e.g.:

  .. code-block:: python

      needspace_amount = r"4\baselineskip"

  .. versionadded:: 3.0.0


.. versionadded:: 2.10.0
.. extensions:: sphinx_toolbox.tweaks.latex_layout


.. versionchanged:: 3.0.0

	The functionality has moved to :mod:`sphinx_toolbox.latex.toc`.
	Please use that extension instead.

-----

"""
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# 3rd party
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("setup", )


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.latex_layout`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx_toolbox.latex.layout")

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/latex_layout.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  latex_toc.py
"""
Adjusts the default LaTeX output as follows:

* The captions from ``toctree`` directives are converted into document parts.
* The PDF outline has the correct hierarchy, including having the indices as top-level elements.

.. versionadded:: 2.1.0
.. extensions:: sphinx_toolbox.tweaks.latex_toc

.. versionchanged:: 3.0.0

	The functionality has moved to :mod:`sphinx_toolbox.latex.toc`.
	Please use that extension instead.

-----

"""  # noqa: D400
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib

# 3rd party
from sphinx.application import Sphinx

# this package
from sphinx_toolbox.latex import use_package
from sphinx_toolbox.utils import Config, SphinxExtMetadata, metadata_add_version

__all__ = ("setup", "configure")


def configure(app: Sphinx, config: Config) -> None:
	"""
	Configure :mod:`sphinx_toolbox.tweaks.latex_toc`.

	:param app: The Sphinx application.
	:param config:
	"""

	use_package("bookmark", config)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.latex_toc`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx_toolbox.latex.toc")

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/latex_toc.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  param_dash.py
"""
Monkeypatches :class:`sphinx.util.docfields.TypedField`
to only output the endash (--) separating the parameter
name from its description if a description was given.

.. versionadded:: 0.9.0


:bold-title:`Example`

.. rest-example::

	.. class:: MyClass(foo, bar)

		This is my class.

		:param foo: An argument
		:param bar:


.. extensions:: sphinx_toolbox.tweaks.param_dash

-----

"""  # noqa: D400
#
#  Copyright © 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/sphinx-doc/sphinx
#  |  Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# stdlib
from typing import Dict, List, Optional, Tuple

# 3rd party
import sphinx.util.docfields
from docutils import nodes
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("setup", )


def make_field(
		self,
		types: Dict[str, List[nodes.Node]],
		domain: str,
		items: Tuple,
		env: Optional[BuildEnvironment] = None,
		**kwargs,  # To support Sphinx 4.1 and later
		) -> nodes.field:

	def handle_item(fieldarg: str, content: List[nodes.Element]) -> nodes.paragraph:
		par = nodes.paragraph()
		par.extend(
				self.make_xrefs(
						self.rolename,
						domain,
						fieldarg,
						addnodes.literal_strong,
						env=env,
						**kwargs,  # To support Sphinx 4.1 and later
						)
				)

		if fieldarg in types:
			par += nodes.Text(" (")
			# NOTE: using .pop() here to prevent a single type node to be
			# inserted twice into the doctree, which leads to
			# inconsistencies later when references are resolved
			fieldtype = types.pop(fieldarg)

			if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text):
				typename = fieldtype[0].astext()
				par.extend(
						self.make_xrefs(self.typerolename, domain, typename, addnodes.literal_emphasis, env=env)
						)
			else:
				par += fieldtype

			par += nodes.Text(')')

		if (content and len(content) == 1 and isinstance(content[0], nodes.inline) and not content[0].children):
			return par

		par += nodes.Text(" -- ")
		par += content

		return par

	fieldname = nodes.field_name('', self.label)
	bodynode: nodes.Node

	if len(items) == 1 and self.can_collapse:
		fieldarg, content = items[0]
		bodynode = handle_item(fieldarg, content)
	else:
		bodynode = self.list_type()
		for fieldarg, content in items:
			bodynode += nodes.list_item('', handle_item(fieldarg, content))  # type: ignore[assignment,operator]

	fieldbody = nodes.field_body('', bodynode)

	return nodes.field('', fieldname, fieldbody)


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.param_dash`.

	:param app: The Sphinx application.
	"""

	sphinx.util.docfields.TypedField.make_field = make_field  # type: ignore[assignment]

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/param_dash.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  revert_footnote_style.py
"""
Reverts the docutils footnote behaviour from ``>=0.18`` to that of ``<=0.17``.

.. versionadded:: 3.1.2
.. extensions:: sphinx_toolbox.tweaks.revert_footnote_style

-----

"""
#
#  Copyright © 2022 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on docutils
#  |  Copyright © 2016 David Goodger, Günter Milde
#  |  BSD Licensed
#  |  All rights reserved.
#  |
#  |  Redistribution and use in source and binary forms, with or without
#  |  modification, are permitted provided that the following conditions are
#  |  met:
#  |
#  |  * Redistributions of source code must retain the above copyright
#  |   notice, this list of conditions and the following disclaimer.
#  |
#  |  * Redistributions in binary form must reproduce the above copyright
#  |   notice, this list of conditions and the following disclaimer in the
#  |   documentation and/or other materials provided with the distribution.
#  |
#  |  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  |  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  |  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  |  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  |  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  |  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  |  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  |  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  |  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  |  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  |  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

# 3rd party
import docutils
from sphinx.application import Sphinx
from sphinx.writers.html import HTMLTranslator
from sphinx.writers.html5 import HTML5Translator

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ["setup"]


def visit_footnote(self: HTMLTranslator, node: docutils.nodes.footnote) -> None:  # pragma: no cover
	if not self.in_footnote_list:
		listnode = node.copy()
		listnode["ids"] = []
		classes = [node.tagname, self.settings.footnote_references]
		self.body.append(self.starttag(listnode, "dl", CLASS=' '.join(classes)))  # role="note"
		self.in_footnote_list = True


def depart_footnote(self, node: docutils.nodes.footnote) -> None:  # pragma: no cover
	self.body.append('</dd>\n')
	if not isinstance(node.next_node(descend=False, siblings=True), docutils.nodes.footnote):
		self.body.append('</dl>\n')
		self.in_footnote_list = False


def visit_footnote_reference(self, node: docutils.nodes.footnote_reference) -> None:  # pragma: no cover
	href = '#' + node["refid"]
	classes = ["footnote-reference", self.settings.footnote_references]
	self.body.append(self.starttag(node, 'a', suffix='', CLASS=' '.join(classes), href=href))  # role='doc-noteref'


def depart_footnote_reference(self, node: docutils.nodes.footnote_reference) -> None:  # pragma: no cover
	self.body.append("</a>")


# footnote and citation labels:
def visit_label(self, node: docutils.nodes.label) -> None:  # pragma: no cover
	if (isinstance(node.parent, docutils.nodes.footnote)):
		classes = self.settings.footnote_references
	else:
		classes = "brackets"

	# pass parent node to get id into starttag:
	self.body.append(self.starttag(node.parent, "dt", '', CLASS="label"))
	self.body.append(self.starttag(node, "span", '', CLASS=classes))

	# footnote/citation backrefs:
	if self.settings.footnote_backlinks:
		backrefs = node.parent.get("backrefs", [])
		if len(backrefs) == 1:
			self.body.append('<a class="fn-backref" href="#%s">' % backrefs[0])


def depart_label(self, node: docutils.nodes.label) -> None:  # pragma: no cover
	if self.settings.footnote_backlinks:
		backrefs = node.parent["backrefs"]
		if len(backrefs) == 1:
			self.body.append("</a>")
	self.body.append("</span>")
	if self.settings.footnote_backlinks and len(backrefs) > 1:
		backlinks = [f'<a href="#{ref}">{i}</a>' for (i, ref) in enumerate(backrefs, 1)]
		self.body.append('<span class="fn-backref">(%s)</span>' % ','.join(backlinks))
	self.body.append('</dt>\n<dd>')


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.revert_footnote_style`.

	:param app: The Sphinx application.
	"""

	if docutils.__version_info__ >= (0, 18):  # pragma: no cover
		app.add_node(
				docutils.nodes.footnote,
				html=(visit_footnote, depart_footnote),
				override=True,
				)
		app.add_node(
				docutils.nodes.footnote_reference,
				html=(visit_footnote_reference, depart_footnote_reference),
				override=True,
				)
		app.add_node(
				docutils.nodes.label,
				html=(visit_label, depart_label),
				override=True,
				)
		HTMLTranslator.in_footnote_list = False
		HTML5Translator.in_footnote_list = False

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/revert_footnote_style.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  sphinx_panels_tabs.py
"""
Tweak to :github:repo:`executablebooks/sphinx-tabs`
to fix a CSS conflict with :github:repo:`executablebooks/sphinx-panels`.

Fix for :github:issue:`51 <executablebooks/sphinx-panels>`.

.. versionadded:: 1.9.0
.. extensions:: sphinx_toolbox.tweaks.sphinx_panels_tabs

-----

"""  # noqa: D400
#
#  Copyright © 2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Parts based on https://github.com/executablebooks/sphinx-panels
#  Copyright (c) 2020 Executable Books
#  MIT Licensed
#

# stdlib
from typing import Optional

# 3rd party
import dict2css
from docutils import nodes
from domdf_python_tools.paths import PathPlus
from sphinx.application import Sphinx
from sphinx.writers.html import HTMLTranslator

# this package
from sphinx_toolbox import _css
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("copy_asset_files", "setup")


def visit_container(self: HTMLTranslator, node: nodes.container) -> None:
	classes = "docutils container"
	if node.get("is_div", False):
		# we don't want the CSS for container for these nodes
		classes = "docutils"

	if any(c.startswith("sphinx-data-tab-") for c in node["classes"]):
		classes = "docutils"

	self.body.append(self.starttag(node, "div", CLASS=classes))


def depart_container(self: HTMLTranslator, node: nodes.Node) -> None:
	self.body.append("</div>\n")


def copy_asset_files(app: Sphinx, exception: Optional[Exception] = None) -> None:
	"""
	Copy asset files to the output.

	:param app: The Sphinx application.
	:param exception: Any exception which occurred and caused Sphinx to abort.

	.. versionchanged:: 2.7.0

		Renamed from ``copy_assets``. The former name was kept as an alias until version 3.0.0
	"""

	if exception:  # pragma: no cover
		return

	if app.builder is None or app.builder.format.lower() != "html":  # pragma: no cover
		return

	# style = StringList([
	# 		".docutils.container {",
	# 		"    padding-left: 0 !important;",
	# 		"    padding-right: 0 !important;",
	# 		'}',
	# 		'',
	# 		# "div.sphinx-tabs.docutils.container {",
	# 		# "    padding-left: 0 !important;",
	# 		# "    padding-right: 0 !important;",
	# 		# "}",
	# 		# '',
	# 		"div.ui.top.attached.tabular.menu.sphinx-menu.docutils.container {",
	# 		# "    padding-left: 0 !important;",
	# 		# "    padding-right: 0 !important;",
	# 		"    margin-left: 0 !important;",
	# 		"    margin-right: 0 !important;",
	# 		'}',
	# 		])

	css_static_dir = PathPlus(app.builder.outdir) / "_static" / "css"
	css_static_dir.maybe_make(parents=True)
	dict2css.dump(_css.tweaks_sphinx_panels_tabs_styles, css_static_dir / "tabs_customise.css")


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.sphinx_panels_tabs`.

	:param app: The Sphinx application.
	"""

	app.setup_extension("sphinx_tabs.tabs")
	app.setup_extension("sphinx_toolbox._css")

	app.add_node(nodes.container, override=True, html=(visit_container, depart_container))

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/sphinx_panels_tabs.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  tabsize.py
r"""
Hack to get the docutils tab size, as there doesn't appear to be any other way.

.. versionadded:: 1.0.0


You probably don't need to use this extension directly, but if you're developing an
extension of your own you can enable it like so:

.. code-block::

	def setup(app: Sphinx) -> Dict[str, Any]:
		app.setup_extension('sphinx_toolbox.github')
		return {}

This will guarantee that the following value will be available via
:attr:`app.config <sphinx.config.Config>`:

* **docutils_tab_width** (:class:`int`\) -- The number of spaces that correspond to a tab when Docutils parses source files.

"""
#
#  Copyright © 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#

# stdlib
from typing import Union

# 3rd party
from docutils.nodes import document
from docutils.statemachine import StringList
from sphinx.application import Sphinx
from sphinx.parsers import RSTParser

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("setup", )


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.tweaks.tabsize`.

	:param app: The Sphinx application.
	"""

	class CustomRSTParser(RSTParser):

		def parse(self, inputstring: Union[str, StringList], document: document) -> None:
			app.config.docutils_tab_width = document.settings.tab_width  # type: ignore[attr-defined]
			super().parse(inputstring, document)

	app.add_source_parser(CustomRSTParser, override=True)

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/tweaks/tabsize.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  utils.py
"""
General utility functions.
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  singleton function based on attrs
#  https://github.com/python-attrs/attrs
#  Copyright (c) 2015 Hynek Schlawack
#  MIT Licensed
#

# stdlib
import atexit
import functools
import re
import sys
from typing import (
		TYPE_CHECKING,
		Any,
		Callable,
		Dict,
		Iterable,
		List,
		Mapping,
		Optional,
		Pattern,
		Set,
		Tuple,
		Type,
		TypeVar,
		cast
		)

# 3rd party
import sphinx.config
from apeye.requests_url import RequestsURL
from docutils.nodes import Node
from domdf_python_tools.doctools import prettify_docstrings
from sphinx.addnodes import desc_content
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.ext.autodoc import Documenter, logger
from sphinx.locale import __
from typing_extensions import TypedDict

__all__ = (
		"add_nbsp_substitution",
		"allow_subclass_add",
		"baseclass_is_private",
		"code_repr",
		"escape_trailing__",
		"filter_members_warning",
		"flag",
		"get_first_matching",
		"GITHUB_COM",
		"is_namedtuple",
		"make_github_url",
		"metadata_add_version",
		"NoMatchError",
		"OptionSpec",
		"Param",
		"parse_parameters",
		"Purger",
		"SetupFunc",
		"SphinxExtMetadata",
		"typed_flag_regex",
		"typed_param_regex",
		"unknown_module_warning",
		"untyped_param_regex",
		"add_fallback_css_class",
		)

#: Instance of :class:`apeye.requests_url.RequestsURL` that points to the GitHub website.
GITHUB_COM: RequestsURL = RequestsURL("https://github.com")

#: Type hint for the ``option_spec`` variable of Docutils directives.
OptionSpec = Mapping[str, Callable[[str], Any]]

_T = TypeVar("_T")

atexit.register(GITHUB_COM.session.close)


@functools.lru_cache()
def make_github_url(username: str, repository: str) -> RequestsURL:
	"""
	Construct a URL to a GitHub repository from a username and repository name.

	:param username: The username of the GitHub account that owns the repository.
	:param repository: The name of the repository.
	"""

	return GITHUB_COM / username / repository


def flag(argument: Any) -> bool:
	"""
	Check for a valid flag option (no argument) and return :py:obj:`True`.

	Used in the ``option_spec`` of directives.

	.. seealso::

		:class:`docutils.parsers.rst.directives.flag`, which returns :py:obj:`None` instead of :py:obj:`True`.

	:raises: :exc:`ValueError` if an argument is given.
	"""

	if argument and argument.strip():
		raise ValueError(f"No argument is allowed; {argument!r} supplied")
	else:
		return True


@prettify_docstrings
class Purger:
	"""
	Class to purge redundant nodes.

	:param attr_name: The name of the build environment's attribute that stores the list of nodes,
		e.g. ``all_installation_nodes``.

	.. autosummary-widths:: 55/100
	"""

	def __init__(self, attr_name: str):
		self.attr_name = str(attr_name)

	def __repr__(self) -> str:
		return f"{self.__class__.__name__}({self.attr_name!r})"

	def purge_nodes(  # pragma: no cover
			self,
			app: Sphinx,
			env: BuildEnvironment,
			docname: str,
			) -> None:
		"""
		Remove all redundant nodes.

		This function can be configured for the :event:`env-purge-doc` event:

		.. code-block::

			my_node_purger = Purger("all_my_node_nodes")

			def setup(app: Sphinx):
				app.connect("env-purge-doc", my_node_purger.purge_nodes)

		:param app: The Sphinx application.
		:param env: The Sphinx build environment.
		:param docname: The name of the document to remove nodes for.
		"""

		if not hasattr(env, self.attr_name):
			return

		all_nodes = [todo for todo in getattr(env, self.attr_name) if todo["docname"] != docname]
		setattr(env, self.attr_name, all_nodes)

	def get_outdated_docnames(
			self,
			app: Sphinx,
			env: BuildEnvironment,
			added: Set[str],
			changed: Set[str],
			removed: Set[str],
			) -> List[str]:
		"""
		Returns a list of all docnames containing one or more nodes this :class:`~.Purger` is aware of.

		This function can be configured for the :event:`env-get-outdated` event:

		.. code-block::

			my_node_purger = Purger("all_my_node_nodes")

			def setup(app: Sphinx):
				app.connect("env-get-outdated", my_node_purger.get_outdated_docnames)

		.. versionadded:: 2.7.0

		:param app: The Sphinx application.
		:param env: The Sphinx build environment.
		:param added: A set of newly added documents.
		:param changed: A set of document names whose content has changed.
		:param removed: A set of document names which have been removed.
		"""

		if not hasattr(env, self.attr_name):
			return []

		return list({todo["docname"] for todo in getattr(env, self.attr_name)})

	def add_node(self, env: BuildEnvironment, node: Node, targetnode: Node, lineno: int) -> None:
		"""
		Add a node.

		:param env: The Sphinx build environment.
		:param node:
		:param targetnode:
		:param lineno:
		"""

		if not hasattr(env, self.attr_name):
			setattr(env, self.attr_name, [])

		all_nodes = getattr(env, self.attr_name)

		all_nodes.append({
				"docname": env.docname,
				"lineno": lineno,
				"installation_node": node.deepcopy(),
				"target": targetnode,
				})


def singleton(name: str) -> object:
	"""
	Factory function to return a string singleton.

	:param name: The name of the singleton.
	"""

	name = str(name)

	class Singleton:
		_singleton = None

		def __new__(cls):
			if Singleton._singleton is None:
				Singleton._singleton = super().__new__(cls)
			return Singleton._singleton

		def __repr__(self) -> str:
			return name

		def __str__(self) -> str:
			return name

	Singleton.__name__ = name
	Singleton.__doc__ = f"Singleton {name}"
	return Singleton()


no_default = singleton("no_default")


class NoMatchError(ValueError):
	"""
	Raised when no matching values were found in :func:`~.get_first_matching`.

	.. versionadded:: 0.7.0
	"""


def get_first_matching(
		condition: Callable[[Any], bool],
		iterable: Iterable[_T],
		default: _T = no_default  # type: ignore[assignment]
		) -> _T:
	"""
	Returns the first value in ``iterable`` that meets ``condition``, or ``default`` if none match.

	.. versionadded:: 0.7.0

	:param condition: The condition to evaluate.
	:param iterable:
	:param default: The default value to return if no values in ``iterable`` match.
	"""

	if default is not no_default:
		if not condition(default):
			raise ValueError("The condition must evaluate to True for the default value.")

		iterable = (*iterable, default)

	for match in iterable:
		if condition(match):
			return match

	raise NoMatchError(f"No matching values for '{condition}' in {iterable}")


def escape_trailing__(string: str) -> str:
	"""
	Returns the given string with trailing underscores escaped to prevent Sphinx treating them as references.

	.. versionadded:: 0.8.0

	:param string:
	"""

	if string.endswith('_'):
		return f"{string[:-1]}\\_"
	return string


def code_repr(obj: Any) -> str:
	"""
	Returns the repr of the given object as reStructuredText inline code.

	.. versionadded:: 0.9.0

	:param obj:
	"""

	return f"``{obj!r}``"


class SphinxExtMetadata(TypedDict, total=False):
	"""
	:class:`typing.TypedDict` representing the metadata dictionary returned by
	Sphinx extensions' ``setup`` functions.

	This is treated by Sphinx as metadata of the extension.
	"""  # noqa: D400

	version: str
	"""
	A string that identifies the extension version.

	It is used for extension version requirement checking and informational purposes.

	If not given, ``'unknown version'`` is substituted.
	"""

	env_version: int
	"""
	An integer that identifies the version of env data structure if the extension stores any data to environment.

	It is used to detect the data structure has been changed from last build.
	Extensions have to increment the version when data structure has changed.

	If not given, Sphinx considers the extension does not stores any data to environment.
	"""

	parallel_read_safe: bool
	"""
	A boolean that specifies if parallel reading of source files can be used when the extension is loaded.

	It defaults to :py:obj:`False`, i.e. you have to explicitly specify your extension
	to be parallel-read-safe after checking that it is.
	"""

	parallel_write_safe: bool
	"""
	A boolean that specifies if parallel writing of output files can be used when the extension is loaded.

	Since extensions usually don’t negatively influence the process, this defaults to :py:obj:`True`.
	"""


SetupFunc = Callable[[Sphinx], Optional[SphinxExtMetadata]]
"""
Type annotation for Sphinx extensions' ``setup`` functions.

.. versionadded:: 1.9.0
"""


def unknown_module_warning(documenter: Documenter) -> None:
	"""
	Log a warning that the module to import the object from is unknown.

	.. versionadded:: 0.2.0

	:param documenter:
	"""

	msg = __(
			"don't know which module to import for autodocumenting %r "
			'(try placing a "module" or "currentmodule" directive in the document, '
			"or giving an explicit module name)"
			)

	logger.warning(msg % documenter.name, type="autodoc")


def filter_members_warning(member: Any, exception: Exception) -> None:
	"""
	Log a warning when filtering members.

	.. versionadded:: 0.2.0

	:param member:
	:param exception:
	"""

	logger.warning(
			__("autodoc: failed to determine %r to be documented, the following exception was raised:\n%s"),
			member,
			exception,
			type="autodoc"
			)


class Param(TypedDict):
	"""
	:class:`~typing.TypedDict` to represent a parameter parsed from a class or function's docstring.

	.. versionadded:: 0.8.0
	"""

	#: The docstring of the parameter.
	doc: List[str]

	#: The type of the parameter.
	type: str  # noqa: A003  # pylint: disable=redefined-builtin


_identifier_pattern = r"[A-Za-z_]\w*"

typed_param_regex: Pattern[str] = re.compile(
		fr"^:(param|parameter|arg|argument)\s*({_identifier_pattern}\s+)({_identifier_pattern}\s*):\s*(.*)",
		flags=re.ASCII,
		)
"""
Regex to match ``:param <type> <name>: <docstring>`` flags.

.. versionadded:: 0.8.0
"""

untyped_param_regex: Pattern[str] = re.compile(
		fr"^:(param|parameter|arg|argument)\s*({_identifier_pattern}\s*):\s*(.*)",
		flags=re.ASCII,
		)
"""
Regex to match ``:param <name>: <docstring>`` flags.

.. versionadded:: 0.8.0
"""

typed_flag_regex: Pattern[str] = re.compile(
		fr"^:(paramtype|type)\s*({_identifier_pattern}\s*):\s*(.*)",
		flags=re.ASCII,
		)
"""
Regex to match ``:type <name>: <type>`` flags.

.. versionadded:: 0.8.0
"""


def parse_parameters(lines: List[str], tab_size: int = 8) -> Tuple[Dict[str, Param], List[str], List[str]]:
	"""
	Parse parameters from the docstring of a class/function.

	.. versionadded:: 0.8.0

	:param lines: The lines of the docstring
	:param tab_size:

	:return: A mapping of parameter names to their docstrings and types, a list of docstring lines that
		appeared before the parameters, and the list of docstring lines that appear after the parameters.
	"""

	a_tab = ' ' * tab_size

	params: Dict[str, Param] = {}
	last_arg: Optional[str] = None

	pre_output: List[str] = []
	post_output: List[str] = []

	def add_empty(param_name: str) -> None:
		if param_name not in params:
			params[param_name] = {"doc": [], "type": ''}

	for line in lines:

		if post_output:
			post_output.append(line)
			continue

		# pylint: disable=loop-global-usage
		typed_m = typed_param_regex.match(line)
		untyped_m = untyped_param_regex.match(line)
		type_only_m = typed_flag_regex.match(line)
		# pylint: enable=loop-global-usage

		if typed_m:
			last_arg = typed_m.group(3).strip()
			add_empty(cast(str, last_arg))
			params[last_arg]["doc"] = [typed_m.group(4)]
			params[last_arg]["type"] = typed_m.group(2).strip()

		elif untyped_m:
			last_arg = untyped_m.group(2).strip()
			add_empty(cast(str, last_arg))
			params[last_arg]["doc"] = [untyped_m.group(3)]

		elif type_only_m:
			add_empty(type_only_m.group(2))
			params[type_only_m.group(2)]["type"] = type_only_m.group(3)

		elif line.startswith(a_tab) and last_arg is not None:
			params[last_arg]["doc"].append(line)

		elif last_arg is None:
			pre_output.append(line)

		else:
			post_output.append(line)

	return params, pre_output, post_output


def is_namedtuple(obj: Any) -> bool:
	"""
	Returns whether the given object is a :func:`collections.namedtuple` class.

	.. versionadded:: 0.8.0

	:param obj:
	"""

	return isinstance(obj, type) and issubclass(obj, tuple) and hasattr(obj, "_fields")


def allow_subclass_add(app: Sphinx, *documenters: Type[Documenter]) -> None:
	"""
	Add the given autodocumenters, but only if a subclass of it is not already registered.

	This allows other libraries to extend the autodocumenters.

	.. versionadded:: 0.8.0

	:param app: The Sphinx application.
	:param documenters:
	"""

	for cls in documenters:
		existing_documenter = app.registry.documenters.get(cls.objtype)
		if existing_documenter is None or not issubclass(existing_documenter, cls):
			app.add_autodocumenter(cls, override=True)


def baseclass_is_private(obj: Type) -> bool:
	"""
	Returns :py:obj:`True` if the first and only base class starts with a double underscore.

	:param obj:
	"""

	if hasattr(obj, "__bases__") and len(obj.__bases__) == 1:
		return obj.__bases__[0].__name__.startswith("__")
	return False


def metadata_add_version(func: SetupFunc) -> SetupFunc:
	"""
	Internal decorator for Sphinx ``setup`` functions to add the ``sphinx-toolbox``
	version number to the returned metadata dict.

	.. versionadded:: 1.9.0

	:param func:
	"""  # noqa: D400

	@functools.wraps(func)
	def wrapper(app: Sphinx) -> SphinxExtMetadata:

		# this package
		from sphinx_toolbox import __version__

		ret = func(app) or {}
		ret["version"] = __version__
		return ret

	return wrapper


def add_nbsp_substitution(config: sphinx.config.Config) -> None:
	"""
	Adds the ``|nbsp|`` substitution directive to the reStructuredText prolog.

	.. versionadded:: 2.1.0

	:param config:
	"""

	nbsp_sub = ".. |nbsp| unicode:: 0xA0\n   :trim:"

	if not config.rst_prolog:
		config.rst_prolog = ''  # type: ignore[attr-defined]

	if nbsp_sub not in config.rst_prolog:
		config.rst_prolog = '\n'.join([config.rst_prolog, '', nbsp_sub])  # type: ignore[attr-defined]


_OBJTYPES_CSS_FALLBACKS = {
		"namedtuple": "class",
		"protocol": "class",
		"typeddict": "class",
		}

# From https://github.com/mansenfranzen/autodoc_pydantic/pull/86/files
# MIT Licensed
# Copyright (c) 2021 Franz Wöllert


def add_fallback_css_class(
		objtypes_css_fallbacks: Dict[str, str]
		) -> Callable[[Sphinx, str, str, desc_content], None]:
	"""
	Registers a transform which will edit the CSS classes of documented objects based on their ``objtype``.

	:param objtypes_css_fallbacks: A mapping of Sphinx objtypes to the CSS class which should be added to them.
		The class is usually the ``objtype`` attribute from the documenter's parent class.

	.. versionadded:: 2.16.0

	Used as follows:

	.. code-block:: python

		app.connect("object-description-transform", add_fallback_css_class({"typeddict": "class"}))

	This will apply the transformation to documented objects with the ``typeddict`` CSS class
	by adding the ``class`` CSS class.

	:param objtypes_css_fallbacks:
	"""

	def func(
			app: Sphinx,
			domain: str,
			objtype: str,
			contentnode: desc_content,
			) -> None:

		if objtype not in objtypes_css_fallbacks:
			return

		classes = contentnode.parent.attributes["classes"]

		# for older sphinx versions, add objtype explicitly
		if sphinx.version_info < (3, 6):
			classes.append(objtype)

		idx = classes.index(objtype)
		fallback = objtypes_css_fallbacks[objtype]
		classes.insert(idx, fallback)

	return func


if TYPE_CHECKING:

	class Config(sphinx.config.Config):
		project: Any
		author: Any
		project_copyright: str
		copyright: str  # noqa: A003  # pylint: disable=redefined-builtin
		version: Any
		release: Any
		today: Any
		today_fmt: str
		language: str
		locale_dirs: Any
		figure_language_filename: str
		gettext_allow_fuzzy_translations: Any
		master_doc: Any
		root_doc: Any
		source_suffix: Any
		source_encoding: Any
		exclude_patterns: Any
		default_role: str
		add_function_parentheses: Any
		add_module_names: Any
		trim_footnote_reference_space: Any
		show_authors: Any
		pygments_style: Any
		highlight_language: Any
		highlight_options: Any
		templates_path: Any
		template_bridge: Any
		keep_warnings: Any
		suppress_warnings: Any
		modindex_common_prefix: Any
		rst_epilog: str
		rst_prolog: str
		trim_doctest_flags: Any
		primary_domain: Any
		needs_sphinx: str
		needs_extensions: Any
		manpages_url: Any
		nitpicky: Any
		nitpick_ignore: Any
		nitpick_ignore_regex: Any
		numfig: Any
		numfig_secnum_depth: Any
		numfig_format: Any
		math_number_all: Any
		math_eqref_format: str
		math_numfig: Any
		tls_verify: Any
		tls_cacerts: Any
		user_agent: str
		smartquotes: Any
		smartquotes_action: Any
		smartquotes_excludes: Any
		config_values: Dict[str, Tuple]
		overrides: Dict
		values: Dict[str, Tuple]
		setup: Optional[Callable]
		extensions: List[str]
		latex_engine: str
		latex_documents: Any
		latex_logo: str
		latex_appendices: Any
		latex_use_latex_multicolumn: Any
		latex_use_xindy: bool
		latex_toplevel_sectioning: str
		latex_domain_indices: List
		latex_show_urls: Any
		latex_show_pagerefs: Any
		latex_elements: Any
		latex_additional_files: Any
		latex_theme: str
		latex_theme_options: Any
		latex_theme_path: Any
		latex_docclass: Any
		html_theme: Any
		html_theme_path: Any
		html_theme_options: Any
		html_title: str
		html_short_title: Any
		html_style: str
		html_logo: str
		html_favicon: str
		html_css_files: Any
		html_js_files: Any
		html_static_path: Any
		html_extra_path: Any
		html_last_updated_fmt: str
		html_sidebars: Any
		html_additional_pages: Any
		html_domain_indices: List
		html_add_permalinks: Any
		html_permalinks: Any
		html_permalinks_icon: Any
		html_use_index: Any
		html_split_index: Any
		html_copy_source: Any
		html_show_sourcelink: Any
		html_sourcelink_suffix: Any
		html_use_opensearch: Any
		html_file_suffix: str
		html_link_suffix: str
		html_show_copyright: Any
		html_show_search_summary: Any
		html_show_sphinx: Any
		html_context: Any
		html_output_encoding: Any
		html_compact_lists: Any
		html_secnumber_suffix: Any
		html_search_language: str
		html_search_options: Any
		html_search_scorer: Any
		html_scaled_image_link: Any
		html_baseurl: Any
		html_codeblock_linenos_style: str
		html_math_renderer: Any
		html4_writer: Any
else:
	Config = sphinx.config.Config

if sphinx.version_info[0] >= 5 or TYPE_CHECKING:

	class RemovedInSphinx50Warning(RuntimeError):
		pass

	def prepare_docstring(s: str, ignore: Optional[int] = None, tabsize: int = 8) -> List[str]:

		if ignore is None:
			ignore = 1
		else:
			raise TypeError("The 'ignore' argument to prepare_docstring() was removed in Sphinx 5.0")

		lines = s.expandtabs(tabsize).splitlines()

		# Find minimum indentation of any non-blank lines after ignored lines.
		margin = sys.maxsize
		for line in lines[ignore:]:
			content = len(line.lstrip())
			if content:
				indent = len(line) - content
				margin = min(margin, indent)

		# Remove indentation from ignored lines.
		for i in range(ignore):
			if i < len(lines):
				lines[i] = lines[i].lstrip()

		if margin < sys.maxsize:
			for i in range(ignore, len(lines)):
				lines[i] = lines[i][margin:]

		# Remove any leading blank lines.
		while lines and not lines[0]:
			lines.pop(0)

		# make sure there is an empty line at the end
		if lines and lines[-1]:
			lines.append('')

		return lines

else:
	# 3rd party
	from sphinx.deprecation import RemovedInSphinx50Warning  # type: ignore[attr-defined,no-redef]  # noqa: F401
	from sphinx.util.docstrings import prepare_docstring  # noqa: F401

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/utils.py
NikolaBlagojevic/pyrecodes
#!/usr/bin/env python3
#
#  wikipedia.py
"""
Sphinx extension to create links to Wikipedia articles.

.. versionadded:: 0.2.0
.. extensions:: sphinx_toolbox.wikipedia


Configuration
--------------

.. latex:vspace:: -5px

.. confval:: wikipedia_lang
	:type: :class:`str`
	:required: False
	:default: ``'en'``

	The Wikipedia language to use for :rst:role:`wikipedia` roles.

	.. versionadded:: 0.2.0


Usage
------

.. latex:vspace:: -5px

.. rst:role:: wikipedia

	Role which shows a link to the given article on Wikipedia.

	The title and language can be customised.


	:bold-title:`Example`

	.. rest-example::

		:wikipedia:`Sphinx`

		:wikipedia:`mythical creature <Sphinx>`

		:wikipedia:`Answer to the Ultimate Question of Life, the Universe, and Everything <:de:42 (Antwort)>`

	.. only:: html

		.. rest-example::

			:wikipedia:`:zh:斯芬克斯`


API Reference
----------------
"""
#
#  Copyright © 2020-2021 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
#  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
#  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
#  OR OTHER DEALINGS IN THE SOFTWARE.
#
#  Based on https://github.com/quiver/sphinx-ext-wikipedia
#  BSD Licensed
#
#  Parts of the docstrings based on https://docutils.sourceforge.io/docs/howto/rst-roles.html
#

# stdlib
import re
from typing import Dict, List, Tuple
from urllib.parse import quote

# 3rd party
from apeye.url import URL
from docutils import nodes
from docutils.nodes import system_message
from docutils.parsers.rst.states import Inliner
from sphinx.application import Sphinx
from sphinx.util.nodes import split_explicit_title

# this package
from sphinx_toolbox.utils import SphinxExtMetadata, metadata_add_version

__all__ = ("make_wikipedia_link", "setup")

base_url = "https://%s.wikipedia.org/wiki"
_wiki_lang_re = re.compile(":(.*?):(.*)")


def _get_wikipedia_lang(inliner: Inliner) -> str:  # pragma: no cover
	return inliner.document.settings.env.config.wikipedia_lang


def make_wikipedia_link(
		name: str,
		rawtext: str,
		text: str,
		lineno: int,
		inliner: Inliner,
		options: Dict = {},
		content: List[str] = []
		) -> Tuple[List[nodes.reference], List[system_message]]:
	"""
	Adds a link to the given article on :wikipedia:`Wikipedia`.

	:param name: The local name of the interpreted role, the role name actually used in the document.
	:param rawtext: A string containing the entire interpreted text input, including the role and markup.
	:param text: The interpreted text content.
	:param lineno: The line number where the interpreted text begins.
	:param inliner: The :class:`docutils.parsers.rst.states.Inliner` object that called :func:`~.source_role`.
		It contains the several attributes useful for error reporting and document tree access.
	:param options: A dictionary of directive options for customization (from the ``role`` directive),
		to be interpreted by the function.
		Used for additional attributes for the generated elements and other functionality.
	:param content: A list of strings, the directive content for customization (from the ``role`` directive).
		To be interpreted by the function.

	:return: A list containing the created node, and a list containing any messages generated during the function.
	"""

	text = nodes.unescape(text)
	has_explicit, title, target = split_explicit_title(text)

	m = _wiki_lang_re.match(target)

	if m:
		lang, target = m.groups()
		if not has_explicit:
			title = target
	else:
		lang = _get_wikipedia_lang(inliner)

	ref = URL(base_url % lang) / quote(target.replace(' ', '_'), safe='')

	node = nodes.reference(rawtext, title, refuri=str(ref), **options)
	return [node], []


@metadata_add_version
def setup(app: Sphinx) -> SphinxExtMetadata:
	"""
	Setup :mod:`sphinx_toolbox.wikipedia`.

	.. versionadded:: 1.0.0

	:param app: The Sphinx application.
	"""

	app.add_role("wikipedia", make_wikipedia_link)
	app.add_config_value("wikipedia_lang", "en", "env", [str])

	return {"parallel_read_safe": True}

env_pyrecodes/lib/python3.9/site-packages/sphinx_toolbox/wikipedia.py
NikolaBlagojevic/pyrecodes
"""
    sphinxcontrib.applehelp
    ~~~~~~~~~~~~~~~~~~~~~~~

    Build Apple help books.

    :copyright: Copyright 2007-2019 by the Sphinx team, see README.
    :license: BSD, see LICENSE for details.
"""

import plistlib
import shlex
import subprocess
from os import environ
from os import path
from subprocess import CalledProcessError, PIPE, STDOUT
from typing import Any, Dict

import sphinx
from sphinx.application import Sphinx
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.errors import SphinxError
from sphinx.locale import get_translation
from sphinx.util import logging
from sphinx.util.fileutil import copy_asset, copy_asset_file
from sphinx.util.matching import Matcher
from sphinx.util.osutil import ensuredir, make_filename

if sphinx.version_info[:2] >= (6, 1):
    from sphinx.util.display import SkipProgressMessage, progress_message
else:
    from sphinx.util import (  # type: ignore[attr-defined,no-redef]
        SkipProgressMessage, progress_message
    )

__version__ = '1.0.4'
__version_info__ = (1, 0, 4)

package_dir = path.abspath(path.dirname(__file__))
template_dir = path.join(package_dir, 'templates')

__ = get_translation(__name__, 'console')
logger = logging.getLogger(__name__)


class AppleHelpIndexerFailed(SphinxError):
    category = __('Help indexer failed')


class AppleHelpCodeSigningFailed(SphinxError):
    category = __('Code signing failed')


class AppleHelpBuilder(StandaloneHTMLBuilder):
    """
    Builder that outputs an Apple help book.  Requires Mac OS X as it relies
    on the ``hiutil`` command line tool.
    """
    name = 'applehelp'
    epilog = __('The help book is in %(outdir)s.\n'
                'Note that won\'t be able to view it unless you put it in '
                '~/Library/Documentation/Help or install it in your application '
                'bundle.')

    # don't copy the reST source
    copysource = False
    supported_image_types = ['image/png', 'image/gif', 'image/jpeg',
                             'image/tiff', 'image/jp2', 'image/svg+xml']

    # don't add links
    add_permalinks = False

    # this is an embedded HTML format
    embedded = True

    # don't generate the search index or include the search page
    search = False

    def init(self) -> None:
        super().init()
        # the output files for HTML help must be .html only
        self.out_suffix = '.html'
        self.link_suffix = '.html'

        if self.config.applehelp_bundle_id is None:
            raise SphinxError(__('You must set applehelp_bundle_id before '
                                 'building Apple Help output'))

        self.bundle_path = path.join(self.outdir,  # type: ignore
                                     self.config.applehelp_bundle_name + '.help')
        self.outdir = path.join(self.bundle_path,
                                'Contents',
                                'Resources',
                                self.config.applehelp_locale + '.lproj')

    def handle_finish(self) -> None:
        super().handle_finish()

        self.finish_tasks.add_task(self.copy_localized_files)
        self.finish_tasks.add_task(self.build_helpbook)

    @progress_message(__('copying localized files'))
    def copy_localized_files(self) -> None:
        source_dir = path.join(self.confdir, self.config.applehelp_locale + '.lproj')  # type: ignore  # NOQA
        target_dir = self.outdir

        if path.isdir(source_dir):
            excluded = Matcher(self.config.exclude_patterns + ['**/.*'])
            copy_asset(source_dir, target_dir, excluded,
                       context=self.globalcontext, renderer=self.templates)

    def build_helpbook(self) -> None:
        contents_dir = path.join(self.bundle_path, 'Contents')
        resources_dir = path.join(contents_dir, 'Resources')
        language_dir = path.join(resources_dir,
                                 self.config.applehelp_locale + '.lproj')
        ensuredir(language_dir)

        self.build_info_plist(contents_dir)
        self.copy_applehelp_icon(resources_dir)
        self.build_access_page(language_dir)
        self.build_helpindex(language_dir)

        if self.config.applehelp_codesign_identity:
            self.do_codesign()

    @progress_message(__('writing Info.plist'))
    def build_info_plist(self, contents_dir: str) -> None:
        """Construct the Info.plist file."""
        info_plist = {
            'CFBundleDevelopmentRegion': self.config.applehelp_dev_region,
            'CFBundleIdentifier': self.config.applehelp_bundle_id,
            'CFBundleInfoDictionaryVersion': '6.0',
            'CFBundlePackageType': 'BNDL',
            'CFBundleShortVersionString': self.config.release,
            'CFBundleSignature': 'hbwr',
            'CFBundleVersion': self.config.applehelp_bundle_version,
            'HPDBookAccessPath': '_access.html',
            'HPDBookIndexPath': 'search.helpindex',
            'HPDBookTitle': self.config.applehelp_title,
            'HPDBookType': '3',
            'HPDBookUsesExternalViewer': False,
        }

        if self.config.applehelp_icon is not None:
            info_plist['HPDBookIconPath'] = path.basename(self.config.applehelp_icon)

        if self.config.applehelp_kb_url is not None:
            info_plist['HPDBookKBProduct'] = self.config.applehelp_kb_product
            info_plist['HPDBookKBURL'] = self.config.applehelp_kb_url

        if self.config.applehelp_remote_url is not None:
            info_plist['HPDBookRemoteURL'] = self.config.applehelp_remote_url

        with open(path.join(contents_dir, 'Info.plist'), 'wb') as f:
            plistlib.dump(info_plist, f)

    def copy_applehelp_icon(self, resources_dir: str) -> None:
        """Copy the icon, if one is supplied."""
        if self.config.applehelp_icon:

            try:
                with progress_message(__('copying icon... ')):
                    applehelp_icon = path.join(self.srcdir, self.config.applehelp_icon)
                    copy_asset_file(applehelp_icon, resources_dir)
            except Exception as err:
                logger.warning(__('cannot copy icon file %r: %s'), applehelp_icon, err)

    @progress_message(__('building access page'))
    def build_access_page(self, language_dir: str) -> None:
        """Build the access page."""
        context = {
            'toc': self.config.master_doc + self.out_suffix,
            'title': self.config.applehelp_title,
        }
        copy_asset_file(path.join(template_dir, '_access.html_t'), language_dir, context)

    @progress_message(__('generating help index'))
    def build_helpindex(self, language_dir: str) -> None:
        """Generate the help index."""
        args = [
            self.config.applehelp_indexer_path,
            '-Cf',
            path.join(language_dir, 'search.helpindex'),
            language_dir
        ]

        if self.config.applehelp_index_anchors is not None:
            args.append('-a')

        if self.config.applehelp_min_term_length is not None:
            args += ['-m', '%s' % self.config.applehelp_min_term_length]

        if self.config.applehelp_stopwords is not None:
            args += ['-s', self.config.applehelp_stopwords]

        if self.config.applehelp_locale is not None:
            args += ['-l', self.config.applehelp_locale]

        if self.config.applehelp_disable_external_tools:
            raise SkipProgressMessage(__('you will need to index this help book with:\n  %s'),
                                      ' '.join([shlex.quote(arg) for arg in args]))
        else:
            try:
                subprocess.run(args, stdout=PIPE, stderr=STDOUT, check=True)
            except OSError:
                raise AppleHelpIndexerFailed(__('Command not found: %s') % args[0])
            except CalledProcessError as exc:
                raise AppleHelpIndexerFailed(exc.stdout)

    @progress_message(__('signing help book'))
    def do_codesign(self) -> None:
        """If we've been asked to, sign the bundle."""
        args = [
            self.config.applehelp_codesign_path,
            '-s', self.config.applehelp_codesign_identity,
            '-f'
        ]

        args += self.config.applehelp_codesign_flags

        args.append(self.bundle_path)

        if self.config.applehelp_disable_external_tools:
            raise SkipProgressMessage(__('you will need to sign this help book with:\n  %s'),
                                      ' '.join([shlex.quote(arg) for arg in args]))
        else:
            try:
                subprocess.run(args, stdout=PIPE, stderr=STDOUT, check=True)
            except OSError:
                raise AppleHelpCodeSigningFailed(__('Command not found: %s') % args[0])
            except CalledProcessError as exc:
                raise AppleHelpCodeSigningFailed(exc.stdout)


def setup(app: Sphinx) -> Dict[str, Any]:
    app.setup_extension('sphinx.builders.html')
    app.add_builder(AppleHelpBuilder)
    app.add_message_catalog(__name__, path.join(package_dir, 'locales'))

    app.add_config_value('applehelp_bundle_name',
                         lambda self: make_filename(self.project), 'applehelp')
    app.add_config_value('applehelp_bundle_id', None, 'applehelp', [str])
    app.add_config_value('applehelp_dev_region', 'en-us', 'applehelp')
    app.add_config_value('applehelp_bundle_version', '1', 'applehelp')
    app.add_config_value('applehelp_icon', None, 'applehelp', [str])
    app.add_config_value('applehelp_kb_product',
                         lambda self: '%s-%s' % (make_filename(self.project), self.release),
                         'applehelp')
    app.add_config_value('applehelp_kb_url', None, 'applehelp', [str])
    app.add_config_value('applehelp_remote_url', None, 'applehelp', [str])
    app.add_config_value('applehelp_index_anchors', False, 'applehelp', [str])
    app.add_config_value('applehelp_min_term_length', None, 'applehelp', [str])
    app.add_config_value('applehelp_stopwords',
                         lambda self: self.language or 'en', 'applehelp')
    app.add_config_value('applehelp_locale', lambda self: self.language or 'en', 'applehelp')
    app.add_config_value('applehelp_title', lambda self: self.project + ' Help', 'applehelp')
    app.add_config_value('applehelp_codesign_identity',
                         lambda self: environ.get('CODE_SIGN_IDENTITY', None),
                         'applehelp')
    app.add_config_value('applehelp_codesign_flags',
                         lambda self: shlex.split(environ.get('OTHER_CODE_SIGN_FLAGS', '')),
                         'applehelp')
    app.add_config_value('applehelp_indexer_path', '/usr/bin/hiutil', 'applehelp')
    app.add_config_value('applehelp_codesign_path', '/usr/bin/codesign', 'applehelp')
    app.add_config_value('applehelp_disable_external_tools', False, 'applehelp')

    return {
        'version': __version__,
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinxcontrib/applehelp/__init__.py
NikolaBlagojevic/pyrecodes
"""
    sphinxcontrib.htmlhelp
    ~~~~~~~~~~~~~~~~~~~~~~

    Build HTML help support files.

    :copyright: Copyright 2007-2019 by the Sphinx team, see README.
    :license: BSD, see LICENSE for details.
"""

import html
import os
from os import path
from typing import Any, Dict, List, Set, Tuple, Type

from docutils import nodes
from docutils.nodes import Element, Node, document

import sphinx
from sphinx import addnodes
from sphinx.application import Sphinx
from sphinx.builders.html import StandaloneHTMLBuilder
from sphinx.config import Config
from sphinx.environment.adapters.indexentries import IndexEntries
from sphinx.locale import get_translation
from sphinx.util import logging
from sphinx.util.fileutil import copy_asset_file
from sphinx.util.nodes import NodeMatcher
from sphinx.util.osutil import make_filename_from_project, relpath
from sphinx.util.template import SphinxRenderer
from sphinx.writers.html import HTMLTranslator

if sphinx.version_info[:2] >= (6, 1):
    from sphinx.util.display import progress_message
else:
    from sphinx.util import progress_message  # type: ignore[attr-defined,no-redef]

__version__ = '2.0.1'
__version_info__ = (2, 0, 1)

logger = logging.getLogger(__name__)
__ = get_translation(__name__, 'console')

package_dir = path.abspath(path.dirname(__file__))
template_dir = path.join(package_dir, 'templates')


# The following list includes only languages supported by Sphinx. See
# https://docs.microsoft.com/en-us/previous-versions/windows/embedded/ms930130(v=msdn.10)
# for more.
chm_locales = {
    # lang:   LCID,  encoding
    'ca':    (0x403, 'cp1252'),
    'cs':    (0x405, 'cp1250'),
    'da':    (0x406, 'cp1252'),
    'de':    (0x407, 'cp1252'),
    'en':    (0x409, 'cp1252'),
    'es':    (0x40a, 'cp1252'),
    'et':    (0x425, 'cp1257'),
    'fa':    (0x429, 'cp1256'),
    'fi':    (0x40b, 'cp1252'),
    'fr':    (0x40c, 'cp1252'),
    'hr':    (0x41a, 'cp1250'),
    'hu':    (0x40e, 'cp1250'),
    'it':    (0x410, 'cp1252'),
    'ja':    (0x411, 'cp932'),
    'ko':    (0x412, 'cp949'),
    'lt':    (0x427, 'cp1257'),
    'lv':    (0x426, 'cp1257'),
    'nl':    (0x413, 'cp1252'),
    'no_NB': (0x414, 'cp1252'),
    'pl':    (0x415, 'cp1250'),
    'pt_BR': (0x416, 'cp1252'),
    'ru':    (0x419, 'windows-1251'),  # emit as <meta chaset='...'>
    'sk':    (0x41b, 'cp1250'),
    'sl':    (0x424, 'cp1250'),
    'sv':    (0x41d, 'cp1252'),
    'tr':    (0x41f, 'cp1254'),
    'uk_UA': (0x422, 'cp1251'),
    'zh_CN': (0x804, 'cp936'),
    'zh_TW': (0x404, 'cp950'),
}


def chm_htmlescape(s: str, quote: bool = True) -> str:
    """
    chm_htmlescape() is a wrapper of html.escape().
    .hhc/.hhk files don't recognize hex escaping, we need convert
    hex escaping to decimal escaping. for example: ``&#x27;`` -> ``&#39;``
    html.escape() may generates a hex escaping ``&#x27;`` for single
    quote ``'``, this wrapper fixes this.
    """
    s = html.escape(s, quote)
    s = s.replace('&#x27;', '&#39;')    # re-escape as decimal
    return s


class ToCTreeVisitor(nodes.NodeVisitor):
    def __init__(self, document: document) -> None:
        super().__init__(document)
        self.body = []  # type: List[str]
        self.depth = 0

    def append(self, text: str) -> None:
        self.body.append(text)

    def astext(self) -> str:
        return '\n'.join(self.body)

    def unknown_visit(self, node: Node) -> None:
        pass

    def unknown_departure(self, node: Node) -> None:
        pass

    def visit_bullet_list(self, node: Element) -> None:
        if self.depth > 0:
            self.append('<UL>')

        self.depth += 1

    def depart_bullet_list(self, node: Element) -> None:
        self.depth -= 1
        if self.depth > 0:
            self.append('</UL>')

    def visit_list_item(self, node: Element) -> None:
        self.append('<LI> <OBJECT type="text/sitemap">')
        self.depth += 1

    def depart_list_item(self, node: Element) -> None:
        self.depth -= 1

    def visit_reference(self, node: Element) -> None:
        title = chm_htmlescape(node.astext(), True)
        self.append('    <param name="Name" value="%s">' % title)
        self.append('    <param name="Local" value="%s">' % node['refuri'])
        self.append('</OBJECT>')
        raise nodes.SkipNode


class HTMLHelpBuilder(StandaloneHTMLBuilder):
    """
    Builder that also outputs Windows HTML help project, contents and
    index files.  Adapted from the original Doc/tools/prechm.py.
    """
    name = 'htmlhelp'
    epilog = __('You can now run HTML Help Workshop with the .htp file in '
                '%(outdir)s.')

    # don't copy the reST source
    copysource = False
    supported_image_types = ['image/png', 'image/gif', 'image/jpeg']

    # don't add links
    add_permalinks = False
    # don't add sidebar etc.
    embedded = True

    # don't generate search index or include search page
    search = False

    lcid = 0x409
    encoding = 'cp1252'

    def init(self) -> None:
        # the output files for HTML help is .html by default
        self.out_suffix = '.html'
        self.link_suffix = '.html'
        super().init()
        # determine the correct locale setting
        locale = chm_locales.get(self.config.language)
        if locale is not None:
            self.lcid, self.encoding = locale

    @property
    def default_translator_class(self) -> "Type[nodes.NodeVisitor]":  # type: ignore
        # Use HTML4 writer always
        return HTMLTranslator

    def prepare_writing(self, docnames: Set[str]) -> None:
        super().prepare_writing(docnames)
        self.globalcontext['html5_doctype'] = False

    def update_page_context(self, pagename: str, templatename: str, ctx: Dict, event_arg: str) -> None:  # NOQA
        ctx['encoding'] = self.encoding

    def handle_finish(self) -> None:
        self.copy_stopword_list()
        self.build_project_file()
        self.build_toc_file()
        self.build_hhx(self.outdir, self.config.htmlhelp_basename)

    def write_doc(self, docname: str, doctree: document) -> None:
        for node in doctree.traverse(nodes.reference):
            # add ``target=_blank`` attributes to external links
            if node.get('internal') is None and 'refuri' in node:
                node['target'] = '_blank'

        super().write_doc(docname, doctree)

    def render(self, name: str, context: Dict) -> str:
        template = SphinxRenderer(template_dir)
        return template.render(name, context)

    @progress_message(__('copying stopword list'))
    def copy_stopword_list(self) -> None:
        """Copy a stopword list (.stp) to outdir.

        The stopword list contains a list of words the full text search facility
        shouldn't index.  Note that this list must be pretty small.  Different
        versions of the MS docs claim the file has a maximum size of 256 or 512
        bytes (including \r\n at the end of each line).  Note that "and", "or",
        "not" and "near" are operators in the search language, so no point
        indexing them even if we wanted to.
        """
        template = path.join(template_dir, 'project.stp')
        filename = path.join(self.outdir, self.config.htmlhelp_basename + '.stp')
        copy_asset_file(template, filename)

    @progress_message(__('writing project file'))
    def build_project_file(self) -> None:
        """Create a project file (.hhp) on outdir."""
        # scan project files
        project_files = []  # type: List[str]
        for root, dirs, files in os.walk(self.outdir):
            dirs.sort()
            files.sort()
            in_staticdir = root.startswith(path.join(self.outdir, '_static'))
            for fn in sorted(files):
                if (in_staticdir and not fn.endswith('.js')) or fn.endswith('.html'):
                    fn = relpath(path.join(root, fn), self.outdir)
                    project_files.append(fn.replace(os.sep, '\\'))

        filename = path.join(self.outdir, self.config.htmlhelp_basename + '.hhp')
        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
            context = {
                'outname': self.config.htmlhelp_basename,
                'title': self.config.html_title,
                'version': self.config.version,
                'project': self.config.project,
                'lcid': self.lcid,
                'master_doc': self.config.master_doc + self.out_suffix,
                'files': project_files,
            }
            body = self.render('project.hhp', context)
            f.write(body)

    @progress_message(__('writing TOC file'))
    def build_toc_file(self) -> None:
        """Create a ToC file (.hhp) on outdir."""
        filename = path.join(self.outdir, self.config.htmlhelp_basename + '.hhc')
        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
            toctree = self.env.get_and_resolve_doctree(self.config.master_doc, self,
                                                       prune_toctrees=False)
            visitor = ToCTreeVisitor(toctree)
            matcher = NodeMatcher(addnodes.compact_paragraph, toctree=True)
            for node in toctree.traverse(matcher):  # type: addnodes.compact_paragraph
                node.walkabout(visitor)

            context = {
                'body': visitor.astext(),
                'suffix': self.out_suffix,
                'short_title': self.config.html_short_title,
                'master_doc': self.config.master_doc,
                'domain_indices': self.domain_indices,
            }
            f.write(self.render('project.hhc', context))

    def build_hhx(self, outdir: str, outname: str) -> None:
        logger.info(__('writing index file...'))
        index = IndexEntries(self.env).create_index(self)
        filename = path.join(outdir, outname + '.hhk')
        with open(filename, 'w', encoding=self.encoding, errors='xmlcharrefreplace') as f:
            f.write('<UL>\n')

            def write_index(title: str, refs: List[Tuple[str, str]], subitems: List[Tuple[str, List[Tuple[str, str]]]]) -> None:  # NOQA
                def write_param(name: str, value: str) -> None:
                    item = '    <param name="%s" value="%s">\n' % (name, value)
                    f.write(item)
                title = chm_htmlescape(title, True)
                f.write('<LI> <OBJECT type="text/sitemap">\n')
                write_param('Keyword', title)
                if len(refs) == 0:
                    write_param('See Also', title)
                elif len(refs) == 1:
                    write_param('Local', refs[0][1])
                else:
                    for i, ref in enumerate(refs):
                        # XXX: better title?
                        write_param('Name', '[%d] %s' % (i, ref[1]))
                        write_param('Local', ref[1])
                f.write('</OBJECT>\n')
                if subitems:
                    f.write('<UL> ')
                    for subitem in subitems:
                        write_index(subitem[0], subitem[1], [])
                    f.write('</UL>')
            for (key, group) in index:
                for title, (refs, subitems, key_) in group:
                    write_index(title, refs, subitems)
            f.write('</UL>\n')


def default_htmlhelp_basename(config: Config) -> str:
    """Better default htmlhelp_basename setting."""
    return make_filename_from_project(config.project) + 'doc'


def setup(app: Sphinx) -> Dict[str, Any]:
    app.setup_extension('sphinx.builders.html')
    app.add_builder(HTMLHelpBuilder)
    app.add_message_catalog(__name__, path.join(package_dir, 'locales'))

    app.add_config_value('htmlhelp_basename', default_htmlhelp_basename, '')
    app.add_config_value('htmlhelp_file_suffix', None, 'html', [str])
    app.add_config_value('htmlhelp_link_suffix', None, 'html', [str])

    return {
        'version': __version__,
        'parallel_read_safe': True,
        'parallel_write_safe': True,
    }

env_pyrecodes/lib/python3.9/site-packages/sphinxcontrib/htmlhelp/__init__.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Structured Logging for Python
"""

from structlog import dev, processors, stdlib, testing, threadlocal, types
from structlog._base import BoundLoggerBase, get_context
from structlog._config import (
    configure,
    configure_once,
    get_config,
    get_logger,
    getLogger,
    is_configured,
    reset_defaults,
    wrap_logger,
)
from structlog._generic import BoundLogger
from structlog._log_levels import make_filtering_bound_logger
from structlog._loggers import (
    BytesLogger,
    BytesLoggerFactory,
    PrintLogger,
    PrintLoggerFactory,
)
from structlog.exceptions import DropEvent
from structlog.testing import ReturnLogger, ReturnLoggerFactory


try:
    from structlog import twisted
except ImportError:
    twisted = None  # type: ignore

try:
    from structlog import contextvars
except ImportError:
    contextvars = None  # type: ignore


__version__ = "21.5.0"

__title__ = "structlog"
# __doc__ is None when running with PYTHONOPTIMIZE=2 / -OO
__description__ = (__doc__ or "").strip()

__uri__ = "https://www.structlog.org/"

__author__ = "Hynek Schlawack"
__email__ = "hs@ox.cx"

__license__ = "MIT or Apache License, Version 2.0"
__copyright__ = "Copyright (c) 2013 " + __author__


__all__ = [
    "BoundLogger",
    "BoundLoggerBase",
    "BytesLogger",
    "BytesLoggerFactory",
    "DropEvent",
    "PrintLogger",
    "PrintLoggerFactory",
    "ReturnLogger",
    "ReturnLoggerFactory",
    "configure",
    "configure_once",
    "contextvars",
    "dev",
    "getLogger",
    "get_config",
    "get_context",
    "get_logger",
    "is_configured",
    "make_filtering_bound_logger",
    "processors",
    "reset_defaults",
    "stdlib",
    "testing",
    "threadlocal",
    "twisted",
    "types",
    "wrap_logger",
]

env_pyrecodes/lib/python3.9/site-packages/structlog/__init__.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Global state department.  Don't reload this module or everything breaks.
"""


import sys
import warnings

from typing import (
    Any,
    Callable,
    Dict,
    Iterable,
    Optional,
    Sequence,
    Type,
    cast,
)

from ._log_levels import make_filtering_bound_logger
from ._loggers import PrintLoggerFactory
from .dev import ConsoleRenderer, _use_colors, set_exc_info
from .processors import StackInfoRenderer, TimeStamper, add_log_level
from .types import BindableLogger, Context, Processor, WrappedLogger


"""
.. note::

   Any changes to these defaults must be reflected in `getting-started`.
"""
_BUILTIN_DEFAULT_PROCESSORS: Sequence[Processor] = [
    add_log_level,
    StackInfoRenderer(),
    set_exc_info,
    TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False),
    ConsoleRenderer(
        colors=_use_colors and sys.stdout is not None and sys.stdout.isatty()
    ),
]
_BUILTIN_DEFAULT_CONTEXT_CLASS = cast(Type[Context], dict)
_BUILTIN_DEFAULT_WRAPPER_CLASS = make_filtering_bound_logger(0)
_BUILTIN_DEFAULT_LOGGER_FACTORY = PrintLoggerFactory()
_BUILTIN_CACHE_LOGGER_ON_FIRST_USE = False


class _Configuration:
    """
    Global defaults.
    """

    is_configured: bool = False
    default_processors: Iterable[Processor] = _BUILTIN_DEFAULT_PROCESSORS[:]
    default_context_class: Type[Context] = _BUILTIN_DEFAULT_CONTEXT_CLASS
    default_wrapper_class: Any = _BUILTIN_DEFAULT_WRAPPER_CLASS
    logger_factory: Callable[
        ..., WrappedLogger
    ] = _BUILTIN_DEFAULT_LOGGER_FACTORY
    cache_logger_on_first_use: bool = _BUILTIN_CACHE_LOGGER_ON_FIRST_USE


_CONFIG = _Configuration()
"""
Global defaults used when arguments to `wrap_logger` are omitted.
"""


def is_configured() -> bool:
    """
    Return whether ``structlog`` has been configured.

    If `False`, ``structlog`` is running with builtin defaults.

    .. versionadded: 18.1
    """
    return _CONFIG.is_configured


def get_config() -> Dict[str, Any]:
    """
    Get a dictionary with the current configuration.

    .. note::

       Changes to the returned dictionary do *not* affect ``structlog``.

    .. versionadded: 18.1
    """
    return {
        "processors": _CONFIG.default_processors,
        "context_class": _CONFIG.default_context_class,
        "wrapper_class": _CONFIG.default_wrapper_class,
        "logger_factory": _CONFIG.logger_factory,
        "cache_logger_on_first_use": _CONFIG.cache_logger_on_first_use,
    }


def get_logger(*args: Any, **initial_values: Any) -> Any:
    """
    Convenience function that returns a logger according to configuration.

    >>> from structlog import get_logger
    >>> log = get_logger(y=23)
    >>> log.msg("hello", x=42)
    y=23 x=42 event='hello'

    :param args: *Optional* positional arguments that are passed unmodified to
        the logger factory.  Therefore it depends on the factory what they
        mean.
    :param initial_values: Values that are used to pre-populate your contexts.

    :returns: A proxy that creates a correctly configured bound logger when
        necessary. The type of that bound logger depends on your configuration
        and is `structlog.BoundLogger` by default.

    See `configuration` for details.

    If you prefer CamelCase, there's an alias for your reading pleasure:
    `structlog.getLogger`.

    .. versionadded:: 0.4.0
        *args*
    """
    return wrap_logger(None, logger_factory_args=args, **initial_values)


getLogger = get_logger
"""
CamelCase alias for `structlog.get_logger`.

This function is supposed to be in every source file -- we don't want it to
stick out like a sore thumb in frameworks like Twisted or Zope.
"""


def wrap_logger(
    logger: WrappedLogger,
    processors: Optional[Iterable[Processor]] = None,
    wrapper_class: Optional[Type[BindableLogger]] = None,
    context_class: Optional[Type[Context]] = None,
    cache_logger_on_first_use: Optional[bool] = None,
    logger_factory_args: Optional[Iterable[Any]] = None,
    **initial_values: Any
) -> Any:
    """
    Create a new bound logger for an arbitrary *logger*.

    Default values for *processors*, *wrapper_class*, and *context_class* can
    be set using `configure`.

    If you set an attribute here, `configure` calls have *no* effect for
    the *respective* attribute.

    In other words: selective overwriting of the defaults while keeping some
    *is* possible.

    :param initial_values: Values that are used to pre-populate your contexts.
    :param logger_factory_args: Values that are passed unmodified as
        ``*logger_factory_args`` to the logger factory if not `None`.

    :returns: A proxy that creates a correctly configured bound logger when
        necessary.

    See `configure` for the meaning of the rest of the arguments.

    .. versionadded:: 0.4.0
        *logger_factory_args*
    """
    return BoundLoggerLazyProxy(
        logger,
        wrapper_class=wrapper_class,
        processors=processors,
        context_class=context_class,
        cache_logger_on_first_use=cache_logger_on_first_use,
        initial_values=initial_values,
        logger_factory_args=logger_factory_args,
    )


def configure(
    processors: Optional[Iterable[Processor]] = None,
    wrapper_class: Optional[Type[BindableLogger]] = None,
    context_class: Optional[Type[Context]] = None,
    logger_factory: Optional[Callable[..., WrappedLogger]] = None,
    cache_logger_on_first_use: Optional[bool] = None,
) -> None:
    """
    Configures the **global** defaults.

    They are used if `wrap_logger` or `get_logger` are called without
    arguments.

    Can be called several times, keeping an argument at `None` leaves it
    unchanged from the current setting.

    After calling for the first time, `is_configured` starts returning
    `True`.

    Use `reset_defaults` to undo your changes.

    :param processors: The processor chain. See :doc:`processors` for details.
    :param wrapper_class: Class to use for wrapping loggers instead of
        `structlog.BoundLogger`.  See `standard-library`, :doc:`twisted`, and
        `custom-wrappers`.
    :param context_class: Class to be used for internal context keeping.
    :param logger_factory: Factory to be called to create a new
        logger that shall be wrapped.
    :param cache_logger_on_first_use: `wrap_logger` doesn't return an
        actual wrapped logger but a proxy that assembles one when it's first
        used.  If this option is set to `True`, this assembled logger is
        cached.  See `performance`.

    .. versionadded:: 0.3.0
        *cache_logger_on_first_use*
    """
    _CONFIG.is_configured = True

    if processors is not None:
        _CONFIG.default_processors = processors
    if wrapper_class is not None:
        _CONFIG.default_wrapper_class = wrapper_class
    if context_class is not None:
        _CONFIG.default_context_class = context_class
    if logger_factory is not None:
        _CONFIG.logger_factory = logger_factory  # type: ignore
    if cache_logger_on_first_use is not None:
        _CONFIG.cache_logger_on_first_use = cache_logger_on_first_use


def configure_once(
    processors: Optional[Iterable[Processor]] = None,
    wrapper_class: Optional[Type[BindableLogger]] = None,
    context_class: Optional[Type[Context]] = None,
    logger_factory: Optional[Callable[..., WrappedLogger]] = None,
    cache_logger_on_first_use: Optional[bool] = None,
) -> None:
    """
    Configures if structlog isn't configured yet.

    It does *not* matter whether it was configured using `configure` or
    `configure_once` before.

    Raises a `RuntimeWarning` if repeated configuration is attempted.
    """
    if not _CONFIG.is_configured:
        configure(
            processors=processors,
            wrapper_class=wrapper_class,
            context_class=context_class,
            logger_factory=logger_factory,
            cache_logger_on_first_use=cache_logger_on_first_use,
        )
    else:
        warnings.warn("Repeated configuration attempted.", RuntimeWarning)


def reset_defaults() -> None:
    """
    Resets global default values to builtin defaults.

    `is_configured` starts returning `False` afterwards.
    """
    _CONFIG.is_configured = False
    _CONFIG.default_processors = _BUILTIN_DEFAULT_PROCESSORS[:]
    _CONFIG.default_wrapper_class = _BUILTIN_DEFAULT_WRAPPER_CLASS
    _CONFIG.default_context_class = _BUILTIN_DEFAULT_CONTEXT_CLASS
    _CONFIG.logger_factory = _BUILTIN_DEFAULT_LOGGER_FACTORY  # type: ignore
    _CONFIG.cache_logger_on_first_use = _BUILTIN_CACHE_LOGGER_ON_FIRST_USE


class BoundLoggerLazyProxy:
    """
    Instantiates a ``BoundLogger`` on first usage.

    Takes both configuration and instantiation parameters into account.

    The only points where a BoundLogger changes state are ``bind()``,
    ``unbind()``, and ``new()`` and that return the actual ``BoundLogger``.

    If and only if configuration says so, that actual ``BoundLogger`` is
    cached on first usage.

    .. versionchanged:: 0.4.0
        Added support for *logger_factory_args*.
    """

    def __init__(
        self,
        logger: WrappedLogger,
        wrapper_class: Optional[Type[BindableLogger]] = None,
        processors: Optional[Iterable[Processor]] = None,
        context_class: Optional[Type[Context]] = None,
        cache_logger_on_first_use: Optional[bool] = None,
        initial_values: Optional[Dict[str, Any]] = None,
        logger_factory_args: Any = None,
    ) -> None:
        self._logger = logger
        self._wrapper_class = wrapper_class
        self._processors = processors
        self._context_class = context_class
        self._cache_logger_on_first_use = cache_logger_on_first_use
        self._initial_values = initial_values or {}
        self._logger_factory_args = logger_factory_args or ()

    def __repr__(self) -> str:
        return (
            "<BoundLoggerLazyProxy(logger={0._logger!r}, wrapper_class="
            "{0._wrapper_class!r}, processors={0._processors!r}, "
            "context_class={0._context_class!r}, "
            "initial_values={0._initial_values!r}, "
            "logger_factory_args={0._logger_factory_args!r})>".format(self)
        )

    def bind(self, **new_values: Any) -> BindableLogger:
        """
        Assemble a new BoundLogger from arguments and configuration.
        """
        if self._context_class:
            ctx = self._context_class(self._initial_values)
        else:
            ctx = _CONFIG.default_context_class(self._initial_values)

        _logger = self._logger
        if not _logger:
            _logger = _CONFIG.logger_factory(*self._logger_factory_args)

        if self._processors is None:
            procs = _CONFIG.default_processors
        else:
            procs = self._processors

        cls = self._wrapper_class or _CONFIG.default_wrapper_class
        # Looks like Protocols ignore definitions of __init__ so we have to
        # silence Mypy here.
        logger = cls(_logger, processors=procs, context=ctx)  # type: ignore

        def finalized_bind(**new_values: Any) -> BindableLogger:
            """
            Use cached assembled logger to bind potentially new values.
            """
            if new_values:
                return logger.bind(**new_values)
            else:
                return logger

        if self._cache_logger_on_first_use is True or (
            self._cache_logger_on_first_use is None
            and _CONFIG.cache_logger_on_first_use is True
        ):
            self.bind = finalized_bind  # type: ignore

        return finalized_bind(**new_values)

    def unbind(self, *keys: str) -> BindableLogger:
        """
        Same as bind, except unbind *keys* first.

        In our case that could be only initial values.
        """
        return self.bind().unbind(*keys)

    def try_unbind(self, *keys: str) -> BindableLogger:
        return self.bind().try_unbind(*keys)

    def new(self, **new_values: Any) -> BindableLogger:
        """
        Clear context, then bind.
        """
        if self._context_class:
            self._context_class().clear()
        else:
            _CONFIG.default_context_class().clear()

        bl = self.bind(**new_values)

        return bl

    def __getattr__(self, name: str) -> Any:
        """
        If a logging method if called on a lazy proxy, we have to create an
        ephemeral BoundLogger first.
        """
        if name == "__isabstractmethod__":
            raise AttributeError

        bl = self.bind()

        return getattr(bl, name)

    def __getstate__(self) -> Dict[str, Any]:
        """
        Our __getattr__ magic makes this necessary.
        """
        return self.__dict__

    def __setstate__(self, state: Dict[str, Any]) -> None:
        """
        Our __getattr__ magic makes this necessary.
        """
        for k, v in state.items():
            setattr(self, k, v)

env_pyrecodes/lib/python3.9/site-packages/structlog/_config.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Extracted log level data used by both stdlib and native log level filters.
"""

import logging

from typing import Any, Callable, Type

from ._base import BoundLoggerBase
from .types import EventDict, FilteringBoundLogger


# Adapted from the stdlib
CRITICAL = 50
FATAL = CRITICAL
ERROR = 40
WARNING = 30
WARN = WARNING
INFO = 20
DEBUG = 10
NOTSET = 0

_NAME_TO_LEVEL = {
    "critical": CRITICAL,
    "exception": ERROR,
    "error": ERROR,
    "warn": WARNING,
    "warning": WARNING,
    "info": INFO,
    "debug": DEBUG,
    "notset": NOTSET,
}

_LEVEL_TO_NAME = {
    v: k
    for k, v in _NAME_TO_LEVEL.items()
    if k not in ("warn", "exception", "notset")
}


def add_log_level(
    logger: logging.Logger, method_name: str, event_dict: EventDict
) -> EventDict:
    """
    Add the log level to the event dict under the ``level`` key.

    Since that's just the log method name, this processor works with non-stdlib
    logging as well. Therefore it's importable both from `structlog.processors`
    as well as from `structlog.stdlib`.

    .. versionadded:: 15.0.0
    .. versionchanged:: 20.2.0
       Importable from `structlog.processors` (additionally to
       `structlog.stdlib`).
    """
    if method_name == "warn":
        # The stdlib has an alias
        method_name = "warning"

    event_dict["level"] = method_name

    return event_dict


def _nop(*args: Any, **kw: Any) -> Any:
    return None


def exception(self: FilteringBoundLogger, event: str, **kw: Any) -> Any:
    kw.setdefault("exc_info", True)

    return self.error(event, **kw)


def make_filtering_bound_logger(min_level: int) -> Type[FilteringBoundLogger]:
    """
    Create a new `FilteringBoundLogger` that only logs *min_level* or higher.

    The logger is optimized such that log levels below *min_level* only consist
    of a ``return None``.

    Compared to using ``structlog``'s standard library integration and the
    `structlog.stdlib.filter_by_level` processor:

    - It's faster because once the logger is built at program start, it's a
      static class.
    - For the same reason you can't change the log level once configured.
      Use the dynamic approach of `standard-library` instead, if you need this
      feature.

    :param min_level: The log level as an integer. You can use the constants
        from `logging` like ``logging.INFO`` or pass the values directly. See
        `this table from the logging docs
        <https://docs.python.org/3/library/logging.html#levels>`_ for possible
        values.

    .. versionadded:: 20.2.0
    .. versionchanged:: 21.1.0 The returned loggers are now pickleable.
    """

    return _LEVEL_TO_FILTERING_LOGGER[min_level]


def _make_filtering_bound_logger(min_level: int) -> Type[FilteringBoundLogger]:
    """
    Create a new `FilteringBoundLogger` that only logs *min_level* or higher.

    The logger is optimized such that log levels below *min_level* only consist
    of a ``return None``.
    """

    def make_method(level: int) -> Callable[..., Any]:
        if level < min_level:
            return _nop

        name = _LEVEL_TO_NAME[level]

        def meth(self: Any, event: str, **kw: Any) -> Any:
            return self._proxy_to_logger(name, event, **kw)

        meth.__name__ = name

        return meth

    meths = {}
    for lvl, name in _LEVEL_TO_NAME.items():
        meths[name] = make_method(lvl)

    meths["exception"] = exception
    meths["fatal"] = meths["error"]
    meths["warn"] = meths["warning"]
    meths["msg"] = meths["info"]

    return type(
        "BoundLoggerFilteringAt%s"
        % (_LEVEL_TO_NAME.get(min_level, "Notset").capitalize()),
        (BoundLoggerBase,),
        meths,
    )


# Pre-create all possible filters to make them pickleable.
BoundLoggerFilteringAtNotset = _make_filtering_bound_logger(NOTSET)
BoundLoggerFilteringAtDebug = _make_filtering_bound_logger(DEBUG)
BoundLoggerFilteringAtInfo = _make_filtering_bound_logger(INFO)
BoundLoggerFilteringAtWarning = _make_filtering_bound_logger(WARNING)
BoundLoggerFilteringAtError = _make_filtering_bound_logger(ERROR)
BoundLoggerFilteringAtCritical = _make_filtering_bound_logger(CRITICAL)

_LEVEL_TO_FILTERING_LOGGER = {
    CRITICAL: BoundLoggerFilteringAtCritical,
    ERROR: BoundLoggerFilteringAtError,
    WARNING: BoundLoggerFilteringAtWarning,
    INFO: BoundLoggerFilteringAtInfo,
    DEBUG: BoundLoggerFilteringAtDebug,
    NOTSET: BoundLoggerFilteringAtNotset,
}

env_pyrecodes/lib/python3.9/site-packages/structlog/_log_levels.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Logger wrapper and helper class.
"""

import copy
import sys
import threading

from pickle import PicklingError
from typing import IO, Any, BinaryIO, Dict, Optional, TextIO

from structlog._utils import until_not_interrupted


WRITE_LOCKS: Dict[IO[Any], threading.Lock] = {}


def _get_lock_for_file(file: IO[Any]) -> threading.Lock:
    global WRITE_LOCKS

    lock = WRITE_LOCKS.get(file)
    if lock is None:
        lock = threading.Lock()
        WRITE_LOCKS[file] = lock

    return lock


class PrintLogger:
    """
    Print events into a file.

    :param file: File to print to. (default: `sys.stdout`)

    >>> from structlog import PrintLogger
    >>> PrintLogger().msg("hello")
    hello

    Useful if you follow
    `current logging best practices <logging-best-practices>`.

    Also very useful for testing and examples since logging is finicky in
    doctests.
    """

    def __init__(self, file: Optional[TextIO] = None):
        self._file = file or sys.stdout
        self._write = self._file.write
        self._flush = self._file.flush

        self._lock = _get_lock_for_file(self._file)

    def __getstate__(self) -> str:
        """
        Our __getattr__ magic makes this necessary.
        """
        if self._file is sys.stdout:
            return "stdout"

        elif self._file is sys.stderr:
            return "stderr"

        raise PicklingError(
            "Only PrintLoggers to sys.stdout and sys.stderr can be pickled."
        )

    def __setstate__(self, state: Any) -> None:
        """
        Our __getattr__ magic makes this necessary.
        """
        if state == "stdout":
            self._file = sys.stdout
        else:
            self._file = sys.stderr

        self._write = self._file.write
        self._flush = self._file.flush
        self._lock = _get_lock_for_file(self._file)

    def __deepcopy__(self, memodict: Dict[Any, Any] = {}) -> "PrintLogger":
        """
        Create a new PrintLogger with the same attributes. Similar to pickling.
        """
        if self._file not in (sys.stdout, sys.stderr):
            raise copy.error(
                "Only PrintLoggers to sys.stdout and sys.stderr "
                "can be deepcopied."
            )

        newself = self.__class__(self._file)

        newself._write = newself._file.write
        newself._flush = newself._file.flush
        newself._lock = _get_lock_for_file(newself._file)

        return newself

    def __repr__(self) -> str:
        return f"<PrintLogger(file={self._file!r})>"

    def msg(self, message: str) -> None:
        """
        Print *message*.
        """
        with self._lock:
            until_not_interrupted(self._write, message + "\n")
            until_not_interrupted(self._flush)

    log = debug = info = warn = warning = msg
    fatal = failure = err = error = critical = exception = msg


class PrintLoggerFactory:
    r"""
    Produce `PrintLogger`\ s.

    To be used with `structlog.configure`\ 's ``logger_factory``.

    :param file: File to print to. (default: `sys.stdout`)

    Positional arguments are silently ignored.

    .. versionadded:: 0.4.0
    """

    def __init__(self, file: Optional[TextIO] = None):
        self._file = file

    def __call__(self, *args: Any) -> PrintLogger:
        return PrintLogger(self._file)


class BytesLogger:
    r"""
    Writes bytes into a file.

    :param file: File to print to. (default: `sys.stdout`\ ``.buffer``)

    Useful if you follow
    `current logging best practices <logging-best-practices>` together with
    a formatter that returns bytes (e.g. `orjson
    <https://github.com/ijl/orjson>`_).

    .. versionadded:: 20.2.0
    """
    __slots__ = ("_file", "_write", "_flush", "_lock")

    def __init__(self, file: Optional[BinaryIO] = None):
        self._file = file or sys.stdout.buffer
        self._write = self._file.write
        self._flush = self._file.flush

        self._lock = _get_lock_for_file(self._file)

    def __getstate__(self) -> str:
        """
        Our __getattr__ magic makes this necessary.
        """
        if self._file is sys.stdout.buffer:
            return "stdout"

        elif self._file is sys.stderr.buffer:
            return "stderr"

        raise PicklingError(
            "Only BytesLoggers to sys.stdout and sys.stderr can be pickled."
        )

    def __setstate__(self, state: Any) -> None:
        """
        Our __getattr__ magic makes this necessary.
        """
        if state == "stdout":
            self._file = sys.stdout.buffer
        else:
            self._file = sys.stderr.buffer

        self._write = self._file.write
        self._flush = self._file.flush
        self._lock = _get_lock_for_file(self._file)

    def __deepcopy__(self, memodict: Dict[Any, Any] = {}) -> "BytesLogger":
        """
        Create a new BytesLogger with the same attributes. Similar to pickling.
        """
        if self._file not in (sys.stdout.buffer, sys.stderr.buffer):
            raise copy.error(
                "Only BytesLoggers to sys.stdout and sys.stderr "
                "can be deepcopied."
            )

        newself = self.__class__(self._file)

        newself._write = newself._file.write
        newself._flush = newself._file.flush
        newself._lock = _get_lock_for_file(newself._file)

        return newself

    def __repr__(self) -> str:
        return f"<BytesLogger(file={self._file!r})>"

    def msg(self, message: bytes) -> None:
        """
        Write *message*.
        """
        with self._lock:
            until_not_interrupted(self._write, message + b"\n")
            until_not_interrupted(self._flush)

    log = debug = info = warn = warning = msg
    fatal = failure = err = error = critical = exception = msg


class BytesLoggerFactory:
    r"""
    Produce `BytesLogger`\ s.

    To be used with `structlog.configure`\ 's ``logger_factory``.

    :param file: File to print to. (default: `sys.stdout`\ ``.buffer``)

    Positional arguments are silently ignored.

    .. versionadded:: 20.2.0
    """
    __slots__ = ("_file",)

    def __init__(self, file: Optional[BinaryIO] = None):
        self._file = file

    def __call__(self, *args: Any) -> BytesLogger:
        return BytesLogger(self._file)

env_pyrecodes/lib/python3.9/site-packages/structlog/_loggers.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Helpers that make development with ``structlog`` more pleasant.

See also the narrative documentation in `development`.
"""

import sys
import warnings

from io import StringIO
from typing import Any, Iterable, Optional, TextIO, Type, Union

from ._frames import _format_exception
from .types import (
    EventDict,
    ExceptionFormatter,
    ExcInfo,
    Protocol,
    WrappedLogger,
)


try:
    import colorama
except ImportError:
    colorama = None

try:
    import better_exceptions
except ImportError:
    better_exceptions = None

try:
    import rich

    from rich.console import Console
    from rich.traceback import Traceback
except ImportError:
    rich = None  # type: ignore


__all__ = [
    "ConsoleRenderer",
    "plain_traceback",
    "rich_traceback",
    "better_traceback",
]

_IS_WINDOWS = sys.platform == "win32"

_MISSING = "{who} requires the {package} package installed.  "
_EVENT_WIDTH = 30  # pad the event name to so many characters


def _pad(s: str, length: int) -> str:
    """
    Pads *s* to length *length*.
    """
    missing = length - len(s)

    return s + " " * (missing if missing > 0 else 0)


if colorama is not None:
    RESET_ALL = colorama.Style.RESET_ALL
    BRIGHT = colorama.Style.BRIGHT
    DIM = colorama.Style.DIM
    RED = colorama.Fore.RED
    BLUE = colorama.Fore.BLUE
    CYAN = colorama.Fore.CYAN
    MAGENTA = colorama.Fore.MAGENTA
    YELLOW = colorama.Fore.YELLOW
    GREEN = colorama.Fore.GREEN
    RED_BACK = colorama.Back.RED
else:
    # These are the same values as the colorama color codes. Redefining them
    # here allows users to specify that they want color without having to
    # install colorama, which is only supposed to be necessary in Windows.
    RESET_ALL = "\033[0m"
    BRIGHT = "\033[1m"
    DIM = "\033[2m"
    RED = "\033[31m"
    BLUE = "\033[34m"
    CYAN = "\033[36m"
    MAGENTA = "\033[35m"
    YELLOW = "\033[33m"
    GREEN = "\033[32m"
    RED_BACK = "\033[41m"


if _IS_WINDOWS:  # pragma: no cover
    # On Windows, use colors by default only if colorama is installed.
    _use_colors = colorama is not None
else:
    # On other OSes, use colors by default.
    _use_colors = True


class _Styles(Protocol):
    reset: str
    bright: str
    level_critical: str
    level_exception: str
    level_error: str
    level_warn: str
    level_info: str
    level_debug: str
    level_notset: str

    timestamp: str
    logger_name: str
    kv_key: str
    kv_value: str


Styles = Union[_Styles, Type[_Styles]]


class _ColorfulStyles:
    reset = RESET_ALL
    bright = BRIGHT

    level_critical = RED
    level_exception = RED
    level_error = RED
    level_warn = YELLOW
    level_info = GREEN
    level_debug = GREEN
    level_notset = RED_BACK

    timestamp = DIM
    logger_name = BLUE
    kv_key = CYAN
    kv_value = MAGENTA


class _PlainStyles:
    reset = ""
    bright = ""

    level_critical = ""
    level_exception = ""
    level_error = ""
    level_warn = ""
    level_info = ""
    level_debug = ""
    level_notset = ""

    timestamp = ""
    logger_name = ""
    kv_key = ""
    kv_value = ""


def plain_traceback(sio: TextIO, exc_info: ExcInfo) -> None:
    """
    "Pretty"-print *exc_info* to *sio* using our own plain formatter.

    To be passed into `ConsoleRenderer`'s ``exception_formatter`` argument.

    Used by default if neither ``rich`` not ``better-exceptions`` are present.

    .. versionadded:: 21.2
    """
    sio.write("\n" + _format_exception(exc_info))


def rich_traceback(sio: TextIO, exc_info: ExcInfo) -> None:
    """
    Pretty-print *exc_info* to *sio* using the ``rich`` package.

    To be passed into `ConsoleRenderer`'s ``exception_formatter`` argument.

    Used by default if ``rich`` is installed.

    .. versionadded:: 21.2
    """
    sio.write("\n")
    Console(file=sio, color_system="truecolor").print(
        Traceback.from_exception(*exc_info, show_locals=True)
    )


def better_traceback(sio: TextIO, exc_info: ExcInfo) -> None:
    """
    Pretty-print *exc_info* to *sio* using the ``better-exceptions`` package.

    To be passed into `ConsoleRenderer`'s ``exception_formatter`` argument.

    Used by default if ``better-exceptions`` is installed and ``rich`` is
    absent.

    .. versionadded:: 21.2
    """
    sio.write("\n" + "".join(better_exceptions.format_exception(*exc_info)))


if rich is not None:
    default_exception_formatter = rich_traceback
elif better_exceptions is not None:  # type: ignore
    default_exception_formatter = better_traceback
else:
    default_exception_formatter = plain_traceback


class ConsoleRenderer:
    """
    Render ``event_dict`` nicely aligned, possibly in colors, and ordered.

    If ``event_dict`` contains a true-ish ``exc_info`` key, it will be
    rendered *after* the log line. If rich_ or better-exceptions_ are present,
    in colors and with extra context.

    :param pad_event: Pad the event to this many characters.
    :param colors: Use colors for a nicer output. `True` by default if
        colorama_ is installed.
    :param force_colors: Force colors even for non-tty destinations.
        Use this option if your logs are stored in a file that is meant
        to be streamed to the console.
    :param repr_native_str: When `True`, `repr` is also applied
        to native strings (i.e. unicode on Python 3 and bytes on Python 2).
        Setting this to `False` is useful if you want to have human-readable
        non-ASCII output on Python 2.  The ``event`` key is *never*
        `repr` -ed.
    :param level_styles: When present, use these styles for colors. This
        must be a dict from level names (strings) to colorama styles. The
        default can be obtained by calling
        `ConsoleRenderer.get_default_level_styles`
    :param exception_formatter: A callable to render ``exc_infos``. If rich_
        or better-exceptions_ are installed, they are used for pretty-printing
        by default (rich_ taking precedence). You can also manually set it to
        `plain_traceback`, `better_traceback`, `rich_traceback`, or implement
        your own.
    :param sort_keys: Whether to sort keys when formatting. `True` by default.

    Requires the colorama_ package if *colors* is `True` **on Windows**.

    .. _colorama: https://pypi.org/project/colorama/
    .. _better-exceptions: https://pypi.org/project/better-exceptions/
    .. _rich: https://pypi.org/project/rich/

    .. versionadded:: 16.0
    .. versionadded:: 16.1 *colors*
    .. versionadded:: 17.1 *repr_native_str*
    .. versionadded:: 18.1 *force_colors*
    .. versionadded:: 18.1 *level_styles*
    .. versionchanged:: 19.2
       ``colorama`` now initializes lazily to avoid unwanted initializations as
       ``ConsoleRenderer`` is used by default.
    .. versionchanged:: 19.2 Can be pickled now.
    .. versionchanged:: 20.1 ``colorama`` does not initialize lazily on Windows
       anymore because it breaks rendering.
    .. versionchanged:: 21.1 It is additionally possible to set the logger name
       using the ``logger_name`` key in the ``event_dict``.
    .. versionadded:: 21.2 *exception_formatter*
    .. versionchanged:: 21.2 `ConsoleRenderer` now handles the ``exc_info``
       event dict key itself. Do **not** use the
       `structlog.processors.format_exc_info` processor together with
       `ConsoleRenderer` anymore! It will keep working, but you can't have
       customize exception formatting and a warning will be raised if you ask
       for it.
    .. versionchanged:: 21.2 The colors keyword now defaults to True on
       non-Windows systems, and either True or False in Windows depending on
       whether colorama is installed.
    .. versionadded:: 21.3.0 *sort_keys*
    """

    def __init__(
        self,
        pad_event: int = _EVENT_WIDTH,
        colors: bool = _use_colors,
        force_colors: bool = False,
        repr_native_str: bool = False,
        level_styles: Optional[Styles] = None,
        exception_formatter: ExceptionFormatter = default_exception_formatter,
        sort_keys: bool = True,
    ):
        styles: Styles
        if colors:
            if _IS_WINDOWS:  # pragma: no cover
                # On Windows, we can't do colorful output without colorama.
                if colorama is None:
                    classname = self.__class__.__name__
                    raise SystemError(
                        _MISSING.format(
                            who=classname + " with `colors=True`",
                            package="colorama",
                        )
                    )
                # Colorama must be init'd on Windows, but must NOT be
                # init'd on other OSes, because it can break colors.
                if force_colors:
                    colorama.deinit()
                    colorama.init(strip=False)
                else:
                    colorama.init()

            styles = _ColorfulStyles
        else:
            styles = _PlainStyles

        self._styles = styles
        self._pad_event = pad_event

        if level_styles is None:
            self._level_to_color = self.get_default_level_styles(colors)
        else:
            self._level_to_color = level_styles

        for key in self._level_to_color.keys():
            self._level_to_color[key] += styles.bright
        self._longest_level = len(
            max(self._level_to_color.keys(), key=lambda e: len(e))
        )

        self._repr_native_str = repr_native_str
        self._exception_formatter = exception_formatter
        self._sort_keys = sort_keys

    def _repr(self, val: Any) -> str:
        """
        Determine representation of *val* depending on its type &
        self._repr_native_str.
        """
        if self._repr_native_str is True:
            return repr(val)

        if isinstance(val, str):
            return val
        else:
            return repr(val)

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> str:

        sio = StringIO()

        ts = event_dict.pop("timestamp", None)
        if ts is not None:
            sio.write(
                # can be a number if timestamp is UNIXy
                self._styles.timestamp
                + str(ts)
                + self._styles.reset
                + " "
            )
        level = event_dict.pop("level", None)
        if level is not None:
            sio.write(
                "["
                + self._level_to_color.get(level, "")
                + _pad(level, self._longest_level)
                + self._styles.reset
                + "] "
            )

        # force event to str for compatibility with standard library
        event = event_dict.pop("event", None)
        if not isinstance(event, str):
            event = str(event)

        if event_dict:
            event = _pad(event, self._pad_event) + self._styles.reset + " "
        else:
            event += self._styles.reset
        sio.write(self._styles.bright + event)

        logger_name = event_dict.pop("logger", None)
        if logger_name is None:
            logger_name = event_dict.pop("logger_name", None)

        if logger_name is not None:
            sio.write(
                "["
                + self._styles.logger_name
                + self._styles.bright
                + logger_name
                + self._styles.reset
                + "] "
            )

        stack = event_dict.pop("stack", None)
        exc = event_dict.pop("exception", None)
        exc_info = event_dict.pop("exc_info", None)

        event_dict_keys: Iterable[str] = event_dict.keys()
        if self._sort_keys:
            event_dict_keys = sorted(event_dict_keys)

        sio.write(
            " ".join(
                self._styles.kv_key
                + key
                + self._styles.reset
                + "="
                + self._styles.kv_value
                + self._repr(event_dict[key])
                + self._styles.reset
                for key in event_dict_keys
            )
        )

        if stack is not None:
            sio.write("\n" + stack)
            if exc_info or exc is not None:
                sio.write("\n\n" + "=" * 79 + "\n")

        if exc_info:
            if not isinstance(exc_info, tuple):
                exc_info = sys.exc_info()

            self._exception_formatter(sio, exc_info)
        elif exc is not None:
            if self._exception_formatter is not plain_traceback:
                warnings.warn(
                    "Remove `format_exc_info` from your processor chain "
                    "if you want pretty exceptions."
                )
            sio.write("\n" + exc)

        return sio.getvalue()

    @staticmethod
    def get_default_level_styles(colors: bool = True) -> Any:
        """
        Get the default styles for log levels

        This is intended to be used with `ConsoleRenderer`'s ``level_styles``
        parameter.  For example, if you are adding custom levels in your
        home-grown :func:`~structlog.stdlib.add_log_level` you could do::

            my_styles = ConsoleRenderer.get_default_level_styles()
            my_styles["EVERYTHING_IS_ON_FIRE"] = my_styles["critical"]
            renderer = ConsoleRenderer(level_styles=my_styles)

        :param colors: Whether to use colorful styles. This must match the
            *colors* parameter to `ConsoleRenderer`. Default: `True`.
        """
        styles: Styles
        if colors:
            styles = _ColorfulStyles
        else:
            styles = _PlainStyles
        return {
            "critical": styles.level_critical,
            "exception": styles.level_exception,
            "error": styles.level_error,
            "warn": styles.level_warn,
            "warning": styles.level_warn,
            "info": styles.level_info,
            "debug": styles.level_debug,
            "notset": styles.level_notset,
        }


_SENTINEL = object()


def set_exc_info(
    logger: WrappedLogger, method_name: str, event_dict: EventDict
) -> EventDict:

    """
    Set ``event_dict["exc_info"] = True`` if *method_name* is ``"exception"``.

    Do nothing if the name is different or ``exc_info`` is already set.
    """
    if (
        method_name != "exception"
        or event_dict.get("exc_info", _SENTINEL) is not _SENTINEL
    ):
        return event_dict

    event_dict["exc_info"] = True

    return event_dict

env_pyrecodes/lib/python3.9/site-packages/structlog/dev.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Processors useful regardless of the logging framework.
"""

import datetime
import enum
import inspect
import json
import logging
import operator
import os
import sys
import threading
import time

from typing import (
    Any,
    Callable,
    ClassVar,
    Collection,
    Dict,
    List,
    NamedTuple,
    Optional,
    Sequence,
    Set,
    TextIO,
    Tuple,
    Union,
)

from ._frames import (
    _find_first_app_frame_and_name,
    _format_exception,
    _format_stack,
)
from ._log_levels import _NAME_TO_LEVEL, add_log_level
from ._utils import get_processname
from .types import EventDict, ExcInfo, WrappedLogger


__all__ = [
    "_NAME_TO_LEVEL",  # some people rely on it being here
    "KeyValueRenderer",
    "TimeStamper",
    "add_log_level",
    "UnicodeEncoder",
    "UnicodeDecoder",
    "JSONRenderer",
    "format_exc_info",
    "ExceptionPrettyPrinter",
    "StackInfoRenderer",
    "CallsiteParameter",
    "CallsiteParameterAdder",
]


class KeyValueRenderer:
    """
    Render ``event_dict`` as a list of ``Key=repr(Value)`` pairs.

    :param sort_keys: Whether to sort keys when formatting.
    :param key_order: List of keys that should be rendered in this exact
        order.  Missing keys will be rendered as ``None``, extra keys depending
        on *sort_keys* and the dict class.
    :param drop_missing: When ``True``, extra keys in *key_order* will be
        dropped rather than rendered as ``None``.
    :param repr_native_str: When ``True``, :func:`repr()` is also applied
        to native strings.
        Setting this to ``False`` is useful if you want to have human-readable
        non-ASCII output on Python 2.

    .. versionadded:: 0.2.0 *key_order*
    .. versionadded:: 16.1.0 *drop_missing*
    .. versionadded:: 17.1.0 *repr_native_str*
    """

    def __init__(
        self,
        sort_keys: bool = False,
        key_order: Optional[Sequence[str]] = None,
        drop_missing: bool = False,
        repr_native_str: bool = True,
    ):
        self._ordered_items = _items_sorter(sort_keys, key_order, drop_missing)

        if repr_native_str is True:
            self._repr = repr
        else:

            def _repr(inst: Any) -> str:
                if isinstance(inst, str):
                    return inst
                else:
                    return repr(inst)

            self._repr = _repr

    def __call__(
        self, _: WrappedLogger, __: str, event_dict: EventDict
    ) -> str:
        return " ".join(
            k + "=" + self._repr(v) for k, v in self._ordered_items(event_dict)
        )


class LogfmtRenderer:
    """
    Render ``event_dict`` using the logfmt_ format.

    .. _logfmt: https://brandur.org/logfmt

    :param sort_keys: Whether to sort keys when formatting.
    :param key_order: List of keys that should be rendered in this exact
        order. Missing keys are rendered with empty values, extra keys
        depending on *sort_keys* and the dict class.
    :param drop_missing: When ``True``, extra keys in *key_order* will be
        dropped rather than rendered with empty values.
    :param bool_as_flag: When ``True``, render ``{"flag": True}`` as
        ``flag``, instead of ``flag=true``. ``{"flag": False}`` is
        always rendered as ``flag=false``.

    :raises ValueError: If a key contains non printable or space characters.

    .. versionadded:: 21.5.0
    """

    def __init__(
        self,
        sort_keys: bool = False,
        key_order: Optional[Sequence[str]] = None,
        drop_missing: bool = False,
        bool_as_flag: bool = True,
    ):
        self._ordered_items = _items_sorter(sort_keys, key_order, drop_missing)
        self.bool_as_flag = bool_as_flag

    def __call__(
        self, _: WrappedLogger, __: str, event_dict: EventDict
    ) -> str:

        elements: List[str] = []
        for key, value in self._ordered_items(event_dict):
            if any(c <= " " for c in key):
                raise ValueError(f'Invalid key: "{key}"')

            if value is None:
                elements.append(f"{key}=")
                continue

            if isinstance(value, bool):
                if self.bool_as_flag and value:
                    elements.append(f"{key}")
                    continue
                value = "true" if value else "false"

            value = f"{value}".replace('"', '\\"')

            if " " in value or "=" in value:
                value = f'"{value}"'

            elements.append(f"{key}={value}")

        return " ".join(elements)


def _items_sorter(
    sort_keys: bool,
    key_order: Optional[Sequence[str]],
    drop_missing: bool,
) -> Callable[[EventDict], List[Tuple[str, Any]]]:
    """
    Return a function to sort items from an ``event_dict``.

    See `KeyValueRenderer` for an explanation of the parameters.
    """
    # Use an optimized version for each case.
    if key_order and sort_keys:

        def ordered_items(event_dict: EventDict) -> List[Tuple[str, Any]]:
            items = []
            for key in key_order:  # type: ignore
                value = event_dict.pop(key, None)
                if value is not None or not drop_missing:
                    items.append((key, value))

            items += sorted(event_dict.items())

            return items

    elif key_order:

        def ordered_items(event_dict: EventDict) -> List[Tuple[str, Any]]:
            items = []
            for key in key_order:  # type: ignore
                value = event_dict.pop(key, None)
                if value is not None or not drop_missing:
                    items.append((key, value))

            items += event_dict.items()

            return items

    elif sort_keys:

        def ordered_items(event_dict: EventDict) -> List[Tuple[str, Any]]:
            return sorted(event_dict.items())

    else:
        ordered_items = operator.methodcaller("items")  # type: ignore

    return ordered_items


class UnicodeEncoder:
    """
    Encode unicode values in ``event_dict``.

    :param encoding: Encoding to encode to (default: ``"utf-8"``).
    :param errors: How to cope with encoding errors (default
        ``"backslashreplace"``).

    Useful if you're running Python 2 as otherwise ``u"abc"`` will be rendered
    as ``'u"abc"'``.

    Just put it in the processor chain before the renderer.
    """

    _encoding: str
    _errors: str

    def __init__(
        self, encoding: str = "utf-8", errors: str = "backslashreplace"
    ) -> None:
        self._encoding = encoding
        self._errors = errors

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> EventDict:
        for key, value in event_dict.items():
            if isinstance(value, str):
                event_dict[key] = value.encode(self._encoding, self._errors)

        return event_dict


class UnicodeDecoder:
    """
    Decode byte string values in ``event_dict``.

    :param encoding: Encoding to decode from (default: ``"utf-8"``).
    :param errors: How to cope with encoding errors (default:
        ``"replace"``).

    Useful if you're running Python 3 as otherwise ``b"abc"`` will be rendered
    as ``'b"abc"'``.

    Just put it in the processor chain before the renderer.

    .. versionadded:: 15.4.0
    """

    _encoding: str
    _errors: str

    def __init__(
        self, encoding: str = "utf-8", errors: str = "replace"
    ) -> None:
        self._encoding = encoding
        self._errors = errors

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> EventDict:
        for key, value in event_dict.items():
            if isinstance(value, bytes):
                event_dict[key] = value.decode(self._encoding, self._errors)

        return event_dict


class JSONRenderer:
    """
    Render the ``event_dict`` using ``serializer(event_dict, **json_kw)``.

    :param json_kw: Are passed unmodified to *serializer*.  If *default*
        is passed, it will disable support for ``__structlog__``-based
        serialization.
    :param serializer: A :func:`json.dumps`-compatible callable that
        will be used to format the string.  This can be used to use alternative
        JSON encoders like `simplejson
        <https://pypi.org/project/simplejson/>`_ or `RapidJSON
        <https://pypi.org/project/python-rapidjson/>`_ (faster but Python
        3-only) (default: :func:`json.dumps`).

    .. versionadded:: 0.2.0
        Support for ``__structlog__`` serialization method.

    .. versionadded:: 15.4.0
        *serializer* parameter.

    .. versionadded:: 18.2.0
       Serializer's *default* parameter can be overwritten now.

    """

    def __init__(
        self,
        serializer: Callable[..., Union[str, bytes]] = json.dumps,
        **dumps_kw: Any,
    ) -> None:
        dumps_kw.setdefault("default", _json_fallback_handler)
        self._dumps_kw = dumps_kw
        self._dumps = serializer

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> Union[str, bytes]:
        """
        The return type of this depends on the return type of self._dumps.
        """
        return self._dumps(event_dict, **self._dumps_kw)


def _json_fallback_handler(obj: Any) -> Any:
    """
    Serialize custom datatypes and pass the rest to __structlog__ & repr().
    """
    # circular imports :(
    from structlog.threadlocal import _ThreadLocalDictWrapper

    if isinstance(obj, _ThreadLocalDictWrapper):
        return obj._dict
    else:
        try:
            return obj.__structlog__()
        except AttributeError:
            return repr(obj)


def format_exc_info(
    logger: WrappedLogger, name: str, event_dict: EventDict
) -> EventDict:
    """
    Replace an ``exc_info`` field by an ``exception`` string field:

    If *event_dict* contains the key ``exc_info``, there are two possible
    behaviors:

    - If the value is a tuple, render it into the key ``exception``.
    - If the value is an Exception render it into the key ``exception``.
    - If the value true but no tuple, obtain exc_info ourselves and render
      that.

    If there is no ``exc_info`` key, the *event_dict* is not touched.
    This behavior is analogue to the one of the stdlib's logging.
    """
    exc_info = event_dict.pop("exc_info", None)
    if exc_info:
        event_dict["exception"] = _format_exception(
            _figure_out_exc_info(exc_info)
        )

    return event_dict


class TimeStamper:
    """
    Add a timestamp to ``event_dict``.

    :param fmt: strftime format string, or ``"iso"`` for `ISO 8601
        <https://en.wikipedia.org/wiki/ISO_8601>`_, or `None` for a `UNIX
        timestamp <https://en.wikipedia.org/wiki/Unix_time>`_.
    :param utc: Whether timestamp should be in UTC or local time.
    :param key: Target key in *event_dict* for added timestamps.

    .. versionchanged:: 19.2 Can be pickled now.
    """

    __slots__ = ("_stamper", "fmt", "utc", "key")

    def __init__(
        self,
        fmt: Optional[str] = None,
        utc: bool = True,
        key: str = "timestamp",
    ) -> None:
        self.fmt, self.utc, self.key = fmt, utc, key

        self._stamper = _make_stamper(fmt, utc, key)

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> EventDict:
        return self._stamper(event_dict)

    def __getstate__(self) -> Dict[str, Any]:
        return {"fmt": self.fmt, "utc": self.utc, "key": self.key}

    def __setstate__(self, state: Dict[str, Any]) -> None:
        self.fmt = state["fmt"]
        self.utc = state["utc"]
        self.key = state["key"]

        self._stamper = _make_stamper(**state)


def _make_stamper(
    fmt: Optional[str], utc: bool, key: str
) -> Callable[[EventDict], EventDict]:
    """
    Create a stamper function.
    """
    if fmt is None and not utc:
        raise ValueError("UNIX timestamps are always UTC.")

    now: Callable[[], datetime.datetime]

    if utc:

        def now() -> datetime.datetime:
            return datetime.datetime.utcnow()

    else:

        def now() -> datetime.datetime:
            return datetime.datetime.now()

    if fmt is None:

        def stamper_unix(event_dict: EventDict) -> EventDict:
            event_dict[key] = time.time()

            return event_dict

        return stamper_unix
    elif fmt.upper() == "ISO":

        def stamper_iso_local(event_dict: EventDict) -> EventDict:
            event_dict[key] = now().isoformat()
            return event_dict

        def stamper_iso_utc(event_dict: EventDict) -> EventDict:
            event_dict[key] = now().isoformat() + "Z"
            return event_dict

        if utc:
            return stamper_iso_utc

        return stamper_iso_local

    def stamper_fmt(event_dict: EventDict) -> EventDict:
        event_dict[key] = now().strftime(fmt)  # type: ignore

        return event_dict

    return stamper_fmt


def _figure_out_exc_info(v: Any) -> ExcInfo:
    """
    Depending on the Python version will try to do the smartest thing possible
    to transform *v* into an ``exc_info`` tuple.
    """
    if isinstance(v, BaseException):
        return (v.__class__, v, v.__traceback__)
    elif isinstance(v, tuple):
        return v  # type: ignore
    elif v:
        return sys.exc_info()  # type: ignore

    return v


class ExceptionPrettyPrinter:
    """
    Pretty print exceptions and remove them from the ``event_dict``.

    :param file: Target file for output (default: ``sys.stdout``).

    This processor is mostly for development and testing so you can read
    exceptions properly formatted.

    It behaves like `format_exc_info` except it removes the exception
    data from the event dictionary after printing it.

    It's tolerant to having `format_exc_info` in front of itself in the
    processor chain but doesn't require it.  In other words, it handles both
    ``exception`` as well as ``exc_info`` keys.

    .. versionadded:: 0.4.0

    .. versionchanged:: 16.0.0
       Added support for passing exceptions as ``exc_info`` on Python 3.
    """

    def __init__(self, file: Optional[TextIO] = None) -> None:
        if file is not None:
            self._file = file
        else:
            self._file = sys.stdout

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> EventDict:
        exc = event_dict.pop("exception", None)
        if exc is None:
            exc_info = _figure_out_exc_info(event_dict.pop("exc_info", None))
            if exc_info:
                exc = _format_exception(exc_info)

        if exc:
            print(exc, file=self._file)

        return event_dict


class StackInfoRenderer:
    """
    Add stack information with key ``stack`` if ``stack_info`` is `True`.

    Useful when you want to attach a stack dump to a log entry without
    involving an exception.

    It works analogously to the *stack_info* argument of the Python 3 standard
    library logging.

    .. versionadded:: 0.4.0
    """

    def __call__(
        self, logger: WrappedLogger, name: str, event_dict: EventDict
    ) -> EventDict:
        if event_dict.pop("stack_info", None):
            event_dict["stack"] = _format_stack(
                _find_first_app_frame_and_name()[0]
            )

        return event_dict


class CallsiteParameter(enum.Enum):
    """
    Callsite parameters that can be added to an event dictionary with the
    `structlog.processors.CallsiteParameterAdder` processor class.

    The string values of the members of this enum will be used as the keys for
    the callsite parameters in the event dictionary.

    .. versionadded:: 21.5.0
    """

    #: The full path to the python source file of the callsite.
    PATHNAME = "pathname"
    #: The basename part of the full path to the python source file of the
    #: callsite.
    FILENAME = "filename"
    #: The python module the callsite was in. This mimicks the module attribute
    #: of `logging.LogRecord` objects and will be the basename, without
    #: extension, of the full path to the python source file of the callsite.
    MODULE = "module"
    #: The name of the function that the callsite was in.
    FUNC_NAME = "func_name"
    #: The line number of the callsite.
    LINENO = "lineno"
    #: The ID of the thread the callsite was executed in.
    THREAD = "thread"
    #: The name of the thread the callsite was executed in.
    THREAD_NAME = "thread_name"
    #: The ID of the process the callsite was executed in.
    PROCESS = "process"
    #: The name of the process the callsite was executed in.
    PROCESS_NAME = "process_name"


class CallsiteParameterAdder:
    """
    Adds parameters of the callsite that an event dictionary originated from to
    the event dictionary. This processor can be used to enrich events
    dictionaries with information such as the function name, line number and
    filename that an event dictionary originated from.

    .. warning::
        This processor cannot detect the correct callsite for invocation of
        async functions.

    If the event dictionary has an embedded `logging.LogRecord` object and did
    not originate from `structlog` then the callsite information will be
    determined from the `logging.LogRecord` object. For event dictionaries
    without an embedded `logging.LogRecord` object the callsite will be
    determined from the stack trace, ignoring all intra-structlog calls, calls
    from the `logging` module, and stack frames from modules with names that
    start with values in ``additional_ignores``, if it is specified.

    The keys used for callsite parameters in the event dictionary are the
    string values of `CallsiteParameter` enum members.

    :param parameters:
        A collection of `CallsiteParameter` values that should be added to the
        event dictionary.

    :param additional_ignores:
        Additional names with which a stack frame's module name must not
        start for it to be considered when determening the callsite.

    .. note::
        When used with `structlog.stdlib.ProcessorFormatter` the most efficient
        configuration is to either use this processor in ``foreign_pre_chain``
        of `structlog.stdlib.ProcessorFormatter` and in ``processors`` of
        `structlog.configure`, or to use it in ``processors`` of
        `structlog.stdlib.ProcessorFormatter` without using it in
        ``processors`` of `structlog.configure` and ``foreign_pre_chain`` of
        `structlog.stdlib.ProcessorFormatter`.

    .. versionadded:: 21.5.0
    """

    _handlers: ClassVar[
        Dict[CallsiteParameter, Callable[[str, inspect.Traceback], Any]]
    ] = {
        CallsiteParameter.PATHNAME: (
            lambda module, frame_info: frame_info.filename
        ),
        CallsiteParameter.FILENAME: (
            lambda module, frame_info: os.path.basename(frame_info.filename)
        ),
        CallsiteParameter.MODULE: (
            lambda module, frame_info: os.path.splitext(
                os.path.basename(frame_info.filename)
            )[0]
        ),
        CallsiteParameter.FUNC_NAME: (
            lambda module, frame_info: frame_info.function
        ),
        CallsiteParameter.LINENO: (
            lambda module, frame_info: frame_info.lineno
        ),
        CallsiteParameter.THREAD: (
            lambda module, frame_info: threading.get_ident()
        ),
        CallsiteParameter.THREAD_NAME: (
            lambda module, frame_info: threading.current_thread().name
        ),
        CallsiteParameter.PROCESS: (lambda module, frame_info: os.getpid()),
        CallsiteParameter.PROCESS_NAME: (
            lambda module, frame_info: get_processname()
        ),
    }
    _record_attribute_map: ClassVar[Dict[CallsiteParameter, str]] = {
        CallsiteParameter.PATHNAME: "pathname",
        CallsiteParameter.FILENAME: "filename",
        CallsiteParameter.MODULE: "module",
        CallsiteParameter.FUNC_NAME: "funcName",
        CallsiteParameter.LINENO: "lineno",
        CallsiteParameter.THREAD: "thread",
        CallsiteParameter.THREAD_NAME: "threadName",
        CallsiteParameter.PROCESS: "process",
        CallsiteParameter.PROCESS_NAME: "processName",
    }

    _all_parameters: ClassVar[Set[CallsiteParameter]] = set(CallsiteParameter)

    class _RecordMapping(NamedTuple):
        event_dict_key: str
        record_attribute: str

    __slots__ = [
        "_active_handlers",
        "_additional_ignores",
        "_record_mappings",
    ]

    def __init__(
        self,
        parameters: Collection[CallsiteParameter] = _all_parameters,
        additional_ignores: Optional[List[str]] = None,
    ) -> None:
        if additional_ignores is None:
            additional_ignores = []
        # Ignore stack frames from the logging module. They will occur if this
        # processor is used in ProcessorFormatter, and additionally the logging
        # module should not be logging using structlog.
        self._additional_ignores = ["logging", *additional_ignores]
        self._active_handlers: List[
            Tuple[CallsiteParameter, Callable[[str, inspect.Traceback], Any]]
        ] = []
        self._record_mappings: List[
            "CallsiteParameterAdder._RecordMapping"
        ] = []
        for parameter in parameters:
            self._active_handlers.append(
                (parameter, self._handlers[parameter])
            )
            self._record_mappings.append(
                self._RecordMapping(
                    parameter.value,
                    self._record_attribute_map[parameter],
                )
            )

    def __call__(
        self, logger: logging.Logger, name: str, event_dict: EventDict
    ) -> EventDict:
        record: Optional[logging.LogRecord] = event_dict.get("_record")
        from_structlog: Optional[bool] = event_dict.get("_from_structlog")
        # If the event dictionary has a record, but it comes from structlog,
        # then the callsite parameters of the record will not be correct.
        if record is not None and not from_structlog:
            for mapping in self._record_mappings:
                event_dict[mapping.event_dict_key] = record.__dict__[
                    mapping.record_attribute
                ]
        else:
            frame, module = _find_first_app_frame_and_name(
                additional_ignores=self._additional_ignores
            )
            frame_info = inspect.getframeinfo(frame)
            for parameter, handler in self._active_handlers:
                event_dict[parameter.value] = handler(module, frame_info)
        return event_dict

env_pyrecodes/lib/python3.9/site-packages/structlog/processors.py
NikolaBlagojevic/pyrecodes
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Processors and helpers specific to the :mod:`logging` module from the `Python
standard library <https://docs.python.org/>`_.

See also :doc:`structlog's standard library support <standard-library>`.
"""

import asyncio
import functools
import logging
import sys

from functools import partial
from typing import (
    Any,
    Callable,
    Collection,
    Dict,
    Iterable,
    List,
    Optional,
    Sequence,
    Tuple,
)

from ._base import BoundLoggerBase
from ._config import get_logger as _generic_get_logger
from ._frames import _find_first_app_frame_and_name, _format_stack
from ._log_levels import _LEVEL_TO_NAME, _NAME_TO_LEVEL, add_log_level
from .exceptions import DropEvent
from .types import Context, EventDict, ExcInfo, Processor, WrappedLogger


try:
    import contextvars
except ImportError:
    contextvars = None  # type: ignore


__all__ = [
    "add_log_level_number",
    "add_log_level",
    "add_logger_name",
    "ExtraAdder",
    "BoundLogger",
    "filter_by_level",
    "get_logger",
    "LoggerFactory",
    "PositionalArgumentsFormatter",
    "ProcessorFormatter",
    "render_to_log_kwargs",
]


_SENTINEL = object()


class _FixedFindCallerLogger(logging.Logger):
    """
    Change the behavior of `logging.Logger.findCaller` to cope with
    ``structlog``'s extra frames.
    """

    def findCaller(
        self, stack_info: bool = False, stacklevel: int = 1
    ) -> Tuple[str, int, str, Optional[str]]:
        """
        Finds the first caller frame outside of structlog so that the caller
        info is populated for wrapping stdlib.

        This logger gets set as the default one when using LoggerFactory.
        """
        sinfo: Optional[str]
        f, name = _find_first_app_frame_and_name(["logging"])
        if stack_info:
            sinfo = _format_stack(f)
        else:
            sinfo = None

        return f.f_code.co_filename, f.f_lineno, f.f_code.co_name, sinfo


class BoundLogger(BoundLoggerBase):
    """
    Python Standard Library version of `structlog.BoundLogger`.

    Works exactly like the generic one except that it takes advantage of
    knowing the logging methods in advance.

    Use it like::

        structlog.configure(
            wrapper_class=structlog.stdlib.BoundLogger,
        )

    It also contains a bunch of properties that pass-through to the wrapped
    `logging.Logger` which should make it work as a drop-in replacement.
    """

    _logger: logging.Logger

    def bind(self, **new_values: Any) -> "BoundLogger":
        """
        Return a new logger with *new_values* added to the existing ones.
        """
        return super().bind(**new_values)  # type: ignore

    def unbind(self, *keys: str) -> "BoundLogger":
        """
        Return a new logger with *keys* removed from the context.

        :raises KeyError: If the key is not part of the context.
        """
        return super().unbind(*keys)  # type: ignore

    def try_unbind(self, *keys: str) -> "BoundLogger":
        """
        Like :meth:`unbind`, but best effort: missing keys are ignored.

        .. versionadded:: 18.2.0
        """
        return super().try_unbind(*keys)  # type: ignore

    def new(self, **new_values: Any) -> "BoundLogger":
        """
        Clear context and binds *initial_values* using `bind`.

        Only necessary with dict implementations that keep global state like
        those wrapped by `structlog.threadlocal.wrap_dict` when threads
        are re-used.
        """
        return super().new(**new_values)  # type: ignore

    def debug(self, event: Optional[str] = None, *args: Any, **kw: Any) -> Any:
        """
        Process event and call `logging.Logger.debug` with the result.
        """
        return self._proxy_to_logger("debug", event, *args, **kw)

    def info(self, event: Optional[str] = None, *args: Any, **kw: Any) -> Any:
        """
        Process event and call `logging.Logger.info` with the result.
        """
        return self._proxy_to_logger("info", event, *args, **kw)

    def warning(
        self, event: Optional[str] = None, *args: Any, **kw: Any
    ) -> Any:
        """
        Process event and call `logging.Logger.warning` with the result.
        """
        return self._proxy_to_logger("warning", event, *args, **kw)

    warn = warning

    def error(self, event: Optional[str] = None, *args: Any, **kw: Any) -> Any:
        """
        Process event and call `logging.Logger.error` with the result.
        """
        return self._proxy_to_logger("error", event, *args, **kw)

    def critical(
        self, event: Optional[str] = None, *args: Any, **kw: Any
    ) -> Any:
        """
        Process event and call `logging.Logger.critical` with the result.
        """
        return self._proxy_to_logger("critical", event, *args, **kw)

    def exception(
        self, event: Optional[str] = None, *args: Any, **kw: Any
    ) -> Any:
        """
        Process event and call `logging.Logger.error` with the result,
        after setting ``exc_info`` to `True`.
        """
        kw.setdefault("exc_info", True)

        return self.error(event, *args, **kw)

    def log(
        self, level: int, event: Optional[str] = None, *args: Any, **kw: Any
    ) -> Any:
        """
        Process *event* and call the appropriate logging method depending on
        *level*.
        """
        return self._proxy_to_logger(_LEVEL_TO_NAME[level], event, *args, **kw)

    fatal = critical

    def _proxy_to_logger(
        self,
        method_name: str,
        event: Optional[str] = None,
        *event_args: str,
        **event_kw: Any,
    ) -> Any:
        """
        Propagate a method call to the wrapped logger.

        This is the same as the superclass implementation, except that
        it also preserves positional arguments in the ``event_dict`` so
        that the stdlib's support for format strings can be used.
        """
        if event_args:
            event_kw["positional_args"] = event_args

        return super()._proxy_to_logger(method_name, event=event, **event_kw)

    #
    # Pass-through attributes and methods to mimick the stdlib's logger
    # interface.
    #

    @property
    def name(self) -> str:
        """
        Returns :attr:`logging.Logger.name`
        """
        return self._logger.name

    @property
    def level(self) -> int:
        """
        Returns :attr:`logging.Logger.level`
        """
        return self._logger.level

    @property
    def parent(self) -> Any:
        """
        Returns :attr:`logging.Logger.parent`
        """
        return self._logger.parent

    @property
    def propagate(self) -> bool:
        """
        Returns :attr:`logging.Logger.propagate`
        """
        return self._logger.propagate

    @property
    def handlers(self) -> Any:
        """
        Returns :attr:`logging.Logger.handlers`
        """
        return self._logger.handlers

    @property
    def disabled(self) -> int:
        """
        Returns :attr:`logging.Logger.disabled`
        """
        return self._logger.disabled

    def setLevel(self, level: int) -> None:
        """
        Calls :meth:`logging.Logger.setLevel` with unmodified arguments.
        """
        self._logger.setLevel(level)

    def findCaller(
        self, stack_info: bool = False
    ) -> Tuple[str, int, str, Optional[str]]:
        """
        Calls :meth:`logging.Logger.findCaller` with unmodified arguments.
        """
        return self._logger.findCaller(stack_info=stack_info)

    def makeRecord(
        self,
        name: str,
        level: int,
        fn: str,
        lno: int,
        msg: str,
        args: Tuple[Any, ...],
        exc_info: ExcInfo,
        func: Optional[str] = None,
        extra: Any = None,
    ) -> logging.LogRecord:
        """
        Calls :meth:`logging.Logger.makeRecord` with unmodified arguments.
        """
        return self._logger.makeRecord(
            name, level, fn, lno, msg, args, exc_info, func=func, extra=extra
        )

    def handle(self, record: logging.LogRecord) -> None:
        """
        Calls :meth:`logging.Logger.handle` with unmodified arguments.
        """
        self._logger.handle(record)

    def addHandler(self, hdlr: logging.Handler) -> None:
        """
        Calls :meth:`logging.Logger.addHandler` with unmodified arguments.
        """
        self._logger.addHandler(hdlr)

    def removeHandler(self, hdlr: logging.Handler) -> None:
        """
        Calls :meth:`logging.Logger.removeHandler` with unmodified arguments.
        """
        self._logger.removeHandler(hdlr)

    def hasHandlers(self) -> bool:
        """
        Calls :meth:`logging.Logger.hasHandlers` with unmodified arguments.

        Exists only in Python 3.
        """
        return self._logger.hasHandlers()

    def callHandlers(self, record: logging.LogRecord) -> None:
        """
        Calls :meth:`logging.Logger.callHandlers` with unmodified arguments.
        """
        self._logger.callHandlers(record)

    def getEffectiveLevel(self) -> int:
        """
        Calls :meth:`logging.Logger.getEffectiveLevel` with unmodified
        arguments.
        """
        return self._logger.getEffectiveLevel()

    def isEnabledFor(self, level: int) -> bool:
        """
        Calls :meth:`logging.Logger.isEnabledFor` with unmodified arguments.
        """
        return self._logger.isEnabledFor(level)

    def getChild(self, suffix: str) -> logging.Logger:
        """
        Calls :meth:`logging.Logger.getChild` with unmodified arguments.
        """
        return self._logger.getChild(suffix)


def get_logger(*args: Any, **initial_values: Any) -> BoundLogger:
    """
    Only calls `structlog.get_logger`, but has the correct type hints.

    .. warning::

       Does **not** check whether you've configured ``structlog`` correctly!

       See :doc:`standard-library` for details.

    .. versionadded:: 20.2.0
    """
    return _generic_get_logger(*args, **initial_values)


class AsyncBoundLogger:
    """
    Wraps a `BoundLogger` & exposes its logging methods as ``async`` versions.

    Instead of blocking the program, they are run asynchronously in a thread
    pool executor.

    This means more computational overhead per log call. But it also means that
    the processor chain (e.g. JSON serialization) and I/O won't block your
    whole application.

    Only available for Python 3.7 and later.

    :ivar structlog.stdlib.BoundLogger sync_bl: The wrapped synchronous logger.
       It is useful to be able to log synchronously occasionally.

    .. versionadded:: 20.2.0
    .. versionchanged:: 20.2.0 fix _dispatch_to_sync contextvars usage
    """

    __slots__ = ["sync_bl", "_loop"]

    sync_bl: BoundLogger

    # Blatant lie, we use a property for _context. Need this for Protocol
    # though.
    _context: Context

    _executor = None
    _bound_logger_factory = BoundLogger

    def __init__(
        self,
        logger: logging.Logger,
        processors: Iterable[Processor],
        context: Context,
        *,
        # Only as an optimization for binding!
        _sync_bl: Any = None,  # *vroom vroom* over purity.
        _loop: Any = None,
    ):
        if _sync_bl:
            self.sync_bl = _sync_bl
            self._loop = _loop

            return

        self.sync_bl = self._bound_logger_factory(
            logger=logger, processors=processors, context=context
        )
        self._loop = asyncio.get_running_loop()

    # We have to ignore the type because we've already declared it to ensure
    # we're a BindableLogger.
    # Instances would've been correctly recognized as such, however the class
    # not and we need the class in `structlog.configure()`.
    @property  # type: ignore
    def _context(self) -> Context:
        return self.sync_bl._context

    def bind(self, **new_values: Any) -> "AsyncBoundLogger":
        return AsyncBoundLogger(
            # logger, processors and context are within sync_bl. These
            # arguments are ignored if _sync_bl is passed. *vroom vroom* over
            # purity.
            logger=None,  # type: ignore
            processors=(),
            context={},
            _sync_bl=self.sync_bl.bind(**new_values),
            _loop=self._loop,
        )

    def new(self, **new_values: Any) -> "AsyncBoundLogger":
        return AsyncBoundLogger(
            # c.f. comment in bind
            logger=None,  # type: ignore
            processors=(),
            context={},
            _sync_bl=self.sync_bl.new(**new_values),
            _loop=self._loop,
        )

    def unbind(self, *keys: str) -> "AsyncBoundLogger":
        return AsyncBoundLogger(
            # c.f. comment in bind
            logger=None,  # type: ignore
            processors=(),
            context={},
            _sync_bl=self.sync_bl.unbind(*keys),
            _loop=self._loop,
        )

    def try_unbind(self, *keys: str) -> "AsyncBoundLogger":
        return AsyncBoundLogger(
            # c.f. comment in bind
            logger=None,  # type: ignore
            processors=(),
            context={},
            _sync_bl=self.sync_bl.try_unbind(*keys),
            _loop=self._loop,
        )

    async def _dispatch_to_sync(
        self,
        meth: Callable[..., Any],
        event: str,
        args: Tuple[Any, ...],
        kw: Dict[str, Any],
    ) -> None:
        """
        Merge contextvars and log using the sync logger in a thread pool.
        """
        ctx = contextvars.copy_context()

        await asyncio.get_running_loop().run_in_executor(
            self._executor,
            lambda: ctx.run(lambda: meth(event, *args, **kw)),
        )

    async def debug(self, event: str, *args: Any, **kw: Any) -> None:
        await self._dispatch_to_sync(self.sync_bl.debug, event, args, kw)

    async def info(self, event: str, *args: Any, **kw: Any) -> None:
        await self._dispatch_to_sync(self.sync_bl.info, event, args, kw)

    async def warning(self, event: str, *args: Any, **kw: Any) -> None:
        await self._dispatch_to_sync(self.sync_bl.warning, event, args, kw)

    warn = warning

    async def error(self, event: str, *args: Any, **kw: Any) -> None:
        await self._dispatch_to_sync(self.sync_bl.error, event, args, kw)

    async def critical(self, event: str, *args: Any, **kw: Any) -> None:
        await self._dispatch_to_sync(self.sync_bl.critical, event, args, kw)

    fatal = critical

    async def exception(self, event: str, *args: Any, **kw: Any) -> None:
        # To make `log.exception("foo") work, we have to check if the user
        # passed an explicit exc_info and if not, supply our own.
        ei = kw.pop("exc_info", None)
        if ei is None and kw.get("exception") is None:
            ei = sys.exc_info()

        kw["exc_info"] = ei

        await self._dispatch_to_sync(self.sync_bl.exception, event, args, kw)

    async def log(self, level: Any, event: str, *args: Any, **kw: Any) -> None:
        await self._dispatch_to_sync(
            partial(self.sync_bl.log, level), event, args, kw
        )


class LoggerFactory:
    """
    Build a standard library logger when an *instance* is called.

    Sets a custom logger using :func:`logging.setLoggerClass` so variables in
    log format are expanded properly.

    >>> from structlog import configure
    >>> from structlog.stdlib import LoggerFactory
    >>> configure(logger_factory=LoggerFactory())

    :param ignore_frame_names: When guessing the name of a logger, skip frames
        whose names *start* with one of these.  For example, in pyramid
        applications you'll want to set it to
        ``["venusian", "pyramid.config"]``.
    """

    def __init__(self, ignore_frame_names: Optional[List[str]] = None):
        self._ignore = ignore_frame_names
        logging.setLoggerClass(_FixedFindCallerLogger)

    def __call__(self, *args: Any) -> logging.Logger:
        """
        Deduce the caller's module name and create a stdlib logger.

        If an optional argument is passed, it will be used as the logger name
        instead of guesswork.  This optional argument would be passed from the
        :func:`structlog.get_logger` call.  For example
        ``structlog.get_logger("foo")`` would cause this method to be called
        with ``"foo"`` as its first positional argument.

        .. versionchanged:: 0.4.0
            Added support for optional positional arguments.  Using the first
            one for naming the constructed logger.
        """
        if args:
            return logging.getLogger(args[0])

        # We skip all frames that originate from within structlog or one of the
        # configured names.
        _, name = _find_first_app_frame_and_name(self._ignore)

        return logging.getLogger(name)


class PositionalArgumentsFormatter:
    """
    Apply stdlib-like string formatting to the ``event`` key.

    If the ``positional_args`` key in the event dict is set, it must
    contain a tuple that is used for formatting (using the ``%s`` string
    formatting operator) of the value from the ``event`` key.  This works
    in the same way as the stdlib handles arguments to the various log
    methods: if the tuple contains only a single `dict` argument it is
    used for keyword placeholders in the ``event`` string, otherwise it
    will be used for positional placeholders.

    ``positional_args`` is populated by `structlog.stdlib.BoundLogger` or
    can be set manually.

    The *remove_positional_args* flag can be set to `False` to keep the
    ``positional_args`` key in the event dict; by default it will be
    removed from the event dict after formatting a message.
    """

    def __init__(self, remove_positional_args: bool = True) -> None:
        self.remove_positional_args = remove_positional_args

    def __call__(
        self, _: WrappedLogger, __: str, event_dict: EventDict
    ) -> EventDict:
        args = event_dict.get("positional_args")

        # Mimick the formatting behaviour of the stdlib's logging
        # module, which accepts both positional arguments and a single
        # dict argument. The "single dict" check is the same one as the
        # stdlib's logging module performs in LogRecord.__init__().
        if args:
            if len(args) == 1 and isinstance(args[0], dict) and args[0]:
                args = args[0]

            event_dict["event"] %= args

        if self.remove_positional_args and args is not None:
            del event_dict["positional_args"]

        return event_dict


def filter_by_level(
    logger: logging.Logger, method_name: str, event_dict: EventDict
) -> EventDict:
    """
    Check whether logging is configured to accept messages from this log level.

    Should be the first processor if stdlib's filtering by level is used so
    possibly expensive processors like exception formatters are avoided in the
    first place.

    >>> import logging
    >>> from structlog.stdlib import filter_by_level
    >>> logging.basicConfig(level=logging.WARN)
    >>> logger = logging.getLogger()
    >>> filter_by_level(logger, 'warn', {})
    {}
    >>> filter_by_level(logger, 'debug', {})
    Traceback (most recent call last):
    ...
    DropEvent
    """
    if logger.isEnabledFor(_NAME_TO_LEVEL[method_name]):
        return event_dict
    else:
        raise DropEvent


def add_log_level_number(
    logger: logging.Logger, method_name: str, event_dict: EventDict
) -> EventDict:
    """
    Add the log level number to the event dict.

    Log level numbers map to the log level names. The Python stdlib uses them
    for filtering logic. This adds the same numbers so users can leverage
    similar filtering. Compare::

       level in ("warning", "error", "critical")
       level_number >= 30

    The mapping of names to numbers is in
    ``structlog.stdlib._log_levels._NAME_TO_LEVEL``.

    .. versionadded:: 18.2.0
    """
    event_dict["level_number"] = _NAME_TO_LEVEL[method_name]

    return event_dict


def add_logger_name(
    logger: logging.Logger, method_name: str, event_dict: EventDict
) -> EventDict:
    """
    Add the logger name to the event dict.
    """
    record = event_dict.get("_record")
    if record is None:
        event_dict["logger"] = logger.name
    else:
        event_dict["logger"] = record.name
    return event_dict


_LOG_RECORD_KEYS = logging.LogRecord(
    "name", 0, "pathname", 0, "msg", tuple(), None
).__dict__.keys()


class ExtraAdder:
    """
    Add extra attributes of `logging.LogRecord` objects to the event
    dictionary.

    This processor can be used for adding data passed in the ``extra``
    parameter of the `logging` module's log methods to the event dictionary.

    :param allow: An optional collection of attributes that, if present in
        `logging.LogRecord` objects, will be copied to event dictionaries.

        If ``allow`` is None all attributes of `logging.LogRecord` objects that
        do not exist on a standard `logging.LogRecord` object will be copied to
        event dictionaries.

    .. versionadded:: 21.5.0
    """

    __slots__ = ["_copier"]

    def __init__(self, allow: Optional[Collection[str]] = None) -> None:
        self._copier: Callable[[EventDict, logging.LogRecord], None]
        if allow is not None:
            # The contents of allow is copied to a new list so that changes to
            # the list passed into the constructor does not change the
            # behaviour of this processor.
            self._copier = functools.partial(self._copy_allowed, [*allow])
        else:
            self._copier = self._copy_all

    def __call__(
        self, logger: logging.Logger, name: str, event_dict: EventDict
    ) -> EventDict:
        record: Optional[logging.LogRecord] = event_dict.get("_record")
        if record is not None:
            self._copier(event_dict, record)
        return event_dict

    @classmethod
    def _copy_all(
        cls, event_dict: EventDict, record: logging.LogRecord
    ) -> None:
        for key, value in record.__dict__.items():
            if key not in _LOG_RECORD_KEYS:
                event_dict[key] = value

    @classmethod
    def _copy_allowed(
        cls,
        allow: Collection[str],
        event_dict: EventDict,
        record: logging.LogRecord,
    ) -> None:
        for key in allow:
            if key in record.__dict__:
                event_dict[key] = record.__dict__[key]


def render_to_log_kwargs(
    _: logging.Logger, __: str, event_dict: EventDict
) -> EventDict:
    """
    Render ``event_dict`` into keyword arguments for `logging.log`.

    The ``event`` field is translated into ``msg`` and the rest of the
    *event_dict* is added as ``extra``.

    This allows you to defer formatting to `logging`.

    .. versionadded:: 17.1.0
    """
    return {"msg": event_dict.pop("event"), "extra": event_dict}


class ProcessorFormatter(logging.Formatter):
    r"""
    Call ``structlog`` processors on `logging.LogRecord`\s.

    This is an implementation of a `logging.Formatter` that can be used to
    format log entries from both ``structlog`` and `logging`.

    Its static method `wrap_for_formatter` must be the final processor in
    ``structlog``'s processor chain.

    Please refer to :ref:`processor-formatter` for examples.

    :param foreign_pre_chain:
        If not `None`, it is used as a processor chain that is applied to
        **non**-``structlog`` log entries before the event dictionary is passed
        to *processors*. (default: `None`)
    :param processors:
        A chain of ``structlog`` processors that is used to process **all** log
        entries. The last one must render to a `str` which then gets passed on
        to `logging` for output.

        Compared to ``structlog``'s regular processor chains, there's a few
        differences:

        - The event dictionary contains two additional keys:

          #. ``_record``: a `logging.LogRecord` that either was created using
             `logging` APIs, **or** is a wrapped ``structlog`` log entry
             created by `wrap_for_formatter`.
          #. ``_from_structlog``: a `bool` that indicates whether or not
             ``_record`` was created by a ``structlog`` logger.

          Since you most likely don't want ``_record`` and
          ``_from_structlog`` in your log files,  we've added
          the static method `remove_processors_meta` to ``ProcessorFormatter``
          that you can add just before your renderer.

        - Since this is a `logging` *formatter*, raising `structlog.DropEvent`
          will crash your application.

    :param keep_exc_info: ``exc_info`` on `logging.LogRecord`\ s is
        added to the ``event_dict`` and removed afterwards. Set this to
        ``True`` to keep it on the `logging.LogRecord`. (default: False)
    :param keep_stack_info: Same as *keep_exc_info* except for ``stack_info``.
        (default: False)
    :param logger: Logger which we want to push through the ``structlog``
        processor chain. This parameter is necessary for some of the
        processors like `filter_by_level`. (default: None)
    :param pass_foreign_args: If True, pass a foreign log record's
        ``args`` attribute to the ``event_dict`` under ``positional_args`` key.
        (default: False)
    :param processor:
        A single ``structlog`` processor used for rendering the event
        dictionary before passing it off to `logging`. Must return a `str`.
        The event dictionary does **not** contain ``_record`` and
        ``_from_structlog``.

        This parameter exists for historic reasons. Please consider using
        *processors* instead.

    :raises TypeError: If both or neither *processor* and *processors* are
        passed.

    .. versionadded:: 17.1.0
    .. versionadded:: 17.2.0 *keep_exc_info* and *keep_stack_info*
    .. versionadded:: 19.2.0 *logger*
    .. versionadded:: 19.2.0 *pass_foreign_args*
    .. versionadded:: 21.3.0 *processors*
    .. deprecated:: 21.3.0
       *processor* (singular) in favor of *processors* (plural). Removal is not
       planned.
    """

    def __init__(
        self,
        processor: Optional[Processor] = None,
        processors: Optional[Sequence[Processor]] = (),
        foreign_pre_chain: Optional[Sequence[Processor]] = None,
        keep_exc_info: bool = False,
        keep_stack_info: bool = False,
        logger: Optional[logging.Logger] = None,
        pass_foreign_args: bool = False,
        *args: Any,
        **kwargs: Any,
    ) -> None:
        fmt = kwargs.pop("fmt", "%(message)s")
        super().__init__(*args, fmt=fmt, **kwargs)  # type: ignore

        if processor and processors:
            raise TypeError(
                "The `processor` and `processors` arguments are mutually "
                "exclusive."
            )

        self.processors: Sequence[Processor]
        if processor is not None:
            self.processors = (self.remove_processors_meta, processor)
        elif processors:
            self.processors = processors
        else:
            raise TypeError(
                "Either `processor` or `processors` must be passed."
            )

        self.foreign_pre_chain = foreign_pre_chain
        self.keep_exc_info = keep_exc_info
        self.keep_stack_info = keep_stack_info
        self.logger = logger
        self.pass_foreign_args = pass_foreign_args

    def format(self, record: logging.LogRecord) -> str:
        """
        Extract ``structlog``'s `event_dict` from ``record.msg`` and format it.

        *record* has been patched by `wrap_for_formatter` first though, so the
         type isn't quite right.
        """
        # Make a shallow copy of the record to let other handlers/formatters
        # process the original one
        record = logging.makeLogRecord(record.__dict__)

        logger = getattr(record, "_logger", _SENTINEL)
        meth_name = getattr(record, "_name", _SENTINEL)

        if logger is not _SENTINEL and meth_name is not _SENTINEL:
            # Both attached by wrap_for_formatter
            if self.logger is not None:
                logger = self.logger
            meth_name = record._name  # type: ignore

            # We need to copy because it's possible that the same record gets
            # processed by multiple logging formatters.  LogRecord.getMessage
            # would transform our dict into a str.
            ed = record.msg.copy()  # type: ignore
            ed["_record"] = record
            ed["_from_structlog"] = True
        else:
            logger = self.logger
            meth_name = record.levelname.lower()
            ed = {
                "event": record.getMessage(),
                "_record": record,
                "_from_structlog": False,
            }

            if self.pass_foreign_args:
                ed["positional_args"] = record.args

            record.args = ()

            # Add stack-related attributes to event_dict and unset them
            # on the record copy so that the base implementation wouldn't
            # append stacktraces to the output.
            if record.exc_info:
                ed["exc_info"] = record.exc_info
            if record.stack_info:
                ed["stack_info"] = record.stack_info

            if not self.keep_exc_info:
                record.exc_text = None
                record.exc_info = None
            if not self.keep_stack_info:
                record.stack_info = None

            # Non-structlog allows to run through a chain to prepare it for the
            # final processor (e.g. adding timestamps and log levels).
            for proc in self.foreign_pre_chain or ():
                ed = proc(logger, meth_name, ed)

        for p in self.processors:
            ed = p(logger, meth_name, ed)

        record.msg = ed

        return super().format(record)

    @staticmethod
    def wrap_for_formatter(
        logger: logging.Logger, name: str, event_dict: EventDict
    ) -> Tuple[Tuple[EventDict], Dict[str, Dict[str, Any]]]:
        """
        Wrap *logger*, *name*, and *event_dict*.

        The result is later unpacked by `ProcessorFormatter` when
        formatting log entries.

        Use this static method as the renderer (i.e. final processor) if you
        want to use `ProcessorFormatter` in your `logging` configuration.
        """
        return (event_dict,), {"extra": {"_logger": logger, "_name": name}}

    @staticmethod
    def remove_processors_meta(
        _: WrappedLogger, __: str, event_dict: EventDict
    ) -> EventDict:
        """
        Remove ``_record`` and ``_from_structlog`` from *event_dict*.

        These keys are added to the event dictionary, before
        `ProcessorFormatter`'s *processors* are run.

        .. versionadded:: 21.3.0
        """
        del event_dict["_record"]
        del event_dict["_from_structlog"]

        return event_dict

env_pyrecodes/lib/python3.9/site-packages/structlog/stdlib.py
NikolaBlagojevic/pyrecodes
# SPDX-License-Identifier: MIT OR Apache-2.0
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the MIT License.  See the LICENSE file in the root of this
# repository for complete details.

"""
Type information used throughout ``structlog``.

For now, they are considered provisional. Especially `BindableLogger` will
probably change to something more elegant.

.. versionadded:: 20.2
"""

import sys

from types import TracebackType
from typing import (
    Any,
    Callable,
    Dict,
    Mapping,
    MutableMapping,
    Optional,
    TextIO,
    Tuple,
    Type,
    Union,
)


# This construct works better with Mypy.
# Doing the obvious ImportError route leads to an 'Incompatible import of
# "Protocol"' error.
if sys.version_info >= (3, 8):
    from typing import Protocol, runtime_checkable
else:
    from typing_extensions import Protocol, runtime_checkable


WrappedLogger = Any
"""
A logger that is wrapped by a bound logger and is ultimately responsible for
the output of the log entries.

``structlog`` makes *no* assumptions about it.

.. versionadded:: 20.2
"""


Context = Union[Dict[str, Any], Dict[Any, Any]]
"""
A dict-like context carrier.

.. versionadded:: 20.2
"""


EventDict = MutableMapping[str, Any]
"""
An event dictionary as it is passed into processors.

It's created by copying the configured `Context` but doesn't need to support
copy itself.

.. versionadded:: 20.2
"""

Processor = Callable[
    [WrappedLogger, str, EventDict],
    Union[Mapping[str, Any], str, bytes, bytearray, Tuple[Any, ...]],
]
"""
A callable that is part of the processor chain.

See :doc:`processors`.

.. versionadded:: 20.2
"""

ExcInfo = Tuple[Type[BaseException], BaseException, Optional[TracebackType]]
"""
An exception info tuple as returned by `sys.exc_info`.

.. versionadded:: 20.2
"""


ExceptionFormatter = Callable[[TextIO, ExcInfo], None]
"""
A callable that pretty-prints an `ExcInfo` into a file-like object.

Used by `structlog.dev.ConsoleRenderer`.

.. versionadded:: 21.2
"""


@runtime_checkable
class BindableLogger(Protocol):
    """
    **Protocol**: Methods shared among all bound loggers and that are relied on
    by ``structlog``.

    .. versionadded:: 20.2
    """

    _context: Context

    def bind(self, **new_values: Any) -> "BindableLogger":
        ...

    def unbind(self, *keys: str) -> "BindableLogger":
        ...

    def try_unbind(self, *keys: str) -> "BindableLogger":
        ...

    def new(self, **new_values: Any) -> "BindableLogger":
        ...


class FilteringBoundLogger(BindableLogger, Protocol):
    """
    **Protocol**: A `BindableLogger` that filters by a level.

    The only way to instantiate one is using `make_filtering_bound_logger`.

    .. versionadded:: 20.2.0
    """

    def debug(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **debug** level.
        """

    def info(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **info** level.
        """

    def warning(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **warn** level.
        """

    def warn(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **warn** level.
        """

    def error(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **error** level.
        """

    def err(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **error** level.
        """

    def fatal(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **critical** level.
        """

    def exception(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **error** level and ensure that ``exc_info``
        is set in the event dictionary.
        """

    def critical(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **critical** level.
        """

    def msg(self, event: str, **kw: Any) -> Any:
        """
        Log *event* with **kw** at **info** level.
        """

env_pyrecodes/lib/python3.9/site-packages/structlog/types.py
NikolaBlagojevic/pyrecodes
from tornado.concurrent import Future
from tornado import gen
from tornado.escape import (
    json_decode,
    utf8,
    to_unicode,
    recursive_unicode,
    native_str,
    to_basestring,
)
from tornado.httpclient import HTTPClientError
from tornado.httputil import format_timestamp
from tornado.iostream import IOStream
from tornado import locale
from tornado.locks import Event
from tornado.log import app_log, gen_log
from tornado.simple_httpclient import SimpleAsyncHTTPClient
from tornado.template import DictLoader
from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, ExpectLog, gen_test
from tornado.util import ObjectDict, unicode_type
from tornado.web import (
    Application,
    RequestHandler,
    StaticFileHandler,
    RedirectHandler as WebRedirectHandler,
    HTTPError,
    MissingArgumentError,
    ErrorHandler,
    authenticated,
    url,
    _create_signature_v1,
    create_signed_value,
    decode_signed_value,
    get_signature_key_version,
    UIModule,
    Finish,
    stream_request_body,
    removeslash,
    addslash,
    GZipContentEncoding,
)

import binascii
import contextlib
import copy
import datetime
import email.utils
import gzip
from io import BytesIO
import itertools
import logging
import os
import re
import socket
import typing  # noqa: F401
import unittest
import urllib.parse


def relpath(*a):
    return os.path.join(os.path.dirname(__file__), *a)


class WebTestCase(AsyncHTTPTestCase):
    """Base class for web tests that also supports WSGI mode.

    Override get_handlers and get_app_kwargs instead of get_app.
    This class is deprecated since WSGI mode is no longer supported.
    """

    def get_app(self):
        self.app = Application(self.get_handlers(), **self.get_app_kwargs())
        return self.app

    def get_handlers(self):
        raise NotImplementedError()

    def get_app_kwargs(self):
        return {}


class SimpleHandlerTestCase(WebTestCase):
    """Simplified base class for tests that work with a single handler class.

    To use, define a nested class named ``Handler``.
    """

    Handler = None

    def get_handlers(self):
        return [("/", self.Handler)]


class HelloHandler(RequestHandler):
    def get(self):
        self.write("hello")


class CookieTestRequestHandler(RequestHandler):
    # stub out enough methods to make the signed_cookie functions work
    def __init__(self, cookie_secret="0123456789", key_version=None):
        # don't call super.__init__
        self._cookies = {}  # type: typing.Dict[str, bytes]
        if key_version is None:
            self.application = ObjectDict(  # type: ignore
                settings=dict(cookie_secret=cookie_secret)
            )
        else:
            self.application = ObjectDict(  # type: ignore
                settings=dict(cookie_secret=cookie_secret, key_version=key_version)
            )

    def get_cookie(self, name):
        return self._cookies.get(name)

    def set_cookie(self, name, value, expires_days=None):
        self._cookies[name] = value


# See SignedValueTest below for more.
class SecureCookieV1Test(unittest.TestCase):
    def test_round_trip(self):
        handler = CookieTestRequestHandler()
        handler.set_signed_cookie("foo", b"bar", version=1)
        self.assertEqual(handler.get_signed_cookie("foo", min_version=1), b"bar")

    def test_cookie_tampering_future_timestamp(self):
        handler = CookieTestRequestHandler()
        # this string base64-encodes to '12345678'
        handler.set_signed_cookie("foo", binascii.a2b_hex(b"d76df8e7aefc"), version=1)
        cookie = handler._cookies["foo"]
        match = re.match(rb"12345678\|([0-9]+)\|([0-9a-f]+)", cookie)
        assert match is not None
        timestamp = match.group(1)
        sig = match.group(2)
        self.assertEqual(
            _create_signature_v1(
                handler.application.settings["cookie_secret"],
                "foo",
                "12345678",
                timestamp,
            ),
            sig,
        )
        # shifting digits from payload to timestamp doesn't alter signature
        # (this is not desirable behavior, just confirming that that's how it
        # works)
        self.assertEqual(
            _create_signature_v1(
                handler.application.settings["cookie_secret"],
                "foo",
                "1234",
                b"5678" + timestamp,
            ),
            sig,
        )
        # tamper with the cookie
        handler._cookies["foo"] = utf8(
            "1234|5678%s|%s" % (to_basestring(timestamp), to_basestring(sig))
        )
        # it gets rejected
        with ExpectLog(gen_log, "Cookie timestamp in future"):
            self.assertTrue(handler.get_signed_cookie("foo", min_version=1) is None)

    def test_arbitrary_bytes(self):
        # Secure cookies accept arbitrary data (which is base64 encoded).
        # Note that normal cookies accept only a subset of ascii.
        handler = CookieTestRequestHandler()
        handler.set_signed_cookie("foo", b"\xe9", version=1)
        self.assertEqual(handler.get_signed_cookie("foo", min_version=1), b"\xe9")


# See SignedValueTest below for more.
class SecureCookieV2Test(unittest.TestCase):
    KEY_VERSIONS = {0: "ajklasdf0ojaisdf", 1: "aslkjasaolwkjsdf"}

    def test_round_trip(self):
        handler = CookieTestRequestHandler()
        handler.set_signed_cookie("foo", b"bar", version=2)
        self.assertEqual(handler.get_signed_cookie("foo", min_version=2), b"bar")

    def test_key_version_roundtrip(self):
        handler = CookieTestRequestHandler(
            cookie_secret=self.KEY_VERSIONS, key_version=0
        )
        handler.set_signed_cookie("foo", b"bar")
        self.assertEqual(handler.get_signed_cookie("foo"), b"bar")

    def test_key_version_roundtrip_differing_version(self):
        handler = CookieTestRequestHandler(
            cookie_secret=self.KEY_VERSIONS, key_version=1
        )
        handler.set_signed_cookie("foo", b"bar")
        self.assertEqual(handler.get_signed_cookie("foo"), b"bar")

    def test_key_version_increment_version(self):
        handler = CookieTestRequestHandler(
            cookie_secret=self.KEY_VERSIONS, key_version=0
        )
        handler.set_signed_cookie("foo", b"bar")
        new_handler = CookieTestRequestHandler(
            cookie_secret=self.KEY_VERSIONS, key_version=1
        )
        new_handler._cookies = handler._cookies
        self.assertEqual(new_handler.get_signed_cookie("foo"), b"bar")

    def test_key_version_invalidate_version(self):
        handler = CookieTestRequestHandler(
            cookie_secret=self.KEY_VERSIONS, key_version=0
        )
        handler.set_signed_cookie("foo", b"bar")
        new_key_versions = self.KEY_VERSIONS.copy()
        new_key_versions.pop(0)
        new_handler = CookieTestRequestHandler(
            cookie_secret=new_key_versions, key_version=1
        )
        new_handler._cookies = handler._cookies
        self.assertEqual(new_handler.get_signed_cookie("foo"), None)


class FinalReturnTest(WebTestCase):
    final_return = None  # type: Future

    def get_handlers(self):
        test = self

        class FinishHandler(RequestHandler):
            @gen.coroutine
            def get(self):
                test.final_return = self.finish()
                yield test.final_return

            @gen.coroutine
            def post(self):
                self.write("hello,")
                yield self.flush()
                test.final_return = self.finish("world")
                yield test.final_return

        class RenderHandler(RequestHandler):
            def create_template_loader(self, path):
                return DictLoader({"foo.html": "hi"})

            @gen.coroutine
            def get(self):
                test.final_return = self.render("foo.html")

        return [("/finish", FinishHandler), ("/render", RenderHandler)]

    def get_app_kwargs(self):
        return dict(template_path="FinalReturnTest")

    def test_finish_method_return_future(self):
        response = self.fetch(self.get_url("/finish"))
        self.assertEqual(response.code, 200)
        self.assertIsInstance(self.final_return, Future)
        self.assertTrue(self.final_return.done())

        response = self.fetch(self.get_url("/finish"), method="POST", body=b"")
        self.assertEqual(response.code, 200)
        self.assertIsInstance(self.final_return, Future)
        self.assertTrue(self.final_return.done())

    def test_render_method_return_future(self):
        response = self.fetch(self.get_url("/render"))
        self.assertEqual(response.code, 200)
        self.assertIsInstance(self.final_return, Future)


class CookieTest(WebTestCase):
    def get_handlers(self):
        class SetCookieHandler(RequestHandler):
            def get(self):
                # Try setting cookies with different argument types
                # to ensure that everything gets encoded correctly
                self.set_cookie("str", "asdf")
                self.set_cookie("unicode", "qwer")
                self.set_cookie("bytes", b"zxcv")

        class GetCookieHandler(RequestHandler):
            def get(self):
                cookie = self.get_cookie("foo", "default")
                assert cookie is not None
                self.write(cookie)

        class SetCookieDomainHandler(RequestHandler):
            def get(self):
                # unicode domain and path arguments shouldn't break things
                # either (see bug #285)
                self.set_cookie("unicode_args", "blah", domain="foo.com", path="/foo")

        class SetCookieSpecialCharHandler(RequestHandler):
            def get(self):
                self.set_cookie("equals", "a=b")
                self.set_cookie("semicolon", "a;b")
                self.set_cookie("quote", 'a"b')

        class SetCookieOverwriteHandler(RequestHandler):
            def get(self):
                self.set_cookie("a", "b", domain="example.com")
                self.set_cookie("c", "d", domain="example.com")
                # A second call with the same name clobbers the first.
                # Attributes from the first call are not carried over.
                self.set_cookie("a", "e")

        class SetCookieMaxAgeHandler(RequestHandler):
            def get(self):
                self.set_cookie("foo", "bar", max_age=10)

        class SetCookieExpiresDaysHandler(RequestHandler):
            def get(self):
                self.set_cookie("foo", "bar", expires_days=10)

        class SetCookieFalsyFlags(RequestHandler):
            def get(self):
                self.set_cookie("a", "1", secure=True)
                self.set_cookie("b", "1", secure=False)
                self.set_cookie("c", "1", httponly=True)
                self.set_cookie("d", "1", httponly=False)

        return [
            ("/set", SetCookieHandler),
            ("/get", GetCookieHandler),
            ("/set_domain", SetCookieDomainHandler),
            ("/special_char", SetCookieSpecialCharHandler),
            ("/set_overwrite", SetCookieOverwriteHandler),
            ("/set_max_age", SetCookieMaxAgeHandler),
            ("/set_expires_days", SetCookieExpiresDaysHandler),
            ("/set_falsy_flags", SetCookieFalsyFlags),
        ]

    def test_set_cookie(self):
        response = self.fetch("/set")
        self.assertEqual(
            sorted(response.headers.get_list("Set-Cookie")),
            ["bytes=zxcv; Path=/", "str=asdf; Path=/", "unicode=qwer; Path=/"],
        )

    def test_get_cookie(self):
        response = self.fetch("/get", headers={"Cookie": "foo=bar"})
        self.assertEqual(response.body, b"bar")

        response = self.fetch("/get", headers={"Cookie": 'foo="bar"'})
        self.assertEqual(response.body, b"bar")

        response = self.fetch("/get", headers={"Cookie": "/=exception;"})
        self.assertEqual(response.body, b"default")

    def test_set_cookie_domain(self):
        response = self.fetch("/set_domain")
        self.assertEqual(
            response.headers.get_list("Set-Cookie"),
            ["unicode_args=blah; Domain=foo.com; Path=/foo"],
        )

    def test_cookie_special_char(self):
        response = self.fetch("/special_char")
        headers = sorted(response.headers.get_list("Set-Cookie"))
        self.assertEqual(len(headers), 3)
        self.assertEqual(headers[0], 'equals="a=b"; Path=/')
        self.assertEqual(headers[1], 'quote="a\\"b"; Path=/')
        # python 2.7 octal-escapes the semicolon; older versions leave it alone
        self.assertTrue(
            headers[2] in ('semicolon="a;b"; Path=/', 'semicolon="a\\073b"; Path=/'),
            headers[2],
        )

        data = [
            ("foo=a=b", "a=b"),
            ('foo="a=b"', "a=b"),
            ('foo="a;b"', '"a'),  # even quoted, ";" is a delimiter
            ("foo=a\\073b", "a\\073b"),  # escapes only decoded in quotes
            ('foo="a\\073b"', "a;b"),
            ('foo="a\\"b"', 'a"b'),
        ]
        for header, expected in data:
            logging.debug("trying %r", header)
            response = self.fetch("/get", headers={"Cookie": header})
            self.assertEqual(response.body, utf8(expected))

    def test_set_cookie_overwrite(self):
        response = self.fetch("/set_overwrite")
        headers = response.headers.get_list("Set-Cookie")
        self.assertEqual(
            sorted(headers), ["a=e; Path=/", "c=d; Domain=example.com; Path=/"]
        )

    def test_set_cookie_max_age(self):
        response = self.fetch("/set_max_age")
        headers = response.headers.get_list("Set-Cookie")
        self.assertEqual(sorted(headers), ["foo=bar; Max-Age=10; Path=/"])

    def test_set_cookie_expires_days(self):
        response = self.fetch("/set_expires_days")
        header = response.headers.get("Set-Cookie")
        assert header is not None
        match = re.match("foo=bar; expires=(?P<expires>.+); Path=/", header)
        assert match is not None

        expires = datetime.datetime.utcnow() + datetime.timedelta(days=10)
        parsed = email.utils.parsedate(match.groupdict()["expires"])
        assert parsed is not None
        header_expires = datetime.datetime(*parsed[:6])
        self.assertTrue(abs((expires - header_expires).total_seconds()) < 10)

    def test_set_cookie_false_flags(self):
        response = self.fetch("/set_falsy_flags")
        headers = sorted(response.headers.get_list("Set-Cookie"))
        # The secure and httponly headers are capitalized in py35 and
        # lowercase in older versions.
        self.assertEqual(headers[0].lower(), "a=1; path=/; secure")
        self.assertEqual(headers[1].lower(), "b=1; path=/")
        self.assertEqual(headers[2].lower(), "c=1; httponly; path=/")
        self.assertEqual(headers[3].lower(), "d=1; path=/")


class AuthRedirectRequestHandler(RequestHandler):
    def initialize(self, login_url):
        self.login_url = login_url

    def get_login_url(self):
        return self.login_url

    @authenticated
    def get(self):
        # we'll never actually get here because the test doesn't follow redirects
        self.send_error(500)


class AuthRedirectTest(WebTestCase):
    def get_handlers(self):
        return [
            ("/relative", AuthRedirectRequestHandler, dict(login_url="/login")),
            (
                "/absolute",
                AuthRedirectRequestHandler,
                dict(login_url="http://example.com/login"),
            ),
        ]

    def test_relative_auth_redirect(self):
        response = self.fetch(self.get_url("/relative"), follow_redirects=False)
        self.assertEqual(response.code, 302)
        self.assertEqual(response.headers["Location"], "/login?next=%2Frelative")

    def test_absolute_auth_redirect(self):
        response = self.fetch(self.get_url("/absolute"), follow_redirects=False)
        self.assertEqual(response.code, 302)
        self.assertTrue(
            re.match(
                r"http://example.com/login\?next=http%3A%2F%2F127.0.0.1%3A[0-9]+%2Fabsolute",
                response.headers["Location"],
            ),
            response.headers["Location"],
        )


class ConnectionCloseHandler(RequestHandler):
    def initialize(self, test):
        self.test = test

    @gen.coroutine
    def get(self):
        self.test.on_handler_waiting()
        yield self.test.cleanup_event.wait()

    def on_connection_close(self):
        self.test.on_connection_close()


class ConnectionCloseTest(WebTestCase):
    def get_handlers(self):
        self.cleanup_event = Event()
        return [("/", ConnectionCloseHandler, dict(test=self))]

    def test_connection_close(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        s.connect(("127.0.0.1", self.get_http_port()))
        self.stream = IOStream(s)
        self.stream.write(b"GET / HTTP/1.0\r\n\r\n")
        self.wait()
        # Let the hanging coroutine clean up after itself
        self.cleanup_event.set()
        self.io_loop.run_sync(lambda: gen.sleep(0))

    def on_handler_waiting(self):
        logging.debug("handler waiting")
        self.stream.close()

    def on_connection_close(self):
        logging.debug("connection closed")
        self.stop()


class EchoHandler(RequestHandler):
    def get(self, *path_args):
        # Type checks: web.py interfaces convert argument values to
        # unicode strings (by default, but see also decode_argument).
        # In httpserver.py (i.e. self.request.arguments), they're left
        # as bytes.  Keys are always native strings.
        for key in self.request.arguments:
            if type(key) != str:
                raise Exception("incorrect type for key: %r" % type(key))
            for bvalue in self.request.arguments[key]:
                if type(bvalue) != bytes:
                    raise Exception("incorrect type for value: %r" % type(bvalue))
            for svalue in self.get_arguments(key):
                if type(svalue) != unicode_type:
                    raise Exception("incorrect type for value: %r" % type(svalue))
        for arg in path_args:
            if type(arg) != unicode_type:
                raise Exception("incorrect type for path arg: %r" % type(arg))
        self.write(
            dict(
                path=self.request.path,
                path_args=path_args,
                args=recursive_unicode(self.request.arguments),
            )
        )


class RequestEncodingTest(WebTestCase):
    def get_handlers(self):
        return [("/group/(.*)", EchoHandler), ("/slashes/([^/]*)/([^/]*)", EchoHandler)]

    def fetch_json(self, path):
        return json_decode(self.fetch(path).body)

    def test_group_question_mark(self):
        # Ensure that url-encoded question marks are handled properly
        self.assertEqual(
            self.fetch_json("/group/%3F"),
            dict(path="/group/%3F", path_args=["?"], args={}),
        )
        self.assertEqual(
            self.fetch_json("/group/%3F?%3F=%3F"),
            dict(path="/group/%3F", path_args=["?"], args={"?": ["?"]}),
        )

    def test_group_encoding(self):
        # Path components and query arguments should be decoded the same way
        self.assertEqual(
            self.fetch_json("/group/%C3%A9?arg=%C3%A9"),
            {
                "path": "/group/%C3%A9",
                "path_args": ["\u00e9"],
                "args": {"arg": ["\u00e9"]},
            },
        )

    def test_slashes(self):
        # Slashes may be escaped to appear as a single "directory" in the path,
        # but they are then unescaped when passed to the get() method.
        self.assertEqual(
            self.fetch_json("/slashes/foo/bar"),
            dict(path="/slashes/foo/bar", path_args=["foo", "bar"], args={}),
        )
        self.assertEqual(
            self.fetch_json("/slashes/a%2Fb/c%2Fd"),
            dict(path="/slashes/a%2Fb/c%2Fd", path_args=["a/b", "c/d"], args={}),
        )

    def test_error(self):
        # Percent signs (encoded as %25) should not mess up printf-style
        # messages in logs
        with ExpectLog(gen_log, ".*Invalid unicode"):
            self.fetch("/group/?arg=%25%e9")


class TypeCheckHandler(RequestHandler):
    def prepare(self):
        self.errors = {}  # type: typing.Dict[str, str]

        self.check_type("status", self.get_status(), int)

        # get_argument is an exception from the general rule of using
        # type str for non-body data mainly for historical reasons.
        self.check_type("argument", self.get_argument("foo"), unicode_type)
        self.check_type("cookie_key", list(self.cookies.keys())[0], str)
        self.check_type("cookie_value", list(self.cookies.values())[0].value, str)

        # Secure cookies return bytes because they can contain arbitrary
        # data, but regular cookies are native strings.
        if list(self.cookies.keys()) != ["asdf"]:
            raise Exception(
                "unexpected values for cookie keys: %r" % self.cookies.keys()
            )
        self.check_type("get_signed_cookie", self.get_signed_cookie("asdf"), bytes)
        self.check_type("get_cookie", self.get_cookie("asdf"), str)

        self.check_type("xsrf_token", self.xsrf_token, bytes)
        self.check_type("xsrf_form_html", self.xsrf_form_html(), str)

        self.check_type("reverse_url", self.reverse_url("typecheck", "foo"), str)

        self.check_type("request_summary", self._request_summary(), str)

    def get(self, path_component):
        # path_component uses type unicode instead of str for consistency
        # with get_argument()
        self.check_type("path_component", path_component, unicode_type)
        self.write(self.errors)

    def post(self, path_component):
        self.check_type("path_component", path_component, unicode_type)
        self.write(self.errors)

    def check_type(self, name, obj, expected_type):
        actual_type = type(obj)
        if expected_type != actual_type:
            self.errors[name] = "expected %s, got %s" % (expected_type, actual_type)


class DecodeArgHandler(RequestHandler):
    def decode_argument(self, value, name=None):
        if type(value) != bytes:
            raise Exception("unexpected type for value: %r" % type(value))
        # use self.request.arguments directly to avoid recursion
        if "encoding" in self.request.arguments:
            return value.decode(to_unicode(self.request.arguments["encoding"][0]))
        else:
            return value

    def get(self, arg):
        def describe(s):
            if type(s) == bytes:
                return ["bytes", native_str(binascii.b2a_hex(s))]
            elif type(s) == unicode_type:
                return ["unicode", s]
            raise Exception("unknown type")

        self.write({"path": describe(arg), "query": describe(self.get_argument("foo"))})


class LinkifyHandler(RequestHandler):
    def get(self):
        self.render("linkify.html", message="http://example.com")


class UIModuleResourceHandler(RequestHandler):
    def get(self):
        self.render("page.html", entries=[1, 2])


class OptionalPathHandler(RequestHandler):
    def get(self, path):
        self.write({"path": path})


class MultiHeaderHandler(RequestHandler):
    def get(self):
        self.set_header("x-overwrite", "1")
        self.set_header("X-Overwrite", 2)
        self.add_header("x-multi", 3)
        self.add_header("X-Multi", "4")


class RedirectHandler(RequestHandler):
    def get(self):
        if self.get_argument("permanent", None) is not None:
            self.redirect("/", permanent=bool(int(self.get_argument("permanent"))))
        elif self.get_argument("status", None) is not None:
            self.redirect("/", status=int(self.get_argument("status")))
        else:
            raise Exception("didn't get permanent or status arguments")


class EmptyFlushCallbackHandler(RequestHandler):
    @gen.coroutine
    def get(self):
        # Ensure that the flush callback is run whether or not there
        # was any output.  The gen.Task and direct yield forms are
        # equivalent.
        yield self.flush()  # "empty" flush, but writes headers
        yield self.flush()  # empty flush
        self.write("o")
        yield self.flush()  # flushes the "o"
        yield self.flush()  # empty flush
        self.finish("k")


class HeaderInjectionHandler(RequestHandler):
    def get(self):
        try:
            self.set_header("X-Foo", "foo\r\nX-Bar: baz")
            raise Exception("Didn't get expected exception")
        except ValueError as e:
            if "Unsafe header value" in str(e):
                self.finish(b"ok")
            else:
                raise


class GetArgumentHandler(RequestHandler):
    def prepare(self):
        if self.get_argument("source", None) == "query":
            method = self.get_query_argument
        elif self.get_argument("source", None) == "body":
            method = self.get_body_argument
        else:
            method = self.get_argument  # type: ignore
        self.finish(method("foo", "default"))


class GetArgumentsHandler(RequestHandler):
    def prepare(self):
        self.finish(
            dict(
                default=self.get_arguments("foo"),
                query=self.get_query_arguments("foo"),
                body=self.get_body_arguments("foo"),
            )
        )


# This test was shared with wsgi_test.py; now the name is meaningless.
class WSGISafeWebTest(WebTestCase):
    COOKIE_SECRET = "WebTest.COOKIE_SECRET"

    def get_app_kwargs(self):
        loader = DictLoader(
            {
                "linkify.html": "{% module linkify(message) %}",
                "page.html": """\
<html><head></head><body>
{% for e in entries %}
{% module Template("entry.html", entry=e) %}
{% end %}
</body></html>""",
                "entry.html": """\
{{ set_resources(embedded_css=".entry { margin-bottom: 1em; }",
                 embedded_javascript="js_embed()",
                 css_files=["/base.css", "/foo.css"],
                 javascript_files="/common.js",
                 html_head="<meta>",
                 html_body='<script src="/analytics.js"/>') }}
<div class="entry">...</div>""",
            }
        )
        return dict(
            template_loader=loader,
            autoescape="xhtml_escape",
            cookie_secret=self.COOKIE_SECRET,
        )

    def tearDown(self):
        super().tearDown()
        RequestHandler._template_loaders.clear()

    def get_handlers(self):
        urls = [
            url("/typecheck/(.*)", TypeCheckHandler, name="typecheck"),
            url("/decode_arg/(.*)", DecodeArgHandler, name="decode_arg"),
            url("/decode_arg_kw/(?P<arg>.*)", DecodeArgHandler),
            url("/linkify", LinkifyHandler),
            url("/uimodule_resources", UIModuleResourceHandler),
            url("/optional_path/(.+)?", OptionalPathHandler),
            url("/multi_header", MultiHeaderHandler),
            url("/redirect", RedirectHandler),
            url(
                "/web_redirect_permanent",
                WebRedirectHandler,
                {"url": "/web_redirect_newpath"},
            ),
            url(
                "/web_redirect",
                WebRedirectHandler,
                {"url": "/web_redirect_newpath", "permanent": False},
            ),
            url(
                "//web_redirect_double_slash",
                WebRedirectHandler,
                {"url": "/web_redirect_newpath"},
            ),
            url("/header_injection", HeaderInjectionHandler),
            url("/get_argument", GetArgumentHandler),
            url("/get_arguments", GetArgumentsHandler),
        ]
        return urls

    def fetch_json(self, *args, **kwargs):
        response = self.fetch(*args, **kwargs)
        response.rethrow()
        return json_decode(response.body)

    def test_types(self):
        cookie_value = to_unicode(
            create_signed_value(self.COOKIE_SECRET, "asdf", "qwer")
        )
        response = self.fetch(
            "/typecheck/asdf?foo=bar", headers={"Cookie": "asdf=" + cookie_value}
        )
        data = json_decode(response.body)
        self.assertEqual(data, {})

        response = self.fetch(
            "/typecheck/asdf?foo=bar",
            method="POST",
            headers={"Cookie": "asdf=" + cookie_value},
            body="foo=bar",
        )

    def test_decode_argument(self):
        # These urls all decode to the same thing
        urls = [
            "/decode_arg/%C3%A9?foo=%C3%A9&encoding=utf-8",
            "/decode_arg/%E9?foo=%E9&encoding=latin1",
            "/decode_arg_kw/%E9?foo=%E9&encoding=latin1",
        ]
        for req_url in urls:
            response = self.fetch(req_url)
            response.rethrow()
            data = json_decode(response.body)
            self.assertEqual(
                data,
                {"path": ["unicode", "\u00e9"], "query": ["unicode", "\u00e9"]},
            )

        response = self.fetch("/decode_arg/%C3%A9?foo=%C3%A9")
        response.rethrow()
        data = json_decode(response.body)
        self.assertEqual(data, {"path": ["bytes", "c3a9"], "query": ["bytes", "c3a9"]})

    def test_decode_argument_invalid_unicode(self):
        # test that invalid unicode in URLs causes 400, not 500
        with ExpectLog(gen_log, ".*Invalid unicode.*"):
            response = self.fetch("/typecheck/invalid%FF")
            self.assertEqual(response.code, 400)
            response = self.fetch("/typecheck/invalid?foo=%FF")
            self.assertEqual(response.code, 400)

    def test_decode_argument_plus(self):
        # These urls are all equivalent.
        urls = [
            "/decode_arg/1%20%2B%201?foo=1%20%2B%201&encoding=utf-8",
            "/decode_arg/1%20+%201?foo=1+%2B+1&encoding=utf-8",
        ]
        for req_url in urls:
            response = self.fetch(req_url)
            response.rethrow()
            data = json_decode(response.body)
            self.assertEqual(
                data,
                {"path": ["unicode", "1 + 1"], "query": ["unicode", "1 + 1"]},
            )

    def test_reverse_url(self):
        self.assertEqual(self.app.reverse_url("decode_arg", "foo"), "/decode_arg/foo")
        self.assertEqual(self.app.reverse_url("decode_arg", 42), "/decode_arg/42")
        self.assertEqual(self.app.reverse_url("decode_arg", b"\xe9"), "/decode_arg/%E9")
        self.assertEqual(
            self.app.reverse_url("decode_arg", "\u00e9"), "/decode_arg/%C3%A9"
        )
        self.assertEqual(
            self.app.reverse_url("decode_arg", "1 + 1"), "/decode_arg/1%20%2B%201"
        )

    def test_uimodule_unescaped(self):
        response = self.fetch("/linkify")
        self.assertEqual(
            response.body, b'<a href="http://example.com">http://example.com</a>'
        )

    def test_uimodule_resources(self):
        response = self.fetch("/uimodule_resources")
        self.assertEqual(
            response.body,
            b"""\
<html><head><link href="/base.css" type="text/css" rel="stylesheet"/><link href="/foo.css" type="text/css" rel="stylesheet"/>
<style type="text/css">
.entry { margin-bottom: 1em; }
</style>
<meta>
</head><body>


<div class="entry">...</div>


<div class="entry">...</div>

<script src="/common.js" type="text/javascript"></script>
<script type="text/javascript">
//<![CDATA[
js_embed()
//]]>
</script>
<script src="/analytics.js"/>
</body></html>""",  # noqa: E501
        )

    def test_optional_path(self):
        self.assertEqual(self.fetch_json("/optional_path/foo"), {"path": "foo"})
        self.assertEqual(self.fetch_json("/optional_path/"), {"path": None})

    def test_multi_header(self):
        response = self.fetch("/multi_header")
        self.assertEqual(response.headers["x-overwrite"], "2")
        self.assertEqual(response.headers.get_list("x-multi"), ["3", "4"])

    def test_redirect(self):
        response = self.fetch("/redirect?permanent=1", follow_redirects=False)
        self.assertEqual(response.code, 301)
        response = self.fetch("/redirect?permanent=0", follow_redirects=False)
        self.assertEqual(response.code, 302)
        response = self.fetch("/redirect?status=307", follow_redirects=False)
        self.assertEqual(response.code, 307)

    def test_web_redirect(self):
        response = self.fetch("/web_redirect_permanent", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/web_redirect_newpath")
        response = self.fetch("/web_redirect", follow_redirects=False)
        self.assertEqual(response.code, 302)
        self.assertEqual(response.headers["Location"], "/web_redirect_newpath")

    def test_web_redirect_double_slash(self):
        response = self.fetch("//web_redirect_double_slash", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/web_redirect_newpath")

    def test_header_injection(self):
        response = self.fetch("/header_injection")
        self.assertEqual(response.body, b"ok")

    def test_get_argument(self):
        response = self.fetch("/get_argument?foo=bar")
        self.assertEqual(response.body, b"bar")
        response = self.fetch("/get_argument?foo=")
        self.assertEqual(response.body, b"")
        response = self.fetch("/get_argument")
        self.assertEqual(response.body, b"default")

        # Test merging of query and body arguments.
        # In singular form, body arguments take precedence over query arguments.
        body = urllib.parse.urlencode(dict(foo="hello"))
        response = self.fetch("/get_argument?foo=bar", method="POST", body=body)
        self.assertEqual(response.body, b"hello")
        # In plural methods they are merged.
        response = self.fetch("/get_arguments?foo=bar", method="POST", body=body)
        self.assertEqual(
            json_decode(response.body),
            dict(default=["bar", "hello"], query=["bar"], body=["hello"]),
        )

    def test_get_query_arguments(self):
        # send as a post so we can ensure the separation between query
        # string and body arguments.
        body = urllib.parse.urlencode(dict(foo="hello"))
        response = self.fetch(
            "/get_argument?source=query&foo=bar", method="POST", body=body
        )
        self.assertEqual(response.body, b"bar")
        response = self.fetch(
            "/get_argument?source=query&foo=", method="POST", body=body
        )
        self.assertEqual(response.body, b"")
        response = self.fetch("/get_argument?source=query", method="POST", body=body)
        self.assertEqual(response.body, b"default")

    def test_get_body_arguments(self):
        body = urllib.parse.urlencode(dict(foo="bar"))
        response = self.fetch(
            "/get_argument?source=body&foo=hello", method="POST", body=body
        )
        self.assertEqual(response.body, b"bar")

        body = urllib.parse.urlencode(dict(foo=""))
        response = self.fetch(
            "/get_argument?source=body&foo=hello", method="POST", body=body
        )
        self.assertEqual(response.body, b"")

        body = urllib.parse.urlencode(dict())
        response = self.fetch(
            "/get_argument?source=body&foo=hello", method="POST", body=body
        )
        self.assertEqual(response.body, b"default")

    def test_no_gzip(self):
        response = self.fetch("/get_argument")
        self.assertNotIn("Accept-Encoding", response.headers.get("Vary", ""))
        self.assertNotIn("gzip", response.headers.get("Content-Encoding", ""))


class NonWSGIWebTests(WebTestCase):
    def get_handlers(self):
        return [("/empty_flush", EmptyFlushCallbackHandler)]

    def test_empty_flush(self):
        response = self.fetch("/empty_flush")
        self.assertEqual(response.body, b"ok")


class ErrorResponseTest(WebTestCase):
    def get_handlers(self):
        class DefaultHandler(RequestHandler):
            def get(self):
                if self.get_argument("status", None):
                    raise HTTPError(int(self.get_argument("status")))
                1 / 0

        class WriteErrorHandler(RequestHandler):
            def get(self):
                if self.get_argument("status", None):
                    self.send_error(int(self.get_argument("status")))
                else:
                    1 / 0

            def write_error(self, status_code, **kwargs):
                self.set_header("Content-Type", "text/plain")
                if "exc_info" in kwargs:
                    self.write("Exception: %s" % kwargs["exc_info"][0].__name__)
                else:
                    self.write("Status: %d" % status_code)

        class FailedWriteErrorHandler(RequestHandler):
            def get(self):
                1 / 0

            def write_error(self, status_code, **kwargs):
                raise Exception("exception in write_error")

        return [
            url("/default", DefaultHandler),
            url("/write_error", WriteErrorHandler),
            url("/failed_write_error", FailedWriteErrorHandler),
        ]

    def test_default(self):
        with ExpectLog(app_log, "Uncaught exception"):
            response = self.fetch("/default")
            self.assertEqual(response.code, 500)
            self.assertTrue(b"500: Internal Server Error" in response.body)

            response = self.fetch("/default?status=503")
            self.assertEqual(response.code, 503)
            self.assertTrue(b"503: Service Unavailable" in response.body)

            response = self.fetch("/default?status=435")
            self.assertEqual(response.code, 435)
            self.assertTrue(b"435: Unknown" in response.body)

    def test_write_error(self):
        with ExpectLog(app_log, "Uncaught exception"):
            response = self.fetch("/write_error")
            self.assertEqual(response.code, 500)
            self.assertEqual(b"Exception: ZeroDivisionError", response.body)

            response = self.fetch("/write_error?status=503")
            self.assertEqual(response.code, 503)
            self.assertEqual(b"Status: 503", response.body)

    def test_failed_write_error(self):
        with ExpectLog(app_log, "Uncaught exception"):
            response = self.fetch("/failed_write_error")
            self.assertEqual(response.code, 500)
            self.assertEqual(b"", response.body)


class StaticFileTest(WebTestCase):
    # The expected SHA-512 hash of robots.txt, used in tests that call
    # StaticFileHandler.get_version
    robots_txt_hash = (
        b"63a36e950e134b5217e33c763e88840c10a07d80e6057d92b9ac97508de7fb1f"
        b"a6f0e9b7531e169657165ea764e8963399cb6d921ffe6078425aaafe54c04563"
    )
    static_dir = os.path.join(os.path.dirname(__file__), "static")

    def get_handlers(self):
        class StaticUrlHandler(RequestHandler):
            def get(self, path):
                with_v = int(self.get_argument("include_version", "1"))
                self.write(self.static_url(path, include_version=with_v))

        class AbsoluteStaticUrlHandler(StaticUrlHandler):
            include_host = True

        class OverrideStaticUrlHandler(RequestHandler):
            def get(self, path):
                do_include = bool(self.get_argument("include_host"))
                self.include_host = not do_include

                regular_url = self.static_url(path)
                override_url = self.static_url(path, include_host=do_include)
                if override_url == regular_url:
                    return self.write(str(False))

                protocol = self.request.protocol + "://"
                protocol_length = len(protocol)
                check_regular = regular_url.find(protocol, 0, protocol_length)
                check_override = override_url.find(protocol, 0, protocol_length)

                if do_include:
                    result = check_override == 0 and check_regular == -1
                else:
                    result = check_override == -1 and check_regular == 0
                self.write(str(result))

        return [
            ("/static_url/(.*)", StaticUrlHandler),
            ("/abs_static_url/(.*)", AbsoluteStaticUrlHandler),
            ("/override_static_url/(.*)", OverrideStaticUrlHandler),
            ("/root_static/(.*)", StaticFileHandler, dict(path="/")),
        ]

    def get_app_kwargs(self):
        return dict(static_path=relpath("static"))

    def test_static_files(self):
        response = self.fetch("/robots.txt")
        self.assertTrue(b"Disallow: /" in response.body)

        response = self.fetch("/static/robots.txt")
        self.assertTrue(b"Disallow: /" in response.body)
        self.assertEqual(response.headers.get("Content-Type"), "text/plain")

    def test_static_compressed_files(self):
        response = self.fetch("/static/sample.xml.gz")
        self.assertEqual(response.headers.get("Content-Type"), "application/gzip")
        response = self.fetch("/static/sample.xml.bz2")
        self.assertEqual(
            response.headers.get("Content-Type"), "application/octet-stream"
        )
        # make sure the uncompressed file still has the correct type
        response = self.fetch("/static/sample.xml")
        self.assertTrue(
            response.headers.get("Content-Type") in set(("text/xml", "application/xml"))
        )

    def test_static_url(self):
        response = self.fetch("/static_url/robots.txt")
        self.assertEqual(response.body, b"/static/robots.txt?v=" + self.robots_txt_hash)

    def test_absolute_static_url(self):
        response = self.fetch("/abs_static_url/robots.txt")
        self.assertEqual(
            response.body,
            (utf8(self.get_url("/")) + b"static/robots.txt?v=" + self.robots_txt_hash),
        )

    def test_relative_version_exclusion(self):
        response = self.fetch("/static_url/robots.txt?include_version=0")
        self.assertEqual(response.body, b"/static/robots.txt")

    def test_absolute_version_exclusion(self):
        response = self.fetch("/abs_static_url/robots.txt?include_version=0")
        self.assertEqual(response.body, utf8(self.get_url("/") + "static/robots.txt"))

    def test_include_host_override(self):
        self._trigger_include_host_check(False)
        self._trigger_include_host_check(True)

    def _trigger_include_host_check(self, include_host):
        path = "/override_static_url/robots.txt?include_host=%s"
        response = self.fetch(path % int(include_host))
        self.assertEqual(response.body, utf8(str(True)))

    def get_and_head(self, *args, **kwargs):
        """Performs a GET and HEAD request and returns the GET response.

        Fails if any ``Content-*`` headers returned by the two requests
        differ.
        """
        head_response = self.fetch(*args, method="HEAD", **kwargs)
        get_response = self.fetch(*args, method="GET", **kwargs)
        content_headers = set()
        for h in itertools.chain(head_response.headers, get_response.headers):
            if h.startswith("Content-"):
                content_headers.add(h)
        for h in content_headers:
            self.assertEqual(
                head_response.headers.get(h),
                get_response.headers.get(h),
                "%s differs between GET (%s) and HEAD (%s)"
                % (h, head_response.headers.get(h), get_response.headers.get(h)),
            )
        return get_response

    def test_static_304_if_modified_since(self):
        response1 = self.get_and_head("/static/robots.txt")
        response2 = self.get_and_head(
            "/static/robots.txt",
            headers={"If-Modified-Since": response1.headers["Last-Modified"]},
        )
        self.assertEqual(response2.code, 304)
        self.assertTrue("Content-Length" not in response2.headers)

    def test_static_304_if_none_match(self):
        response1 = self.get_and_head("/static/robots.txt")
        response2 = self.get_and_head(
            "/static/robots.txt", headers={"If-None-Match": response1.headers["Etag"]}
        )
        self.assertEqual(response2.code, 304)

    def test_static_304_etag_modified_bug(self):
        response1 = self.get_and_head("/static/robots.txt")
        response2 = self.get_and_head(
            "/static/robots.txt",
            headers={
                "If-None-Match": '"MISMATCH"',
                "If-Modified-Since": response1.headers["Last-Modified"],
            },
        )
        self.assertEqual(response2.code, 200)

    def test_static_if_modified_since_pre_epoch(self):
        # On windows, the functions that work with time_t do not accept
        # negative values, and at least one client (processing.js) seems
        # to use if-modified-since 1/1/1960 as a cache-busting technique.
        response = self.get_and_head(
            "/static/robots.txt",
            headers={"If-Modified-Since": "Fri, 01 Jan 1960 00:00:00 GMT"},
        )
        self.assertEqual(response.code, 200)

    def test_static_if_modified_since_time_zone(self):
        # Instead of the value from Last-Modified, make requests with times
        # chosen just before and after the known modification time
        # of the file to ensure that the right time zone is being used
        # when parsing If-Modified-Since.
        stat = os.stat(relpath("static/robots.txt"))

        response = self.get_and_head(
            "/static/robots.txt",
            headers={"If-Modified-Since": format_timestamp(stat.st_mtime - 1)},
        )
        self.assertEqual(response.code, 200)
        response = self.get_and_head(
            "/static/robots.txt",
            headers={"If-Modified-Since": format_timestamp(stat.st_mtime + 1)},
        )
        self.assertEqual(response.code, 304)

    def test_static_etag(self):
        response = self.get_and_head("/static/robots.txt")
        self.assertEqual(
            utf8(response.headers.get("Etag")), b'"' + self.robots_txt_hash + b'"'
        )

    def test_static_with_range(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=0-9"}
        )
        self.assertEqual(response.code, 206)
        self.assertEqual(response.body, b"User-agent")
        self.assertEqual(
            utf8(response.headers.get("Etag")), b'"' + self.robots_txt_hash + b'"'
        )
        self.assertEqual(response.headers.get("Content-Length"), "10")
        self.assertEqual(response.headers.get("Content-Range"), "bytes 0-9/26")

    def test_static_with_range_full_file(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=0-"}
        )
        # Note: Chrome refuses to play audio if it gets an HTTP 206 in response
        # to ``Range: bytes=0-`` :(
        self.assertEqual(response.code, 200)
        robots_file_path = os.path.join(self.static_dir, "robots.txt")
        with open(robots_file_path) as f:
            self.assertEqual(response.body, utf8(f.read()))
        self.assertEqual(response.headers.get("Content-Length"), "26")
        self.assertEqual(response.headers.get("Content-Range"), None)

    def test_static_with_range_full_past_end(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=0-10000000"}
        )
        self.assertEqual(response.code, 200)
        robots_file_path = os.path.join(self.static_dir, "robots.txt")
        with open(robots_file_path) as f:
            self.assertEqual(response.body, utf8(f.read()))
        self.assertEqual(response.headers.get("Content-Length"), "26")
        self.assertEqual(response.headers.get("Content-Range"), None)

    def test_static_with_range_partial_past_end(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=1-10000000"}
        )
        self.assertEqual(response.code, 206)
        robots_file_path = os.path.join(self.static_dir, "robots.txt")
        with open(robots_file_path) as f:
            self.assertEqual(response.body, utf8(f.read()[1:]))
        self.assertEqual(response.headers.get("Content-Length"), "25")
        self.assertEqual(response.headers.get("Content-Range"), "bytes 1-25/26")

    def test_static_with_range_end_edge(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=22-"}
        )
        self.assertEqual(response.body, b": /\n")
        self.assertEqual(response.headers.get("Content-Length"), "4")
        self.assertEqual(response.headers.get("Content-Range"), "bytes 22-25/26")

    def test_static_with_range_neg_end(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=-4"}
        )
        self.assertEqual(response.body, b": /\n")
        self.assertEqual(response.headers.get("Content-Length"), "4")
        self.assertEqual(response.headers.get("Content-Range"), "bytes 22-25/26")

    def test_static_with_range_neg_past_start(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=-1000000"}
        )
        self.assertEqual(response.code, 200)
        robots_file_path = os.path.join(self.static_dir, "robots.txt")
        with open(robots_file_path) as f:
            self.assertEqual(response.body, utf8(f.read()))
        self.assertEqual(response.headers.get("Content-Length"), "26")
        self.assertEqual(response.headers.get("Content-Range"), None)

    def test_static_invalid_range(self):
        response = self.get_and_head("/static/robots.txt", headers={"Range": "asdf"})
        self.assertEqual(response.code, 200)

    def test_static_unsatisfiable_range_zero_suffix(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=-0"}
        )
        self.assertEqual(response.headers.get("Content-Range"), "bytes */26")
        self.assertEqual(response.code, 416)

    def test_static_unsatisfiable_range_invalid_start(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=26"}
        )
        self.assertEqual(response.code, 416)
        self.assertEqual(response.headers.get("Content-Range"), "bytes */26")

    def test_static_unsatisfiable_range_end_less_than_start(self):
        response = self.get_and_head(
            "/static/robots.txt", headers={"Range": "bytes=10-3"}
        )
        self.assertEqual(response.code, 416)
        self.assertEqual(response.headers.get("Content-Range"), "bytes */26")

    def test_static_head(self):
        response = self.fetch("/static/robots.txt", method="HEAD")
        self.assertEqual(response.code, 200)
        # No body was returned, but we did get the right content length.
        self.assertEqual(response.body, b"")
        self.assertEqual(response.headers["Content-Length"], "26")
        self.assertEqual(
            utf8(response.headers["Etag"]), b'"' + self.robots_txt_hash + b'"'
        )

    def test_static_head_range(self):
        response = self.fetch(
            "/static/robots.txt", method="HEAD", headers={"Range": "bytes=1-4"}
        )
        self.assertEqual(response.code, 206)
        self.assertEqual(response.body, b"")
        self.assertEqual(response.headers["Content-Length"], "4")
        self.assertEqual(
            utf8(response.headers["Etag"]), b'"' + self.robots_txt_hash + b'"'
        )

    def test_static_range_if_none_match(self):
        response = self.get_and_head(
            "/static/robots.txt",
            headers={
                "Range": "bytes=1-4",
                "If-None-Match": b'"' + self.robots_txt_hash + b'"',
            },
        )
        self.assertEqual(response.code, 304)
        self.assertEqual(response.body, b"")
        self.assertTrue("Content-Length" not in response.headers)
        self.assertEqual(
            utf8(response.headers["Etag"]), b'"' + self.robots_txt_hash + b'"'
        )

    def test_static_404(self):
        response = self.get_and_head("/static/blarg")
        self.assertEqual(response.code, 404)

    def test_path_traversal_protection(self):
        # curl_httpclient processes ".." on the client side, so we
        # must test this with simple_httpclient.
        self.http_client.close()
        self.http_client = SimpleAsyncHTTPClient()
        with ExpectLog(gen_log, ".*not in root static directory"):
            response = self.get_and_head("/static/../static_foo.txt")
        # Attempted path traversal should result in 403, not 200
        # (which means the check failed and the file was served)
        # or 404 (which means that the file didn't exist and
        # is probably a packaging error).
        self.assertEqual(response.code, 403)

    @unittest.skipIf(os.name != "posix", "non-posix OS")
    def test_root_static_path(self):
        # Sometimes people set the StaticFileHandler's path to '/'
        # to disable Tornado's path validation (in conjunction with
        # their own validation in get_absolute_path). Make sure
        # that the stricter validation in 4.2.1 doesn't break them.
        path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), "static/robots.txt"
        )
        response = self.get_and_head("/root_static" + urllib.parse.quote(path))
        self.assertEqual(response.code, 200)


class StaticDefaultFilenameTest(WebTestCase):
    def get_app_kwargs(self):
        return dict(
            static_path=relpath("static"),
            static_handler_args=dict(default_filename="index.html"),
        )

    def get_handlers(self):
        return []

    def test_static_default_filename(self):
        response = self.fetch("/static/dir/", follow_redirects=False)
        self.assertEqual(response.code, 200)
        self.assertEqual(b"this is the index\n", response.body)

    def test_static_default_redirect(self):
        response = self.fetch("/static/dir", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertTrue(response.headers["Location"].endswith("/static/dir/"))


class StaticFileWithPathTest(WebTestCase):
    def get_app_kwargs(self):
        return dict(
            static_path=relpath("static"),
            static_handler_args=dict(default_filename="index.html"),
        )

    def get_handlers(self):
        return [("/foo/(.*)", StaticFileHandler, {"path": relpath("templates/")})]

    def test_serve(self):
        response = self.fetch("/foo/utf8.html")
        self.assertEqual(response.body, b"H\xc3\xa9llo\n")


class CustomStaticFileTest(WebTestCase):
    def get_handlers(self):
        class MyStaticFileHandler(StaticFileHandler):
            @classmethod
            def make_static_url(cls, settings, path):
                version_hash = cls.get_version(settings, path)
                extension_index = path.rindex(".")
                before_version = path[:extension_index]
                after_version = path[(extension_index + 1) :]
                return "/static/%s.%s.%s" % (
                    before_version,
                    version_hash,
                    after_version,
                )

            def parse_url_path(self, url_path):
                extension_index = url_path.rindex(".")
                version_index = url_path.rindex(".", 0, extension_index)
                return "%s%s" % (url_path[:version_index], url_path[extension_index:])

            @classmethod
            def get_absolute_path(cls, settings, path):
                return "CustomStaticFileTest:" + path

            def validate_absolute_path(self, root, absolute_path):
                return absolute_path

            @classmethod
            def get_content(self, path, start=None, end=None):
                assert start is None and end is None
                if path == "CustomStaticFileTest:foo.txt":
                    return b"bar"
                raise Exception("unexpected path %r" % path)

            def get_content_size(self):
                if self.absolute_path == "CustomStaticFileTest:foo.txt":
                    return 3
                raise Exception("unexpected path %r" % self.absolute_path)

            def get_modified_time(self):
                return None

            @classmethod
            def get_version(cls, settings, path):
                return "42"

        class StaticUrlHandler(RequestHandler):
            def get(self, path):
                self.write(self.static_url(path))

        self.static_handler_class = MyStaticFileHandler

        return [("/static_url/(.*)", StaticUrlHandler)]

    def get_app_kwargs(self):
        return dict(static_path="dummy", static_handler_class=self.static_handler_class)

    def test_serve(self):
        response = self.fetch("/static/foo.42.txt")
        self.assertEqual(response.body, b"bar")

    def test_static_url(self):
        with ExpectLog(gen_log, "Could not open static file", required=False):
            response = self.fetch("/static_url/foo.txt")
            self.assertEqual(response.body, b"/static/foo.42.txt")


class HostMatchingTest(WebTestCase):
    class Handler(RequestHandler):
        def initialize(self, reply):
            self.reply = reply

        def get(self):
            self.write(self.reply)

    def get_handlers(self):
        return [("/foo", HostMatchingTest.Handler, {"reply": "wildcard"})]

    def test_host_matching(self):
        self.app.add_handlers(
            "www.example.com", [("/foo", HostMatchingTest.Handler, {"reply": "[0]"})]
        )
        self.app.add_handlers(
            r"www\.example\.com", [("/bar", HostMatchingTest.Handler, {"reply": "[1]"})]
        )
        self.app.add_handlers(
            "www.example.com", [("/baz", HostMatchingTest.Handler, {"reply": "[2]"})]
        )
        self.app.add_handlers(
            "www.e.*e.com", [("/baz", HostMatchingTest.Handler, {"reply": "[3]"})]
        )

        response = self.fetch("/foo")
        self.assertEqual(response.body, b"wildcard")
        response = self.fetch("/bar")
        self.assertEqual(response.code, 404)
        response = self.fetch("/baz")
        self.assertEqual(response.code, 404)

        response = self.fetch("/foo", headers={"Host": "www.example.com"})
        self.assertEqual(response.body, b"[0]")
        response = self.fetch("/bar", headers={"Host": "www.example.com"})
        self.assertEqual(response.body, b"[1]")
        response = self.fetch("/baz", headers={"Host": "www.example.com"})
        self.assertEqual(response.body, b"[2]")
        response = self.fetch("/baz", headers={"Host": "www.exe.com"})
        self.assertEqual(response.body, b"[3]")


class DefaultHostMatchingTest(WebTestCase):
    def get_handlers(self):
        return []

    def get_app_kwargs(self):
        return {"default_host": "www.example.com"}

    def test_default_host_matching(self):
        self.app.add_handlers(
            "www.example.com", [("/foo", HostMatchingTest.Handler, {"reply": "[0]"})]
        )
        self.app.add_handlers(
            r"www\.example\.com", [("/bar", HostMatchingTest.Handler, {"reply": "[1]"})]
        )
        self.app.add_handlers(
            "www.test.com", [("/baz", HostMatchingTest.Handler, {"reply": "[2]"})]
        )

        response = self.fetch("/foo")
        self.assertEqual(response.body, b"[0]")
        response = self.fetch("/bar")
        self.assertEqual(response.body, b"[1]")
        response = self.fetch("/baz")
        self.assertEqual(response.code, 404)

        response = self.fetch("/foo", headers={"X-Real-Ip": "127.0.0.1"})
        self.assertEqual(response.code, 404)

        self.app.default_host = "www.test.com"

        response = self.fetch("/baz")
        self.assertEqual(response.body, b"[2]")


class NamedURLSpecGroupsTest(WebTestCase):
    def get_handlers(self):
        class EchoHandler(RequestHandler):
            def get(self, path):
                self.write(path)

        return [
            ("/str/(?P<path>.*)", EchoHandler),
            ("/unicode/(?P<path>.*)", EchoHandler),
        ]

    def test_named_urlspec_groups(self):
        response = self.fetch("/str/foo")
        self.assertEqual(response.body, b"foo")

        response = self.fetch("/unicode/bar")
        self.assertEqual(response.body, b"bar")


class ClearHeaderTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.set_header("h1", "foo")
            self.set_header("h2", "bar")
            self.clear_header("h1")
            self.clear_header("nonexistent")

    def test_clear_header(self):
        response = self.fetch("/")
        self.assertTrue("h1" not in response.headers)
        self.assertEqual(response.headers["h2"], "bar")


class Header204Test(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.set_status(204)
            self.finish()

    def test_204_headers(self):
        response = self.fetch("/")
        self.assertEqual(response.code, 204)
        self.assertNotIn("Content-Length", response.headers)
        self.assertNotIn("Transfer-Encoding", response.headers)


class Header304Test(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.set_header("Content-Language", "en_US")
            self.write("hello")

    def test_304_headers(self):
        response1 = self.fetch("/")
        self.assertEqual(response1.headers["Content-Length"], "5")
        self.assertEqual(response1.headers["Content-Language"], "en_US")

        response2 = self.fetch(
            "/", headers={"If-None-Match": response1.headers["Etag"]}
        )
        self.assertEqual(response2.code, 304)
        self.assertTrue("Content-Length" not in response2.headers)
        self.assertTrue("Content-Language" not in response2.headers)
        # Not an entity header, but should not be added to 304s by chunking
        self.assertTrue("Transfer-Encoding" not in response2.headers)


class StatusReasonTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            reason = self.request.arguments.get("reason", [])
            self.set_status(
                int(self.get_argument("code")),
                reason=to_unicode(reason[0]) if reason else None,
            )

    def get_http_client(self):
        # simple_httpclient only: curl doesn't expose the reason string
        return SimpleAsyncHTTPClient()

    def test_status(self):
        response = self.fetch("/?code=304")
        self.assertEqual(response.code, 304)
        self.assertEqual(response.reason, "Not Modified")
        response = self.fetch("/?code=304&reason=Foo")
        self.assertEqual(response.code, 304)
        self.assertEqual(response.reason, "Foo")
        response = self.fetch("/?code=682&reason=Bar")
        self.assertEqual(response.code, 682)
        self.assertEqual(response.reason, "Bar")
        response = self.fetch("/?code=682")
        self.assertEqual(response.code, 682)
        self.assertEqual(response.reason, "Unknown")


class DateHeaderTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.write("hello")

    def test_date_header(self):
        response = self.fetch("/")
        parsed = email.utils.parsedate(response.headers["Date"])
        assert parsed is not None
        header_date = datetime.datetime(*parsed[:6])
        self.assertTrue(
            header_date - datetime.datetime.utcnow() < datetime.timedelta(seconds=2)
        )


class RaiseWithReasonTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            raise HTTPError(682, reason="Foo")

    def get_http_client(self):
        # simple_httpclient only: curl doesn't expose the reason string
        return SimpleAsyncHTTPClient()

    def test_raise_with_reason(self):
        response = self.fetch("/")
        self.assertEqual(response.code, 682)
        self.assertEqual(response.reason, "Foo")
        self.assertIn(b"682: Foo", response.body)

    def test_httperror_str(self):
        self.assertEqual(str(HTTPError(682, reason="Foo")), "HTTP 682: Foo")

    def test_httperror_str_from_httputil(self):
        self.assertEqual(str(HTTPError(682)), "HTTP 682: Unknown")


class ErrorHandlerXSRFTest(WebTestCase):
    def get_handlers(self):
        # note that if the handlers list is empty we get the default_host
        # redirect fallback instead of a 404, so test with both an
        # explicitly defined error handler and an implicit 404.
        return [("/error", ErrorHandler, dict(status_code=417))]

    def get_app_kwargs(self):
        return dict(xsrf_cookies=True)

    def test_error_xsrf(self):
        response = self.fetch("/error", method="POST", body="")
        self.assertEqual(response.code, 417)

    def test_404_xsrf(self):
        response = self.fetch("/404", method="POST", body="")
        self.assertEqual(response.code, 404)


class GzipTestCase(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            for v in self.get_arguments("vary"):
                self.add_header("Vary", v)
            # Must write at least MIN_LENGTH bytes to activate compression.
            self.write("hello world" + ("!" * GZipContentEncoding.MIN_LENGTH))

    def get_app_kwargs(self):
        return dict(
            gzip=True, static_path=os.path.join(os.path.dirname(__file__), "static")
        )

    def assert_compressed(self, response):
        # simple_httpclient renames the content-encoding header;
        # curl_httpclient doesn't.
        self.assertEqual(
            response.headers.get(
                "Content-Encoding", response.headers.get("X-Consumed-Content-Encoding")
            ),
            "gzip",
        )

    def test_gzip(self):
        response = self.fetch("/")
        self.assert_compressed(response)
        self.assertEqual(response.headers["Vary"], "Accept-Encoding")

    def test_gzip_static(self):
        # The streaming responses in StaticFileHandler have subtle
        # interactions with the gzip output so test this case separately.
        response = self.fetch("/robots.txt")
        self.assert_compressed(response)
        self.assertEqual(response.headers["Vary"], "Accept-Encoding")

    def test_gzip_not_requested(self):
        response = self.fetch("/", use_gzip=False)
        self.assertNotIn("Content-Encoding", response.headers)
        self.assertEqual(response.headers["Vary"], "Accept-Encoding")

    def test_vary_already_present(self):
        response = self.fetch("/?vary=Accept-Language")
        self.assert_compressed(response)
        self.assertEqual(
            [s.strip() for s in response.headers["Vary"].split(",")],
            ["Accept-Language", "Accept-Encoding"],
        )

    def test_vary_already_present_multiple(self):
        # Regression test for https://github.com/tornadoweb/tornado/issues/1670
        response = self.fetch("/?vary=Accept-Language&vary=Cookie")
        self.assert_compressed(response)
        self.assertEqual(
            [s.strip() for s in response.headers["Vary"].split(",")],
            ["Accept-Language", "Cookie", "Accept-Encoding"],
        )


class PathArgsInPrepareTest(WebTestCase):
    class Handler(RequestHandler):
        def prepare(self):
            self.write(dict(args=self.path_args, kwargs=self.path_kwargs))

        def get(self, path):
            assert path == "foo"
            self.finish()

    def get_handlers(self):
        return [("/pos/(.*)", self.Handler), ("/kw/(?P<path>.*)", self.Handler)]

    def test_pos(self):
        response = self.fetch("/pos/foo")
        response.rethrow()
        data = json_decode(response.body)
        self.assertEqual(data, {"args": ["foo"], "kwargs": {}})

    def test_kw(self):
        response = self.fetch("/kw/foo")
        response.rethrow()
        data = json_decode(response.body)
        self.assertEqual(data, {"args": [], "kwargs": {"path": "foo"}})


class ClearAllCookiesTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.clear_all_cookies()
            self.write("ok")

    def test_clear_all_cookies(self):
        response = self.fetch("/", headers={"Cookie": "foo=bar; baz=xyzzy"})
        set_cookies = sorted(response.headers.get_list("Set-Cookie"))
        # Python 3.5 sends 'baz="";'; older versions use 'baz=;'
        self.assertTrue(
            set_cookies[0].startswith("baz=;") or set_cookies[0].startswith('baz="";')
        )
        self.assertTrue(
            set_cookies[1].startswith("foo=;") or set_cookies[1].startswith('foo="";')
        )


class PermissionError(Exception):
    pass


class ExceptionHandlerTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            exc = self.get_argument("exc")
            if exc == "http":
                raise HTTPError(410, "no longer here")
            elif exc == "zero":
                1 / 0
            elif exc == "permission":
                raise PermissionError("not allowed")

        def write_error(self, status_code, **kwargs):
            if "exc_info" in kwargs:
                typ, value, tb = kwargs["exc_info"]
                if isinstance(value, PermissionError):
                    self.set_status(403)
                    self.write("PermissionError")
                    return
            RequestHandler.write_error(self, status_code, **kwargs)

        def log_exception(self, typ, value, tb):
            if isinstance(value, PermissionError):
                app_log.warning("custom logging for PermissionError: %s", value.args[0])
            else:
                RequestHandler.log_exception(self, typ, value, tb)

    def test_http_error(self):
        # HTTPErrors are logged as warnings with no stack trace.
        # TODO: extend ExpectLog to test this more precisely
        with ExpectLog(gen_log, ".*no longer here"):
            response = self.fetch("/?exc=http")
            self.assertEqual(response.code, 410)

    def test_unknown_error(self):
        # Unknown errors are logged as errors with a stack trace.
        with ExpectLog(app_log, "Uncaught exception"):
            response = self.fetch("/?exc=zero")
            self.assertEqual(response.code, 500)

    def test_known_error(self):
        # log_exception can override logging behavior, and write_error
        # can override the response.
        with ExpectLog(app_log, "custom logging for PermissionError: not allowed"):
            response = self.fetch("/?exc=permission")
            self.assertEqual(response.code, 403)


class BuggyLoggingTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            1 / 0

        def log_exception(self, typ, value, tb):
            1 / 0

    def test_buggy_log_exception(self):
        # Something gets logged even though the application's
        # logger is broken.
        with ExpectLog(app_log, ".*"):
            self.fetch("/")


class UIMethodUIModuleTest(SimpleHandlerTestCase):
    """Test that UI methods and modules are created correctly and
    associated with the handler.
    """

    class Handler(RequestHandler):
        def get(self):
            self.render("foo.html")

        def value(self):
            return self.get_argument("value")

    def get_app_kwargs(self):
        def my_ui_method(handler, x):
            return "In my_ui_method(%s) with handler value %s." % (x, handler.value())

        class MyModule(UIModule):
            def render(self, x):
                return "In MyModule(%s) with handler value %s." % (
                    x,
                    typing.cast(UIMethodUIModuleTest.Handler, self.handler).value(),
                )

        loader = DictLoader(
            {"foo.html": "{{ my_ui_method(42) }} {% module MyModule(123) %}"}
        )
        return dict(
            template_loader=loader,
            ui_methods={"my_ui_method": my_ui_method},
            ui_modules={"MyModule": MyModule},
        )

    def tearDown(self):
        super().tearDown()
        # TODO: fix template loader caching so this isn't necessary.
        RequestHandler._template_loaders.clear()

    def test_ui_method(self):
        response = self.fetch("/?value=asdf")
        self.assertEqual(
            response.body,
            b"In my_ui_method(42) with handler value asdf. "
            b"In MyModule(123) with handler value asdf.",
        )


class GetArgumentErrorTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            try:
                self.get_argument("foo")
                self.write({})
            except MissingArgumentError as e:
                self.write({"arg_name": e.arg_name, "log_message": e.log_message})

    def test_catch_error(self):
        response = self.fetch("/")
        self.assertEqual(
            json_decode(response.body),
            {"arg_name": "foo", "log_message": "Missing argument foo"},
        )


class SetLazyPropertiesTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def prepare(self):
            self.current_user = "Ben"
            self.locale = locale.get("en_US")

        def get_user_locale(self):
            raise NotImplementedError()

        def get_current_user(self):
            raise NotImplementedError()

        def get(self):
            self.write("Hello %s (%s)" % (self.current_user, self.locale.code))

    def test_set_properties(self):
        # Ensure that current_user can be assigned to normally for apps
        # that want to forgo the lazy get_current_user property
        response = self.fetch("/")
        self.assertEqual(response.body, b"Hello Ben (en_US)")


class GetCurrentUserTest(WebTestCase):
    def get_app_kwargs(self):
        class WithoutUserModule(UIModule):
            def render(self):
                return ""

        class WithUserModule(UIModule):
            def render(self):
                return str(self.current_user)

        loader = DictLoader(
            {
                "without_user.html": "",
                "with_user.html": "{{ current_user }}",
                "without_user_module.html": "{% module WithoutUserModule() %}",
                "with_user_module.html": "{% module WithUserModule() %}",
            }
        )
        return dict(
            template_loader=loader,
            ui_modules={
                "WithUserModule": WithUserModule,
                "WithoutUserModule": WithoutUserModule,
            },
        )

    def tearDown(self):
        super().tearDown()
        RequestHandler._template_loaders.clear()

    def get_handlers(self):
        class CurrentUserHandler(RequestHandler):
            def prepare(self):
                self.has_loaded_current_user = False

            def get_current_user(self):
                self.has_loaded_current_user = True
                return ""

        class WithoutUserHandler(CurrentUserHandler):
            def get(self):
                self.render_string("without_user.html")
                self.finish(str(self.has_loaded_current_user))

        class WithUserHandler(CurrentUserHandler):
            def get(self):
                self.render_string("with_user.html")
                self.finish(str(self.has_loaded_current_user))

        class CurrentUserModuleHandler(CurrentUserHandler):
            def get_template_namespace(self):
                # If RequestHandler.get_template_namespace is called, then
                # get_current_user is evaluated. Until #820 is fixed, this
                # is a small hack to circumvent the issue.
                return self.ui

        class WithoutUserModuleHandler(CurrentUserModuleHandler):
            def get(self):
                self.render_string("without_user_module.html")
                self.finish(str(self.has_loaded_current_user))

        class WithUserModuleHandler(CurrentUserModuleHandler):
            def get(self):
                self.render_string("with_user_module.html")
                self.finish(str(self.has_loaded_current_user))

        return [
            ("/without_user", WithoutUserHandler),
            ("/with_user", WithUserHandler),
            ("/without_user_module", WithoutUserModuleHandler),
            ("/with_user_module", WithUserModuleHandler),
        ]

    @unittest.skip("needs fix")
    def test_get_current_user_is_lazy(self):
        # TODO: Make this test pass. See #820.
        response = self.fetch("/without_user")
        self.assertEqual(response.body, b"False")

    def test_get_current_user_works(self):
        response = self.fetch("/with_user")
        self.assertEqual(response.body, b"True")

    def test_get_current_user_from_ui_module_is_lazy(self):
        response = self.fetch("/without_user_module")
        self.assertEqual(response.body, b"False")

    def test_get_current_user_from_ui_module_works(self):
        response = self.fetch("/with_user_module")
        self.assertEqual(response.body, b"True")


class UnimplementedHTTPMethodsTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        pass

    def test_unimplemented_standard_methods(self):
        for method in ["HEAD", "GET", "DELETE", "OPTIONS"]:
            response = self.fetch("/", method=method)
            self.assertEqual(response.code, 405)
        for method in ["POST", "PUT"]:
            response = self.fetch("/", method=method, body=b"")
            self.assertEqual(response.code, 405)


class UnimplementedNonStandardMethodsTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def other(self):
            # Even though this method exists, it won't get called automatically
            # because it is not in SUPPORTED_METHODS.
            self.write("other")

    def test_unimplemented_patch(self):
        # PATCH is recently standardized; Tornado supports it by default
        # but wsgiref.validate doesn't like it.
        response = self.fetch("/", method="PATCH", body=b"")
        self.assertEqual(response.code, 405)

    def test_unimplemented_other(self):
        response = self.fetch("/", method="OTHER", allow_nonstandard_methods=True)
        self.assertEqual(response.code, 405)


class AllHTTPMethodsTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def method(self):
            assert self.request.method is not None
            self.write(self.request.method)

        get = delete = options = post = put = method  # type: ignore

    def test_standard_methods(self):
        response = self.fetch("/", method="HEAD")
        self.assertEqual(response.body, b"")
        for method in ["GET", "DELETE", "OPTIONS"]:
            response = self.fetch("/", method=method)
            self.assertEqual(response.body, utf8(method))
        for method in ["POST", "PUT"]:
            response = self.fetch("/", method=method, body=b"")
            self.assertEqual(response.body, utf8(method))


class PatchMethodTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        SUPPORTED_METHODS = RequestHandler.SUPPORTED_METHODS + (  # type: ignore
            "OTHER",
        )

        def patch(self):
            self.write("patch")

        def other(self):
            self.write("other")

    def test_patch(self):
        response = self.fetch("/", method="PATCH", body=b"")
        self.assertEqual(response.body, b"patch")

    def test_other(self):
        response = self.fetch("/", method="OTHER", allow_nonstandard_methods=True)
        self.assertEqual(response.body, b"other")


class FinishInPrepareTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def prepare(self):
            self.finish("done")

        def get(self):
            # It's difficult to assert for certain that a method did not
            # or will not be called in an asynchronous context, but this
            # will be logged noisily if it is reached.
            raise Exception("should not reach this method")

    def test_finish_in_prepare(self):
        response = self.fetch("/")
        self.assertEqual(response.body, b"done")


class Default404Test(WebTestCase):
    def get_handlers(self):
        # If there are no handlers at all a default redirect handler gets added.
        return [("/foo", RequestHandler)]

    def test_404(self):
        response = self.fetch("/")
        self.assertEqual(response.code, 404)
        self.assertEqual(
            response.body,
            b"<html><title>404: Not Found</title>"
            b"<body>404: Not Found</body></html>",
        )


class Custom404Test(WebTestCase):
    def get_handlers(self):
        return [("/foo", RequestHandler)]

    def get_app_kwargs(self):
        class Custom404Handler(RequestHandler):
            def get(self):
                self.set_status(404)
                self.write("custom 404 response")

        return dict(default_handler_class=Custom404Handler)

    def test_404(self):
        response = self.fetch("/")
        self.assertEqual(response.code, 404)
        self.assertEqual(response.body, b"custom 404 response")


class DefaultHandlerArgumentsTest(WebTestCase):
    def get_handlers(self):
        return [("/foo", RequestHandler)]

    def get_app_kwargs(self):
        return dict(
            default_handler_class=ErrorHandler,
            default_handler_args=dict(status_code=403),
        )

    def test_403(self):
        response = self.fetch("/")
        self.assertEqual(response.code, 403)


class HandlerByNameTest(WebTestCase):
    def get_handlers(self):
        # All three are equivalent.
        return [
            ("/hello1", HelloHandler),
            ("/hello2", "tornado.test.web_test.HelloHandler"),
            url("/hello3", "tornado.test.web_test.HelloHandler"),
        ]

    def test_handler_by_name(self):
        resp = self.fetch("/hello1")
        self.assertEqual(resp.body, b"hello")
        resp = self.fetch("/hello2")
        self.assertEqual(resp.body, b"hello")
        resp = self.fetch("/hello3")
        self.assertEqual(resp.body, b"hello")


class StreamingRequestBodyTest(WebTestCase):
    def get_handlers(self):
        @stream_request_body
        class StreamingBodyHandler(RequestHandler):
            def initialize(self, test):
                self.test = test

            def prepare(self):
                self.test.prepared.set_result(None)

            def data_received(self, data):
                self.test.data.set_result(data)

            def get(self):
                self.test.finished.set_result(None)
                self.write({})

        @stream_request_body
        class EarlyReturnHandler(RequestHandler):
            def prepare(self):
                # If we finish the response in prepare, it won't continue to
                # the (non-existent) data_received.
                raise HTTPError(401)

        @stream_request_body
        class CloseDetectionHandler(RequestHandler):
            def initialize(self, test):
                self.test = test

            def on_connection_close(self):
                super().on_connection_close()
                self.test.close_future.set_result(None)

        return [
            ("/stream_body", StreamingBodyHandler, dict(test=self)),
            ("/early_return", EarlyReturnHandler),
            ("/close_detection", CloseDetectionHandler, dict(test=self)),
        ]

    def connect(self, url, connection_close):
        # Use a raw connection so we can control the sending of data.
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        s.connect(("127.0.0.1", self.get_http_port()))
        stream = IOStream(s)
        stream.write(b"GET " + url + b" HTTP/1.1\r\n")
        if connection_close:
            stream.write(b"Connection: close\r\n")
        stream.write(b"Transfer-Encoding: chunked\r\n\r\n")
        return stream

    @gen_test
    def test_streaming_body(self):
        self.prepared = Future()  # type: Future[None]
        self.data = Future()  # type: Future[bytes]
        self.finished = Future()  # type: Future[None]

        stream = self.connect(b"/stream_body", connection_close=True)
        yield self.prepared
        stream.write(b"4\r\nasdf\r\n")
        # Ensure the first chunk is received before we send the second.
        data = yield self.data
        self.assertEqual(data, b"asdf")
        self.data = Future()
        stream.write(b"4\r\nqwer\r\n")
        data = yield self.data
        self.assertEqual(data, b"qwer")
        stream.write(b"0\r\n\r\n")
        yield self.finished
        data = yield stream.read_until_close()
        # This would ideally use an HTTP1Connection to read the response.
        self.assertTrue(data.endswith(b"{}"))
        stream.close()

    @gen_test
    def test_early_return(self):
        stream = self.connect(b"/early_return", connection_close=False)
        data = yield stream.read_until_close()
        self.assertTrue(data.startswith(b"HTTP/1.1 401"))

    @gen_test
    def test_early_return_with_data(self):
        stream = self.connect(b"/early_return", connection_close=False)
        stream.write(b"4\r\nasdf\r\n")
        data = yield stream.read_until_close()
        self.assertTrue(data.startswith(b"HTTP/1.1 401"))

    @gen_test
    def test_close_during_upload(self):
        self.close_future = Future()  # type: Future[None]
        stream = self.connect(b"/close_detection", connection_close=False)
        stream.close()
        yield self.close_future


# Each method in this handler returns a yieldable object and yields to the
# IOLoop so the future is not immediately ready.  Ensure that the
# yieldables are respected and no method is called before the previous
# one has completed.
@stream_request_body
class BaseFlowControlHandler(RequestHandler):
    def initialize(self, test):
        self.test = test
        self.method = None
        self.methods = []  # type: typing.List[str]

    @contextlib.contextmanager
    def in_method(self, method):
        if self.method is not None:
            self.test.fail("entered method %s while in %s" % (method, self.method))
        self.method = method
        self.methods.append(method)
        try:
            yield
        finally:
            self.method = None

    @gen.coroutine
    def prepare(self):
        # Note that asynchronous prepare() does not block data_received,
        # so we don't use in_method here.
        self.methods.append("prepare")
        yield gen.moment

    @gen.coroutine
    def post(self):
        with self.in_method("post"):
            yield gen.moment
        self.write(dict(methods=self.methods))


class BaseStreamingRequestFlowControlTest(object):
    def get_httpserver_options(self):
        # Use a small chunk size so flow control is relevant even though
        # all the data arrives at once.
        return dict(chunk_size=10, decompress_request=True)

    def get_http_client(self):
        # simple_httpclient only: curl doesn't support body_producer.
        return SimpleAsyncHTTPClient()

    # Test all the slightly different code paths for fixed, chunked, etc bodies.
    def test_flow_control_fixed_body(self: typing.Any):
        response = self.fetch("/", body="abcdefghijklmnopqrstuvwxyz", method="POST")
        response.rethrow()
        self.assertEqual(
            json_decode(response.body),
            dict(
                methods=[
                    "prepare",
                    "data_received",
                    "data_received",
                    "data_received",
                    "post",
                ]
            ),
        )

    def test_flow_control_chunked_body(self: typing.Any):
        chunks = [b"abcd", b"efgh", b"ijkl"]

        @gen.coroutine
        def body_producer(write):
            for i in chunks:
                yield write(i)

        response = self.fetch("/", body_producer=body_producer, method="POST")
        response.rethrow()
        self.assertEqual(
            json_decode(response.body),
            dict(
                methods=[
                    "prepare",
                    "data_received",
                    "data_received",
                    "data_received",
                    "post",
                ]
            ),
        )

    def test_flow_control_compressed_body(self: typing.Any):
        bytesio = BytesIO()
        gzip_file = gzip.GzipFile(mode="w", fileobj=bytesio)
        gzip_file.write(b"abcdefghijklmnopqrstuvwxyz")
        gzip_file.close()
        compressed_body = bytesio.getvalue()
        response = self.fetch(
            "/",
            body=compressed_body,
            method="POST",
            headers={"Content-Encoding": "gzip"},
        )
        response.rethrow()
        self.assertEqual(
            json_decode(response.body),
            dict(
                methods=[
                    "prepare",
                    "data_received",
                    "data_received",
                    "data_received",
                    "post",
                ]
            ),
        )


class DecoratedStreamingRequestFlowControlTest(
    BaseStreamingRequestFlowControlTest, WebTestCase
):
    def get_handlers(self):
        class DecoratedFlowControlHandler(BaseFlowControlHandler):
            @gen.coroutine
            def data_received(self, data):
                with self.in_method("data_received"):
                    yield gen.moment

        return [("/", DecoratedFlowControlHandler, dict(test=self))]


class NativeStreamingRequestFlowControlTest(
    BaseStreamingRequestFlowControlTest, WebTestCase
):
    def get_handlers(self):
        class NativeFlowControlHandler(BaseFlowControlHandler):
            async def data_received(self, data):
                with self.in_method("data_received"):
                    import asyncio

                    await asyncio.sleep(0)

        return [("/", NativeFlowControlHandler, dict(test=self))]


class IncorrectContentLengthTest(SimpleHandlerTestCase):
    def get_handlers(self):
        test = self
        self.server_error = None

        # Manually set a content-length that doesn't match the actual content.
        class TooHigh(RequestHandler):
            def get(self):
                self.set_header("Content-Length", "42")
                try:
                    self.finish("ok")
                except Exception as e:
                    test.server_error = e
                    raise

        class TooLow(RequestHandler):
            def get(self):
                self.set_header("Content-Length", "2")
                try:
                    self.finish("hello")
                except Exception as e:
                    test.server_error = e
                    raise

        return [("/high", TooHigh), ("/low", TooLow)]

    def test_content_length_too_high(self):
        # When the content-length is too high, the connection is simply
        # closed without completing the response.  An error is logged on
        # the server.
        with ExpectLog(app_log, "(Uncaught exception|Exception in callback)"):
            with ExpectLog(
                gen_log,
                "(Cannot send error response after headers written"
                "|Failed to flush partial response)",
            ):
                with self.assertRaises(HTTPClientError):
                    self.fetch("/high", raise_error=True)
        self.assertEqual(
            str(self.server_error), "Tried to write 40 bytes less than Content-Length"
        )

    def test_content_length_too_low(self):
        # When the content-length is too low, the connection is closed
        # without writing the last chunk, so the client never sees the request
        # complete (which would be a framing error).
        with ExpectLog(app_log, "(Uncaught exception|Exception in callback)"):
            with ExpectLog(
                gen_log,
                "(Cannot send error response after headers written"
                "|Failed to flush partial response)",
            ):
                with self.assertRaises(HTTPClientError):
                    self.fetch("/low", raise_error=True)
        self.assertEqual(
            str(self.server_error), "Tried to write more data than Content-Length"
        )


class ClientCloseTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            if self.request.version.startswith("HTTP/1"):
                # Simulate a connection closed by the client during
                # request processing.  The client will see an error, but the
                # server should respond gracefully (without logging errors
                # because we were unable to write out as many bytes as
                # Content-Length said we would)
                self.request.connection.stream.close()  # type: ignore
                self.write("hello")
            else:
                # TODO: add a HTTP2-compatible version of this test.
                self.write("requires HTTP/1.x")

    def test_client_close(self):
        with self.assertRaises((HTTPClientError, unittest.SkipTest)):  # type: ignore
            response = self.fetch("/", raise_error=True)
            if response.body == b"requires HTTP/1.x":
                self.skipTest("requires HTTP/1.x")
            self.assertEqual(response.code, 599)


class SignedValueTest(unittest.TestCase):
    SECRET = "It's a secret to everybody"
    SECRET_DICT = {0: "asdfbasdf", 1: "12312312", 2: "2342342"}

    def past(self):
        return self.present() - 86400 * 32

    def present(self):
        return 1300000000

    def test_known_values(self):
        signed_v1 = create_signed_value(
            SignedValueTest.SECRET, "key", "value", version=1, clock=self.present
        )
        self.assertEqual(
            signed_v1, b"dmFsdWU=|1300000000|31c934969f53e48164c50768b40cbd7e2daaaa4f"
        )

        signed_v2 = create_signed_value(
            SignedValueTest.SECRET, "key", "value", version=2, clock=self.present
        )
        self.assertEqual(
            signed_v2,
            b"2|1:0|10:1300000000|3:key|8:dmFsdWU=|"
            b"3d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e152",
        )

        signed_default = create_signed_value(
            SignedValueTest.SECRET, "key", "value", clock=self.present
        )
        self.assertEqual(signed_default, signed_v2)

        decoded_v1 = decode_signed_value(
            SignedValueTest.SECRET, "key", signed_v1, min_version=1, clock=self.present
        )
        self.assertEqual(decoded_v1, b"value")

        decoded_v2 = decode_signed_value(
            SignedValueTest.SECRET, "key", signed_v2, min_version=2, clock=self.present
        )
        self.assertEqual(decoded_v2, b"value")

    def test_name_swap(self):
        signed1 = create_signed_value(
            SignedValueTest.SECRET, "key1", "value", clock=self.present
        )
        signed2 = create_signed_value(
            SignedValueTest.SECRET, "key2", "value", clock=self.present
        )
        # Try decoding each string with the other's "name"
        decoded1 = decode_signed_value(
            SignedValueTest.SECRET, "key2", signed1, clock=self.present
        )
        self.assertIs(decoded1, None)
        decoded2 = decode_signed_value(
            SignedValueTest.SECRET, "key1", signed2, clock=self.present
        )
        self.assertIs(decoded2, None)

    def test_expired(self):
        signed = create_signed_value(
            SignedValueTest.SECRET, "key1", "value", clock=self.past
        )
        decoded_past = decode_signed_value(
            SignedValueTest.SECRET, "key1", signed, clock=self.past
        )
        self.assertEqual(decoded_past, b"value")
        decoded_present = decode_signed_value(
            SignedValueTest.SECRET, "key1", signed, clock=self.present
        )
        self.assertIs(decoded_present, None)

    def test_payload_tampering(self):
        # These cookies are variants of the one in test_known_values.
        sig = "3d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e152"

        def validate(prefix):
            return b"value" == decode_signed_value(
                SignedValueTest.SECRET, "key", prefix + sig, clock=self.present
            )

        self.assertTrue(validate("2|1:0|10:1300000000|3:key|8:dmFsdWU=|"))
        # Change key version
        self.assertFalse(validate("2|1:1|10:1300000000|3:key|8:dmFsdWU=|"))
        # length mismatch (field too short)
        self.assertFalse(validate("2|1:0|10:130000000|3:key|8:dmFsdWU=|"))
        # length mismatch (field too long)
        self.assertFalse(validate("2|1:0|10:1300000000|3:keey|8:dmFsdWU=|"))

    def test_signature_tampering(self):
        prefix = "2|1:0|10:1300000000|3:key|8:dmFsdWU=|"

        def validate(sig):
            return b"value" == decode_signed_value(
                SignedValueTest.SECRET, "key", prefix + sig, clock=self.present
            )

        self.assertTrue(
            validate("3d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e152")
        )
        # All zeros
        self.assertFalse(validate("0" * 32))
        # Change one character
        self.assertFalse(
            validate("4d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e152")
        )
        # Change another character
        self.assertFalse(
            validate("3d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e153")
        )
        # Truncate
        self.assertFalse(
            validate("3d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e15")
        )
        # Lengthen
        self.assertFalse(
            validate(
                "3d4e60b996ff9c5d5788e333a0cba6f238a22c6c0f94788870e1a9ecd482e1538"
            )
        )

    def test_non_ascii(self):
        value = b"\xe9"
        signed = create_signed_value(
            SignedValueTest.SECRET, "key", value, clock=self.present
        )
        decoded = decode_signed_value(
            SignedValueTest.SECRET, "key", signed, clock=self.present
        )
        self.assertEqual(value, decoded)

    def test_key_versioning_read_write_default_key(self):
        value = b"\xe9"
        signed = create_signed_value(
            SignedValueTest.SECRET_DICT, "key", value, clock=self.present, key_version=0
        )
        decoded = decode_signed_value(
            SignedValueTest.SECRET_DICT, "key", signed, clock=self.present
        )
        self.assertEqual(value, decoded)

    def test_key_versioning_read_write_non_default_key(self):
        value = b"\xe9"
        signed = create_signed_value(
            SignedValueTest.SECRET_DICT, "key", value, clock=self.present, key_version=1
        )
        decoded = decode_signed_value(
            SignedValueTest.SECRET_DICT, "key", signed, clock=self.present
        )
        self.assertEqual(value, decoded)

    def test_key_versioning_invalid_key(self):
        value = b"\xe9"
        signed = create_signed_value(
            SignedValueTest.SECRET_DICT, "key", value, clock=self.present, key_version=0
        )
        newkeys = SignedValueTest.SECRET_DICT.copy()
        newkeys.pop(0)
        decoded = decode_signed_value(newkeys, "key", signed, clock=self.present)
        self.assertEqual(None, decoded)

    def test_key_version_retrieval(self):
        value = b"\xe9"
        signed = create_signed_value(
            SignedValueTest.SECRET_DICT, "key", value, clock=self.present, key_version=1
        )
        key_version = get_signature_key_version(signed)
        self.assertEqual(1, key_version)


class XSRFTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            version = int(self.get_argument("version", "2"))
            # This would be a bad idea in a real app, but in this test
            # it's fine.
            self.settings["xsrf_cookie_version"] = version
            self.write(self.xsrf_token)

        def post(self):
            self.write("ok")

    def get_app_kwargs(self):
        return dict(xsrf_cookies=True)

    def setUp(self):
        super().setUp()
        self.xsrf_token = self.get_token()

    def get_token(self, old_token=None, version=None):
        if old_token is not None:
            headers = self.cookie_headers(old_token)
        else:
            headers = None
        response = self.fetch(
            "/" if version is None else ("/?version=%d" % version), headers=headers
        )
        response.rethrow()
        return native_str(response.body)

    def cookie_headers(self, token=None):
        if token is None:
            token = self.xsrf_token
        return {"Cookie": "_xsrf=" + token}

    def test_xsrf_fail_no_token(self):
        with ExpectLog(gen_log, ".*'_xsrf' argument missing"):
            response = self.fetch("/", method="POST", body=b"")
        self.assertEqual(response.code, 403)

    def test_xsrf_fail_body_no_cookie(self):
        with ExpectLog(gen_log, ".*XSRF cookie does not match POST"):
            response = self.fetch(
                "/",
                method="POST",
                body=urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
            )
        self.assertEqual(response.code, 403)

    def test_xsrf_fail_argument_invalid_format(self):
        with ExpectLog(gen_log, ".*'_xsrf' argument has invalid format"):
            response = self.fetch(
                "/",
                method="POST",
                headers=self.cookie_headers(),
                body=urllib.parse.urlencode(dict(_xsrf="3|")),
            )
        self.assertEqual(response.code, 403)

    def test_xsrf_fail_cookie_invalid_format(self):
        with ExpectLog(gen_log, ".*XSRF cookie does not match POST"):
            response = self.fetch(
                "/",
                method="POST",
                headers=self.cookie_headers(token="3|"),
                body=urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
            )
        self.assertEqual(response.code, 403)

    def test_xsrf_fail_cookie_no_body(self):
        with ExpectLog(gen_log, ".*'_xsrf' argument missing"):
            response = self.fetch(
                "/", method="POST", body=b"", headers=self.cookie_headers()
            )
        self.assertEqual(response.code, 403)

    def test_xsrf_success_short_token(self):
        response = self.fetch(
            "/",
            method="POST",
            body=urllib.parse.urlencode(dict(_xsrf="deadbeef")),
            headers=self.cookie_headers(token="deadbeef"),
        )
        self.assertEqual(response.code, 200)

    def test_xsrf_success_non_hex_token(self):
        response = self.fetch(
            "/",
            method="POST",
            body=urllib.parse.urlencode(dict(_xsrf="xoxo")),
            headers=self.cookie_headers(token="xoxo"),
        )
        self.assertEqual(response.code, 200)

    def test_xsrf_success_post_body(self):
        response = self.fetch(
            "/",
            method="POST",
            body=urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
            headers=self.cookie_headers(),
        )
        self.assertEqual(response.code, 200)

    def test_xsrf_success_query_string(self):
        response = self.fetch(
            "/?" + urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
            method="POST",
            body=b"",
            headers=self.cookie_headers(),
        )
        self.assertEqual(response.code, 200)

    def test_xsrf_success_header(self):
        response = self.fetch(
            "/",
            method="POST",
            body=b"",
            headers=dict(
                {"X-Xsrftoken": self.xsrf_token},  # type: ignore
                **self.cookie_headers()
            ),
        )
        self.assertEqual(response.code, 200)

    def test_distinct_tokens(self):
        # Every request gets a distinct token.
        NUM_TOKENS = 10
        tokens = set()
        for i in range(NUM_TOKENS):
            tokens.add(self.get_token())
        self.assertEqual(len(tokens), NUM_TOKENS)

    def test_cross_user(self):
        token2 = self.get_token()
        # Each token can be used to authenticate its own request.
        for token in (self.xsrf_token, token2):
            response = self.fetch(
                "/",
                method="POST",
                body=urllib.parse.urlencode(dict(_xsrf=token)),
                headers=self.cookie_headers(token),
            )
            self.assertEqual(response.code, 200)
        # Sending one in the cookie and the other in the body is not allowed.
        for cookie_token, body_token in (
            (self.xsrf_token, token2),
            (token2, self.xsrf_token),
        ):
            with ExpectLog(gen_log, ".*XSRF cookie does not match POST"):
                response = self.fetch(
                    "/",
                    method="POST",
                    body=urllib.parse.urlencode(dict(_xsrf=body_token)),
                    headers=self.cookie_headers(cookie_token),
                )
            self.assertEqual(response.code, 403)

    def test_refresh_token(self):
        token = self.xsrf_token
        tokens_seen = set([token])
        # A user's token is stable over time.  Refreshing the page in one tab
        # might update the cookie while an older tab still has the old cookie
        # in its DOM.  Simulate this scenario by passing a constant token
        # in the body and re-querying for the token.
        for i in range(5):
            token = self.get_token(token)
            # Tokens are encoded uniquely each time
            tokens_seen.add(token)
            response = self.fetch(
                "/",
                method="POST",
                body=urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
                headers=self.cookie_headers(token),
            )
            self.assertEqual(response.code, 200)
        self.assertEqual(len(tokens_seen), 6)

    def test_versioning(self):
        # Version 1 still produces distinct tokens per request.
        self.assertNotEqual(self.get_token(version=1), self.get_token(version=1))

        # Refreshed v1 tokens are all identical.
        v1_token = self.get_token(version=1)
        for i in range(5):
            self.assertEqual(self.get_token(v1_token, version=1), v1_token)

        # Upgrade to a v2 version of the same token
        v2_token = self.get_token(v1_token)
        self.assertNotEqual(v1_token, v2_token)
        # Each v1 token can map to many v2 tokens.
        self.assertNotEqual(v2_token, self.get_token(v1_token))

        # The tokens are cross-compatible.
        for cookie_token, body_token in ((v1_token, v2_token), (v2_token, v1_token)):
            response = self.fetch(
                "/",
                method="POST",
                body=urllib.parse.urlencode(dict(_xsrf=body_token)),
                headers=self.cookie_headers(cookie_token),
            )
            self.assertEqual(response.code, 200)


# A subset of the previous test with a different cookie name
class XSRFCookieNameTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.write(self.xsrf_token)

        def post(self):
            self.write("ok")

    def get_app_kwargs(self):
        return dict(
            xsrf_cookies=True,
            xsrf_cookie_name="__Host-xsrf",
            xsrf_cookie_kwargs={"secure": True},
        )

    def setUp(self):
        super().setUp()
        self.xsrf_token = self.get_token()

    def get_token(self, old_token=None):
        if old_token is not None:
            headers = self.cookie_headers(old_token)
        else:
            headers = None
        response = self.fetch("/", headers=headers)
        response.rethrow()
        return native_str(response.body)

    def cookie_headers(self, token=None):
        if token is None:
            token = self.xsrf_token
        return {"Cookie": "__Host-xsrf=" + token}

    def test_xsrf_fail_no_token(self):
        with ExpectLog(gen_log, ".*'_xsrf' argument missing"):
            response = self.fetch("/", method="POST", body=b"")
        self.assertEqual(response.code, 403)

    def test_xsrf_fail_body_no_cookie(self):
        with ExpectLog(gen_log, ".*XSRF cookie does not match POST"):
            response = self.fetch(
                "/",
                method="POST",
                body=urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
            )
        self.assertEqual(response.code, 403)

    def test_xsrf_success_post_body(self):
        response = self.fetch(
            "/",
            method="POST",
            # Note that renaming the cookie doesn't rename the POST param
            body=urllib.parse.urlencode(dict(_xsrf=self.xsrf_token)),
            headers=self.cookie_headers(),
        )
        self.assertEqual(response.code, 200)


class XSRFCookieKwargsTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.write(self.xsrf_token)

    def get_app_kwargs(self):
        return dict(
            xsrf_cookies=True, xsrf_cookie_kwargs=dict(httponly=True, expires_days=2)
        )

    def test_xsrf_httponly(self):
        response = self.fetch("/")
        self.assertIn("httponly;", response.headers["Set-Cookie"].lower())
        self.assertIn("expires=", response.headers["Set-Cookie"].lower())
        header = response.headers.get("Set-Cookie")
        assert header is not None
        match = re.match(".*; expires=(?P<expires>.+);.*", header)
        assert match is not None

        expires = datetime.datetime.utcnow() + datetime.timedelta(days=2)
        parsed = email.utils.parsedate(match.groupdict()["expires"])
        assert parsed is not None
        header_expires = datetime.datetime(*parsed[:6])
        self.assertTrue(abs((expires - header_expires).total_seconds()) < 10)


class FinishExceptionTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            self.set_status(401)
            self.set_header("WWW-Authenticate", 'Basic realm="something"')
            if self.get_argument("finish_value", ""):
                raise Finish("authentication required")
            else:
                self.write("authentication required")
                raise Finish()

    def test_finish_exception(self):
        for u in ["/", "/?finish_value=1"]:
            response = self.fetch(u)
            self.assertEqual(response.code, 401)
            self.assertEqual(
                'Basic realm="something"', response.headers.get("WWW-Authenticate")
            )
            self.assertEqual(b"authentication required", response.body)


class DecoratorTest(WebTestCase):
    def get_handlers(self):
        class RemoveSlashHandler(RequestHandler):
            @removeslash
            def get(self):
                pass

        class AddSlashHandler(RequestHandler):
            @addslash
            def get(self):
                pass

        return [("/removeslash/", RemoveSlashHandler), ("/addslash", AddSlashHandler)]

    def test_removeslash(self):
        response = self.fetch("/removeslash/", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/removeslash")

        response = self.fetch("/removeslash/?foo=bar", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/removeslash?foo=bar")

    def test_addslash(self):
        response = self.fetch("/addslash", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/addslash/")

        response = self.fetch("/addslash?foo=bar", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/addslash/?foo=bar")


class CacheTest(WebTestCase):
    def get_handlers(self):
        class EtagHandler(RequestHandler):
            def get(self, computed_etag):
                self.write(computed_etag)

            def compute_etag(self):
                return self._write_buffer[0]

        return [("/etag/(.*)", EtagHandler)]

    def test_wildcard_etag(self):
        computed_etag = '"xyzzy"'
        etags = "*"
        self._test_etag(computed_etag, etags, 304)

    def test_strong_etag_match(self):
        computed_etag = '"xyzzy"'
        etags = '"xyzzy"'
        self._test_etag(computed_etag, etags, 304)

    def test_multiple_strong_etag_match(self):
        computed_etag = '"xyzzy1"'
        etags = '"xyzzy1", "xyzzy2"'
        self._test_etag(computed_etag, etags, 304)

    def test_strong_etag_not_match(self):
        computed_etag = '"xyzzy"'
        etags = '"xyzzy1"'
        self._test_etag(computed_etag, etags, 200)

    def test_multiple_strong_etag_not_match(self):
        computed_etag = '"xyzzy"'
        etags = '"xyzzy1", "xyzzy2"'
        self._test_etag(computed_etag, etags, 200)

    def test_weak_etag_match(self):
        computed_etag = '"xyzzy1"'
        etags = 'W/"xyzzy1"'
        self._test_etag(computed_etag, etags, 304)

    def test_multiple_weak_etag_match(self):
        computed_etag = '"xyzzy2"'
        etags = 'W/"xyzzy1", W/"xyzzy2"'
        self._test_etag(computed_etag, etags, 304)

    def test_weak_etag_not_match(self):
        computed_etag = '"xyzzy2"'
        etags = 'W/"xyzzy1"'
        self._test_etag(computed_etag, etags, 200)

    def test_multiple_weak_etag_not_match(self):
        computed_etag = '"xyzzy3"'
        etags = 'W/"xyzzy1", W/"xyzzy2"'
        self._test_etag(computed_etag, etags, 200)

    def _test_etag(self, computed_etag, etags, status_code):
        response = self.fetch(
            "/etag/" + computed_etag, headers={"If-None-Match": etags}
        )
        self.assertEqual(response.code, status_code)


class RequestSummaryTest(SimpleHandlerTestCase):
    class Handler(RequestHandler):
        def get(self):
            # remote_ip is optional, although it's set by
            # both HTTPServer and WSGIAdapter.
            # Clobber it to make sure it doesn't break logging.
            self.request.remote_ip = None
            self.finish(self._request_summary())

    def test_missing_remote_ip(self):
        resp = self.fetch("/")
        self.assertEqual(resp.body, b"GET / (None)")


class HTTPErrorTest(unittest.TestCase):
    def test_copy(self):
        e = HTTPError(403, reason="Go away")
        e2 = copy.copy(e)
        self.assertIsNot(e, e2)
        self.assertEqual(e.status_code, e2.status_code)
        self.assertEqual(e.reason, e2.reason)


class ApplicationTest(AsyncTestCase):
    def test_listen(self):
        app = Application([])
        server = app.listen(0, address="127.0.0.1")
        server.stop()


class URLSpecReverseTest(unittest.TestCase):
    def test_reverse(self):
        self.assertEqual("/favicon.ico", url(r"/favicon\.ico", None).reverse())
        self.assertEqual("/favicon.ico", url(r"^/favicon\.ico$", None).reverse())

    def test_non_reversible(self):
        # URLSpecs are non-reversible if they include non-constant
        # regex features outside capturing groups. Currently, this is
        # only strictly enforced for backslash-escaped character
        # classes.
        paths = [r"^/api/v\d+/foo/(\w+)$"]
        for path in paths:
            # A URLSpec can still be created even if it cannot be reversed.
            url_spec = url(path, None)
            try:
                result = url_spec.reverse()
                self.fail(
                    "did not get expected exception when reversing %s. "
                    "result: %s" % (path, result)
                )
            except ValueError:
                pass

    def test_reverse_arguments(self):
        self.assertEqual(
            "/api/v1/foo/bar", url(r"^/api/v1/foo/(\w+)$", None).reverse("bar")
        )
        self.assertEqual(
            "/api.v1/foo/5/icon.png",
            url(r"/api\.v1/foo/([0-9]+)/icon\.png", None).reverse(5),
        )


class RedirectHandlerTest(WebTestCase):
    def get_handlers(self):
        return [
            ("/src", WebRedirectHandler, {"url": "/dst"}),
            ("/src2", WebRedirectHandler, {"url": "/dst2?foo=bar"}),
            (r"/(.*?)/(.*?)/(.*)", WebRedirectHandler, {"url": "/{1}/{0}/{2}"}),
        ]

    def test_basic_redirect(self):
        response = self.fetch("/src", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/dst")

    def test_redirect_with_argument(self):
        response = self.fetch("/src?foo=bar", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/dst?foo=bar")

    def test_redirect_with_appending_argument(self):
        response = self.fetch("/src2?foo2=bar2", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/dst2?foo=bar&foo2=bar2")

    def test_redirect_pattern(self):
        response = self.fetch("/a/b/c", follow_redirects=False)
        self.assertEqual(response.code, 301)
        self.assertEqual(response.headers["Location"], "/b/a/c")


class AcceptLanguageTest(WebTestCase):
    """Test evaluation of Accept-Language header"""

    def get_handlers(self):
        locale.load_gettext_translations(
            os.path.join(os.path.dirname(__file__), "gettext_translations"),
            "tornado_test",
        )

        class AcceptLanguageHandler(RequestHandler):
            def get(self):
                self.set_header(
                    "Content-Language", self.get_browser_locale().code.replace("_", "-")
                )
                self.finish(b"")

        return [
            ("/", AcceptLanguageHandler),
        ]

    def test_accept_language(self):
        response = self.fetch("/", headers={"Accept-Language": "fr-FR;q=0.9"})
        self.assertEqual(response.headers["Content-Language"], "fr-FR")

        response = self.fetch("/", headers={"Accept-Language": "fr-FR; q=0.9"})
        self.assertEqual(response.headers["Content-Language"], "fr-FR")

    def test_accept_language_ignore(self):
        response = self.fetch("/", headers={"Accept-Language": "fr-FR;q=0"})
        self.assertEqual(response.headers["Content-Language"], "en-US")

    def test_accept_language_invalid(self):
        response = self.fetch("/", headers={"Accept-Language": "fr-FR;q=-1"})
        self.assertEqual(response.headers["Content-Language"], "en-US")

env_pyrecodes/lib/python3.9/site-packages/tornado/test/web_test.py
NikolaBlagojevic/pyrecodes
#
# Copyright 2009 Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""``tornado.web`` provides a simple web framework with asynchronous
features that allow it to scale to large numbers of open connections,
making it ideal for `long polling
<http://en.wikipedia.org/wiki/Push_technology#Long_polling>`_.

Here is a simple "Hello, world" example app:

.. testcode::

    import asyncio
    import tornado

    class MainHandler(tornado.web.RequestHandler):
        def get(self):
            self.write("Hello, world")

    async def main():
        application = tornado.web.Application([
            (r"/", MainHandler),
        ])
        application.listen(8888)
        await asyncio.Event().wait()

    if __name__ == "__main__":
        asyncio.run(main())

.. testoutput::
   :hide:


See the :doc:`guide` for additional information.

Thread-safety notes
-------------------

In general, methods on `RequestHandler` and elsewhere in Tornado are
not thread-safe. In particular, methods such as
`~RequestHandler.write()`, `~RequestHandler.finish()`, and
`~RequestHandler.flush()` must only be called from the main thread. If
you use multiple threads it is important to use `.IOLoop.add_callback`
to transfer control back to the main thread before finishing the
request, or to limit your use of other threads to
`.IOLoop.run_in_executor` and ensure that your callbacks running in
the executor do not refer to Tornado objects.

"""

import base64
import binascii
import datetime
import email.utils
import functools
import gzip
import hashlib
import hmac
import http.cookies
from inspect import isclass
from io import BytesIO
import mimetypes
import numbers
import os.path
import re
import socket
import sys
import threading
import time
import tornado
import traceback
import types
import urllib.parse
from urllib.parse import urlencode

from tornado.concurrent import Future, future_set_result_unless_cancelled
from tornado import escape
from tornado import gen
from tornado.httpserver import HTTPServer
from tornado import httputil
from tornado import iostream
from tornado import locale
from tornado.log import access_log, app_log, gen_log
from tornado import template
from tornado.escape import utf8, _unicode
from tornado.routing import (
    AnyMatches,
    DefaultHostMatches,
    HostMatches,
    ReversibleRouter,
    Rule,
    ReversibleRuleRouter,
    URLSpec,
    _RuleList,
)
from tornado.util import ObjectDict, unicode_type, _websocket_mask

url = URLSpec

from typing import (
    Dict,
    Any,
    Union,
    Optional,
    Awaitable,
    Tuple,
    List,
    Callable,
    Iterable,
    Generator,
    Type,
    TypeVar,
    cast,
    overload,
)
from types import TracebackType
import typing

if typing.TYPE_CHECKING:
    from typing import Set  # noqa: F401


# The following types are accepted by RequestHandler.set_header
# and related methods.
_HeaderTypes = Union[bytes, unicode_type, int, numbers.Integral, datetime.datetime]

_CookieSecretTypes = Union[str, bytes, Dict[int, str], Dict[int, bytes]]


MIN_SUPPORTED_SIGNED_VALUE_VERSION = 1
"""The oldest signed value version supported by this version of Tornado.

Signed values older than this version cannot be decoded.

.. versionadded:: 3.2.1
"""

MAX_SUPPORTED_SIGNED_VALUE_VERSION = 2
"""The newest signed value version supported by this version of Tornado.

Signed values newer than this version cannot be decoded.

.. versionadded:: 3.2.1
"""

DEFAULT_SIGNED_VALUE_VERSION = 2
"""The signed value version produced by `.RequestHandler.create_signed_value`.

May be overridden by passing a ``version`` keyword argument.

.. versionadded:: 3.2.1
"""

DEFAULT_SIGNED_VALUE_MIN_VERSION = 1
"""The oldest signed value accepted by `.RequestHandler.get_signed_cookie`.

May be overridden by passing a ``min_version`` keyword argument.

.. versionadded:: 3.2.1
"""


class _ArgDefaultMarker:
    pass


_ARG_DEFAULT = _ArgDefaultMarker()


class RequestHandler(object):
    """Base class for HTTP request handlers.

    Subclasses must define at least one of the methods defined in the
    "Entry points" section below.

    Applications should not construct `RequestHandler` objects
    directly and subclasses should not override ``__init__`` (override
    `~RequestHandler.initialize` instead).

    """

    SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PATCH", "PUT", "OPTIONS")

    _template_loaders = {}  # type: Dict[str, template.BaseLoader]
    _template_loader_lock = threading.Lock()
    _remove_control_chars_regex = re.compile(r"[\x00-\x08\x0e-\x1f]")

    _stream_request_body = False

    # Will be set in _execute.
    _transforms = None  # type: List[OutputTransform]
    path_args = None  # type: List[str]
    path_kwargs = None  # type: Dict[str, str]

    def __init__(
        self,
        application: "Application",
        request: httputil.HTTPServerRequest,
        **kwargs: Any,
    ) -> None:
        super().__init__()

        self.application = application
        self.request = request
        self._headers_written = False
        self._finished = False
        self._auto_finish = True
        self._prepared_future = None
        self.ui = ObjectDict(
            (n, self._ui_method(m)) for n, m in application.ui_methods.items()
        )
        # UIModules are available as both `modules` and `_tt_modules` in the
        # template namespace.  Historically only `modules` was available
        # but could be clobbered by user additions to the namespace.
        # The template {% module %} directive looks in `_tt_modules` to avoid
        # possible conflicts.
        self.ui["_tt_modules"] = _UIModuleNamespace(self, application.ui_modules)
        self.ui["modules"] = self.ui["_tt_modules"]
        self.clear()
        assert self.request.connection is not None
        # TODO: need to add set_close_callback to HTTPConnection interface
        self.request.connection.set_close_callback(  # type: ignore
            self.on_connection_close
        )
        self.initialize(**kwargs)  # type: ignore

    def _initialize(self) -> None:
        pass

    initialize = _initialize  # type: Callable[..., None]
    """Hook for subclass initialization. Called for each request.

    A dictionary passed as the third argument of a ``URLSpec`` will be
    supplied as keyword arguments to ``initialize()``.

    Example::

        class ProfileHandler(RequestHandler):
            def initialize(self, database):
                self.database = database

            def get(self, username):
                ...

        app = Application([
            (r'/user/(.*)', ProfileHandler, dict(database=database)),
            ])
    """

    @property
    def settings(self) -> Dict[str, Any]:
        """An alias for `self.application.settings <Application.settings>`."""
        return self.application.settings

    def _unimplemented_method(self, *args: str, **kwargs: str) -> None:
        raise HTTPError(405)

    head = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]
    get = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]
    post = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]
    delete = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]
    patch = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]
    put = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]
    options = _unimplemented_method  # type: Callable[..., Optional[Awaitable[None]]]

    def prepare(self) -> Optional[Awaitable[None]]:
        """Called at the beginning of a request before  `get`/`post`/etc.

        Override this method to perform common initialization regardless
        of the request method.

        Asynchronous support: Use ``async def`` or decorate this method with
        `.gen.coroutine` to make it asynchronous.
        If this method returns an  ``Awaitable`` execution will not proceed
        until the ``Awaitable`` is done.

        .. versionadded:: 3.1
           Asynchronous support.
        """
        pass

    def on_finish(self) -> None:
        """Called after the end of a request.

        Override this method to perform cleanup, logging, etc.
        This method is a counterpart to `prepare`.  ``on_finish`` may
        not produce any output, as it is called after the response
        has been sent to the client.
        """
        pass

    def on_connection_close(self) -> None:
        """Called in async handlers if the client closed the connection.

        Override this to clean up resources associated with
        long-lived connections.  Note that this method is called only if
        the connection was closed during asynchronous processing; if you
        need to do cleanup after every request override `on_finish`
        instead.

        Proxies may keep a connection open for a time (perhaps
        indefinitely) after the client has gone away, so this method
        may not be called promptly after the end user closes their
        connection.
        """
        if _has_stream_request_body(self.__class__):
            if not self.request._body_future.done():
                self.request._body_future.set_exception(iostream.StreamClosedError())
                self.request._body_future.exception()

    def clear(self) -> None:
        """Resets all headers and content for this response."""
        self._headers = httputil.HTTPHeaders(
            {
                "Server": "TornadoServer/%s" % tornado.version,
                "Content-Type": "text/html; charset=UTF-8",
                "Date": httputil.format_timestamp(time.time()),
            }
        )
        self.set_default_headers()
        self._write_buffer = []  # type: List[bytes]
        self._status_code = 200
        self._reason = httputil.responses[200]

    def set_default_headers(self) -> None:
        """Override this to set HTTP headers at the beginning of the request.

        For example, this is the place to set a custom ``Server`` header.
        Note that setting such headers in the normal flow of request
        processing may not do what you want, since headers may be reset
        during error handling.
        """
        pass

    def set_status(self, status_code: int, reason: Optional[str] = None) -> None:
        """Sets the status code for our response.

        :arg int status_code: Response status code.
        :arg str reason: Human-readable reason phrase describing the status
            code. If ``None``, it will be filled in from
            `http.client.responses` or "Unknown".

        .. versionchanged:: 5.0

           No longer validates that the response code is in
           `http.client.responses`.
        """
        self._status_code = status_code
        if reason is not None:
            self._reason = escape.native_str(reason)
        else:
            self._reason = httputil.responses.get(status_code, "Unknown")

    def get_status(self) -> int:
        """Returns the status code for our response."""
        return self._status_code

    def set_header(self, name: str, value: _HeaderTypes) -> None:
        """Sets the given response header name and value.

        All header values are converted to strings (`datetime` objects
        are formatted according to the HTTP specification for the
        ``Date`` header).

        """
        self._headers[name] = self._convert_header_value(value)

    def add_header(self, name: str, value: _HeaderTypes) -> None:
        """Adds the given response header and value.

        Unlike `set_header`, `add_header` may be called multiple times
        to return multiple values for the same header.
        """
        self._headers.add(name, self._convert_header_value(value))

    def clear_header(self, name: str) -> None:
        """Clears an outgoing header, undoing a previous `set_header` call.

        Note that this method does not apply to multi-valued headers
        set by `add_header`.
        """
        if name in self._headers:
            del self._headers[name]

    _INVALID_HEADER_CHAR_RE = re.compile(r"[\x00-\x1f]")

    def _convert_header_value(self, value: _HeaderTypes) -> str:
        # Convert the input value to a str. This type check is a bit
        # subtle: The bytes case only executes on python 3, and the
        # unicode case only executes on python 2, because the other
        # cases are covered by the first match for str.
        if isinstance(value, str):
            retval = value
        elif isinstance(value, bytes):
            # Non-ascii characters in headers are not well supported,
            # but if you pass bytes, use latin1 so they pass through as-is.
            retval = value.decode("latin1")
        elif isinstance(value, numbers.Integral):
            # return immediately since we know the converted value will be safe
            return str(value)
        elif isinstance(value, datetime.datetime):
            return httputil.format_timestamp(value)
        else:
            raise TypeError("Unsupported header value %r" % value)
        # If \n is allowed into the header, it is possible to inject
        # additional headers or split the request.
        if RequestHandler._INVALID_HEADER_CHAR_RE.search(retval):
            raise ValueError("Unsafe header value %r", retval)
        return retval

    @overload
    def get_argument(self, name: str, default: str, strip: bool = True) -> str:
        pass

    @overload
    def get_argument(  # noqa: F811
        self, name: str, default: _ArgDefaultMarker = _ARG_DEFAULT, strip: bool = True
    ) -> str:
        pass

    @overload
    def get_argument(  # noqa: F811
        self, name: str, default: None, strip: bool = True
    ) -> Optional[str]:
        pass

    def get_argument(  # noqa: F811
        self,
        name: str,
        default: Union[None, str, _ArgDefaultMarker] = _ARG_DEFAULT,
        strip: bool = True,
    ) -> Optional[str]:
        """Returns the value of the argument with the given name.

        If default is not provided, the argument is considered to be
        required, and we raise a `MissingArgumentError` if it is missing.

        If the argument appears in the request more than once, we return the
        last value.

        This method searches both the query and body arguments.
        """
        return self._get_argument(name, default, self.request.arguments, strip)

    def get_arguments(self, name: str, strip: bool = True) -> List[str]:
        """Returns a list of the arguments with the given name.

        If the argument is not present, returns an empty list.

        This method searches both the query and body arguments.
        """

        # Make sure `get_arguments` isn't accidentally being called with a
        # positional argument that's assumed to be a default (like in
        # `get_argument`.)
        assert isinstance(strip, bool)

        return self._get_arguments(name, self.request.arguments, strip)

    def get_body_argument(
        self,
        name: str,
        default: Union[None, str, _ArgDefaultMarker] = _ARG_DEFAULT,
        strip: bool = True,
    ) -> Optional[str]:
        """Returns the value of the argument with the given name
        from the request body.

        If default is not provided, the argument is considered to be
        required, and we raise a `MissingArgumentError` if it is missing.

        If the argument appears in the url more than once, we return the
        last value.

        .. versionadded:: 3.2
        """
        return self._get_argument(name, default, self.request.body_arguments, strip)

    def get_body_arguments(self, name: str, strip: bool = True) -> List[str]:
        """Returns a list of the body arguments with the given name.

        If the argument is not present, returns an empty list.

        .. versionadded:: 3.2
        """
        return self._get_arguments(name, self.request.body_arguments, strip)

    def get_query_argument(
        self,
        name: str,
        default: Union[None, str, _ArgDefaultMarker] = _ARG_DEFAULT,
        strip: bool = True,
    ) -> Optional[str]:
        """Returns the value of the argument with the given name
        from the request query string.

        If default is not provided, the argument is considered to be
        required, and we raise a `MissingArgumentError` if it is missing.

        If the argument appears in the url more than once, we return the
        last value.

        .. versionadded:: 3.2
        """
        return self._get_argument(name, default, self.request.query_arguments, strip)

    def get_query_arguments(self, name: str, strip: bool = True) -> List[str]:
        """Returns a list of the query arguments with the given name.

        If the argument is not present, returns an empty list.

        .. versionadded:: 3.2
        """
        return self._get_arguments(name, self.request.query_arguments, strip)

    def _get_argument(
        self,
        name: str,
        default: Union[None, str, _ArgDefaultMarker],
        source: Dict[str, List[bytes]],
        strip: bool = True,
    ) -> Optional[str]:
        args = self._get_arguments(name, source, strip=strip)
        if not args:
            if isinstance(default, _ArgDefaultMarker):
                raise MissingArgumentError(name)
            return default
        return args[-1]

    def _get_arguments(
        self, name: str, source: Dict[str, List[bytes]], strip: bool = True
    ) -> List[str]:
        values = []
        for v in source.get(name, []):
            s = self.decode_argument(v, name=name)
            if isinstance(s, unicode_type):
                # Get rid of any weird control chars (unless decoding gave
                # us bytes, in which case leave it alone)
                s = RequestHandler._remove_control_chars_regex.sub(" ", s)
            if strip:
                s = s.strip()
            values.append(s)
        return values

    def decode_argument(self, value: bytes, name: Optional[str] = None) -> str:
        """Decodes an argument from the request.

        The argument has been percent-decoded and is now a byte string.
        By default, this method decodes the argument as utf-8 and returns
        a unicode string, but this may be overridden in subclasses.

        This method is used as a filter for both `get_argument()` and for
        values extracted from the url and passed to `get()`/`post()`/etc.

        The name of the argument is provided if known, but may be None
        (e.g. for unnamed groups in the url regex).
        """
        try:
            return _unicode(value)
        except UnicodeDecodeError:
            raise HTTPError(
                400, "Invalid unicode in %s: %r" % (name or "url", value[:40])
            )

    @property
    def cookies(self) -> Dict[str, http.cookies.Morsel]:
        """An alias for
        `self.request.cookies <.httputil.HTTPServerRequest.cookies>`."""
        return self.request.cookies

    def get_cookie(self, name: str, default: Optional[str] = None) -> Optional[str]:
        """Returns the value of the request cookie with the given name.

        If the named cookie is not present, returns ``default``.

        This method only returns cookies that were present in the request.
        It does not see the outgoing cookies set by `set_cookie` in this
        handler.
        """
        if self.request.cookies is not None and name in self.request.cookies:
            return self.request.cookies[name].value
        return default

    def set_cookie(
        self,
        name: str,
        value: Union[str, bytes],
        domain: Optional[str] = None,
        expires: Optional[Union[float, Tuple, datetime.datetime]] = None,
        path: str = "/",
        expires_days: Optional[float] = None,
        # Keyword-only args start here for historical reasons.
        *,
        max_age: Optional[int] = None,
        httponly: bool = False,
        secure: bool = False,
        samesite: Optional[str] = None,
    ) -> None:
        """Sets an outgoing cookie name/value with the given options.

        Newly-set cookies are not immediately visible via `get_cookie`;
        they are not present until the next request.

        Most arguments are passed directly to `http.cookies.Morsel` directly.
        See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie
        for more information.

        ``expires`` may be a numeric timestamp as returned by `time.time`,
        a time tuple as returned by `time.gmtime`, or a
        `datetime.datetime` object. ``expires_days`` is provided as a convenience
        to set an expiration time in days from today (if both are set, ``expires``
        is used).

        """
        # The cookie library only accepts type str, in both python 2 and 3
        name = escape.native_str(name)
        value = escape.native_str(value)
        if re.search(r"[\x00-\x20]", name + value):
            # Don't let us accidentally inject bad stuff
            raise ValueError("Invalid cookie %r: %r" % (name, value))
        if not hasattr(self, "_new_cookie"):
            self._new_cookie = (
                http.cookies.SimpleCookie()
            )  # type: http.cookies.SimpleCookie
        if name in self._new_cookie:
            del self._new_cookie[name]
        self._new_cookie[name] = value
        morsel = self._new_cookie[name]
        if domain:
            morsel["domain"] = domain
        if expires_days is not None and not expires:
            expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
        if expires:
            morsel["expires"] = httputil.format_timestamp(expires)
        if path:
            morsel["path"] = path
        if max_age:
            # Note change from _ to -.
            morsel["max-age"] = str(max_age)
        if httponly:
            # Note that SimpleCookie ignores the value here. The presense of an
            # httponly (or secure) key is treated as true.
            morsel["httponly"] = True
        if secure:
            morsel["secure"] = True
        if samesite:
            morsel["samesite"] = samesite

    def clear_cookie(self, name: str, **kwargs: Any) -> None:
        """Deletes the cookie with the given name.

        This method accepts the same arguments as `set_cookie`, except for
        ``expires`` and ``max_age``. Clearing a cookie requires the same
        ``domain`` and ``path`` arguments as when it was set. In some cases the
        ``samesite`` and ``secure`` arguments are also required to match. Other
        arguments are ignored.

        Similar to `set_cookie`, the effect of this method will not be
        seen until the following request.

        .. versionchanged:: 6.3

           Now accepts all keyword arguments that ``set_cookie`` does.
           The ``samesite`` and ``secure`` flags have recently become
           required for clearing ``samesite="none"`` cookies.
        """
        for excluded_arg in ["expires", "max_age"]:
            if excluded_arg in kwargs:
                raise TypeError(
                    f"clear_cookie() got an unexpected keyword argument '{excluded_arg}'"
                )
        expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
        self.set_cookie(name, value="", expires=expires, **kwargs)

    def clear_all_cookies(self, **kwargs: Any) -> None:
        """Attempt to delete all the cookies the user sent with this request.

        See `clear_cookie` for more information on keyword arguments. Due to
        limitations of the cookie protocol, it is impossible to determine on the
        server side which values are necessary for the ``domain``, ``path``,
        ``samesite``, or ``secure`` arguments, this method can only be
        successful if you consistently use the same values for these arguments
        when setting cookies.

        Similar to `set_cookie`, the effect of this method will not be seen
        until the following request.

        .. versionchanged:: 3.2

           Added the ``path`` and ``domain`` parameters.

        .. versionchanged:: 6.3

           Now accepts all keyword arguments that ``set_cookie`` does.

        .. deprecated:: 6.3

           The increasingly complex rules governing cookies have made it
           impossible for a ``clear_all_cookies`` method to work reliably
           since all we know about cookies are their names. Applications
           should generally use ``clear_cookie`` one at a time instead.
        """
        for name in self.request.cookies:
            self.clear_cookie(name, **kwargs)

    def set_signed_cookie(
        self,
        name: str,
        value: Union[str, bytes],
        expires_days: Optional[float] = 30,
        version: Optional[int] = None,
        **kwargs: Any,
    ) -> None:
        """Signs and timestamps a cookie so it cannot be forged.

        You must specify the ``cookie_secret`` setting in your Application
        to use this method. It should be a long, random sequence of bytes
        to be used as the HMAC secret for the signature.

        To read a cookie set with this method, use `get_signed_cookie()`.

        Note that the ``expires_days`` parameter sets the lifetime of the
        cookie in the browser, but is independent of the ``max_age_days``
        parameter to `get_signed_cookie`.
        A value of None limits the lifetime to the current browser session.

        Secure cookies may contain arbitrary byte values, not just unicode
        strings (unlike regular cookies)

        Similar to `set_cookie`, the effect of this method will not be
        seen until the following request.

        .. versionchanged:: 3.2.1

           Added the ``version`` argument.  Introduced cookie version 2
           and made it the default.

        .. versionchanged:: 6.3

           Renamed from ``set_secure_cookie`` to ``set_signed_cookie`` to
           avoid confusion with other uses of "secure" in cookie attributes
           and prefixes. The old name remains as an alias.
        """
        self.set_cookie(
            name,
            self.create_signed_value(name, value, version=version),
            expires_days=expires_days,
            **kwargs,
        )

    set_secure_cookie = set_signed_cookie

    def create_signed_value(
        self, name: str, value: Union[str, bytes], version: Optional[int] = None
    ) -> bytes:
        """Signs and timestamps a string so it cannot be forged.

        Normally used via set_signed_cookie, but provided as a separate
        method for non-cookie uses.  To decode a value not stored
        as a cookie use the optional value argument to get_signed_cookie.

        .. versionchanged:: 3.2.1

           Added the ``version`` argument.  Introduced cookie version 2
           and made it the default.
        """
        self.require_setting("cookie_secret", "secure cookies")
        secret = self.application.settings["cookie_secret"]
        key_version = None
        if isinstance(secret, dict):
            if self.application.settings.get("key_version") is None:
                raise Exception("key_version setting must be used for secret_key dicts")
            key_version = self.application.settings["key_version"]

        return create_signed_value(
            secret, name, value, version=version, key_version=key_version
        )

    def get_signed_cookie(
        self,
        name: str,
        value: Optional[str] = None,
        max_age_days: float = 31,
        min_version: Optional[int] = None,
    ) -> Optional[bytes]:
        """Returns the given signed cookie if it validates, or None.

        The decoded cookie value is returned as a byte string (unlike
        `get_cookie`).

        Similar to `get_cookie`, this method only returns cookies that
        were present in the request. It does not see outgoing cookies set by
        `set_signed_cookie` in this handler.

        .. versionchanged:: 3.2.1

           Added the ``min_version`` argument.  Introduced cookie version 2;
           both versions 1 and 2 are accepted by default.

         .. versionchanged:: 6.3

           Renamed from ``get_secure_cookie`` to ``get_signed_cookie`` to
           avoid confusion with other uses of "secure" in cookie attributes
           and prefixes. The old name remains as an alias.

        """
        self.require_setting("cookie_secret", "secure cookies")
        if value is None:
            value = self.get_cookie(name)
        return decode_signed_value(
            self.application.settings["cookie_secret"],
            name,
            value,
            max_age_days=max_age_days,
            min_version=min_version,
        )

    get_secure_cookie = get_signed_cookie

    def get_signed_cookie_key_version(
        self, name: str, value: Optional[str] = None
    ) -> Optional[int]:
        """Returns the signing key version of the secure cookie.

        The version is returned as int.

        .. versionchanged:: 6.3

           Renamed from ``get_secure_cookie_key_version`` to
           ``set_signed_cookie_key_version`` to avoid confusion with other
           uses of "secure" in cookie attributes and prefixes. The old name
           remains as an alias.

        """
        self.require_setting("cookie_secret", "secure cookies")
        if value is None:
            value = self.get_cookie(name)
        if value is None:
            return None
        return get_signature_key_version(value)

    get_secure_cookie_key_version = get_signed_cookie_key_version

    def redirect(
        self, url: str, permanent: bool = False, status: Optional[int] = None
    ) -> None:
        """Sends a redirect to the given (optionally relative) URL.

        If the ``status`` argument is specified, that value is used as the
        HTTP status code; otherwise either 301 (permanent) or 302
        (temporary) is chosen based on the ``permanent`` argument.
        The default is 302 (temporary).
        """
        if self._headers_written:
            raise Exception("Cannot redirect after headers have been written")
        if status is None:
            status = 301 if permanent else 302
        else:
            assert isinstance(status, int) and 300 <= status <= 399
        self.set_status(status)
        self.set_header("Location", utf8(url))
        self.finish()

    def write(self, chunk: Union[str, bytes, dict]) -> None:
        """Writes the given chunk to the output buffer.

        To write the output to the network, use the `flush()` method below.

        If the given chunk is a dictionary, we write it as JSON and set
        the Content-Type of the response to be ``application/json``.
        (if you want to send JSON as a different ``Content-Type``, call
        ``set_header`` *after* calling ``write()``).

        Note that lists are not converted to JSON because of a potential
        cross-site security vulnerability.  All JSON output should be
        wrapped in a dictionary.  More details at
        http://haacked.com/archive/2009/06/25/json-hijacking.aspx/ and
        https://github.com/facebook/tornado/issues/1009
        """
        if self._finished:
            raise RuntimeError("Cannot write() after finish()")
        if not isinstance(chunk, (bytes, unicode_type, dict)):
            message = "write() only accepts bytes, unicode, and dict objects"
            if isinstance(chunk, list):
                message += (
                    ". Lists not accepted for security reasons; see "
                    + "http://www.tornadoweb.org/en/stable/web.html#tornado.web.RequestHandler.write"  # noqa: E501
                )
            raise TypeError(message)
        if isinstance(chunk, dict):
            chunk = escape.json_encode(chunk)
            self.set_header("Content-Type", "application/json; charset=UTF-8")
        chunk = utf8(chunk)
        self._write_buffer.append(chunk)

    def render(self, template_name: str, **kwargs: Any) -> "Future[None]":
        """Renders the template with the given arguments as the response.

        ``render()`` calls ``finish()``, so no other output methods can be called
        after it.

        Returns a `.Future` with the same semantics as the one returned by `finish`.
        Awaiting this `.Future` is optional.

        .. versionchanged:: 5.1

           Now returns a `.Future` instead of ``None``.
        """
        if self._finished:
            raise RuntimeError("Cannot render() after finish()")
        html = self.render_string(template_name, **kwargs)

        # Insert the additional JS and CSS added by the modules on the page
        js_embed = []
        js_files = []
        css_embed = []
        css_files = []
        html_heads = []
        html_bodies = []
        for module in getattr(self, "_active_modules", {}).values():
            embed_part = module.embedded_javascript()
            if embed_part:
                js_embed.append(utf8(embed_part))
            file_part = module.javascript_files()
            if file_part:
                if isinstance(file_part, (unicode_type, bytes)):
                    js_files.append(_unicode(file_part))
                else:
                    js_files.extend(file_part)
            embed_part = module.embedded_css()
            if embed_part:
                css_embed.append(utf8(embed_part))
            file_part = module.css_files()
            if file_part:
                if isinstance(file_part, (unicode_type, bytes)):
                    css_files.append(_unicode(file_part))
                else:
                    css_files.extend(file_part)
            head_part = module.html_head()
            if head_part:
                html_heads.append(utf8(head_part))
            body_part = module.html_body()
            if body_part:
                html_bodies.append(utf8(body_part))

        if js_files:
            # Maintain order of JavaScript files given by modules
            js = self.render_linked_js(js_files)
            sloc = html.rindex(b"</body>")
            html = html[:sloc] + utf8(js) + b"\n" + html[sloc:]
        if js_embed:
            js_bytes = self.render_embed_js(js_embed)
            sloc = html.rindex(b"</body>")
            html = html[:sloc] + js_bytes + b"\n" + html[sloc:]
        if css_files:
            css = self.render_linked_css(css_files)
            hloc = html.index(b"</head>")
            html = html[:hloc] + utf8(css) + b"\n" + html[hloc:]
        if css_embed:
            css_bytes = self.render_embed_css(css_embed)
            hloc = html.index(b"</head>")
            html = html[:hloc] + css_bytes + b"\n" + html[hloc:]
        if html_heads:
            hloc = html.index(b"</head>")
            html = html[:hloc] + b"".join(html_heads) + b"\n" + html[hloc:]
        if html_bodies:
            hloc = html.index(b"</body>")
            html = html[:hloc] + b"".join(html_bodies) + b"\n" + html[hloc:]
        return self.finish(html)

    def render_linked_js(self, js_files: Iterable[str]) -> str:
        """Default method used to render the final js links for the
        rendered webpage.

        Override this method in a sub-classed controller to change the output.
        """
        paths = []
        unique_paths = set()  # type: Set[str]

        for path in js_files:
            if not is_absolute(path):
                path = self.static_url(path)
            if path not in unique_paths:
                paths.append(path)
                unique_paths.add(path)

        return "".join(
            '<script src="'
            + escape.xhtml_escape(p)
            + '" type="text/javascript"></script>'
            for p in paths
        )

    def render_embed_js(self, js_embed: Iterable[bytes]) -> bytes:
        """Default method used to render the final embedded js for the
        rendered webpage.

        Override this method in a sub-classed controller to change the output.
        """
        return (
            b'<script type="text/javascript">\n//<![CDATA[\n'
            + b"\n".join(js_embed)
            + b"\n//]]>\n</script>"
        )

    def render_linked_css(self, css_files: Iterable[str]) -> str:
        """Default method used to render the final css links for the
        rendered webpage.

        Override this method in a sub-classed controller to change the output.
        """
        paths = []
        unique_paths = set()  # type: Set[str]

        for path in css_files:
            if not is_absolute(path):
                path = self.static_url(path)
            if path not in unique_paths:
                paths.append(path)
                unique_paths.add(path)

        return "".join(
            '<link href="' + escape.xhtml_escape(p) + '" '
            'type="text/css" rel="stylesheet"/>'
            for p in paths
        )

    def render_embed_css(self, css_embed: Iterable[bytes]) -> bytes:
        """Default method used to render the final embedded css for the
        rendered webpage.

        Override this method in a sub-classed controller to change the output.
        """
        return b'<style type="text/css">\n' + b"\n".join(css_embed) + b"\n</style>"

    def render_string(self, template_name: str, **kwargs: Any) -> bytes:
        """Generate the given template with the given arguments.

        We return the generated byte string (in utf8). To generate and
        write a template as a response, use render() above.
        """
        # If no template_path is specified, use the path of the calling file
        template_path = self.get_template_path()
        if not template_path:
            frame = sys._getframe(0)
            web_file = frame.f_code.co_filename
            while frame.f_code.co_filename == web_file and frame.f_back is not None:
                frame = frame.f_back
            assert frame.f_code.co_filename is not None
            template_path = os.path.dirname(frame.f_code.co_filename)
        with RequestHandler._template_loader_lock:
            if template_path not in RequestHandler._template_loaders:
                loader = self.create_template_loader(template_path)
                RequestHandler._template_loaders[template_path] = loader
            else:
                loader = RequestHandler._template_loaders[template_path]
        t = loader.load(template_name)
        namespace = self.get_template_namespace()
        namespace.update(kwargs)
        return t.generate(**namespace)

    def get_template_namespace(self) -> Dict[str, Any]:
        """Returns a dictionary to be used as the default template namespace.

        May be overridden by subclasses to add or modify values.

        The results of this method will be combined with additional
        defaults in the `tornado.template` module and keyword arguments
        to `render` or `render_string`.
        """
        namespace = dict(
            handler=self,
            request=self.request,
            current_user=self.current_user,
            locale=self.locale,
            _=self.locale.translate,
            pgettext=self.locale.pgettext,
            static_url=self.static_url,
            xsrf_form_html=self.xsrf_form_html,
            reverse_url=self.reverse_url,
        )
        namespace.update(self.ui)
        return namespace

    def create_template_loader(self, template_path: str) -> template.BaseLoader:
        """Returns a new template loader for the given path.

        May be overridden by subclasses.  By default returns a
        directory-based loader on the given path, using the
        ``autoescape`` and ``template_whitespace`` application
        settings.  If a ``template_loader`` application setting is
        supplied, uses that instead.
        """
        settings = self.application.settings
        if "template_loader" in settings:
            return settings["template_loader"]
        kwargs = {}
        if "autoescape" in settings:
            # autoescape=None means "no escaping", so we have to be sure
            # to only pass this kwarg if the user asked for it.
            kwargs["autoescape"] = settings["autoescape"]
        if "template_whitespace" in settings:
            kwargs["whitespace"] = settings["template_whitespace"]
        return template.Loader(template_path, **kwargs)

    def flush(self, include_footers: bool = False) -> "Future[None]":
        """Flushes the current output buffer to the network.

        .. versionchanged:: 4.0
           Now returns a `.Future` if no callback is given.

        .. versionchanged:: 6.0

           The ``callback`` argument was removed.
        """
        assert self.request.connection is not None
        chunk = b"".join(self._write_buffer)
        self._write_buffer = []
        if not self._headers_written:
            self._headers_written = True
            for transform in self._transforms:
                assert chunk is not None
                (
                    self._status_code,
                    self._headers,
                    chunk,
                ) = transform.transform_first_chunk(
                    self._status_code, self._headers, chunk, include_footers
                )
            # Ignore the chunk and only write the headers for HEAD requests
            if self.request.method == "HEAD":
                chunk = b""

            # Finalize the cookie headers (which have been stored in a side
            # object so an outgoing cookie could be overwritten before it
            # is sent).
            if hasattr(self, "_new_cookie"):
                for cookie in self._new_cookie.values():
                    self.add_header("Set-Cookie", cookie.OutputString(None))

            start_line = httputil.ResponseStartLine("", self._status_code, self._reason)
            return self.request.connection.write_headers(
                start_line, self._headers, chunk
            )
        else:
            for transform in self._transforms:
                chunk = transform.transform_chunk(chunk, include_footers)
            # Ignore the chunk and only write the headers for HEAD requests
            if self.request.method != "HEAD":
                return self.request.connection.write(chunk)
            else:
                future = Future()  # type: Future[None]
                future.set_result(None)
                return future

    def finish(self, chunk: Optional[Union[str, bytes, dict]] = None) -> "Future[None]":
        """Finishes this response, ending the HTTP request.

        Passing a ``chunk`` to ``finish()`` is equivalent to passing that
        chunk to ``write()`` and then calling ``finish()`` with no arguments.

        Returns a `.Future` which may optionally be awaited to track the sending
        of the response to the client. This `.Future` resolves when all the response
        data has been sent, and raises an error if the connection is closed before all
        data can be sent.

        .. versionchanged:: 5.1

           Now returns a `.Future` instead of ``None``.
        """
        if self._finished:
            raise RuntimeError("finish() called twice")

        if chunk is not None:
            self.write(chunk)

        # Automatically support ETags and add the Content-Length header if
        # we have not flushed any content yet.
        if not self._headers_written:
            if (
                self._status_code == 200
                and self.request.method in ("GET", "HEAD")
                and "Etag" not in self._headers
            ):
                self.set_etag_header()
                if self.check_etag_header():
                    self._write_buffer = []
                    self.set_status(304)
            if self._status_code in (204, 304) or (100 <= self._status_code < 200):
                assert not self._write_buffer, (
                    "Cannot send body with %s" % self._status_code
                )
                self._clear_representation_headers()
            elif "Content-Length" not in self._headers:
                content_length = sum(len(part) for part in self._write_buffer)
                self.set_header("Content-Length", content_length)

        assert self.request.connection is not None
        # Now that the request is finished, clear the callback we
        # set on the HTTPConnection (which would otherwise prevent the
        # garbage collection of the RequestHandler when there
        # are keepalive connections)
        self.request.connection.set_close_callback(None)  # type: ignore

        future = self.flush(include_footers=True)
        self.request.connection.finish()
        self._log()
        self._finished = True
        self.on_finish()
        self._break_cycles()
        return future

    def detach(self) -> iostream.IOStream:
        """Take control of the underlying stream.

        Returns the underlying `.IOStream` object and stops all
        further HTTP processing. Intended for implementing protocols
        like websockets that tunnel over an HTTP handshake.

        This method is only supported when HTTP/1.1 is used.

        .. versionadded:: 5.1
        """
        self._finished = True
        # TODO: add detach to HTTPConnection?
        return self.request.connection.detach()  # type: ignore

    def _break_cycles(self) -> None:
        # Break up a reference cycle between this handler and the
        # _ui_module closures to allow for faster GC on CPython.
        self.ui = None  # type: ignore

    def send_error(self, status_code: int = 500, **kwargs: Any) -> None:
        """Sends the given HTTP error code to the browser.

        If `flush()` has already been called, it is not possible to send
        an error, so this method will simply terminate the response.
        If output has been written but not yet flushed, it will be discarded
        and replaced with the error page.

        Override `write_error()` to customize the error page that is returned.
        Additional keyword arguments are passed through to `write_error`.
        """
        if self._headers_written:
            gen_log.error("Cannot send error response after headers written")
            if not self._finished:
                # If we get an error between writing headers and finishing,
                # we are unlikely to be able to finish due to a
                # Content-Length mismatch. Try anyway to release the
                # socket.
                try:
                    self.finish()
                except Exception:
                    gen_log.error("Failed to flush partial response", exc_info=True)
            return
        self.clear()

        reason = kwargs.get("reason")
        if "exc_info" in kwargs:
            exception = kwargs["exc_info"][1]
            if isinstance(exception, HTTPError) and exception.reason:
                reason = exception.reason
        self.set_status(status_code, reason=reason)
        try:
            self.write_error(status_code, **kwargs)
        except Exception:
            app_log.error("Uncaught exception in write_error", exc_info=True)
        if not self._finished:
            self.finish()

    def write_error(self, status_code: int, **kwargs: Any) -> None:
        """Override to implement custom error pages.

        ``write_error`` may call `write`, `render`, `set_header`, etc
        to produce output as usual.

        If this error was caused by an uncaught exception (including
        HTTPError), an ``exc_info`` triple will be available as
        ``kwargs["exc_info"]``.  Note that this exception may not be
        the "current" exception for purposes of methods like
        ``sys.exc_info()`` or ``traceback.format_exc``.
        """
        if self.settings.get("serve_traceback") and "exc_info" in kwargs:
            # in debug mode, try to send a traceback
            self.set_header("Content-Type", "text/plain")
            for line in traceback.format_exception(*kwargs["exc_info"]):
                self.write(line)
            self.finish()
        else:
            self.finish(
                "<html><title>%(code)d: %(message)s</title>"
                "<body>%(code)d: %(message)s</body></html>"
                % {"code": status_code, "message": self._reason}
            )

    @property
    def locale(self) -> tornado.locale.Locale:
        """The locale for the current session.

        Determined by either `get_user_locale`, which you can override to
        set the locale based on, e.g., a user preference stored in a
        database, or `get_browser_locale`, which uses the ``Accept-Language``
        header.

        .. versionchanged: 4.1
           Added a property setter.
        """
        if not hasattr(self, "_locale"):
            loc = self.get_user_locale()
            if loc is not None:
                self._locale = loc
            else:
                self._locale = self.get_browser_locale()
                assert self._locale
        return self._locale

    @locale.setter
    def locale(self, value: tornado.locale.Locale) -> None:
        self._locale = value

    def get_user_locale(self) -> Optional[tornado.locale.Locale]:
        """Override to determine the locale from the authenticated user.

        If None is returned, we fall back to `get_browser_locale()`.

        This method should return a `tornado.locale.Locale` object,
        most likely obtained via a call like ``tornado.locale.get("en")``
        """
        return None

    def get_browser_locale(self, default: str = "en_US") -> tornado.locale.Locale:
        """Determines the user's locale from ``Accept-Language`` header.

        See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
        """
        if "Accept-Language" in self.request.headers:
            languages = self.request.headers["Accept-Language"].split(",")
            locales = []
            for language in languages:
                parts = language.strip().split(";")
                if len(parts) > 1 and parts[1].strip().startswith("q="):
                    try:
                        score = float(parts[1].strip()[2:])
                        if score < 0:
                            raise ValueError()
                    except (ValueError, TypeError):
                        score = 0.0
                else:
                    score = 1.0
                if score > 0:
                    locales.append((parts[0], score))
            if locales:
                locales.sort(key=lambda pair: pair[1], reverse=True)
                codes = [loc[0] for loc in locales]
                return locale.get(*codes)
        return locale.get(default)

    @property
    def current_user(self) -> Any:
        """The authenticated user for this request.

        This is set in one of two ways:

        * A subclass may override `get_current_user()`, which will be called
          automatically the first time ``self.current_user`` is accessed.
          `get_current_user()` will only be called once per request,
          and is cached for future access::

              def get_current_user(self):
                  user_cookie = self.get_signed_cookie("user")
                  if user_cookie:
                      return json.loads(user_cookie)
                  return None

        * It may be set as a normal variable, typically from an overridden
          `prepare()`::

              @gen.coroutine
              def prepare(self):
                  user_id_cookie = self.get_signed_cookie("user_id")
                  if user_id_cookie:
                      self.current_user = yield load_user(user_id_cookie)

        Note that `prepare()` may be a coroutine while `get_current_user()`
        may not, so the latter form is necessary if loading the user requires
        asynchronous operations.

        The user object may be any type of the application's choosing.
        """
        if not hasattr(self, "_current_user"):
            self._current_user = self.get_current_user()
        return self._current_user

    @current_user.setter
    def current_user(self, value: Any) -> None:
        self._current_user = value

    def get_current_user(self) -> Any:
        """Override to determine the current user from, e.g., a cookie.

        This method may not be a coroutine.
        """
        return None

    def get_login_url(self) -> str:
        """Override to customize the login URL based on the request.

        By default, we use the ``login_url`` application setting.
        """
        self.require_setting("login_url", "@tornado.web.authenticated")
        return self.application.settings["login_url"]

    def get_template_path(self) -> Optional[str]:
        """Override to customize template path for each handler.

        By default, we use the ``template_path`` application setting.
        Return None to load templates relative to the calling file.
        """
        return self.application.settings.get("template_path")

    @property
    def xsrf_token(self) -> bytes:
        """The XSRF-prevention token for the current user/session.

        To prevent cross-site request forgery, we set an '_xsrf' cookie
        and include the same '_xsrf' value as an argument with all POST
        requests. If the two do not match, we reject the form submission
        as a potential forgery.

        See http://en.wikipedia.org/wiki/Cross-site_request_forgery

        This property is of type `bytes`, but it contains only ASCII
        characters. If a character string is required, there is no
        need to base64-encode it; just decode the byte string as
        UTF-8.

        .. versionchanged:: 3.2.2
           The xsrf token will now be have a random mask applied in every
           request, which makes it safe to include the token in pages
           that are compressed.  See http://breachattack.com for more
           information on the issue fixed by this change.  Old (version 1)
           cookies will be converted to version 2 when this method is called
           unless the ``xsrf_cookie_version`` `Application` setting is
           set to 1.

        .. versionchanged:: 4.3
           The ``xsrf_cookie_kwargs`` `Application` setting may be
           used to supply additional cookie options (which will be
           passed directly to `set_cookie`). For example,
           ``xsrf_cookie_kwargs=dict(httponly=True, secure=True)``
           will set the ``secure`` and ``httponly`` flags on the
           ``_xsrf`` cookie.
        """
        if not hasattr(self, "_xsrf_token"):
            version, token, timestamp = self._get_raw_xsrf_token()
            output_version = self.settings.get("xsrf_cookie_version", 2)
            cookie_kwargs = self.settings.get("xsrf_cookie_kwargs", {})
            if output_version == 1:
                self._xsrf_token = binascii.b2a_hex(token)
            elif output_version == 2:
                mask = os.urandom(4)
                self._xsrf_token = b"|".join(
                    [
                        b"2",
                        binascii.b2a_hex(mask),
                        binascii.b2a_hex(_websocket_mask(mask, token)),
                        utf8(str(int(timestamp))),
                    ]
                )
            else:
                raise ValueError("unknown xsrf cookie version %d", output_version)
            if version is None:
                if self.current_user and "expires_days" not in cookie_kwargs:
                    cookie_kwargs["expires_days"] = 30
                cookie_name = self.settings.get("xsrf_cookie_name", "_xsrf")
                self.set_cookie(cookie_name, self._xsrf_token, **cookie_kwargs)
        return self._xsrf_token

    def _get_raw_xsrf_token(self) -> Tuple[Optional[int], bytes, float]:
        """Read or generate the xsrf token in its raw form.

        The raw_xsrf_token is a tuple containing:

        * version: the version of the cookie from which this token was read,
          or None if we generated a new token in this request.
        * token: the raw token data; random (non-ascii) bytes.
        * timestamp: the time this token was generated (will not be accurate
          for version 1 cookies)
        """
        if not hasattr(self, "_raw_xsrf_token"):
            cookie_name = self.settings.get("xsrf_cookie_name", "_xsrf")
            cookie = self.get_cookie(cookie_name)
            if cookie:
                version, token, timestamp = self._decode_xsrf_token(cookie)
            else:
                version, token, timestamp = None, None, None
            if token is None:
                version = None
                token = os.urandom(16)
                timestamp = time.time()
            assert token is not None
            assert timestamp is not None
            self._raw_xsrf_token = (version, token, timestamp)
        return self._raw_xsrf_token

    def _decode_xsrf_token(
        self, cookie: str
    ) -> Tuple[Optional[int], Optional[bytes], Optional[float]]:
        """Convert a cookie string into a the tuple form returned by
        _get_raw_xsrf_token.
        """

        try:
            m = _signed_value_version_re.match(utf8(cookie))

            if m:
                version = int(m.group(1))
                if version == 2:
                    _, mask_str, masked_token, timestamp_str = cookie.split("|")

                    mask = binascii.a2b_hex(utf8(mask_str))
                    token = _websocket_mask(mask, binascii.a2b_hex(utf8(masked_token)))
                    timestamp = int(timestamp_str)
                    return version, token, timestamp
                else:
                    # Treat unknown versions as not present instead of failing.
                    raise Exception("Unknown xsrf cookie version")
            else:
                version = 1
                try:
                    token = binascii.a2b_hex(utf8(cookie))
                except (binascii.Error, TypeError):
                    token = utf8(cookie)
                # We don't have a usable timestamp in older versions.
                timestamp = int(time.time())
                return (version, token, timestamp)
        except Exception:
            # Catch exceptions and return nothing instead of failing.
            gen_log.debug("Uncaught exception in _decode_xsrf_token", exc_info=True)
            return None, None, None

    def check_xsrf_cookie(self) -> None:
        """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.

        To prevent cross-site request forgery, we set an ``_xsrf``
        cookie and include the same value as a non-cookie
        field with all ``POST`` requests. If the two do not match, we
        reject the form submission as a potential forgery.

        The ``_xsrf`` value may be set as either a form field named ``_xsrf``
        or in a custom HTTP header named ``X-XSRFToken`` or ``X-CSRFToken``
        (the latter is accepted for compatibility with Django).

        See http://en.wikipedia.org/wiki/Cross-site_request_forgery

        .. versionchanged:: 3.2.2
           Added support for cookie version 2.  Both versions 1 and 2 are
           supported.
        """
        # Prior to release 1.1.1, this check was ignored if the HTTP header
        # ``X-Requested-With: XMLHTTPRequest`` was present.  This exception
        # has been shown to be insecure and has been removed.  For more
        # information please see
        # http://www.djangoproject.com/weblog/2011/feb/08/security/
        # http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails
        token = (
            self.get_argument("_xsrf", None)
            or self.request.headers.get("X-Xsrftoken")
            or self.request.headers.get("X-Csrftoken")
        )
        if not token:
            raise HTTPError(403, "'_xsrf' argument missing from POST")
        _, token, _ = self._decode_xsrf_token(token)
        _, expected_token, _ = self._get_raw_xsrf_token()
        if not token:
            raise HTTPError(403, "'_xsrf' argument has invalid format")
        if not hmac.compare_digest(utf8(token), utf8(expected_token)):
            raise HTTPError(403, "XSRF cookie does not match POST argument")

    def xsrf_form_html(self) -> str:
        """An HTML ``<input/>`` element to be included with all POST forms.

        It defines the ``_xsrf`` input value, which we check on all POST
        requests to prevent cross-site request forgery. If you have set
        the ``xsrf_cookies`` application setting, you must include this
        HTML within all of your HTML forms.

        In a template, this method should be called with ``{% module
        xsrf_form_html() %}``

        See `check_xsrf_cookie()` above for more information.
        """
        return (
            '<input type="hidden" name="_xsrf" value="'
            + escape.xhtml_escape(self.xsrf_token)
            + '"/>'
        )

    def static_url(
        self, path: str, include_host: Optional[bool] = None, **kwargs: Any
    ) -> str:
        """Returns a static URL for the given relative static file path.

        This method requires you set the ``static_path`` setting in your
        application (which specifies the root directory of your static
        files).

        This method returns a versioned url (by default appending
        ``?v=<signature>``), which allows the static files to be
        cached indefinitely.  This can be disabled by passing
        ``include_version=False`` (in the default implementation;
        other static file implementations are not required to support
        this, but they may support other options).

        By default this method returns URLs relative to the current
        host, but if ``include_host`` is true the URL returned will be
        absolute.  If this handler has an ``include_host`` attribute,
        that value will be used as the default for all `static_url`
        calls that do not pass ``include_host`` as a keyword argument.

        """
        self.require_setting("static_path", "static_url")
        get_url = self.settings.get(
            "static_handler_class", StaticFileHandler
        ).make_static_url

        if include_host is None:
            include_host = getattr(self, "include_host", False)

        if include_host:
            base = self.request.protocol + "://" + self.request.host
        else:
            base = ""

        return base + get_url(self.settings, path, **kwargs)

    def require_setting(self, name: str, feature: str = "this feature") -> None:
        """Raises an exception if the given app setting is not defined."""
        if not self.application.settings.get(name):
            raise Exception(
                "You must define the '%s' setting in your "
                "application to use %s" % (name, feature)
            )

    def reverse_url(self, name: str, *args: Any) -> str:
        """Alias for `Application.reverse_url`."""
        return self.application.reverse_url(name, *args)

    def compute_etag(self) -> Optional[str]:
        """Computes the etag header to be used for this request.

        By default uses a hash of the content written so far.

        May be overridden to provide custom etag implementations,
        or may return None to disable tornado's default etag support.
        """
        hasher = hashlib.sha1()
        for part in self._write_buffer:
            hasher.update(part)
        return '"%s"' % hasher.hexdigest()

    def set_etag_header(self) -> None:
        """Sets the response's Etag header using ``self.compute_etag()``.

        Note: no header will be set if ``compute_etag()`` returns ``None``.

        This method is called automatically when the request is finished.
        """
        etag = self.compute_etag()
        if etag is not None:
            self.set_header("Etag", etag)

    def check_etag_header(self) -> bool:
        """Checks the ``Etag`` header against requests's ``If-None-Match``.

        Returns ``True`` if the request's Etag matches and a 304 should be
        returned. For example::

            self.set_etag_header()
            if self.check_etag_header():
                self.set_status(304)
                return

        This method is called automatically when the request is finished,
        but may be called earlier for applications that override
        `compute_etag` and want to do an early check for ``If-None-Match``
        before completing the request.  The ``Etag`` header should be set
        (perhaps with `set_etag_header`) before calling this method.
        """
        computed_etag = utf8(self._headers.get("Etag", ""))
        # Find all weak and strong etag values from If-None-Match header
        # because RFC 7232 allows multiple etag values in a single header.
        etags = re.findall(
            rb'\*|(?:W/)?"[^"]*"', utf8(self.request.headers.get("If-None-Match", ""))
        )
        if not computed_etag or not etags:
            return False

        match = False
        if etags[0] == b"*":
            match = True
        else:
            # Use a weak comparison when comparing entity-tags.
            def val(x: bytes) -> bytes:
                return x[2:] if x.startswith(b"W/") else x

            for etag in etags:
                if val(etag) == val(computed_etag):
                    match = True
                    break
        return match

    async def _execute(
        self, transforms: List["OutputTransform"], *args: bytes, **kwargs: bytes
    ) -> None:
        """Executes this request with the given output transforms."""
        self._transforms = transforms
        try:
            if self.request.method not in self.SUPPORTED_METHODS:
                raise HTTPError(405)
            self.path_args = [self.decode_argument(arg) for arg in args]
            self.path_kwargs = dict(
                (k, self.decode_argument(v, name=k)) for (k, v) in kwargs.items()
            )
            # If XSRF cookies are turned on, reject form submissions without
            # the proper cookie
            if self.request.method not in (
                "GET",
                "HEAD",
                "OPTIONS",
            ) and self.application.settings.get("xsrf_cookies"):
                self.check_xsrf_cookie()

            result = self.prepare()
            if result is not None:
                result = await result  # type: ignore
            if self._prepared_future is not None:
                # Tell the Application we've finished with prepare()
                # and are ready for the body to arrive.
                future_set_result_unless_cancelled(self._prepared_future, None)
            if self._finished:
                return

            if _has_stream_request_body(self.__class__):
                # In streaming mode request.body is a Future that signals
                # the body has been completely received.  The Future has no
                # result; the data has been passed to self.data_received
                # instead.
                try:
                    await self.request._body_future
                except iostream.StreamClosedError:
                    return

            method = getattr(self, self.request.method.lower())
            result = method(*self.path_args, **self.path_kwargs)
            if result is not None:
                result = await result
            if self._auto_finish and not self._finished:
                self.finish()
        except Exception as e:
            try:
                self._handle_request_exception(e)
            except Exception:
                app_log.error("Exception in exception handler", exc_info=True)
            finally:
                # Unset result to avoid circular references
                result = None
            if self._prepared_future is not None and not self._prepared_future.done():
                # In case we failed before setting _prepared_future, do it
                # now (to unblock the HTTP server).  Note that this is not
                # in a finally block to avoid GC issues prior to Python 3.4.
                self._prepared_future.set_result(None)

    def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]:
        """Implement this method to handle streamed request data.

        Requires the `.stream_request_body` decorator.

        May be a coroutine for flow control.
        """
        raise NotImplementedError()

    def _log(self) -> None:
        """Logs the current request.

        Sort of deprecated since this functionality was moved to the
        Application, but left in place for the benefit of existing apps
        that have overridden this method.
        """
        self.application.log_request(self)

    def _request_summary(self) -> str:
        return "%s %s (%s)" % (
            self.request.method,
            self.request.uri,
            self.request.remote_ip,
        )

    def _handle_request_exception(self, e: BaseException) -> None:
        if isinstance(e, Finish):
            # Not an error; just finish the request without logging.
            if not self._finished:
                self.finish(*e.args)
            return
        try:
            self.log_exception(*sys.exc_info())
        except Exception:
            # An error here should still get a best-effort send_error()
            # to avoid leaking the connection.
            app_log.error("Error in exception logger", exc_info=True)
        if self._finished:
            # Extra errors after the request has been finished should
            # be logged, but there is no reason to continue to try and
            # send a response.
            return
        if isinstance(e, HTTPError):
            self.send_error(e.status_code, exc_info=sys.exc_info())
        else:
            self.send_error(500, exc_info=sys.exc_info())

    def log_exception(
        self,
        typ: "Optional[Type[BaseException]]",
        value: Optional[BaseException],
        tb: Optional[TracebackType],
    ) -> None:
        """Override to customize logging of uncaught exceptions.

        By default logs instances of `HTTPError` as warnings without
        stack traces (on the ``tornado.general`` logger), and all
        other exceptions as errors with stack traces (on the
        ``tornado.application`` logger).

        .. versionadded:: 3.1
        """
        if isinstance(value, HTTPError):
            if value.log_message:
                format = "%d %s: " + value.log_message
                args = [value.status_code, self._request_summary()] + list(value.args)
                gen_log.warning(format, *args)
        else:
            app_log.error(
                "Uncaught exception %s\n%r",
                self._request_summary(),
                self.request,
                exc_info=(typ, value, tb),  # type: ignore
            )

    def _ui_module(self, name: str, module: Type["UIModule"]) -> Callable[..., str]:
        def render(*args, **kwargs) -> str:  # type: ignore
            if not hasattr(self, "_active_modules"):
                self._active_modules = {}  # type: Dict[str, UIModule]
            if name not in self._active_modules:
                self._active_modules[name] = module(self)
            rendered = self._active_modules[name].render(*args, **kwargs)
            return rendered

        return render

    def _ui_method(self, method: Callable[..., str]) -> Callable[..., str]:
        return lambda *args, **kwargs: method(self, *args, **kwargs)

    def _clear_representation_headers(self) -> None:
        # 304 responses should not contain representation metadata
        # headers (defined in
        # https://tools.ietf.org/html/rfc7231#section-3.1)
        # not explicitly allowed by
        # https://tools.ietf.org/html/rfc7232#section-4.1
        headers = ["Content-Encoding", "Content-Language", "Content-Type"]
        for h in headers:
            self.clear_header(h)


_RequestHandlerType = TypeVar("_RequestHandlerType", bound=RequestHandler)


def stream_request_body(cls: Type[_RequestHandlerType]) -> Type[_RequestHandlerType]:
    """Apply to `RequestHandler` subclasses to enable streaming body support.

    This decorator implies the following changes:

    * `.HTTPServerRequest.body` is undefined, and body arguments will not
      be included in `RequestHandler.get_argument`.
    * `RequestHandler.prepare` is called when the request headers have been
      read instead of after the entire body has been read.
    * The subclass must define a method ``data_received(self, data):``, which
      will be called zero or more times as data is available.  Note that
      if the request has an empty body, ``data_received`` may not be called.
    * ``prepare`` and ``data_received`` may return Futures (such as via
      ``@gen.coroutine``, in which case the next method will not be called
      until those futures have completed.
    * The regular HTTP method (``post``, ``put``, etc) will be called after
      the entire body has been read.

    See the `file receiver demo <https://github.com/tornadoweb/tornado/tree/stable/demos/file_upload/>`_
    for example usage.
    """  # noqa: E501
    if not issubclass(cls, RequestHandler):
        raise TypeError("expected subclass of RequestHandler, got %r", cls)
    cls._stream_request_body = True
    return cls


def _has_stream_request_body(cls: Type[RequestHandler]) -> bool:
    if not issubclass(cls, RequestHandler):
        raise TypeError("expected subclass of RequestHandler, got %r", cls)
    return cls._stream_request_body


def removeslash(
    method: Callable[..., Optional[Awaitable[None]]]
) -> Callable[..., Optional[Awaitable[None]]]:
    """Use this decorator to remove trailing slashes from the request path.

    For example, a request to ``/foo/`` would redirect to ``/foo`` with this
    decorator. Your request handler mapping should use a regular expression
    like ``r'/foo/*'`` in conjunction with using the decorator.
    """

    @functools.wraps(method)
    def wrapper(  # type: ignore
        self: RequestHandler, *args, **kwargs
    ) -> Optional[Awaitable[None]]:
        if self.request.path.endswith("/"):
            if self.request.method in ("GET", "HEAD"):
                uri = self.request.path.rstrip("/")
                if uri:  # don't try to redirect '/' to ''
                    if self.request.query:
                        uri += "?" + self.request.query
                    self.redirect(uri, permanent=True)
                    return None
            else:
                raise HTTPError(404)
        return method(self, *args, **kwargs)

    return wrapper


def addslash(
    method: Callable[..., Optional[Awaitable[None]]]
) -> Callable[..., Optional[Awaitable[None]]]:
    """Use this decorator to add a missing trailing slash to the request path.

    For example, a request to ``/foo`` would redirect to ``/foo/`` with this
    decorator. Your request handler mapping should use a regular expression
    like ``r'/foo/?'`` in conjunction with using the decorator.
    """

    @functools.wraps(method)
    def wrapper(  # type: ignore
        self: RequestHandler, *args, **kwargs
    ) -> Optional[Awaitable[None]]:
        if not self.request.path.endswith("/"):
            if self.request.method in ("GET", "HEAD"):
                uri = self.request.path + "/"
                if self.request.query:
                    uri += "?" + self.request.query
                self.redirect(uri, permanent=True)
                return None
            raise HTTPError(404)
        return method(self, *args, **kwargs)

    return wrapper


class _ApplicationRouter(ReversibleRuleRouter):
    """Routing implementation used internally by `Application`.

    Provides a binding between `Application` and `RequestHandler`.
    This implementation extends `~.routing.ReversibleRuleRouter` in a couple of ways:
        * it allows to use `RequestHandler` subclasses as `~.routing.Rule` target and
        * it allows to use a list/tuple of rules as `~.routing.Rule` target.
        ``process_rule`` implementation will substitute this list with an appropriate
        `_ApplicationRouter` instance.
    """

    def __init__(
        self, application: "Application", rules: Optional[_RuleList] = None
    ) -> None:
        assert isinstance(application, Application)
        self.application = application
        super().__init__(rules)

    def process_rule(self, rule: Rule) -> Rule:
        rule = super().process_rule(rule)

        if isinstance(rule.target, (list, tuple)):
            rule.target = _ApplicationRouter(
                self.application, rule.target  # type: ignore
            )

        return rule

    def get_target_delegate(
        self, target: Any, request: httputil.HTTPServerRequest, **target_params: Any
    ) -> Optional[httputil.HTTPMessageDelegate]:
        if isclass(target) and issubclass(target, RequestHandler):
            return self.application.get_handler_delegate(
                request, target, **target_params
            )

        return super().get_target_delegate(target, request, **target_params)


class Application(ReversibleRouter):
    r"""A collection of request handlers that make up a web application.

    Instances of this class are callable and can be passed directly to
    HTTPServer to serve the application::

        application = web.Application([
            (r"/", MainPageHandler),
        ])
        http_server = httpserver.HTTPServer(application)
        http_server.listen(8080)

    The constructor for this class takes in a list of `~.routing.Rule`
    objects or tuples of values corresponding to the arguments of
    `~.routing.Rule` constructor: ``(matcher, target, [target_kwargs], [name])``,
    the values in square brackets being optional. The default matcher is
    `~.routing.PathMatches`, so ``(regexp, target)`` tuples can also be used
    instead of ``(PathMatches(regexp), target)``.

    A common routing target is a `RequestHandler` subclass, but you can also
    use lists of rules as a target, which create a nested routing configuration::

        application = web.Application([
            (HostMatches("example.com"), [
                (r"/", MainPageHandler),
                (r"/feed", FeedHandler),
            ]),
        ])

    In addition to this you can use nested `~.routing.Router` instances,
    `~.httputil.HTTPMessageDelegate` subclasses and callables as routing targets
    (see `~.routing` module docs for more information).

    When we receive requests, we iterate over the list in order and
    instantiate an instance of the first request class whose regexp
    matches the request path. The request class can be specified as
    either a class object or a (fully-qualified) name.

    A dictionary may be passed as the third element (``target_kwargs``)
    of the tuple, which will be used as keyword arguments to the handler's
    constructor and `~RequestHandler.initialize` method. This pattern
    is used for the `StaticFileHandler` in this example (note that a
    `StaticFileHandler` can be installed automatically with the
    static_path setting described below)::

        application = web.Application([
            (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
        ])

    We support virtual hosts with the `add_handlers` method, which takes in
    a host regular expression as the first argument::

        application.add_handlers(r"www\.myhost\.com", [
            (r"/article/([0-9]+)", ArticleHandler),
        ])

    If there's no match for the current request's host, then ``default_host``
    parameter value is matched against host regular expressions.


    .. warning::

       Applications that do not use TLS may be vulnerable to :ref:`DNS
       rebinding <dnsrebinding>` attacks. This attack is especially
       relevant to applications that only listen on ``127.0.0.1`` or
       other private networks. Appropriate host patterns must be used
       (instead of the default of ``r'.*'``) to prevent this risk. The
       ``default_host`` argument must not be used in applications that
       may be vulnerable to DNS rebinding.

    You can serve static files by sending the ``static_path`` setting
    as a keyword argument. We will serve those files from the
    ``/static/`` URI (this is configurable with the
    ``static_url_prefix`` setting), and we will serve ``/favicon.ico``
    and ``/robots.txt`` from the same directory.  A custom subclass of
    `StaticFileHandler` can be specified with the
    ``static_handler_class`` setting.

    .. versionchanged:: 4.5
       Integration with the new `tornado.routing` module.

    """

    def __init__(
        self,
        handlers: Optional[_RuleList] = None,
        default_host: Optional[str] = None,
        transforms: Optional[List[Type["OutputTransform"]]] = None,
        **settings: Any,
    ) -> None:
        if transforms is None:
            self.transforms = []  # type: List[Type[OutputTransform]]
            if settings.get("compress_response") or settings.get("gzip"):
                self.transforms.append(GZipContentEncoding)
        else:
            self.transforms = transforms
        self.default_host = default_host
        self.settings = settings
        self.ui_modules = {
            "linkify": _linkify,
            "xsrf_form_html": _xsrf_form_html,
            "Template": TemplateModule,
        }
        self.ui_methods = {}  # type: Dict[str, Callable[..., str]]
        self._load_ui_modules(settings.get("ui_modules", {}))
        self._load_ui_methods(settings.get("ui_methods", {}))
        if self.settings.get("static_path"):
            path = self.settings["static_path"]
            handlers = list(handlers or [])
            static_url_prefix = settings.get("static_url_prefix", "/static/")
            static_handler_class = settings.get(
                "static_handler_class", StaticFileHandler
            )
            static_handler_args = settings.get("static_handler_args", {})
            static_handler_args["path"] = path
            for pattern in [
                re.escape(static_url_prefix) + r"(.*)",
                r"/(favicon\.ico)",
                r"/(robots\.txt)",
            ]:
                handlers.insert(0, (pattern, static_handler_class, static_handler_args))

        if self.settings.get("debug"):
            self.settings.setdefault("autoreload", True)
            self.settings.setdefault("compiled_template_cache", False)
            self.settings.setdefault("static_hash_cache", False)
            self.settings.setdefault("serve_traceback", True)

        self.wildcard_router = _ApplicationRouter(self, handlers)
        self.default_router = _ApplicationRouter(
            self, [Rule(AnyMatches(), self.wildcard_router)]
        )

        # Automatically reload modified modules
        if self.settings.get("autoreload"):
            from tornado import autoreload

            autoreload.start()

    def listen(
        self,
        port: int,
        address: Optional[str] = None,
        *,
        family: socket.AddressFamily = socket.AF_UNSPEC,
        backlog: int = tornado.netutil._DEFAULT_BACKLOG,
        flags: Optional[int] = None,
        reuse_port: bool = False,
        **kwargs: Any,
    ) -> HTTPServer:
        """Starts an HTTP server for this application on the given port.

        This is a convenience alias for creating an `.HTTPServer` object and
        calling its listen method.  Keyword arguments not supported by
        `HTTPServer.listen <.TCPServer.listen>` are passed to the `.HTTPServer`
        constructor.  For advanced uses (e.g. multi-process mode), do not use
        this method; create an `.HTTPServer` and call its
        `.TCPServer.bind`/`.TCPServer.start` methods directly.

        Note that after calling this method you still need to call
        ``IOLoop.current().start()`` (or run within ``asyncio.run``) to start
        the server.

        Returns the `.HTTPServer` object.

        .. versionchanged:: 4.3
           Now returns the `.HTTPServer` object.

        .. versionchanged:: 6.2
           Added support for new keyword arguments in `.TCPServer.listen`,
           including ``reuse_port``.
        """
        server = HTTPServer(self, **kwargs)
        server.listen(
            port,
            address=address,
            family=family,
            backlog=backlog,
            flags=flags,
            reuse_port=reuse_port,
        )
        return server

    def add_handlers(self, host_pattern: str, host_handlers: _RuleList) -> None:
        """Appends the given handlers to our handler list.

        Host patterns are processed sequentially in the order they were
        added. All matching patterns will be considered.
        """
        host_matcher = HostMatches(host_pattern)
        rule = Rule(host_matcher, _ApplicationRouter(self, host_handlers))

        self.default_router.rules.insert(-1, rule)

        if self.default_host is not None:
            self.wildcard_router.add_rules(
                [(DefaultHostMatches(self, host_matcher.host_pattern), host_handlers)]
            )

    def add_transform(self, transform_class: Type["OutputTransform"]) -> None:
        self.transforms.append(transform_class)

    def _load_ui_methods(self, methods: Any) -> None:
        if isinstance(methods, types.ModuleType):
            self._load_ui_methods(dict((n, getattr(methods, n)) for n in dir(methods)))
        elif isinstance(methods, list):
            for m in methods:
                self._load_ui_methods(m)
        else:
            for name, fn in methods.items():
                if (
                    not name.startswith("_")
                    and hasattr(fn, "__call__")
                    and name[0].lower() == name[0]
                ):
                    self.ui_methods[name] = fn

    def _load_ui_modules(self, modules: Any) -> None:
        if isinstance(modules, types.ModuleType):
            self._load_ui_modules(dict((n, getattr(modules, n)) for n in dir(modules)))
        elif isinstance(modules, list):
            for m in modules:
                self._load_ui_modules(m)
        else:
            assert isinstance(modules, dict)
            for name, cls in modules.items():
                try:
                    if issubclass(cls, UIModule):
                        self.ui_modules[name] = cls
                except TypeError:
                    pass

    def __call__(
        self, request: httputil.HTTPServerRequest
    ) -> Optional[Awaitable[None]]:
        # Legacy HTTPServer interface
        dispatcher = self.find_handler(request)
        return dispatcher.execute()

    def find_handler(
        self, request: httputil.HTTPServerRequest, **kwargs: Any
    ) -> "_HandlerDelegate":
        route = self.default_router.find_handler(request)
        if route is not None:
            return cast("_HandlerDelegate", route)

        if self.settings.get("default_handler_class"):
            return self.get_handler_delegate(
                request,
                self.settings["default_handler_class"],
                self.settings.get("default_handler_args", {}),
            )

        return self.get_handler_delegate(request, ErrorHandler, {"status_code": 404})

    def get_handler_delegate(
        self,
        request: httputil.HTTPServerRequest,
        target_class: Type[RequestHandler],
        target_kwargs: Optional[Dict[str, Any]] = None,
        path_args: Optional[List[bytes]] = None,
        path_kwargs: Optional[Dict[str, bytes]] = None,
    ) -> "_HandlerDelegate":
        """Returns `~.httputil.HTTPMessageDelegate` that can serve a request
        for application and `RequestHandler` subclass.

        :arg httputil.HTTPServerRequest request: current HTTP request.
        :arg RequestHandler target_class: a `RequestHandler` class.
        :arg dict target_kwargs: keyword arguments for ``target_class`` constructor.
        :arg list path_args: positional arguments for ``target_class`` HTTP method that
            will be executed while handling a request (``get``, ``post`` or any other).
        :arg dict path_kwargs: keyword arguments for ``target_class`` HTTP method.
        """
        return _HandlerDelegate(
            self, request, target_class, target_kwargs, path_args, path_kwargs
        )

    def reverse_url(self, name: str, *args: Any) -> str:
        """Returns a URL path for handler named ``name``

        The handler must be added to the application as a named `URLSpec`.

        Args will be substituted for capturing groups in the `URLSpec` regex.
        They will be converted to strings if necessary, encoded as utf8,
        and url-escaped.
        """
        reversed_url = self.default_router.reverse_url(name, *args)
        if reversed_url is not None:
            return reversed_url

        raise KeyError("%s not found in named urls" % name)

    def log_request(self, handler: RequestHandler) -> None:
        """Writes a completed HTTP request to the logs.

        By default writes to the python root logger.  To change
        this behavior either subclass Application and override this method,
        or pass a function in the application settings dictionary as
        ``log_function``.
        """
        if "log_function" in self.settings:
            self.settings["log_function"](handler)
            return
        if handler.get_status() < 400:
            log_method = access_log.info
        elif handler.get_status() < 500:
            log_method = access_log.warning
        else:
            log_method = access_log.error
        request_time = 1000.0 * handler.request.request_time()
        log_method(
            "%d %s %.2fms",
            handler.get_status(),
            handler._request_summary(),
            request_time,
        )


class _HandlerDelegate(httputil.HTTPMessageDelegate):
    def __init__(
        self,
        application: Application,
        request: httputil.HTTPServerRequest,
        handler_class: Type[RequestHandler],
        handler_kwargs: Optional[Dict[str, Any]],
        path_args: Optional[List[bytes]],
        path_kwargs: Optional[Dict[str, bytes]],
    ) -> None:
        self.application = application
        self.connection = request.connection
        self.request = request
        self.handler_class = handler_class
        self.handler_kwargs = handler_kwargs or {}
        self.path_args = path_args or []
        self.path_kwargs = path_kwargs or {}
        self.chunks = []  # type: List[bytes]
        self.stream_request_body = _has_stream_request_body(self.handler_class)

    def headers_received(
        self,
        start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine],
        headers: httputil.HTTPHeaders,
    ) -> Optional[Awaitable[None]]:
        if self.stream_request_body:
            self.request._body_future = Future()
            return self.execute()
        return None

    def data_received(self, data: bytes) -> Optional[Awaitable[None]]:
        if self.stream_request_body:
            return self.handler.data_received(data)
        else:
            self.chunks.append(data)
            return None

    def finish(self) -> None:
        if self.stream_request_body:
            future_set_result_unless_cancelled(self.request._body_future, None)
        else:
            self.request.body = b"".join(self.chunks)
            self.request._parse_body()
            self.execute()

    def on_connection_close(self) -> None:
        if self.stream_request_body:
            self.handler.on_connection_close()
        else:
            self.chunks = None  # type: ignore

    def execute(self) -> Optional[Awaitable[None]]:
        # If template cache is disabled (usually in the debug mode),
        # re-compile templates and reload static files on every
        # request so you don't need to restart to see changes
        if not self.application.settings.get("compiled_template_cache", True):
            with RequestHandler._template_loader_lock:
                for loader in RequestHandler._template_loaders.values():
                    loader.reset()
        if not self.application.settings.get("static_hash_cache", True):
            static_handler_class = self.application.settings.get(
                "static_handler_class", StaticFileHandler
            )
            static_handler_class.reset()

        self.handler = self.handler_class(
            self.application, self.request, **self.handler_kwargs
        )
        transforms = [t(self.request) for t in self.application.transforms]

        if self.stream_request_body:
            self.handler._prepared_future = Future()
        # Note that if an exception escapes handler._execute it will be
        # trapped in the Future it returns (which we are ignoring here,
        # leaving it to be logged when the Future is GC'd).
        # However, that shouldn't happen because _execute has a blanket
        # except handler, and we cannot easily access the IOLoop here to
        # call add_future (because of the requirement to remain compatible
        # with WSGI)
        fut = gen.convert_yielded(
            self.handler._execute(transforms, *self.path_args, **self.path_kwargs)
        )
        fut.add_done_callback(lambda f: f.result())
        # If we are streaming the request body, then execute() is finished
        # when the handler has prepared to receive the body.  If not,
        # it doesn't matter when execute() finishes (so we return None)
        return self.handler._prepared_future


class HTTPError(Exception):
    """An exception that will turn into an HTTP error response.

    Raising an `HTTPError` is a convenient alternative to calling
    `RequestHandler.send_error` since it automatically ends the
    current function.

    To customize the response sent with an `HTTPError`, override
    `RequestHandler.write_error`.

    :arg int status_code: HTTP status code.  Must be listed in
        `httplib.responses <http.client.responses>` unless the ``reason``
        keyword argument is given.
    :arg str log_message: Message to be written to the log for this error
        (will not be shown to the user unless the `Application` is in debug
        mode).  May contain ``%s``-style placeholders, which will be filled
        in with remaining positional parameters.
    :arg str reason: Keyword-only argument.  The HTTP "reason" phrase
        to pass in the status line along with ``status_code``.  Normally
        determined automatically from ``status_code``, but can be used
        to use a non-standard numeric code.
    """

    def __init__(
        self,
        status_code: int = 500,
        log_message: Optional[str] = None,
        *args: Any,
        **kwargs: Any,
    ) -> None:
        self.status_code = status_code
        self.log_message = log_message
        self.args = args
        self.reason = kwargs.get("reason", None)
        if log_message and not args:
            self.log_message = log_message.replace("%", "%%")

    def __str__(self) -> str:
        message = "HTTP %d: %s" % (
            self.status_code,
            self.reason or httputil.responses.get(self.status_code, "Unknown"),
        )
        if self.log_message:
            return message + " (" + (self.log_message % self.args) + ")"
        else:
            return message


class Finish(Exception):
    """An exception that ends the request without producing an error response.

    When `Finish` is raised in a `RequestHandler`, the request will
    end (calling `RequestHandler.finish` if it hasn't already been
    called), but the error-handling methods (including
    `RequestHandler.write_error`) will not be called.

    If `Finish()` was created with no arguments, the pending response
    will be sent as-is. If `Finish()` was given an argument, that
    argument will be passed to `RequestHandler.finish()`.

    This can be a more convenient way to implement custom error pages
    than overriding ``write_error`` (especially in library code)::

        if self.current_user is None:
            self.set_status(401)
            self.set_header('WWW-Authenticate', 'Basic realm="something"')
            raise Finish()

    .. versionchanged:: 4.3
       Arguments passed to ``Finish()`` will be passed on to
       `RequestHandler.finish`.
    """

    pass


class MissingArgumentError(HTTPError):
    """Exception raised by `RequestHandler.get_argument`.

    This is a subclass of `HTTPError`, so if it is uncaught a 400 response
    code will be used instead of 500 (and a stack trace will not be logged).

    .. versionadded:: 3.1
    """

    def __init__(self, arg_name: str) -> None:
        super().__init__(400, "Missing argument %s" % arg_name)
        self.arg_name = arg_name


class ErrorHandler(RequestHandler):
    """Generates an error response with ``status_code`` for all requests."""

    def initialize(self, status_code: int) -> None:
        self.set_status(status_code)

    def prepare(self) -> None:
        raise HTTPError(self._status_code)

    def check_xsrf_cookie(self) -> None:
        # POSTs to an ErrorHandler don't actually have side effects,
        # so we don't need to check the xsrf token.  This allows POSTs
        # to the wrong url to return a 404 instead of 403.
        pass


class RedirectHandler(RequestHandler):
    """Redirects the client to the given URL for all GET requests.

    You should provide the keyword argument ``url`` to the handler, e.g.::

        application = web.Application([
            (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}),
        ])

    `RedirectHandler` supports regular expression substitutions. E.g., to
    swap the first and second parts of a path while preserving the remainder::

        application = web.Application([
            (r"/(.*?)/(.*?)/(.*)", web.RedirectHandler, {"url": "/{1}/{0}/{2}"}),
        ])

    The final URL is formatted with `str.format` and the substrings that match
    the capturing groups. In the above example, a request to "/a/b/c" would be
    formatted like::

        str.format("/{1}/{0}/{2}", "a", "b", "c")  # -> "/b/a/c"

    Use Python's :ref:`format string syntax <formatstrings>` to customize how
    values are substituted.

    .. versionchanged:: 4.5
       Added support for substitutions into the destination URL.

    .. versionchanged:: 5.0
       If any query arguments are present, they will be copied to the
       destination URL.
    """

    def initialize(self, url: str, permanent: bool = True) -> None:
        self._url = url
        self._permanent = permanent

    def get(self, *args: Any, **kwargs: Any) -> None:
        to_url = self._url.format(*args, **kwargs)
        if self.request.query_arguments:
            # TODO: figure out typing for the next line.
            to_url = httputil.url_concat(
                to_url,
                list(httputil.qs_to_qsl(self.request.query_arguments)),  # type: ignore
            )
        self.redirect(to_url, permanent=self._permanent)


class StaticFileHandler(RequestHandler):
    """A simple handler that can serve static content from a directory.

    A `StaticFileHandler` is configured automatically if you pass the
    ``static_path`` keyword argument to `Application`.  This handler
    can be customized with the ``static_url_prefix``, ``static_handler_class``,
    and ``static_handler_args`` settings.

    To map an additional path to this handler for a static data directory
    you would add a line to your application like::

        application = web.Application([
            (r"/content/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
        ])

    The handler constructor requires a ``path`` argument, which specifies the
    local root directory of the content to be served.

    Note that a capture group in the regex is required to parse the value for
    the ``path`` argument to the get() method (different than the constructor
    argument above); see `URLSpec` for details.

    To serve a file like ``index.html`` automatically when a directory is
    requested, set ``static_handler_args=dict(default_filename="index.html")``
    in your application settings, or add ``default_filename`` as an initializer
    argument for your ``StaticFileHandler``.

    To maximize the effectiveness of browser caching, this class supports
    versioned urls (by default using the argument ``?v=``).  If a version
    is given, we instruct the browser to cache this file indefinitely.
    `make_static_url` (also available as `RequestHandler.static_url`) can
    be used to construct a versioned url.

    This handler is intended primarily for use in development and light-duty
    file serving; for heavy traffic it will be more efficient to use
    a dedicated static file server (such as nginx or Apache).  We support
    the HTTP ``Accept-Ranges`` mechanism to return partial content (because
    some browsers require this functionality to be present to seek in
    HTML5 audio or video).

    **Subclassing notes**

    This class is designed to be extensible by subclassing, but because
    of the way static urls are generated with class methods rather than
    instance methods, the inheritance patterns are somewhat unusual.
    Be sure to use the ``@classmethod`` decorator when overriding a
    class method.  Instance methods may use the attributes ``self.path``
    ``self.absolute_path``, and ``self.modified``.

    Subclasses should only override methods discussed in this section;
    overriding other methods is error-prone.  Overriding
    ``StaticFileHandler.get`` is particularly problematic due to the
    tight coupling with ``compute_etag`` and other methods.

    To change the way static urls are generated (e.g. to match the behavior
    of another server or CDN), override `make_static_url`, `parse_url_path`,
    `get_cache_time`, and/or `get_version`.

    To replace all interaction with the filesystem (e.g. to serve
    static content from a database), override `get_content`,
    `get_content_size`, `get_modified_time`, `get_absolute_path`, and
    `validate_absolute_path`.

    .. versionchanged:: 3.1
       Many of the methods for subclasses were added in Tornado 3.1.
    """

    CACHE_MAX_AGE = 86400 * 365 * 10  # 10 years

    _static_hashes = {}  # type: Dict[str, Optional[str]]
    _lock = threading.Lock()  # protects _static_hashes

    def initialize(self, path: str, default_filename: Optional[str] = None) -> None:
        self.root = path
        self.default_filename = default_filename

    @classmethod
    def reset(cls) -> None:
        with cls._lock:
            cls._static_hashes = {}

    def head(self, path: str) -> Awaitable[None]:
        return self.get(path, include_body=False)

    async def get(self, path: str, include_body: bool = True) -> None:
        # Set up our path instance variables.
        self.path = self.parse_url_path(path)
        del path  # make sure we don't refer to path instead of self.path again
        absolute_path = self.get_absolute_path(self.root, self.path)
        self.absolute_path = self.validate_absolute_path(self.root, absolute_path)
        if self.absolute_path is None:
            return

        self.modified = self.get_modified_time()
        self.set_headers()

        if self.should_return_304():
            self.set_status(304)
            return

        request_range = None
        range_header = self.request.headers.get("Range")
        if range_header:
            # As per RFC 2616 14.16, if an invalid Range header is specified,
            # the request will be treated as if the header didn't exist.
            request_range = httputil._parse_request_range(range_header)

        size = self.get_content_size()
        if request_range:
            start, end = request_range
            if start is not None and start < 0:
                start += size
                if start < 0:
                    start = 0
            if (
                start is not None
                and (start >= size or (end is not None and start >= end))
            ) or end == 0:
                # As per RFC 2616 14.35.1, a range is not satisfiable only: if
                # the first requested byte is equal to or greater than the
                # content, or when a suffix with length 0 is specified.
                # https://tools.ietf.org/html/rfc7233#section-2.1
                # A byte-range-spec is invalid if the last-byte-pos value is present
                # and less than the first-byte-pos.
                self.set_status(416)  # Range Not Satisfiable
                self.set_header("Content-Type", "text/plain")
                self.set_header("Content-Range", "bytes */%s" % (size,))
                return
            if end is not None and end > size:
                # Clients sometimes blindly use a large range to limit their
                # download size; cap the endpoint at the actual file size.
                end = size
            # Note: only return HTTP 206 if less than the entire range has been
            # requested. Not only is this semantically correct, but Chrome
            # refuses to play audio if it gets an HTTP 206 in response to
            # ``Range: bytes=0-``.
            if size != (end or size) - (start or 0):
                self.set_status(206)  # Partial Content
                self.set_header(
                    "Content-Range", httputil._get_content_range(start, end, size)
                )
        else:
            start = end = None

        if start is not None and end is not None:
            content_length = end - start
        elif end is not None:
            content_length = end
        elif start is not None:
            content_length = size - start
        else:
            content_length = size
        self.set_header("Content-Length", content_length)

        if include_body:
            content = self.get_content(self.absolute_path, start, end)
            if isinstance(content, bytes):
                content = [content]
            for chunk in content:
                try:
                    self.write(chunk)
                    await self.flush()
                except iostream.StreamClosedError:
                    return
        else:
            assert self.request.method == "HEAD"

    def compute_etag(self) -> Optional[str]:
        """Sets the ``Etag`` header based on static url version.

        This allows efficient ``If-None-Match`` checks against cached
        versions, and sends the correct ``Etag`` for a partial response
        (i.e. the same ``Etag`` as the full file).

        .. versionadded:: 3.1
        """
        assert self.absolute_path is not None
        version_hash = self._get_cached_version(self.absolute_path)
        if not version_hash:
            return None
        return '"%s"' % (version_hash,)

    def set_headers(self) -> None:
        """Sets the content and caching headers on the response.

        .. versionadded:: 3.1
        """
        self.set_header("Accept-Ranges", "bytes")
        self.set_etag_header()

        if self.modified is not None:
            self.set_header("Last-Modified", self.modified)

        content_type = self.get_content_type()
        if content_type:
            self.set_header("Content-Type", content_type)

        cache_time = self.get_cache_time(self.path, self.modified, content_type)
        if cache_time > 0:
            self.set_header(
                "Expires",
                datetime.datetime.utcnow() + datetime.timedelta(seconds=cache_time),
            )
            self.set_header("Cache-Control", "max-age=" + str(cache_time))

        self.set_extra_headers(self.path)

    def should_return_304(self) -> bool:
        """Returns True if the headers indicate that we should return 304.

        .. versionadded:: 3.1
        """
        # If client sent If-None-Match, use it, ignore If-Modified-Since
        if self.request.headers.get("If-None-Match"):
            return self.check_etag_header()

        # Check the If-Modified-Since, and don't send the result if the
        # content has not been modified
        ims_value = self.request.headers.get("If-Modified-Since")
        if ims_value is not None:
            date_tuple = email.utils.parsedate(ims_value)
            if date_tuple is not None:
                if_since = datetime.datetime(*date_tuple[:6])
                assert self.modified is not None
                if if_since >= self.modified:
                    return True

        return False

    @classmethod
    def get_absolute_path(cls, root: str, path: str) -> str:
        """Returns the absolute location of ``path`` relative to ``root``.

        ``root`` is the path configured for this `StaticFileHandler`
        (in most cases the ``static_path`` `Application` setting).

        This class method may be overridden in subclasses.  By default
        it returns a filesystem path, but other strings may be used
        as long as they are unique and understood by the subclass's
        overridden `get_content`.

        .. versionadded:: 3.1
        """
        abspath = os.path.abspath(os.path.join(root, path))
        return abspath

    def validate_absolute_path(self, root: str, absolute_path: str) -> Optional[str]:
        """Validate and return the absolute path.

        ``root`` is the configured path for the `StaticFileHandler`,
        and ``path`` is the result of `get_absolute_path`

        This is an instance method called during request processing,
        so it may raise `HTTPError` or use methods like
        `RequestHandler.redirect` (return None after redirecting to
        halt further processing).  This is where 404 errors for missing files
        are generated.

        This method may modify the path before returning it, but note that
        any such modifications will not be understood by `make_static_url`.

        In instance methods, this method's result is available as
        ``self.absolute_path``.

        .. versionadded:: 3.1
        """
        # os.path.abspath strips a trailing /.
        # We must add it back to `root` so that we only match files
        # in a directory named `root` instead of files starting with
        # that prefix.
        root = os.path.abspath(root)
        if not root.endswith(os.path.sep):
            # abspath always removes a trailing slash, except when
            # root is '/'. This is an unusual case, but several projects
            # have independently discovered this technique to disable
            # Tornado's path validation and (hopefully) do their own,
            # so we need to support it.
            root += os.path.sep
        # The trailing slash also needs to be temporarily added back
        # the requested path so a request to root/ will match.
        if not (absolute_path + os.path.sep).startswith(root):
            raise HTTPError(403, "%s is not in root static directory", self.path)
        if os.path.isdir(absolute_path) and self.default_filename is not None:
            # need to look at the request.path here for when path is empty
            # but there is some prefix to the path that was already
            # trimmed by the routing
            if not self.request.path.endswith("/"):
                self.redirect(self.request.path + "/", permanent=True)
                return None
            absolute_path = os.path.join(absolute_path, self.default_filename)
        if not os.path.exists(absolute_path):
            raise HTTPError(404)
        if not os.path.isfile(absolute_path):
            raise HTTPError(403, "%s is not a file", self.path)
        return absolute_path

    @classmethod
    def get_content(
        cls, abspath: str, start: Optional[int] = None, end: Optional[int] = None
    ) -> Generator[bytes, None, None]:
        """Retrieve the content of the requested resource which is located
        at the given absolute path.

        This class method may be overridden by subclasses.  Note that its
        signature is different from other overridable class methods
        (no ``settings`` argument); this is deliberate to ensure that
        ``abspath`` is able to stand on its own as a cache key.

        This method should either return a byte string or an iterator
        of byte strings.  The latter is preferred for large files
        as it helps reduce memory fragmentation.

        .. versionadded:: 3.1
        """
        with open(abspath, "rb") as file:
            if start is not None:
                file.seek(start)
            if end is not None:
                remaining = end - (start or 0)  # type: Optional[int]
            else:
                remaining = None
            while True:
                chunk_size = 64 * 1024
                if remaining is not None and remaining < chunk_size:
                    chunk_size = remaining
                chunk = file.read(chunk_size)
                if chunk:
                    if remaining is not None:
                        remaining -= len(chunk)
                    yield chunk
                else:
                    if remaining is not None:
                        assert remaining == 0
                    return

    @classmethod
    def get_content_version(cls, abspath: str) -> str:
        """Returns a version string for the resource at the given path.

        This class method may be overridden by subclasses.  The
        default implementation is a SHA-512 hash of the file's contents.

        .. versionadded:: 3.1
        """
        data = cls.get_content(abspath)
        hasher = hashlib.sha512()
        if isinstance(data, bytes):
            hasher.update(data)
        else:
            for chunk in data:
                hasher.update(chunk)
        return hasher.hexdigest()

    def _stat(self) -> os.stat_result:
        assert self.absolute_path is not None
        if not hasattr(self, "_stat_result"):
            self._stat_result = os.stat(self.absolute_path)
        return self._stat_result

    def get_content_size(self) -> int:
        """Retrieve the total size of the resource at the given path.

        This method may be overridden by subclasses.

        .. versionadded:: 3.1

        .. versionchanged:: 4.0
           This method is now always called, instead of only when
           partial results are requested.
        """
        stat_result = self._stat()
        return stat_result.st_size

    def get_modified_time(self) -> Optional[datetime.datetime]:
        """Returns the time that ``self.absolute_path`` was last modified.

        May be overridden in subclasses.  Should return a `~datetime.datetime`
        object or None.

        .. versionadded:: 3.1
        """
        stat_result = self._stat()
        # NOTE: Historically, this used stat_result[stat.ST_MTIME],
        # which truncates the fractional portion of the timestamp. It
        # was changed from that form to stat_result.st_mtime to
        # satisfy mypy (which disallows the bracket operator), but the
        # latter form returns a float instead of an int. For
        # consistency with the past (and because we have a unit test
        # that relies on this), we truncate the float here, although
        # I'm not sure that's the right thing to do.
        modified = datetime.datetime.utcfromtimestamp(int(stat_result.st_mtime))
        return modified

    def get_content_type(self) -> str:
        """Returns the ``Content-Type`` header to be used for this request.

        .. versionadded:: 3.1
        """
        assert self.absolute_path is not None
        mime_type, encoding = mimetypes.guess_type(self.absolute_path)
        # per RFC 6713, use the appropriate type for a gzip compressed file
        if encoding == "gzip":
            return "application/gzip"
        # As of 2015-07-21 there is no bzip2 encoding defined at
        # http://www.iana.org/assignments/media-types/media-types.xhtml
        # So for that (and any other encoding), use octet-stream.
        elif encoding is not None:
            return "application/octet-stream"
        elif mime_type is not None:
            return mime_type
        # if mime_type not detected, use application/octet-stream
        else:
            return "application/octet-stream"

    def set_extra_headers(self, path: str) -> None:
        """For subclass to add extra headers to the response"""
        pass

    def get_cache_time(
        self, path: str, modified: Optional[datetime.datetime], mime_type: str
    ) -> int:
        """Override to customize cache control behavior.

        Return a positive number of seconds to make the result
        cacheable for that amount of time or 0 to mark resource as
        cacheable for an unspecified amount of time (subject to
        browser heuristics).

        By default returns cache expiry of 10 years for resources requested
        with ``v`` argument.
        """
        return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0

    @classmethod
    def make_static_url(
        cls, settings: Dict[str, Any], path: str, include_version: bool = True
    ) -> str:
        """Constructs a versioned url for the given path.

        This method may be overridden in subclasses (but note that it
        is a class method rather than an instance method).  Subclasses
        are only required to implement the signature
        ``make_static_url(cls, settings, path)``; other keyword
        arguments may be passed through `~RequestHandler.static_url`
        but are not standard.

        ``settings`` is the `Application.settings` dictionary.  ``path``
        is the static path being requested.  The url returned should be
        relative to the current host.

        ``include_version`` determines whether the generated URL should
        include the query string containing the version hash of the
        file corresponding to the given ``path``.

        """
        url = settings.get("static_url_prefix", "/static/") + path
        if not include_version:
            return url

        version_hash = cls.get_version(settings, path)
        if not version_hash:
            return url

        return "%s?v=%s" % (url, version_hash)

    def parse_url_path(self, url_path: str) -> str:
        """Converts a static URL path into a filesystem path.

        ``url_path`` is the path component of the URL with
        ``static_url_prefix`` removed.  The return value should be
        filesystem path relative to ``static_path``.

        This is the inverse of `make_static_url`.
        """
        if os.path.sep != "/":
            url_path = url_path.replace("/", os.path.sep)
        return url_path

    @classmethod
    def get_version(cls, settings: Dict[str, Any], path: str) -> Optional[str]:
        """Generate the version string to be used in static URLs.

        ``settings`` is the `Application.settings` dictionary and ``path``
        is the relative location of the requested asset on the filesystem.
        The returned value should be a string, or ``None`` if no version
        could be determined.

        .. versionchanged:: 3.1
           This method was previously recommended for subclasses to override;
           `get_content_version` is now preferred as it allows the base
           class to handle caching of the result.
        """
        abs_path = cls.get_absolute_path(settings["static_path"], path)
        return cls._get_cached_version(abs_path)

    @classmethod
    def _get_cached_version(cls, abs_path: str) -> Optional[str]:
        with cls._lock:
            hashes = cls._static_hashes
            if abs_path not in hashes:
                try:
                    hashes[abs_path] = cls.get_content_version(abs_path)
                except Exception:
                    gen_log.error("Could not open static file %r", abs_path)
                    hashes[abs_path] = None
            hsh = hashes.get(abs_path)
            if hsh:
                return hsh
        return None


class FallbackHandler(RequestHandler):
    """A `RequestHandler` that wraps another HTTP server callback.

    The fallback is a callable object that accepts an
    `~.httputil.HTTPServerRequest`, such as an `Application` or
    `tornado.wsgi.WSGIContainer`.  This is most useful to use both
    Tornado ``RequestHandlers`` and WSGI in the same server.  Typical
    usage::

        wsgi_app = tornado.wsgi.WSGIContainer(
            django.core.handlers.wsgi.WSGIHandler())
        application = tornado.web.Application([
            (r"/foo", FooHandler),
            (r".*", FallbackHandler, dict(fallback=wsgi_app),
        ])
    """

    def initialize(
        self, fallback: Callable[[httputil.HTTPServerRequest], None]
    ) -> None:
        self.fallback = fallback

    def prepare(self) -> None:
        self.fallback(self.request)
        self._finished = True
        self.on_finish()


class OutputTransform(object):
    """A transform modifies the result of an HTTP request (e.g., GZip encoding)

    Applications are not expected to create their own OutputTransforms
    or interact with them directly; the framework chooses which transforms
    (if any) to apply.
    """

    def __init__(self, request: httputil.HTTPServerRequest) -> None:
        pass

    def transform_first_chunk(
        self,
        status_code: int,
        headers: httputil.HTTPHeaders,
        chunk: bytes,
        finishing: bool,
    ) -> Tuple[int, httputil.HTTPHeaders, bytes]:
        return status_code, headers, chunk

    def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes:
        return chunk


class GZipContentEncoding(OutputTransform):
    """Applies the gzip content encoding to the response.

    See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11

    .. versionchanged:: 4.0
        Now compresses all mime types beginning with ``text/``, instead
        of just a whitelist. (the whitelist is still used for certain
        non-text mime types).
    """

    # Whitelist of compressible mime types (in addition to any types
    # beginning with "text/").
    CONTENT_TYPES = set(
        [
            "application/javascript",
            "application/x-javascript",
            "application/xml",
            "application/atom+xml",
            "application/json",
            "application/xhtml+xml",
            "image/svg+xml",
        ]
    )
    # Python's GzipFile defaults to level 9, while most other gzip
    # tools (including gzip itself) default to 6, which is probably a
    # better CPU/size tradeoff.
    GZIP_LEVEL = 6
    # Responses that are too short are unlikely to benefit from gzipping
    # after considering the "Content-Encoding: gzip" header and the header
    # inside the gzip encoding.
    # Note that responses written in multiple chunks will be compressed
    # regardless of size.
    MIN_LENGTH = 1024

    def __init__(self, request: httputil.HTTPServerRequest) -> None:
        self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "")

    def _compressible_type(self, ctype: str) -> bool:
        return ctype.startswith("text/") or ctype in self.CONTENT_TYPES

    def transform_first_chunk(
        self,
        status_code: int,
        headers: httputil.HTTPHeaders,
        chunk: bytes,
        finishing: bool,
    ) -> Tuple[int, httputil.HTTPHeaders, bytes]:
        # TODO: can/should this type be inherited from the superclass?
        if "Vary" in headers:
            headers["Vary"] += ", Accept-Encoding"
        else:
            headers["Vary"] = "Accept-Encoding"
        if self._gzipping:
            ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
            self._gzipping = (
                self._compressible_type(ctype)
                and (not finishing or len(chunk) >= self.MIN_LENGTH)
                and ("Content-Encoding" not in headers)
            )
        if self._gzipping:
            headers["Content-Encoding"] = "gzip"
            self._gzip_value = BytesIO()
            self._gzip_file = gzip.GzipFile(
                mode="w", fileobj=self._gzip_value, compresslevel=self.GZIP_LEVEL
            )
            chunk = self.transform_chunk(chunk, finishing)
            if "Content-Length" in headers:
                # The original content length is no longer correct.
                # If this is the last (and only) chunk, we can set the new
                # content-length; otherwise we remove it and fall back to
                # chunked encoding.
                if finishing:
                    headers["Content-Length"] = str(len(chunk))
                else:
                    del headers["Content-Length"]
        return status_code, headers, chunk

    def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes:
        if self._gzipping:
            self._gzip_file.write(chunk)
            if finishing:
                self._gzip_file.close()
            else:
                self._gzip_file.flush()
            chunk = self._gzip_value.getvalue()
            self._gzip_value.truncate(0)
            self._gzip_value.seek(0)
        return chunk


def authenticated(
    method: Callable[..., Optional[Awaitable[None]]]
) -> Callable[..., Optional[Awaitable[None]]]:
    """Decorate methods with this to require that the user be logged in.

    If the user is not logged in, they will be redirected to the configured
    `login url <RequestHandler.get_login_url>`.

    If you configure a login url with a query parameter, Tornado will
    assume you know what you're doing and use it as-is.  If not, it
    will add a `next` parameter so the login page knows where to send
    you once you're logged in.
    """

    @functools.wraps(method)
    def wrapper(  # type: ignore
        self: RequestHandler, *args, **kwargs
    ) -> Optional[Awaitable[None]]:
        if not self.current_user:
            if self.request.method in ("GET", "HEAD"):
                url = self.get_login_url()
                if "?" not in url:
                    if urllib.parse.urlsplit(url).scheme:
                        # if login url is absolute, make next absolute too
                        next_url = self.request.full_url()
                    else:
                        assert self.request.uri is not None
                        next_url = self.request.uri
                    url += "?" + urlencode(dict(next=next_url))
                self.redirect(url)
                return None
            raise HTTPError(403)
        return method(self, *args, **kwargs)

    return wrapper


class UIModule(object):
    """A re-usable, modular UI unit on a page.

    UI modules often execute additional queries, and they can include
    additional CSS and JavaScript that will be included in the output
    page, which is automatically inserted on page render.

    Subclasses of UIModule must override the `render` method.
    """

    def __init__(self, handler: RequestHandler) -> None:
        self.handler = handler
        self.request = handler.request
        self.ui = handler.ui
        self.locale = handler.locale

    @property
    def current_user(self) -> Any:
        return self.handler.current_user

    def render(self, *args: Any, **kwargs: Any) -> str:
        """Override in subclasses to return this module's output."""
        raise NotImplementedError()

    def embedded_javascript(self) -> Optional[str]:
        """Override to return a JavaScript string
        to be embedded in the page."""
        return None

    def javascript_files(self) -> Optional[Iterable[str]]:
        """Override to return a list of JavaScript files needed by this module.

        If the return values are relative paths, they will be passed to
        `RequestHandler.static_url`; otherwise they will be used as-is.
        """
        return None

    def embedded_css(self) -> Optional[str]:
        """Override to return a CSS string
        that will be embedded in the page."""
        return None

    def css_files(self) -> Optional[Iterable[str]]:
        """Override to returns a list of CSS files required by this module.

        If the return values are relative paths, they will be passed to
        `RequestHandler.static_url`; otherwise they will be used as-is.
        """
        return None

    def html_head(self) -> Optional[str]:
        """Override to return an HTML string that will be put in the <head/>
        element.
        """
        return None

    def html_body(self) -> Optional[str]:
        """Override to return an HTML string that will be put at the end of
        the <body/> element.
        """
        return None

    def render_string(self, path: str, **kwargs: Any) -> bytes:
        """Renders a template and returns it as a string."""
        return self.handler.render_string(path, **kwargs)


class _linkify(UIModule):
    def render(self, text: str, **kwargs: Any) -> str:  # type: ignore
        return escape.linkify(text, **kwargs)


class _xsrf_form_html(UIModule):
    def render(self) -> str:  # type: ignore
        return self.handler.xsrf_form_html()


class TemplateModule(UIModule):
    """UIModule that simply renders the given template.

    {% module Template("foo.html") %} is similar to {% include "foo.html" %},
    but the module version gets its own namespace (with kwargs passed to
    Template()) instead of inheriting the outer template's namespace.

    Templates rendered through this module also get access to UIModule's
    automatic JavaScript/CSS features.  Simply call set_resources
    inside the template and give it keyword arguments corresponding to
    the methods on UIModule: {{ set_resources(js_files=static_url("my.js")) }}
    Note that these resources are output once per template file, not once
    per instantiation of the template, so they must not depend on
    any arguments to the template.
    """

    def __init__(self, handler: RequestHandler) -> None:
        super().__init__(handler)
        # keep resources in both a list and a dict to preserve order
        self._resource_list = []  # type: List[Dict[str, Any]]
        self._resource_dict = {}  # type: Dict[str, Dict[str, Any]]

    def render(self, path: str, **kwargs: Any) -> bytes:  # type: ignore
        def set_resources(**kwargs) -> str:  # type: ignore
            if path not in self._resource_dict:
                self._resource_list.append(kwargs)
                self._resource_dict[path] = kwargs
            else:
                if self._resource_dict[path] != kwargs:
                    raise ValueError(
                        "set_resources called with different "
                        "resources for the same template"
                    )
            return ""

        return self.render_string(path, set_resources=set_resources, **kwargs)

    def _get_resources(self, key: str) -> Iterable[str]:
        return (r[key] for r in self._resource_list if key in r)

    def embedded_javascript(self) -> str:
        return "\n".join(self._get_resources("embedded_javascript"))

    def javascript_files(self) -> Iterable[str]:
        result = []
        for f in self._get_resources("javascript_files"):
            if isinstance(f, (unicode_type, bytes)):
                result.append(f)
            else:
                result.extend(f)
        return result

    def embedded_css(self) -> str:
        return "\n".join(self._get_resources("embedded_css"))

    def css_files(self) -> Iterable[str]:
        result = []
        for f in self._get_resources("css_files"):
            if isinstance(f, (unicode_type, bytes)):
                result.append(f)
            else:
                result.extend(f)
        return result

    def html_head(self) -> str:
        return "".join(self._get_resources("html_head"))

    def html_body(self) -> str:
        return "".join(self._get_resources("html_body"))


class _UIModuleNamespace(object):
    """Lazy namespace which creates UIModule proxies bound to a handler."""

    def __init__(
        self, handler: RequestHandler, ui_modules: Dict[str, Type[UIModule]]
    ) -> None:
        self.handler = handler
        self.ui_modules = ui_modules

    def __getitem__(self, key: str) -> Callable[..., str]:
        return self.handler._ui_module(key, self.ui_modules[key])

    def __getattr__(self, key: str) -> Callable[..., str]:
        try:
            return self[key]
        except KeyError as e:
            raise AttributeError(str(e))


def create_signed_value(
    secret: _CookieSecretTypes,
    name: str,
    value: Union[str, bytes],
    version: Optional[int] = None,
    clock: Optional[Callable[[], float]] = None,
    key_version: Optional[int] = None,
) -> bytes:
    if version is None:
        version = DEFAULT_SIGNED_VALUE_VERSION
    if clock is None:
        clock = time.time

    timestamp = utf8(str(int(clock())))
    value = base64.b64encode(utf8(value))
    if version == 1:
        assert not isinstance(secret, dict)
        signature = _create_signature_v1(secret, name, value, timestamp)
        value = b"|".join([value, timestamp, signature])
        return value
    elif version == 2:
        # The v2 format consists of a version number and a series of
        # length-prefixed fields "%d:%s", the last of which is a
        # signature, all separated by pipes.  All numbers are in
        # decimal format with no leading zeros.  The signature is an
        # HMAC-SHA256 of the whole string up to that point, including
        # the final pipe.
        #
        # The fields are:
        # - format version (i.e. 2; no length prefix)
        # - key version (integer, default is 0)
        # - timestamp (integer seconds since epoch)
        # - name (not encoded; assumed to be ~alphanumeric)
        # - value (base64-encoded)
        # - signature (hex-encoded; no length prefix)
        def format_field(s: Union[str, bytes]) -> bytes:
            return utf8("%d:" % len(s)) + utf8(s)

        to_sign = b"|".join(
            [
                b"2",
                format_field(str(key_version or 0)),
                format_field(timestamp),
                format_field(name),
                format_field(value),
                b"",
            ]
        )

        if isinstance(secret, dict):
            assert (
                key_version is not None
            ), "Key version must be set when sign key dict is used"
            assert version >= 2, "Version must be at least 2 for key version support"
            secret = secret[key_version]

        signature = _create_signature_v2(secret, to_sign)
        return to_sign + signature
    else:
        raise ValueError("Unsupported version %d" % version)


# A leading version number in decimal
# with no leading zeros, followed by a pipe.
_signed_value_version_re = re.compile(rb"^([1-9][0-9]*)\|(.*)$")


def _get_version(value: bytes) -> int:
    # Figures out what version value is.  Version 1 did not include an
    # explicit version field and started with arbitrary base64 data,
    # which makes this tricky.
    m = _signed_value_version_re.match(value)
    if m is None:
        version = 1
    else:
        try:
            version = int(m.group(1))
            if version > 999:
                # Certain payloads from the version-less v1 format may
                # be parsed as valid integers.  Due to base64 padding
                # restrictions, this can only happen for numbers whose
                # length is a multiple of 4, so we can treat all
                # numbers up to 999 as versions, and for the rest we
                # fall back to v1 format.
                version = 1
        except ValueError:
            version = 1
    return version


def decode_signed_value(
    secret: _CookieSecretTypes,
    name: str,
    value: Union[None, str, bytes],
    max_age_days: float = 31,
    clock: Optional[Callable[[], float]] = None,
    min_version: Optional[int] = None,
) -> Optional[bytes]:
    if clock is None:
        clock = time.time
    if min_version is None:
        min_version = DEFAULT_SIGNED_VALUE_MIN_VERSION
    if min_version > 2:
        raise ValueError("Unsupported min_version %d" % min_version)
    if not value:
        return None

    value = utf8(value)
    version = _get_version(value)

    if version < min_version:
        return None
    if version == 1:
        assert not isinstance(secret, dict)
        return _decode_signed_value_v1(secret, name, value, max_age_days, clock)
    elif version == 2:
        return _decode_signed_value_v2(secret, name, value, max_age_days, clock)
    else:
        return None


def _decode_signed_value_v1(
    secret: Union[str, bytes],
    name: str,
    value: bytes,
    max_age_days: float,
    clock: Callable[[], float],
) -> Optional[bytes]:
    parts = utf8(value).split(b"|")
    if len(parts) != 3:
        return None
    signature = _create_signature_v1(secret, name, parts[0], parts[1])
    if not hmac.compare_digest(parts[2], signature):
        gen_log.warning("Invalid cookie signature %r", value)
        return None
    timestamp = int(parts[1])
    if timestamp < clock() - max_age_days * 86400:
        gen_log.warning("Expired cookie %r", value)
        return None
    if timestamp > clock() + 31 * 86400:
        # _cookie_signature does not hash a delimiter between the
        # parts of the cookie, so an attacker could transfer trailing
        # digits from the payload to the timestamp without altering the
        # signature.  For backwards compatibility, sanity-check timestamp
        # here instead of modifying _cookie_signature.
        gen_log.warning("Cookie timestamp in future; possible tampering %r", value)
        return None
    if parts[1].startswith(b"0"):
        gen_log.warning("Tampered cookie %r", value)
        return None
    try:
        return base64.b64decode(parts[0])
    except Exception:
        return None


def _decode_fields_v2(value: bytes) -> Tuple[int, bytes, bytes, bytes, bytes]:
    def _consume_field(s: bytes) -> Tuple[bytes, bytes]:
        length, _, rest = s.partition(b":")
        n = int(length)
        field_value = rest[:n]
        # In python 3, indexing bytes returns small integers; we must
        # use a slice to get a byte string as in python 2.
        if rest[n : n + 1] != b"|":
            raise ValueError("malformed v2 signed value field")
        rest = rest[n + 1 :]
        return field_value, rest

    rest = value[2:]  # remove version number
    key_version, rest = _consume_field(rest)
    timestamp, rest = _consume_field(rest)
    name_field, rest = _consume_field(rest)
    value_field, passed_sig = _consume_field(rest)
    return int(key_version), timestamp, name_field, value_field, passed_sig


def _decode_signed_value_v2(
    secret: _CookieSecretTypes,
    name: str,
    value: bytes,
    max_age_days: float,
    clock: Callable[[], float],
) -> Optional[bytes]:
    try:
        (
            key_version,
            timestamp_bytes,
            name_field,
            value_field,
            passed_sig,
        ) = _decode_fields_v2(value)
    except ValueError:
        return None
    signed_string = value[: -len(passed_sig)]

    if isinstance(secret, dict):
        try:
            secret = secret[key_version]
        except KeyError:
            return None

    expected_sig = _create_signature_v2(secret, signed_string)
    if not hmac.compare_digest(passed_sig, expected_sig):
        return None
    if name_field != utf8(name):
        return None
    timestamp = int(timestamp_bytes)
    if timestamp < clock() - max_age_days * 86400:
        # The signature has expired.
        return None
    try:
        return base64.b64decode(value_field)
    except Exception:
        return None


def get_signature_key_version(value: Union[str, bytes]) -> Optional[int]:
    value = utf8(value)
    version = _get_version(value)
    if version < 2:
        return None
    try:
        key_version, _, _, _, _ = _decode_fields_v2(value)
    except ValueError:
        return None

    return key_version


def _create_signature_v1(secret: Union[str, bytes], *parts: Union[str, bytes]) -> bytes:
    hash = hmac.new(utf8(secret), digestmod=hashlib.sha1)
    for part in parts:
        hash.update(utf8(part))
    return utf8(hash.hexdigest())


def _create_signature_v2(secret: Union[str, bytes], s: bytes) -> bytes:
    hash = hmac.new(utf8(secret), digestmod=hashlib.sha256)
    hash.update(utf8(s))
    return utf8(hash.hexdigest())


def is_absolute(path: str) -> bool:
    return any(path.startswith(x) for x in ["/", "http:", "https:"])

env_pyrecodes/lib/python3.9/site-packages/tornado/web.py
NikolaBlagojevic/pyrecodes
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import logging.config
from typing import Any, List, Tuple

import importlib_metadata
import rich
import rich.highlighter
import rich.logging
import rich.theme

import twine

args = argparse.Namespace()


def configure_output() -> None:
    # Configure the global Console, available via rich.get_console().
    # https://rich.readthedocs.io/en/latest/reference/init.html
    # https://rich.readthedocs.io/en/latest/console.html
    rich.reconfigure(
        # Setting force_terminal makes testing easier by ensuring color codes. This
        # could be based on FORCE_COLORS or PY_COLORS in os.environ, since Rich
        # doesn't support that (https://github.com/Textualize/rich/issues/343).
        force_terminal=True,
        no_color=getattr(args, "no_color", False),
        highlight=False,
        theme=rich.theme.Theme(
            {
                "logging.level.debug": "green",
                "logging.level.info": "blue",
                "logging.level.warning": "yellow",
                "logging.level.error": "red",
                "logging.level.critical": "reverse red",
            }
        ),
    )

    # Using dictConfig to override existing loggers, which prevents failures in
    # test_main.py due to capsys not being cleared.
    logging.config.dictConfig(
        {
            "disable_existing_loggers": False,
            "version": 1,
            "handlers": {
                "console": {
                    "class": "rich.logging.RichHandler",
                    "show_time": False,
                    "show_path": False,
                    "highlighter": rich.highlighter.NullHighlighter(),
                }
            },
            "root": {
                "handlers": ["console"],
            },
        }
    )


def list_dependencies_and_versions() -> List[Tuple[str, str]]:
    deps = (
        "importlib-metadata",
        "keyring",
        "pkginfo",
        "requests",
        "requests-toolbelt",
        "urllib3",
    )
    return [(dep, importlib_metadata.version(dep)) for dep in deps]  # type: ignore[no-untyped-call] # python/importlib_metadata#288  # noqa: E501


def dep_versions() -> str:
    return ", ".join(
        "{}: {}".format(*dependency) for dependency in list_dependencies_and_versions()
    )


def dispatch(argv: List[str]) -> Any:
    registered_commands = importlib_metadata.entry_points(
        group="twine.registered_commands"
    )

    parser = argparse.ArgumentParser(prog="twine")
    parser.add_argument(
        "--version",
        action="version",
        version=f"%(prog)s version {twine.__version__} ({dep_versions()})",
    )
    parser.add_argument(
        "--no-color",
        default=False,
        required=False,
        action="store_true",
        help="disable colored output",
    )
    parser.add_argument(
        "command",
        choices=registered_commands.names,
    )
    parser.add_argument(
        "args",
        help=argparse.SUPPRESS,
        nargs=argparse.REMAINDER,
    )
    parser.parse_args(argv, namespace=args)

    configure_output()

    main = registered_commands[args.command].load()

    return main(args.args)

env_pyrecodes/lib/python3.9/site-packages/twine/cli.py
NikolaBlagojevic/pyrecodes
"""Module containing the logic for ``twine check``."""
# Copyright 2018 Dustin Ingram
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import cgi
import io
import logging
import re
from typing import List, Optional, Tuple, cast

import readme_renderer.rst
from rich import print

from twine import commands
from twine import package as package_file

logger = logging.getLogger(__name__)


_RENDERERS = {
    None: readme_renderer.rst,  # Default if description_content_type is None
    "text/plain": None,  # Rendering cannot fail
    "text/x-rst": readme_renderer.rst,
    "text/markdown": None,  # Rendering cannot fail
}


# Regular expression used to capture and reformat docutils warnings into
# something that a human can understand. This is loosely borrowed from
# Sphinx: https://github.com/sphinx-doc/sphinx/blob
# /c35eb6fade7a3b4a6de4183d1dd4196f04a5edaf/sphinx/util/docutils.py#L199
_REPORT_RE = re.compile(
    r"^<string>:(?P<line>(?:\d+)?): "
    r"\((?P<level>DEBUG|INFO|WARNING|ERROR|SEVERE)/(\d+)?\) "
    r"(?P<message>.*)",
    re.DOTALL | re.MULTILINE,
)


class _WarningStream(io.StringIO):
    def write(self, text: str) -> int:
        matched = _REPORT_RE.search(text)
        if matched:
            line = matched.group("line")
            level_text = matched.group("level").capitalize()
            message = matched.group("message").rstrip("\r\n")
            text = f"line {line}: {level_text}: {message}\n"

        return super().write(text)

    def __str__(self) -> str:
        return self.getvalue().strip()


def _check_file(
    filename: str, render_warning_stream: _WarningStream
) -> Tuple[List[str], bool]:
    """Check given distribution."""
    warnings = []
    is_ok = True

    package = package_file.PackageFile.from_filename(filename, comment=None)

    metadata = package.metadata_dictionary()
    description = cast(Optional[str], metadata["description"])
    description_content_type = cast(Optional[str], metadata["description_content_type"])

    if description_content_type is None:
        warnings.append(
            "`long_description_content_type` missing. defaulting to `text/x-rst`."
        )
        description_content_type = "text/x-rst"

    content_type, params = cgi.parse_header(description_content_type)
    renderer = _RENDERERS.get(content_type, _RENDERERS[None])

    if description is None or description.rstrip() == "UNKNOWN":
        warnings.append("`long_description` missing.")
    elif renderer:
        rendering_result = renderer.render(
            description, stream=render_warning_stream, **params
        )
        if rendering_result is None:
            is_ok = False

    return warnings, is_ok


def check(
    dists: List[str],
    strict: bool = False,
) -> bool:
    """Check that a distribution will render correctly on PyPI and display the results.

    This is currently only validates ``long_description``, but more checks could be
    added; see https://github.com/pypa/twine/projects/2.

    :param dists:
        The distribution files to check.
    :param output_stream:
        The destination of the resulting output.
    :param strict:
        If ``True``, treat warnings as errors.

    :return:
        ``True`` if there are rendering errors, otherwise ``False``.
    """
    uploads = [i for i in commands._find_dists(dists) if not i.endswith(".asc")]
    if not uploads:  # Return early, if there are no files to check.
        logger.error("No files to check.")
        return False

    failure = False

    for filename in uploads:
        print(f"Checking {filename}: ", end="")
        render_warning_stream = _WarningStream()
        warnings, is_ok = _check_file(filename, render_warning_stream)

        # Print the status and/or error
        if not is_ok:
            failure = True
            print("[red]FAILED[/red]")
            logger.error(
                "`long_description` has syntax errors in markup"
                " and would not be rendered on PyPI."
                f"\n{render_warning_stream}"
            )
        elif warnings:
            if strict:
                failure = True
                print("[red]FAILED due to warnings[/red]")
            else:
                print("[yellow]PASSED with warnings[/yellow]")
        else:
            print("[green]PASSED[/green]")

        # Print warnings after the status and/or error
        for message in warnings:
            logger.warning(message)

    return failure


def main(args: List[str]) -> bool:
    """Execute the ``check`` command.

    :param args:
        The command-line arguments.

    :return:
        The exit status of the ``check`` command.
    """
    parser = argparse.ArgumentParser(prog="twine check")
    parser.add_argument(
        "dists",
        nargs="+",
        metavar="dist",
        help="The distribution files to check, usually dist/*",
    )
    parser.add_argument(
        "--strict",
        action="store_true",
        default=False,
        required=False,
        help="Fail on warnings",
    )

    parsed_args = parser.parse_args(args)

    # Call the check function with the arguments from the command line
    return check(parsed_args.dists, strict=parsed_args.strict)

env_pyrecodes/lib/python3.9/site-packages/twine/commands/check.py
NikolaBlagojevic/pyrecodes
"""Module containing the logic for ``twine upload``."""
# Copyright 2013 Donald Stufft
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import logging
import os.path
from typing import Dict, List, cast

import requests
from rich import print

from twine import commands
from twine import exceptions
from twine import package as package_file
from twine import settings
from twine import utils

logger = logging.getLogger(__name__)


def skip_upload(
    response: requests.Response, skip_existing: bool, package: package_file.PackageFile
) -> bool:
    """Determine if a failed upload is an error or can be safely ignored.

    :param response:
        The response from attempting to upload ``package`` to a repository.
    :param skip_existing:
        If ``True``, use the status and content of ``response`` to determine if the
        package already exists on the repository. If so, then a failed upload is safe
        to ignore.
    :param package:
        The package that was being uploaded.

    :return:
        ``True`` if a failed upload can be safely ignored, otherwise ``False``.
    """
    if not skip_existing:
        return False

    status = response.status_code
    reason = getattr(response, "reason", "").lower()
    text = getattr(response, "text", "").lower()

    # NOTE(sigmavirus24): PyPI presently returns a 400 status code with the
    # error message in the reason attribute. Other implementations return a
    # 403 or 409 status code.
    return (
        # pypiserver (https://pypi.org/project/pypiserver)
        status == 409
        # PyPI / TestPyPI / GCP Artifact Registry
        or (status == 400 and any("already exist" in x for x in [reason, text]))
        # Nexus Repository OSS (https://www.sonatype.com/nexus-repository-oss)
        or (status == 400 and any("updating asset" in x for x in [reason, text]))
        # Artifactory (https://jfrog.com/artifactory/)
        or (status == 403 and "overwrite artifact" in text)
        # Gitlab Enterprise Edition (https://about.gitlab.com)
        or (status == 400 and "already been taken" in text)
    )


def _make_package(
    filename: str, signatures: Dict[str, str], upload_settings: settings.Settings
) -> package_file.PackageFile:
    """Create and sign a package, based off of filename, signatures and settings."""
    package = package_file.PackageFile.from_filename(filename, upload_settings.comment)

    signed_name = package.signed_basefilename
    if signed_name in signatures:
        package.add_gpg_signature(signatures[signed_name], signed_name)
    elif upload_settings.sign:
        package.sign(upload_settings.sign_with, upload_settings.identity)

    file_size = utils.get_file_size(package.filename)
    logger.info(f"{package.filename} ({file_size})")
    if package.gpg_signature:
        logger.info(f"Signed with {package.signed_filename}")

    return package


def upload(upload_settings: settings.Settings, dists: List[str]) -> None:
    """Upload one or more distributions to a repository, and display the progress.

    If a package already exists on the repository, most repositories will return an
    error response. However, if ``upload_settings.skip_existing`` is ``True``, a message
    will be displayed and any remaining distributions will be uploaded.

    For known repositories (like PyPI), the web URLs of successfully uploaded packages
    will be displayed.

    :param upload_settings:
        The configured options related to uploading to a repository.
    :param dists:
        The distribution files to upload to the repository. This can also include
        ``.asc`` files; the GPG signatures will be added to the corresponding uploads.

    :raises twine.exceptions.TwineException:
        The upload failed due to a configuration error.
    :raises requests.HTTPError:
        The repository responded with an error.
    """
    dists = commands._find_dists(dists)
    # Determine if the user has passed in pre-signed distributions
    signatures = {os.path.basename(d): d for d in dists if d.endswith(".asc")}
    uploads = [i for i in dists if not i.endswith(".asc")]

    upload_settings.check_repository_url()
    repository_url = cast(str, upload_settings.repository_config["repository"])
    print(f"Uploading distributions to {repository_url}")

    packages_to_upload = [
        _make_package(filename, signatures, upload_settings) for filename in uploads
    ]

    repository = upload_settings.create_repository()
    uploaded_packages = []

    for package in packages_to_upload:
        skip_message = (
            f"Skipping {package.basefilename} because it appears to already exist"
        )

        # Note: The skip_existing check *needs* to be first, because otherwise
        #       we're going to generate extra HTTP requests against a hardcoded
        #       URL for no reason.
        if upload_settings.skip_existing and repository.package_is_uploaded(package):
            logger.warning(skip_message)
            continue

        resp = repository.upload(package)
        logger.info(f"Response from {resp.url}:\n{resp.status_code} {resp.reason}")
        if resp.text:
            logger.info(resp.text)

        # Bug 92. If we get a redirect we should abort because something seems
        # funky. The behaviour is not well defined and redirects being issued
        # by PyPI should never happen in reality. This should catch malicious
        # redirects as well.
        if resp.is_redirect:
            raise exceptions.RedirectDetected.from_args(
                repository_url,
                resp.headers["location"],
            )

        if skip_upload(resp, upload_settings.skip_existing, package):
            logger.warning(skip_message)
            continue

        utils.check_status_code(resp, upload_settings.verbose)

        uploaded_packages.append(package)

    release_urls = repository.release_urls(uploaded_packages)
    if release_urls:
        print("\n[green]View at:")
        for url in release_urls:
            print(url)

    # Bug 28. Try to silence a ResourceWarning by clearing the connection
    # pool.
    repository.close()


def main(args: List[str]) -> None:
    """Execute the ``upload`` command.

    :param args:
        The command-line arguments.
    """
    parser = argparse.ArgumentParser(prog="twine upload")
    settings.Settings.register_argparse_arguments(parser)
    parser.add_argument(
        "dists",
        nargs="+",
        metavar="dist",
        help="The distribution files to upload to the repository "
        "(package index). Usually dist/* . May additionally contain "
        "a .asc file to include an existing signature with the "
        "file upload.",
    )

    parsed_args = parser.parse_args(args)
    upload_settings = settings.Settings.from_argparse(parsed_args)

    # Call the upload function with the arguments from the command line
    return upload(upload_settings, parsed_args.dists)

env_pyrecodes/lib/python3.9/site-packages/twine/commands/upload.py
NikolaBlagojevic/pyrecodes
# Copyright 2015 Ian Cordasco
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import io
import logging
import os
import re
import subprocess
from typing import Dict, NamedTuple, Optional, Sequence, Tuple, Union

import importlib_metadata
import pkginfo
from rich import print

from twine import exceptions
from twine import wheel
from twine import wininst

DIST_TYPES = {
    "bdist_wheel": wheel.Wheel,
    "bdist_wininst": wininst.WinInst,
    "bdist_egg": pkginfo.BDist,
    "sdist": pkginfo.SDist,
}

DIST_EXTENSIONS = {
    ".whl": "bdist_wheel",
    ".exe": "bdist_wininst",
    ".egg": "bdist_egg",
    ".tar.bz2": "sdist",
    ".tar.gz": "sdist",
    ".zip": "sdist",
}

MetadataValue = Union[str, Sequence[str]]

logger = logging.getLogger(__name__)


def _safe_name(name: str) -> str:
    """Convert an arbitrary string to a standard distribution name.

    Any runs of non-alphanumeric/. characters are replaced with a single '-'.

    Copied from pkg_resources.safe_name for compatibility with warehouse.
    See https://github.com/pypa/twine/issues/743.
    """
    return re.sub("[^A-Za-z0-9.]+", "-", name)


class PackageFile:
    def __init__(
        self,
        filename: str,
        comment: Optional[str],
        metadata: pkginfo.Distribution,
        python_version: Optional[str],
        filetype: Optional[str],
    ) -> None:
        self.filename = filename
        self.basefilename = os.path.basename(filename)
        self.comment = comment
        self.metadata = metadata
        self.python_version = python_version
        self.filetype = filetype
        self.safe_name = _safe_name(metadata.name)
        self.signed_filename = self.filename + ".asc"
        self.signed_basefilename = self.basefilename + ".asc"
        self.gpg_signature: Optional[Tuple[str, bytes]] = None

        hasher = HashManager(filename)
        hasher.hash()
        hexdigest = hasher.hexdigest()

        self.md5_digest = hexdigest.md5
        self.sha2_digest = hexdigest.sha2
        self.blake2_256_digest = hexdigest.blake2

    @classmethod
    def from_filename(cls, filename: str, comment: Optional[str]) -> "PackageFile":
        # Extract the metadata from the package
        for ext, dtype in DIST_EXTENSIONS.items():
            if filename.endswith(ext):
                try:
                    meta = DIST_TYPES[dtype](filename)
                except EOFError:
                    raise exceptions.InvalidDistribution(
                        "Invalid distribution file: '%s'" % os.path.basename(filename)
                    )
                else:
                    break
        else:
            raise exceptions.InvalidDistribution(
                "Unknown distribution format: '%s'" % os.path.basename(filename)
            )

        # If pkginfo encounters a metadata version it doesn't support, it may give us
        # back empty metadata. At the very least, we should have a name and version,
        # which could also be empty if, for example, a MANIFEST.in doesn't include
        # setup.cfg.
        missing_fields = [
            f.capitalize() for f in ["name", "version"] if not getattr(meta, f)
        ]
        if missing_fields:
            supported_metadata = list(pkginfo.distribution.HEADER_ATTRS)
            raise exceptions.InvalidDistribution(
                "Metadata is missing required fields: "
                f"{', '.join(missing_fields)}.\n"
                "Make sure the distribution includes the files where those fields "
                "are specified, and is using a supported Metadata-Version: "
                f"{', '.join(supported_metadata)}."
            )

        py_version: Optional[str]
        if dtype == "bdist_egg":
            (dist,) = importlib_metadata.Distribution.discover(  # type: ignore[no-untyped-call] # python/importlib_metadata#288  # noqa: E501
                path=[filename]
            )
            py_version = dist.metadata["Version"]
        elif dtype == "bdist_wheel":
            py_version = meta.py_version
        elif dtype == "bdist_wininst":
            py_version = meta.py_version
        else:
            py_version = None

        return cls(filename, comment, meta, py_version, dtype)

    def metadata_dictionary(self) -> Dict[str, MetadataValue]:
        """Merge multiple sources of metadata into a single dictionary.

        Includes values from filename, PKG-INFO, hashers, and signature.
        """
        meta = self.metadata
        data = {
            # identify release
            "name": self.safe_name,
            "version": meta.version,
            # file content
            "filetype": self.filetype,
            "pyversion": self.python_version,
            # additional meta-data
            "metadata_version": meta.metadata_version,
            "summary": meta.summary,
            "home_page": meta.home_page,
            "author": meta.author,
            "author_email": meta.author_email,
            "maintainer": meta.maintainer,
            "maintainer_email": meta.maintainer_email,
            "license": meta.license,
            "description": meta.description,
            "keywords": meta.keywords,
            "platform": meta.platforms,
            "classifiers": meta.classifiers,
            "download_url": meta.download_url,
            "supported_platform": meta.supported_platforms,
            "comment": self.comment,
            "sha256_digest": self.sha2_digest,
            # PEP 314
            "provides": meta.provides,
            "requires": meta.requires,
            "obsoletes": meta.obsoletes,
            # Metadata 1.2
            "project_urls": meta.project_urls,
            "provides_dist": meta.provides_dist,
            "obsoletes_dist": meta.obsoletes_dist,
            "requires_dist": meta.requires_dist,
            "requires_external": meta.requires_external,
            "requires_python": meta.requires_python,
            # Metadata 2.1
            "provides_extras": meta.provides_extras,
            "description_content_type": meta.description_content_type,
            # Metadata 2.2
            "dynamic": meta.dynamic,
        }

        if self.gpg_signature is not None:
            data["gpg_signature"] = self.gpg_signature

        # FIPS disables MD5 and Blake2, making the digest values None. Some package
        # repositories don't allow null values, so this only sends non-null values.
        # See also: https://github.com/pypa/twine/issues/775
        if self.md5_digest:
            data["md5_digest"] = self.md5_digest

        if self.blake2_256_digest:
            data["blake2_256_digest"] = self.blake2_256_digest

        return data

    def add_gpg_signature(
        self, signature_filepath: str, signature_filename: str
    ) -> None:
        if self.gpg_signature is not None:
            raise exceptions.InvalidDistribution("GPG Signature can only be added once")

        with open(signature_filepath, "rb") as gpg:
            self.gpg_signature = (signature_filename, gpg.read())

    def sign(self, sign_with: str, identity: Optional[str]) -> None:
        print(f"Signing {self.basefilename}")
        gpg_args: Tuple[str, ...] = (sign_with, "--detach-sign")
        if identity:
            gpg_args += ("--local-user", identity)
        gpg_args += ("-a", self.filename)
        self.run_gpg(gpg_args)

        self.add_gpg_signature(self.signed_filename, self.signed_basefilename)

    @classmethod
    def run_gpg(cls, gpg_args: Tuple[str, ...]) -> None:
        try:
            subprocess.check_call(gpg_args)
            return
        except FileNotFoundError:
            if gpg_args[0] != "gpg":
                raise exceptions.InvalidSigningExecutable(
                    f"{gpg_args[0]} executable not available."
                )

        logger.warning("gpg executable not available. Attempting fallback to gpg2.")
        try:
            subprocess.check_call(("gpg2",) + gpg_args[1:])
        except FileNotFoundError:
            raise exceptions.InvalidSigningExecutable(
                "'gpg' or 'gpg2' executables not available.\n"
                "Try installing one of these or specifying an executable "
                "with the --sign-with flag."
            )


class Hexdigest(NamedTuple):
    md5: Optional[str]
    sha2: Optional[str]
    blake2: Optional[str]


class HashManager:
    """Manage our hashing objects for simplicity.

    This will also allow us to better test this logic.
    """

    def __init__(self, filename: str) -> None:
        """Initialize our manager and hasher objects."""
        self.filename = filename

        self._md5_hasher = None
        try:
            self._md5_hasher = hashlib.md5()
        except ValueError:
            # FIPs mode disables MD5
            pass

        self._sha2_hasher = hashlib.sha256()

        self._blake_hasher = None
        try:
            self._blake_hasher = hashlib.blake2b(digest_size=256 // 8)
        except (ValueError, TypeError):
            # FIPS mode disables blake2
            pass

    def _md5_update(self, content: bytes) -> None:
        if self._md5_hasher is not None:
            self._md5_hasher.update(content)

    def _md5_hexdigest(self) -> Optional[str]:
        if self._md5_hasher is not None:
            return self._md5_hasher.hexdigest()
        return None

    def _sha2_update(self, content: bytes) -> None:
        if self._sha2_hasher is not None:
            self._sha2_hasher.update(content)

    def _sha2_hexdigest(self) -> Optional[str]:
        if self._sha2_hasher is not None:
            return self._sha2_hasher.hexdigest()
        return None

    def _blake_update(self, content: bytes) -> None:
        if self._blake_hasher is not None:
            self._blake_hasher.update(content)

    def _blake_hexdigest(self) -> Optional[str]:
        if self._blake_hasher is not None:
            return self._blake_hasher.hexdigest()
        return None

    def hash(self) -> None:
        """Hash the file contents."""
        with open(self.filename, "rb") as fp:
            for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b""):
                self._md5_update(content)
                self._sha2_update(content)
                self._blake_update(content)

    def hexdigest(self) -> Hexdigest:
        """Return the hexdigest for the file."""
        return Hexdigest(
            self._md5_hexdigest(),
            self._sha2_hexdigest(),
            self._blake_hexdigest(),
        )

env_pyrecodes/lib/python3.9/site-packages/twine/package.py
NikolaBlagojevic/pyrecodes
"""Module for URI Template expansion."""

from __future__ import annotations

from .expansions import ExpansionFailedError
from .uritemplate import ExpansionInvalidError, ExpansionReservedError, URITemplate
from .variable import Variable, VariableInvalidError


__all__ = (
    'URITemplate',
    'Variable',
    'ExpansionInvalidError',
    'ExpansionReservedError',
    'VariableInvalidError',
    'ExpansionFailedError',
)


def expand(template: str, **kwargs) -> (str | None):
    try:
        templ = URITemplate(template)
        return templ.expand(**kwargs)
    except Exception:
        return None


def partial(template: str, **kwargs) -> (str | None):
    try:
        templ = URITemplate(template)
        return str(templ.partial(**kwargs))
    except Exception:
        return None


def validate(template: str) -> bool:
    try:
        URITemplate(template)
        return True
    except Exception:
        return False

env_pyrecodes/lib/python3.9/site-packages/uri_template/__init__.py
NikolaBlagojevic/pyrecodes
"""Character sets."""

from __future__ import annotations


class Charset:
    """Define character sets used in other classes."""

    ALPHA = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
    DIGIT = '0123456789'
    HEX_DIGIT = '0123456789ABCDEFabcdef'
    GEN_DELIMS = ':/?#[]@'
    SUB_DELIMS = "!$&'()*+,;="
    UNRESERVED = ALPHA + DIGIT + '-._~'
    RESERVED = GEN_DELIMS + SUB_DELIMS
    VAR_START = ALPHA + DIGIT + '_'
    VAR_CHAR = VAR_START + '.'

env_pyrecodes/lib/python3.9/site-packages/uri_template/charset.py
NikolaBlagojevic/pyrecodes
"""Process URI templates per http://tools.ietf.org/html/rfc6570."""

from __future__ import annotations

import collections
from typing import Any, TYPE_CHECKING, cast

from .charset import Charset
from .variable import Variable

if (TYPE_CHECKING):
    from collections.abc import Iterable, Mapping


class ExpansionFailedError(Exception):
    """Exception thrown when expansions fail."""

    variable: str

    def __init__(self, variable: str) -> None:
        self.variable = variable

    def __str__(self) -> str:
        """Convert to string."""
        return 'Bad expansion: ' + self.variable


class Expansion:
    """
    Base class for template expansions.

    https://tools.ietf.org/html/rfc6570#section-3
    """

    def __init__(self) -> None:
        pass

    @property
    def variables(self) -> Iterable[Variable]:
        """Get all variables in this expansion."""
        return []

    @property
    def variable_names(self) -> Iterable[str]:
        """Get the names of all variables in this expansion."""
        return []

    def _encode(self, value: str, legal: str, pct_encoded: bool) -> str:
        """Encode a string into legal values."""
        output = ''
        index = 0
        while (index < len(value)):
            codepoint = value[index]
            if (codepoint in legal):
                output += codepoint
            elif (pct_encoded and ('%' == codepoint)
                  and ((index + 2) < len(value))
                  and (value[index + 1] in Charset.HEX_DIGIT)
                  and (value[index + 2] in Charset.HEX_DIGIT)):
                output += value[index:index + 3]
                index += 2
            else:
                utf8 = codepoint.encode('utf8')
                for byte in utf8:
                    output += '%' + Charset.HEX_DIGIT[int(byte / 16)] + Charset.HEX_DIGIT[byte % 16]
            index += 1
        return output

    def _uri_encode_value(self, value: str) -> str:
        """Encode a value into uri encoding."""
        return self._encode(value, Charset.UNRESERVED, False)

    def _uri_encode_name(self, name: (str | int)) -> str:
        """Encode a variable name into uri encoding."""
        return self._encode(str(name), Charset.UNRESERVED + Charset.RESERVED, True) if (name) else ''

    def _join(self, prefix: str, joiner: str, value: str) -> str:
        """Join a prefix to a value."""
        if (prefix):
            return prefix + joiner + value
        return value

    def _encode_str(self, variable: Variable, name: str, value: str, prefix: str, joiner: str, first: bool) -> str:
        """Encode a string value for a variable."""
        if (variable.max_length):
            if (not first):
                raise ExpansionFailedError(str(variable))
            return self._join(prefix, joiner, self._uri_encode_value(value[:variable.max_length]))
        return self._join(prefix, joiner, self._uri_encode_value(value))

    def _encode_dict_item(self, variable: Variable, name: str, key: (int | str), item: Any,
                          delim: str, prefix: str, joiner: str, first: bool) -> (str | None):
        """Encode a dict item for a variable."""
        joiner = '=' if (variable.explode) else ','
        if (variable.array):
            name = self._uri_encode_name(key)
            prefix = (prefix + '[' + name + ']') if (prefix and not first) else name
        else:
            prefix = self._join(prefix, '.', self._uri_encode_name(key))
        return self._encode_var(variable, str(key), item, delim, prefix, joiner, False)

    def _encode_list_item(self, variable: Variable, name: str, index: int, item: Any,
                          delim: str, prefix: str, joiner: str, first: bool) -> (str | None):
        """Encode a list item for a variable."""
        if (variable.array):
            prefix = prefix + '[' + str(index) + ']' if (prefix) else ''
            return self._encode_var(variable, '', item, delim, prefix, joiner, False)
        return self._encode_var(variable, name, item, delim, prefix, '.', False)

    def _encode_var(self, variable: Variable, name: str, value: Any,
                    delim: str = ',', prefix: str = '', joiner: str = '=', first: bool = True) -> (str | None):
        """Encode a variable."""
        if (isinstance(value, str)):
            return self._encode_str(variable, name, value, prefix, joiner, first)
        elif (isinstance(value, collections.abc.Mapping)):
            if (len(value)):
                encoded_items = [self._encode_dict_item(variable, name, key, value[key], delim, prefix, joiner, first)
                                 for key in value.keys()]
                return delim.join([item for item in encoded_items if (item is not None)])
            return None
        elif (isinstance(value, collections.abc.Sequence)):
            if (len(value)):
                encoded_items = [self._encode_list_item(variable, name, index, item, delim, prefix, joiner, first)
                                 for index, item in enumerate(value)]
                return delim.join([item for item in encoded_items if (item is not None)])
            return None
        elif (isinstance(value, bool)):
            return self._encode_str(variable, name, str(value).lower(), prefix, joiner, first)
        else:
            return self._encode_str(variable, name, str(value), prefix, joiner, first)

    def expand(self, values: Mapping[str, Any]) -> (str | None):
        """Expand values."""
        return None

    def partial(self, values: Mapping[str, Any]) -> str:
        """Perform partial expansion."""
        return ''


class Literal(Expansion):
    """
    A literal expansion.

    https://tools.ietf.org/html/rfc6570#section-3.1
    """

    value: str

    def __init__(self, value: str) -> None:
        super().__init__()
        self.value = value

    def expand(self, values: Mapping[str, Any]) -> (str | None):
        """Perform exansion."""
        return self._encode(self.value, (Charset.UNRESERVED + Charset.RESERVED), True)

    def __str__(self) -> str:
        """Convert to string."""
        return self.value


class ExpressionExpansion(Expansion):
    """
    Base class for expression expansions.

    https://tools.ietf.org/html/rfc6570#section-3.2
    """

    operator = ''
    partial_operator = ','
    output_prefix = ''
    var_joiner = ','
    partial_joiner = ','

    vars: list[Variable]
    trailing_joiner: str = ''

    def __init__(self, variables: str) -> None:
        super().__init__()
        if (variables and (variables[-1] in (',', '.', '/', ';', '&'))):
            self.trailing_joiner = variables[-1]
            variables = variables[:-1]
        self.vars = [Variable(var) for var in variables.split(',')]

    @property
    def variables(self) -> Iterable[Variable]:
        """Get all variables."""
        return list(self.vars)

    @property
    def variable_names(self) -> Iterable[str]:
        """Get names of all variables."""
        return [var.name for var in self.vars]

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        return self._encode_var(variable, self._uri_encode_name(variable.name), value)

    def expand(self, values: Mapping[str, Any]) -> (str | None):
        """Expand all variables, skip missing values."""
        expanded_vars: list[str] = []
        for var in self.vars:
            value = values.get(var.key, var.default)
            if (value is not None):
                expanded_var = self._expand_var(var, value)
                if (expanded_var is not None):
                    expanded_vars.append(expanded_var)
        if (expanded_vars):
            return ((self.output_prefix if (not self.trailing_joiner) else '') + self.var_joiner.join(expanded_vars)
                    + self.trailing_joiner)
        return None

    def partial(self, values: Mapping[str, Any]) -> str:
        """Expand all variables, replace missing values with expansions."""
        expanded_vars: list[str] = []
        missing_vars: list[Variable] = []
        result: list[tuple[(list[str] | None), (list[Variable] | None)]] = []
        for var in self.vars:
            value = values.get(var.name, var.default)
            if (value is not None):
                expanded_var = self._expand_var(var, value)
                if (expanded_var is not None):
                    if (missing_vars):
                        result.append((None, missing_vars))
                        missing_vars = []
                    expanded_vars.append(expanded_var)
            else:
                if (expanded_vars):
                    result.append((expanded_vars, None))
                    expanded_vars = []
                missing_vars.append(var)
        if (expanded_vars):
            result.append((expanded_vars, None))
        if (missing_vars):
            result.append((None, missing_vars))

        output: str = ''
        first = True
        for index, (expanded, missing) in enumerate(result):
            last = (index == (len(result) - 1))
            if (expanded):
                output += ((self.output_prefix if (first and (not self.trailing_joiner)) else '')
                           + self.var_joiner.join(expanded) + self.trailing_joiner)
            else:
                output += ((self.output_prefix if (first and not last) else (self.var_joiner if (not last) else ''))
                           + '{' + (self.operator if (first) else self.partial_operator)
                           + ','.join([str(var) for var in cast('list[Variable]', missing)])
                           + (self.partial_joiner if (not last) else '') + '}')
            first = False
        return output

    def __str__(self) -> str:
        """Convert to string."""
        return ('{' + self.operator + ','.join([str(var) for var in self.vars]) + self.trailing_joiner + '}')


class SimpleExpansion(ExpressionExpansion):
    """
    Simple String expansion {var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.2

    """

    def __init__(self, variables: str) -> None:
        super().__init__(variables)


class ReservedExpansion(ExpressionExpansion):
    """
    Reserved Expansion {+var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.3
    """

    operator = '+'
    partial_operator = ',+'

    def __init__(self, variables: str) -> None:
        super().__init__(variables[1:])

    def _uri_encode_value(self, value: str) -> str:
        """Encode a value into uri encoding."""
        return self._encode(value, (Charset.UNRESERVED + Charset.RESERVED), True)


class FragmentExpansion(ReservedExpansion):
    """
    Fragment Expansion {#var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.4
    """

    operator = '#'
    output_prefix = '#'

    def __init__(self, variables: str) -> None:
        super().__init__(variables)


class LabelExpansion(ExpressionExpansion):
    """
    Label Expansion with Dot-Prefix {.var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.5
    """

    operator = '.'
    partial_operator = '.'
    output_prefix = '.'
    var_joiner = '.'
    partial_joiner = '.'

    def __init__(self, variables: str) -> None:
        super().__init__(variables[1:])

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        return self._encode_var(variable, self._uri_encode_name(variable.name), value,
                                delim=('.' if variable.explode else ','))


class PathExpansion(ExpressionExpansion):
    """
    Path Segment Expansion {/var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.6
    """

    operator = '/'
    partial_operator = '/'
    output_prefix = '/'
    var_joiner = '/'
    partial_joiner = '/'

    def __init__(self, variables: str) -> None:
        super().__init__(variables[1:])

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        return self._encode_var(variable, self._uri_encode_name(variable.name), value,
                                delim=('/' if variable.explode else ','))


class PathStyleExpansion(ExpressionExpansion):
    """
    Path-Style Parameter Expansion {;var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.7
    """

    operator = ';'
    partial_operator = ';'
    output_prefix = ';'
    var_joiner = ';'
    partial_joiner = ';'

    def __init__(self, variables: str) -> None:
        super().__init__(variables[1:])

    def _encode_str(self, variable: Variable, name: str, value: Any, prefix: str, joiner: str, first: bool) -> str:
        """Encode a string for a variable."""
        if (variable.array):
            if (name):
                prefix = prefix + '[' + name + ']' if (prefix) else name
        elif (variable.explode):
            prefix = self._join(prefix, '.', name)
        return super()._encode_str(variable, name, value, prefix, joiner, first)

    def _encode_dict_item(self, variable: Variable, name: str, key: (int | str), item: Any,
                          delim: str, prefix: str, joiner: str, first: bool) -> (str | None):
        """Encode a dict item for a variable."""
        if (variable.array):
            if (name):
                prefix = prefix + '[' + name + ']' if (prefix) else name
            if (prefix and not first):
                prefix = (prefix + '[' + self._uri_encode_name(key) + ']')
            else:
                prefix = self._uri_encode_name(key)
        elif (variable.explode):
            prefix = self._join(prefix, '.', name) if (not first) else ''
        else:
            prefix = self._join(prefix, '.', self._uri_encode_name(key))
            joiner = ','
        return self._encode_var(variable, self._uri_encode_name(key) if (not variable.array) else '', item,
                                delim, prefix, joiner, False)

    def _encode_list_item(self, variable: Variable, name: str, index: int, item: Any,
                          delim: str, prefix: str, joiner: str, first: bool) -> (str | None):
        """Encode a list item for a variable."""
        if (variable.array):
            if (name):
                prefix = prefix + '[' + name + ']' if (prefix) else name
            return self._encode_var(variable, str(index), item, delim, prefix, joiner, False)
        return self._encode_var(variable, name, item, delim, prefix, '=' if (variable.explode) else '.', False)

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        if (variable.explode):
            return self._encode_var(variable, self._uri_encode_name(variable.name), value, delim=';')
        value = self._encode_var(variable, self._uri_encode_name(variable.name), value, delim=',')
        return (self._uri_encode_name(variable.name) + '=' + value) if (value) else variable.name


class FormStyleQueryExpansion(PathStyleExpansion):
    """
    Form-Style Query Expansion {?var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.8
    """

    operator = '?'
    partial_operator = '&'
    output_prefix = '?'
    var_joiner = '&'
    partial_joiner = '&'

    def __init__(self, variables: str) -> None:
        super().__init__(variables)

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        if (variable.explode):
            return self._encode_var(variable, self._uri_encode_name(variable.name), value, delim='&')
        value = self._encode_var(variable, self._uri_encode_name(variable.name), value, delim=',')
        return (self._uri_encode_name(variable.name) + '=' + value) if (value is not None) else None


class FormStyleQueryContinuation(FormStyleQueryExpansion):
    """
    Form-Style Query Continuation {&var}.

    https://tools.ietf.org/html/rfc6570#section-3.2.9
    """

    operator = '&'
    output_prefix = '&'

    def __init__(self, variables: str) -> None:
        super().__init__(variables)

# non-standard extension


class CommaExpansion(ExpressionExpansion):
    """
    Label Expansion with Comma-Prefix {,var}.

    Non-standard extension to support partial expansions.
    """

    operator = ','
    output_prefix = ','

    def __init__(self, variables: str) -> None:
        super().__init__(variables[1:])

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        return self._encode_var(variable, self._uri_encode_name(variable.name), value,
                                delim=('.' if variable.explode else ','))


class ReservedCommaExpansion(ReservedExpansion):
    """
    Reserved Expansion with comma prefix {,+var}.

    Non-standard extension to support partial expansions.
    """

    operator = ',+'
    output_prefix = ','

    def __init__(self, variables: str) -> None:
        super().__init__(variables[1:])

    def _expand_var(self, variable: Variable, value: Any) -> (str | None):
        """Expand a single variable."""
        return self._encode_var(variable, self._uri_encode_name(variable.name), value,
                                delim=('.' if variable.explode else ','))

env_pyrecodes/lib/python3.9/site-packages/uri_template/expansions.py
NikolaBlagojevic/pyrecodes
"""Process URI templates per http://tools.ietf.org/html/rfc6570."""

from __future__ import annotations

import re
from typing import TYPE_CHECKING

from .expansions import (CommaExpansion, Expansion,
                         FormStyleQueryContinuation, FormStyleQueryExpansion,
                         FragmentExpansion, LabelExpansion, Literal,
                         PathExpansion, PathStyleExpansion,
                         ReservedCommaExpansion, ReservedExpansion, SimpleExpansion)

if (TYPE_CHECKING):
    from collections.abc import Iterable
    from .variable import Variable


class ExpansionReservedError(Exception):
    """Exception thrown for reserved but unsupported expansions."""

    expansion: str

    def __init__(self, expansion: str) -> None:
        self.expansion = expansion

    def __str__(self) -> str:
        """Convert to string."""
        return 'Unsupported expansion: ' + self.expansion


class ExpansionInvalidError(Exception):
    """Exception thrown for unknown expansions."""

    expansion: str

    def __init__(self, expansion: str) -> None:
        self.expansion = expansion

    def __str__(self) -> str:
        """Convert to string."""
        return 'Bad expansion: ' + self.expansion


class URITemplate:
    """
    URI Template object.

    Constructor may raise ExpansionReservedError, ExpansionInvalidError, or VariableInvalidError.
    """

    expansions: list[Expansion]

    def __init__(self, template: str) -> None:
        self.expansions = []
        parts = re.split(r'(\{[^\}]*\})', template)
        for part in parts:
            if (part):
                if (('{' == part[0]) and ('}' == part[-1])):
                    expansion = part[1:-1]
                    if (re.match('^([a-zA-Z0-9_]|%[0-9a-fA-F][0-9a-fA-F]).*$', expansion)):
                        self.expansions.append(SimpleExpansion(expansion))
                    elif ('+' == part[1]):
                        self.expansions.append(ReservedExpansion(expansion))
                    elif ('#' == part[1]):
                        self.expansions.append(FragmentExpansion(expansion))
                    elif ('.' == part[1]):
                        self.expansions.append(LabelExpansion(expansion))
                    elif ('/' == part[1]):
                        self.expansions.append(PathExpansion(expansion))
                    elif (';' == part[1]):
                        self.expansions.append(PathStyleExpansion(expansion))
                    elif ('?' == part[1]):
                        self.expansions.append(FormStyleQueryExpansion(expansion))
                    elif ('&' == part[1]):
                        self.expansions.append(FormStyleQueryContinuation(expansion))
                    elif (',' == part[1]):
                        if ((1 < len(part)) and ('+' == part[2])):
                            self.expansions.append(ReservedCommaExpansion(expansion))
                        else:
                            self.expansions.append(CommaExpansion(expansion))
                    elif (part[1] in '=!@|'):
                        raise ExpansionReservedError(part)
                    else:
                        raise ExpansionInvalidError(part)
                else:
                    if (('{' not in part) and ('}' not in part)):
                        self.expansions.append(Literal(part))
                    else:
                        raise ExpansionInvalidError(part)

    @property
    def variables(self) -> Iterable[Variable]:
        """Get all variables in template."""
        vars: dict[str, Variable] = {}
        for expansion in self.expansions:
            for var in expansion.variables:
                vars[var.name] = var
        return vars.values()

    @property
    def variable_names(self) -> Iterable[str]:
        """Get names of all variables in template."""
        vars: dict[str, Variable] = {}
        for expansion in self.expansions:
            for var in expansion.variables:
                vars[var.name] = var
        return [var.name for var in vars.values()]

    def expand(self, **kwargs) -> str:
        """
        Expand the template.

        May raise ExpansionFailed if a composite value is passed to a variable with a prefix modifier.
        """
        expanded = [expansion.expand(kwargs) for expansion in self.expansions]
        return ''.join([expansion for expansion in expanded if (expansion is not None)])

    def partial(self, **kwargs) -> URITemplate:
        """
        Expand the template, preserving expansions for missing variables.

        May raise ExpansionFailed if a composite value is passed to a variable with a prefix modifier.
        """
        expanded = [expansion.partial(kwargs) for expansion in self.expansions]
        return URITemplate(''.join(expanded))

    @property
    def expanded(self) -> bool:
        """Determine if template is fully expanded."""
        return (str(self) == self.expand())

    def __str__(self) -> str:
        """Convert to string, returns original template."""
        return ''.join([str(expansion) for expansion in self.expansions])

    def __repr__(self) -> str:
        """Convert to string, returns original template."""
        return str(self)

env_pyrecodes/lib/python3.9/site-packages/uri_template/uritemplate.py
NikolaBlagojevic/pyrecodes
"""Variable class for URITemplate."""

from __future__ import annotations

from .charset import Charset


class VariableInvalidError(Exception):
    """Exception thrown for invalid variables."""

    variable: str

    def __init__(self, variable: str) -> None:
        self.variable = variable

    def __str__(self) -> str:
        """Convert to string."""
        return 'Bad variable: ' + self.variable


class Variable:
    """
    A template variable.

    https://tools.ietf.org/html/rfc6570#section-2.3
    """

    name: str
    key: str
    max_length: int
    explode: bool
    array: bool
    default: (str | None)

    def __init__(self, var_spec: str) -> None:
        self.name = ''
        self.key = ''
        self.max_length = 0
        self.explode = False
        self.array = False
        self.default = None

        if (var_spec[0:1] not in Charset.VAR_START):
            raise VariableInvalidError(var_spec)

        if ('=' in var_spec):
            var_spec, self.default = var_spec.split('=', 1)

        if (':' in var_spec):
            var_spec, max_length = var_spec.split(':', 1)
            if ((0 < len(max_length)) and (len(max_length) < 4)):
                for digit in max_length:
                    if (digit not in Charset.DIGIT):
                        raise VariableInvalidError(var_spec + ':' + max_length)
                self.max_length = int(max_length)
                if (not self.max_length):
                    raise VariableInvalidError(var_spec + ':' + max_length)
            else:
                raise VariableInvalidError(var_spec + ':' + max_length)
        elif ('*' == var_spec[-1]):
            var_spec = var_spec[:-1]
            self.explode = True
        elif ('[]' == var_spec[-2:]):
            var_spec = var_spec[:-2]
            self.array = True
            self.explode = True

        index = 0
        while (index < len(var_spec)):
            codepoint = var_spec[index]
            if (('%' == codepoint)
                    and ((index + 2) < len(var_spec))
                    and (var_spec[index + 1] in Charset.HEX_DIGIT)
                    and (var_spec[index + 2] in Charset.HEX_DIGIT)):
                self.key += var_spec[index:index + 3]
                index += 2
            elif (codepoint in Charset.VAR_CHAR):
                self.key += codepoint
            elif ('/' == codepoint):
                self.name = self.key
                self.key = ''
            else:
                raise VariableInvalidError(var_spec + ((':' + str(self.max_length)) if (self.max_length) else '')
                                           + ('[]' if (self.array) else ('*' if (self.explode) else '')))
            index += 1

        self.name = (self.name or self.key)
        self.key = (self.key or self.name)

    def __str__(self) -> str:
        """Convert to string."""
        return (self.name + (f'/{self.key}' if (self.key and (self.key != self.name)) else '')
                + (f':{self.max_length}' if (self.max_length) else '')
                + ('*' if (self.explode and not self.array) else '') + ('[]' if (self.array) else '')
                + (f'={self.default}' if (self.default is not None) else ''))

env_pyrecodes/lib/python3.9/site-packages/uri_template/variable.py
NikolaBlagojevic/pyrecodes
import array
import os
import struct
import sys

from threading import Lock
from typing import Callable, Union

from ._exceptions import *
from ._utils import validate_utf8

"""
_abnf.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

try:
    # If wsaccel is available, use compiled routines to mask data.
    # wsaccel only provides around a 10% speed boost compared
    # to the websocket-client _mask() implementation.
    # Note that wsaccel is unmaintained.
    from wsaccel.xormask import XorMaskerSimple

    def _mask(_m, _d) -> bytes:
        return XorMaskerSimple(_m).process(_d)

except ImportError:
    # wsaccel is not available, use websocket-client _mask()
    native_byteorder = sys.byteorder

    def _mask(mask_value: array.array, data_value: array.array) -> bytes:
        datalen = len(data_value)
        int_data_value = int.from_bytes(data_value, native_byteorder)
        int_mask_value = int.from_bytes(mask_value * (datalen // 4) + mask_value[: datalen % 4], native_byteorder)
        return (int_data_value ^ int_mask_value).to_bytes(datalen, native_byteorder)


__all__ = [
    'ABNF', 'continuous_frame', 'frame_buffer',
    'STATUS_NORMAL',
    'STATUS_GOING_AWAY',
    'STATUS_PROTOCOL_ERROR',
    'STATUS_UNSUPPORTED_DATA_TYPE',
    'STATUS_STATUS_NOT_AVAILABLE',
    'STATUS_ABNORMAL_CLOSED',
    'STATUS_INVALID_PAYLOAD',
    'STATUS_POLICY_VIOLATION',
    'STATUS_MESSAGE_TOO_BIG',
    'STATUS_INVALID_EXTENSION',
    'STATUS_UNEXPECTED_CONDITION',
    'STATUS_BAD_GATEWAY',
    'STATUS_TLS_HANDSHAKE_ERROR',
]

# closing frame status codes.
STATUS_NORMAL = 1000
STATUS_GOING_AWAY = 1001
STATUS_PROTOCOL_ERROR = 1002
STATUS_UNSUPPORTED_DATA_TYPE = 1003
STATUS_STATUS_NOT_AVAILABLE = 1005
STATUS_ABNORMAL_CLOSED = 1006
STATUS_INVALID_PAYLOAD = 1007
STATUS_POLICY_VIOLATION = 1008
STATUS_MESSAGE_TOO_BIG = 1009
STATUS_INVALID_EXTENSION = 1010
STATUS_UNEXPECTED_CONDITION = 1011
STATUS_SERVICE_RESTART = 1012
STATUS_TRY_AGAIN_LATER = 1013
STATUS_BAD_GATEWAY = 1014
STATUS_TLS_HANDSHAKE_ERROR = 1015

VALID_CLOSE_STATUS = (
    STATUS_NORMAL,
    STATUS_GOING_AWAY,
    STATUS_PROTOCOL_ERROR,
    STATUS_UNSUPPORTED_DATA_TYPE,
    STATUS_INVALID_PAYLOAD,
    STATUS_POLICY_VIOLATION,
    STATUS_MESSAGE_TOO_BIG,
    STATUS_INVALID_EXTENSION,
    STATUS_UNEXPECTED_CONDITION,
    STATUS_SERVICE_RESTART,
    STATUS_TRY_AGAIN_LATER,
    STATUS_BAD_GATEWAY,
)


class ABNF:
    """
    ABNF frame class.
    See http://tools.ietf.org/html/rfc5234
    and http://tools.ietf.org/html/rfc6455#section-5.2
    """

    # operation code values.
    OPCODE_CONT = 0x0
    OPCODE_TEXT = 0x1
    OPCODE_BINARY = 0x2
    OPCODE_CLOSE = 0x8
    OPCODE_PING = 0x9
    OPCODE_PONG = 0xa

    # available operation code value tuple
    OPCODES = (OPCODE_CONT, OPCODE_TEXT, OPCODE_BINARY, OPCODE_CLOSE,
               OPCODE_PING, OPCODE_PONG)

    # opcode human readable string
    OPCODE_MAP = {
        OPCODE_CONT: "cont",
        OPCODE_TEXT: "text",
        OPCODE_BINARY: "binary",
        OPCODE_CLOSE: "close",
        OPCODE_PING: "ping",
        OPCODE_PONG: "pong"
    }

    # data length threshold.
    LENGTH_7 = 0x7e
    LENGTH_16 = 1 << 16
    LENGTH_63 = 1 << 63

    def __init__(self, fin: int = 0, rsv1: int = 0, rsv2: int = 0, rsv3: int = 0,
                 opcode: int = OPCODE_TEXT, mask: int = 1, data: Union[str, bytes] = "") -> None:
        """
        Constructor for ABNF. Please check RFC for arguments.
        """
        self.fin = fin
        self.rsv1 = rsv1
        self.rsv2 = rsv2
        self.rsv3 = rsv3
        self.opcode = opcode
        self.mask = mask
        if data is None:
            data = ""
        self.data = data
        self.get_mask_key = os.urandom

    def validate(self, skip_utf8_validation: bool = False) -> None:
        """
        Validate the ABNF frame.

        Parameters
        ----------
        skip_utf8_validation: skip utf8 validation.
        """
        if self.rsv1 or self.rsv2 or self.rsv3:
            raise WebSocketProtocolException("rsv is not implemented, yet")

        if self.opcode not in ABNF.OPCODES:
            raise WebSocketProtocolException("Invalid opcode %r", self.opcode)

        if self.opcode == ABNF.OPCODE_PING and not self.fin:
            raise WebSocketProtocolException("Invalid ping frame.")

        if self.opcode == ABNF.OPCODE_CLOSE:
            l = len(self.data)
            if not l:
                return
            if l == 1 or l >= 126:
                raise WebSocketProtocolException("Invalid close frame.")
            if l > 2 and not skip_utf8_validation and not validate_utf8(self.data[2:]):
                raise WebSocketProtocolException("Invalid close frame.")

            code = 256 * self.data[0] + self.data[1]
            if not self._is_valid_close_status(code):
                raise WebSocketProtocolException("Invalid close opcode %r", code)

    @staticmethod
    def _is_valid_close_status(code: int) -> bool:
        return code in VALID_CLOSE_STATUS or (3000 <= code < 5000)

    def __str__(self) -> str:
        return "fin=" + str(self.fin) \
            + " opcode=" + str(self.opcode) \
            + " data=" + str(self.data)

    @staticmethod
    def create_frame(data: Union[bytes, str], opcode: int, fin: int = 1) -> 'ABNF':
        """
        Create frame to send text, binary and other data.

        Parameters
        ----------
        data: str
            data to send. This is string value(byte array).
            If opcode is OPCODE_TEXT and this value is unicode,
            data value is converted into unicode string, automatically.
        opcode: int
            operation code. please see OPCODE_MAP.
        fin: int
            fin flag. if set to 0, create continue fragmentation.
        """
        if opcode == ABNF.OPCODE_TEXT and isinstance(data, str):
            data = data.encode("utf-8")
        # mask must be set if send data from client
        return ABNF(fin, 0, 0, 0, opcode, 1, data)

    def format(self) -> bytes:
        """
        Format this object to string(byte array) to send data to server.
        """
        if any(x not in (0, 1) for x in [self.fin, self.rsv1, self.rsv2, self.rsv3]):
            raise ValueError("not 0 or 1")
        if self.opcode not in ABNF.OPCODES:
            raise ValueError("Invalid OPCODE")
        length = len(self.data)
        if length >= ABNF.LENGTH_63:
            raise ValueError("data is too long")

        frame_header = chr(self.fin << 7 |
                           self.rsv1 << 6 | self.rsv2 << 5 | self.rsv3 << 4 |
                           self.opcode).encode('latin-1')
        if length < ABNF.LENGTH_7:
            frame_header += chr(self.mask << 7 | length).encode('latin-1')
        elif length < ABNF.LENGTH_16:
            frame_header += chr(self.mask << 7 | 0x7e).encode('latin-1')
            frame_header += struct.pack("!H", length)
        else:
            frame_header += chr(self.mask << 7 | 0x7f).encode('latin-1')
            frame_header += struct.pack("!Q", length)

        if not self.mask:
            return frame_header + self.data
        else:
            mask_key = self.get_mask_key(4)
            return frame_header + self._get_masked(mask_key)

    def _get_masked(self, mask_key: Union[str, bytes]) -> bytes:
        s = ABNF.mask(mask_key, self.data)

        if isinstance(mask_key, str):
            mask_key = mask_key.encode('utf-8')

        return mask_key + s

    @staticmethod
    def mask(mask_key: Union[str, bytes], data: Union[str, bytes]) -> bytes:
        """
        Mask or unmask data. Just do xor for each byte

        Parameters
        ----------
        mask_key: bytes or str
            4 byte mask.
        data: bytes or str
            data to mask/unmask.
        """
        if data is None:
            data = ""

        if isinstance(mask_key, str):
            mask_key = mask_key.encode('latin-1')

        if isinstance(data, str):
            data = data.encode('latin-1')

        return _mask(array.array("B", mask_key), array.array("B", data))


class frame_buffer:
    _HEADER_MASK_INDEX = 5
    _HEADER_LENGTH_INDEX = 6

    def __init__(self, recv_fn: Callable[[int], int], skip_utf8_validation: bool) -> None:
        self.recv = recv_fn
        self.skip_utf8_validation = skip_utf8_validation
        # Buffers over the packets from the layer beneath until desired amount
        # bytes of bytes are received.
        self.recv_buffer = []
        self.clear()
        self.lock = Lock()

    def clear(self) -> None:
        self.header = None
        self.length = None
        self.mask = None

    def has_received_header(self) -> bool:
        return self.header is None

    def recv_header(self) -> None:
        header = self.recv_strict(2)
        b1 = header[0]
        fin = b1 >> 7 & 1
        rsv1 = b1 >> 6 & 1
        rsv2 = b1 >> 5 & 1
        rsv3 = b1 >> 4 & 1
        opcode = b1 & 0xf
        b2 = header[1]
        has_mask = b2 >> 7 & 1
        length_bits = b2 & 0x7f

        self.header = (fin, rsv1, rsv2, rsv3, opcode, has_mask, length_bits)

    def has_mask(self) -> Union[bool, int]:
        if not self.header:
            return False
        return self.header[frame_buffer._HEADER_MASK_INDEX]

    def has_received_length(self) -> bool:
        return self.length is None

    def recv_length(self) -> None:
        bits = self.header[frame_buffer._HEADER_LENGTH_INDEX]
        length_bits = bits & 0x7f
        if length_bits == 0x7e:
            v = self.recv_strict(2)
            self.length = struct.unpack("!H", v)[0]
        elif length_bits == 0x7f:
            v = self.recv_strict(8)
            self.length = struct.unpack("!Q", v)[0]
        else:
            self.length = length_bits

    def has_received_mask(self) -> bool:
        return self.mask is None

    def recv_mask(self) -> None:
        self.mask = self.recv_strict(4) if self.has_mask() else ""

    def recv_frame(self) -> ABNF:

        with self.lock:
            # Header
            if self.has_received_header():
                self.recv_header()
            (fin, rsv1, rsv2, rsv3, opcode, has_mask, _) = self.header

            # Frame length
            if self.has_received_length():
                self.recv_length()
            length = self.length

            # Mask
            if self.has_received_mask():
                self.recv_mask()
            mask = self.mask

            # Payload
            payload = self.recv_strict(length)
            if has_mask:
                payload = ABNF.mask(mask, payload)

            # Reset for next frame
            self.clear()

            frame = ABNF(fin, rsv1, rsv2, rsv3, opcode, has_mask, payload)
            frame.validate(self.skip_utf8_validation)

        return frame

    def recv_strict(self, bufsize: int) -> bytes:
        shortage = bufsize - sum(map(len, self.recv_buffer))
        while shortage > 0:
            # Limit buffer size that we pass to socket.recv() to avoid
            # fragmenting the heap -- the number of bytes recv() actually
            # reads is limited by socket buffer and is relatively small,
            # yet passing large numbers repeatedly causes lots of large
            # buffers allocated and then shrunk, which results in
            # fragmentation.
            bytes_ = self.recv(min(16384, shortage))
            self.recv_buffer.append(bytes_)
            shortage -= len(bytes_)

        unified = b"".join(self.recv_buffer)

        if shortage == 0:
            self.recv_buffer = []
            return unified
        else:
            self.recv_buffer = [unified[bufsize:]]
            return unified[:bufsize]


class continuous_frame:

    def __init__(self, fire_cont_frame: bool, skip_utf8_validation: bool) -> None:
        self.fire_cont_frame = fire_cont_frame
        self.skip_utf8_validation = skip_utf8_validation
        self.cont_data = None
        self.recving_frames = None

    def validate(self, frame: ABNF) -> None:
        if not self.recving_frames and frame.opcode == ABNF.OPCODE_CONT:
            raise WebSocketProtocolException("Illegal frame")
        if self.recving_frames and \
                frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
            raise WebSocketProtocolException("Illegal frame")

    def add(self, frame: ABNF) -> None:
        if self.cont_data:
            self.cont_data[1] += frame.data
        else:
            if frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY):
                self.recving_frames = frame.opcode
            self.cont_data = [frame.opcode, frame.data]

        if frame.fin:
            self.recving_frames = None

    def is_fire(self, frame: ABNF) -> Union[bool, int]:
        return frame.fin or self.fire_cont_frame

    def extract(self, frame: ABNF) -> list:
        data = self.cont_data
        self.cont_data = None
        frame.data = data[1]
        if not self.fire_cont_frame and data[0] == ABNF.OPCODE_TEXT and not self.skip_utf8_validation and not validate_utf8(frame.data):
            raise WebSocketPayloadException(
                "cannot decode: " + repr(frame.data))

        return [data[0], frame]

env_pyrecodes/lib/python3.9/site-packages/websocket/_abnf.py
NikolaBlagojevic/pyrecodes
import inspect
import selectors
import socket
import sys
import threading
import time
import traceback

from typing import Any, Callable, Optional, Union

from . import _logging
from ._abnf import ABNF
from ._url import parse_url
from ._core import WebSocket, getdefaulttimeout
from ._exceptions import *

"""
_app.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__all__ = ["WebSocketApp"]

RECONNECT = 0


def setReconnect(reconnectInterval: int) -> None:
    global RECONNECT
    RECONNECT = reconnectInterval


class DispatcherBase:
    """
    DispatcherBase
    """
    def __init__(self, app: Any, ping_timeout: float) -> None:
        self.app = app
        self.ping_timeout = ping_timeout

    def timeout(self, seconds: int, callback: Callable) -> None:
        time.sleep(seconds)
        callback()

    def reconnect(self, seconds: int, reconnector: Callable) -> None:
        try:
            _logging.info("reconnect() - retrying in {seconds_count} seconds [{frame_count} frames in stack]".format(
                seconds_count=seconds, frame_count=len(inspect.stack())))
            time.sleep(seconds)
            reconnector(reconnecting=True)
        except KeyboardInterrupt as e:
            _logging.info("User exited {err}".format(err=e))
            raise e


class Dispatcher(DispatcherBase):
    """
    Dispatcher
    """
    def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
        sel = selectors.DefaultSelector()
        sel.register(self.app.sock.sock, selectors.EVENT_READ)
        try:
            while self.app.keep_running:
                r = sel.select(self.ping_timeout)
                if r:
                    if not read_callback():
                        break
                check_callback()
        finally:
            sel.close()


class SSLDispatcher(DispatcherBase):
    """
    SSLDispatcher
    """
    def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
        sock = self.app.sock.sock
        sel = selectors.DefaultSelector()
        sel.register(sock, selectors.EVENT_READ)
        try:
            while self.app.keep_running:
                r = self.select(sock, sel)
                if r:
                    if not read_callback():
                        break
                check_callback()
        finally:
            sel.close()

    def select(self, sock, sel:selectors.DefaultSelector):
        sock = self.app.sock.sock
        if sock.pending():
            return [sock,]

        r = sel.select(self.ping_timeout)

        if len(r) > 0:
            return r[0][0]


class WrappedDispatcher:
    """
    WrappedDispatcher
    """
    def __init__(self, app, ping_timeout: float, dispatcher: Dispatcher) -> None:
        self.app = app
        self.ping_timeout = ping_timeout
        self.dispatcher = dispatcher
        dispatcher.signal(2, dispatcher.abort)  # keyboard interrupt

    def read(self, sock: socket.socket, read_callback: Callable, check_callback: Callable) -> None:
        self.dispatcher.read(sock, read_callback)
        self.ping_timeout and self.timeout(self.ping_timeout, check_callback)

    def timeout(self, seconds: int, callback: Callable) -> None:
        self.dispatcher.timeout(seconds, callback)

    def reconnect(self, seconds: int, reconnector: Callable) -> None:
        self.timeout(seconds, reconnector)


class WebSocketApp:
    """
    Higher level of APIs are provided. The interface is like JavaScript WebSocket object.
    """

    def __init__(self, url: str, header: Union[list, dict, Callable] = None,
                 on_open: Callable = None, on_message: Callable = None, on_error: Callable = None,
                 on_close: Callable = None, on_ping: Callable = None, on_pong: Callable = None,
                 on_cont_message: Callable = None,
                 keep_running: bool = True, get_mask_key: Callable = None, cookie: str = None,
                 subprotocols: list = None,
                 on_data: Callable = None,
                 socket: socket.socket = None) -> None:
        """
        WebSocketApp initialization

        Parameters
        ----------
        url: str
            Websocket url.
        header: list or dict or Callable
            Custom header for websocket handshake.
            If the parameter is a callable object, it is called just before the connection attempt.
            The returned dict or list is used as custom header value.
            This could be useful in order to properly setup timestamp dependent headers.
        on_open: function
            Callback object which is called at opening websocket.
            on_open has one argument.
            The 1st argument is this class object.
        on_message: function
            Callback object which is called when received data.
            on_message has 2 arguments.
            The 1st argument is this class object.
            The 2nd argument is utf-8 data received from the server.
        on_error: function
            Callback object which is called when we get error.
            on_error has 2 arguments.
            The 1st argument is this class object.
            The 2nd argument is exception object.
        on_close: function
            Callback object which is called when connection is closed.
            on_close has 3 arguments.
            The 1st argument is this class object.
            The 2nd argument is close_status_code.
            The 3rd argument is close_msg.
        on_cont_message: function
            Callback object which is called when a continuation
            frame is received.
            on_cont_message has 3 arguments.
            The 1st argument is this class object.
            The 2nd argument is utf-8 string which we get from the server.
            The 3rd argument is continue flag. if 0, the data continue
            to next frame data
        on_data: function
            Callback object which is called when a message received.
            This is called before on_message or on_cont_message,
            and then on_message or on_cont_message is called.
            on_data has 4 argument.
            The 1st argument is this class object.
            The 2nd argument is utf-8 string which we get from the server.
            The 3rd argument is data type. ABNF.OPCODE_TEXT or ABNF.OPCODE_BINARY will be came.
            The 4th argument is continue flag. If 0, the data continue
        keep_running: bool
            This parameter is obsolete and ignored.
        get_mask_key: function
            A callable function to get new mask keys, see the
            WebSocket.set_mask_key's docstring for more information.
        cookie: str
            Cookie value.
        subprotocols: list
            List of available sub protocols. Default is None.
        socket: socket
            Pre-initialized stream socket.
        """
        self.url = url
        self.header = header if header is not None else []
        self.cookie = cookie

        self.on_open = on_open
        self.on_message = on_message
        self.on_data = on_data
        self.on_error = on_error
        self.on_close = on_close
        self.on_ping = on_ping
        self.on_pong = on_pong
        self.on_cont_message = on_cont_message
        self.keep_running = False
        self.get_mask_key = get_mask_key
        self.sock = None
        self.last_ping_tm = 0
        self.last_pong_tm = 0
        self.ping_thread = None
        self.stop_ping = None
        self.ping_interval = 0
        self.ping_timeout = None
        self.ping_payload = ""
        self.subprotocols = subprotocols
        self.prepared_socket = socket
        self.has_errored = False
        self.has_done_teardown = False
        self.has_done_teardown_lock = threading.Lock()

    def send(self, data: str, opcode: int = ABNF.OPCODE_TEXT) -> None:
        """
        send message

        Parameters
        ----------
        data: str
            Message to send. If you set opcode to OPCODE_TEXT,
            data must be utf-8 string or unicode.
        opcode: int
            Operation code of data. Default is OPCODE_TEXT.
        """

        if not self.sock or self.sock.send(data, opcode) == 0:
            raise WebSocketConnectionClosedException(
                "Connection is already closed.")

    def close(self, **kwargs) -> None:
        """
        Close websocket connection.
        """
        self.keep_running = False
        if self.sock:
            self.sock.close(**kwargs)
            self.sock = None

    def _start_ping_thread(self) -> None:
        self.last_ping_tm = self.last_pong_tm = 0
        self.stop_ping = threading.Event()
        self.ping_thread = threading.Thread(target=self._send_ping)
        self.ping_thread.daemon = True
        self.ping_thread.start()

    def _stop_ping_thread(self) -> None:
        if self.stop_ping:
            self.stop_ping.set()
        if self.ping_thread and self.ping_thread.is_alive():
            self.ping_thread.join(3)
        self.last_ping_tm = self.last_pong_tm = 0

    def _send_ping(self) -> None:
        if self.stop_ping.wait(self.ping_interval) or self.keep_running is False:
            return
        while not self.stop_ping.wait(self.ping_interval) and self.keep_running is True:
            if self.sock:
                self.last_ping_tm = time.time()
                try:
                    _logging.debug("Sending ping")
                    self.sock.ping(self.ping_payload)
                except Exception as e:
                    _logging.debug("Failed to send ping: {err}".format(err=e))

    def run_forever(self, sockopt: tuple = None, sslopt: dict = None,
                    ping_interval: float = 0, ping_timeout: Optional[float] = None,
                    ping_payload: str = "",
                    http_proxy_host: str = None, http_proxy_port: Union[int, str] = None,
                    http_no_proxy: list = None, http_proxy_auth: tuple = None,
                    http_proxy_timeout: float = None,
                    skip_utf8_validation: bool = False,
                    host: str = None, origin: str = None, dispatcher: Dispatcher = None,
                    suppress_origin: bool = False, proxy_type: str = None, reconnect: int = None) -> bool:
        """
        Run event loop for WebSocket framework.

        This loop is an infinite loop and is alive while websocket is available.

        Parameters
        ----------
        sockopt: tuple
            Values for socket.setsockopt.
            sockopt must be tuple
            and each element is argument of sock.setsockopt.
        sslopt: dict
            Optional dict object for ssl socket option.
        ping_interval: int or float
            Automatically send "ping" command
            every specified period (in seconds).
            If set to 0, no ping is sent periodically.
        ping_timeout: int or float
            Timeout (in seconds) if the pong message is not received.
        ping_payload: str
            Payload message to send with each ping.
        http_proxy_host: str
            HTTP proxy host name.
        http_proxy_port: int or str
            HTTP proxy port. If not set, set to 80.
        http_no_proxy: list
            Whitelisted host names that don't use the proxy.
        http_proxy_timeout: int or float
            HTTP proxy timeout, default is 60 sec as per python-socks.
        http_proxy_auth: tuple
            HTTP proxy auth information. tuple of username and password. Default is None.
        skip_utf8_validation: bool
            skip utf8 validation.
        host: str
            update host header.
        origin: str
            update origin header.
        dispatcher: Dispatcher object
            customize reading data from socket.
        suppress_origin: bool
            suppress outputting origin header.
        proxy_type: str
            type of proxy from: http, socks4, socks4a, socks5, socks5h
        reconnect: int
            delay interval when reconnecting

        Returns
        -------
        teardown: bool
            False if the `WebSocketApp` is closed or caught KeyboardInterrupt,
            True if any other exception was raised during a loop.
        """

        if reconnect is None:
            reconnect = RECONNECT

        if ping_timeout is not None and ping_timeout <= 0:
            raise WebSocketException("Ensure ping_timeout > 0")
        if ping_interval is not None and ping_interval < 0:
            raise WebSocketException("Ensure ping_interval >= 0")
        if ping_timeout and ping_interval and ping_interval <= ping_timeout:
            raise WebSocketException("Ensure ping_interval > ping_timeout")
        if not sockopt:
            sockopt = []
        if not sslopt:
            sslopt = {}
        if self.sock:
            raise WebSocketException("socket is already opened")

        self.ping_interval = ping_interval
        self.ping_timeout = ping_timeout
        self.ping_payload = ping_payload
        self.keep_running = True

        def teardown(close_frame: ABNF = None):
            """
            Tears down the connection.

            Parameters
            ----------
            close_frame: ABNF frame
                If close_frame is set, the on_close handler is invoked
                with the statusCode and reason from the provided frame.
            """

            # teardown() is called in many code paths to ensure resources are cleaned up and on_close is fired.
            # To ensure the work is only done once, we use this bool and lock.
            with self.has_done_teardown_lock:
                if self.has_done_teardown:
                    return
                self.has_done_teardown = True

            self._stop_ping_thread()
            self.keep_running = False
            if self.sock:
                self.sock.close()
            close_status_code, close_reason = self._get_close_args(
                close_frame if close_frame else None)
            self.sock = None

            # Finally call the callback AFTER all teardown is complete
            self._callback(self.on_close, close_status_code, close_reason)

        def setSock(reconnecting: bool = False) -> None:
            if reconnecting and self.sock:
                self.sock.shutdown()

            self.sock = WebSocket(
                self.get_mask_key, sockopt=sockopt, sslopt=sslopt,
                fire_cont_frame=self.on_cont_message is not None,
                skip_utf8_validation=skip_utf8_validation,
                enable_multithread=True)

            self.sock.settimeout(getdefaulttimeout())
            try:

                header = self.header() if callable(self.header) else self.header

                self.sock.connect(
                    self.url, header=header, cookie=self.cookie,
                    http_proxy_host=http_proxy_host,
                    http_proxy_port=http_proxy_port, http_no_proxy=http_no_proxy,
                    http_proxy_auth=http_proxy_auth, http_proxy_timeout=http_proxy_timeout,
                    subprotocols=self.subprotocols,
                    host=host, origin=origin, suppress_origin=suppress_origin,
                    proxy_type=proxy_type, socket=self.prepared_socket)

                _logging.info("Websocket connected")

                if self.ping_interval:
                    self._start_ping_thread()

                self._callback(self.on_open)

                dispatcher.read(self.sock.sock, read, check)
            except (WebSocketConnectionClosedException, ConnectionRefusedError, KeyboardInterrupt, SystemExit, Exception) as e:
                handleDisconnect(e, reconnecting)

        def read() -> bool:
            if not self.keep_running:
                return teardown()

            try:
                op_code, frame = self.sock.recv_data_frame(True)
            except (WebSocketConnectionClosedException, KeyboardInterrupt) as e:
                if custom_dispatcher:
                    return handleDisconnect(e)
                else:
                    raise e

            if op_code == ABNF.OPCODE_CLOSE:
                return teardown(frame)
            elif op_code == ABNF.OPCODE_PING:
                self._callback(self.on_ping, frame.data)
            elif op_code == ABNF.OPCODE_PONG:
                self.last_pong_tm = time.time()
                self._callback(self.on_pong, frame.data)
            elif op_code == ABNF.OPCODE_CONT and self.on_cont_message:
                self._callback(self.on_data, frame.data,
                               frame.opcode, frame.fin)
                self._callback(self.on_cont_message,
                               frame.data, frame.fin)
            else:
                data = frame.data
                if op_code == ABNF.OPCODE_TEXT and not skip_utf8_validation:
                    data = data.decode("utf-8")
                self._callback(self.on_data, data, frame.opcode, True)
                self._callback(self.on_message, data)

            return True

        def check() -> bool:
            if (self.ping_timeout):
                has_timeout_expired = time.time() - self.last_ping_tm > self.ping_timeout
                has_pong_not_arrived_after_last_ping = self.last_pong_tm - self.last_ping_tm < 0
                has_pong_arrived_too_late = self.last_pong_tm - self.last_ping_tm > self.ping_timeout

                if (self.last_ping_tm and
                        has_timeout_expired and
                        (has_pong_not_arrived_after_last_ping or has_pong_arrived_too_late)):
                    raise WebSocketTimeoutException("ping/pong timed out")
            return True

        def handleDisconnect(e: Exception, reconnecting: bool = False) -> bool:
            self.has_errored = True
            self._stop_ping_thread()
            if not reconnecting:
                self._callback(self.on_error, e)

            if isinstance(e, (KeyboardInterrupt, SystemExit)):
                teardown()
                # Propagate further
                raise

            if reconnect:
                _logging.info("{err} - reconnect".format(err=e))
                if custom_dispatcher:
                    _logging.debug("Calling custom dispatcher reconnect [{frame_count} frames in stack]".format(frame_count=len(inspect.stack())))
                    dispatcher.reconnect(reconnect, setSock)
            else:
                _logging.error("{err} - goodbye".format(err=e))
                teardown()

        custom_dispatcher = bool(dispatcher)
        dispatcher = self.create_dispatcher(ping_timeout, dispatcher, parse_url(self.url)[3])

        try:
            setSock()
            if not custom_dispatcher and reconnect:
                while self.keep_running:
                    _logging.debug("Calling dispatcher reconnect [{frame_count} frames in stack]".format(frame_count=len(inspect.stack())))
                    dispatcher.reconnect(reconnect, setSock)
        except (KeyboardInterrupt, Exception) as e:
            _logging.info("tearing down on exception {err}".format(err=e))
            teardown()
        finally:
            if not custom_dispatcher:
                # Ensure teardown was called before returning from run_forever
                teardown()

        return self.has_errored

    def create_dispatcher(self, ping_timeout: int, dispatcher: Dispatcher = None, is_ssl: bool = False) -> DispatcherBase:
        if dispatcher:  # If custom dispatcher is set, use WrappedDispatcher
            return WrappedDispatcher(self, ping_timeout, dispatcher)
        timeout = ping_timeout or 10
        if is_ssl:
            return SSLDispatcher(self, timeout)

        return Dispatcher(self, timeout)

    def _get_close_args(self, close_frame: ABNF) -> list:
        """
        _get_close_args extracts the close code and reason from the close body
        if it exists (RFC6455 says WebSocket Connection Close Code is optional)
        """
        # Need to catch the case where close_frame is None
        # Otherwise the following if statement causes an error
        if not self.on_close or not close_frame:
            return [None, None]

        # Extract close frame status code
        if close_frame.data and len(close_frame.data) >= 2:
            close_status_code = 256 * close_frame.data[0] + close_frame.data[1]
            reason = close_frame.data[2:].decode('utf-8')
            return [close_status_code, reason]
        else:
            # Most likely reached this because len(close_frame_data.data) < 2
            return [None, None]

    def _callback(self, callback, *args) -> None:
        if callback:
            try:
                callback(self, *args)

            except Exception as e:
                _logging.error("error from callback {callback}: {err}".format(callback=callback, err=e))
                if self.on_error:
                    self.on_error(self, e)

env_pyrecodes/lib/python3.9/site-packages/websocket/_app.py
NikolaBlagojevic/pyrecodes
import http.cookies

from typing import Optional

"""
_cookiejar.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""


class SimpleCookieJar:
    def __init__(self) -> None:
        self.jar = dict()

    def add(self, set_cookie: Optional[str]) -> None:
        if set_cookie:
            simpleCookie = http.cookies.SimpleCookie(set_cookie)

            for k, v in simpleCookie.items():
                domain = v.get("domain")
                if domain:
                    if not domain.startswith("."):
                        domain = "." + domain
                    cookie = self.jar.get(domain) if self.jar.get(domain) else http.cookies.SimpleCookie()
                    cookie.update(simpleCookie)
                    self.jar[domain.lower()] = cookie

    def set(self, set_cookie: str) -> None:
        if set_cookie:
            simpleCookie = http.cookies.SimpleCookie(set_cookie)

            for k, v in simpleCookie.items():
                domain = v.get("domain")
                if domain:
                    if not domain.startswith("."):
                        domain = "." + domain
                    self.jar[domain.lower()] = simpleCookie

    def get(self, host: str) -> str:
        if not host:
            return ""

        cookies = []
        for domain, simpleCookie in self.jar.items():
            host = host.lower()
            if host.endswith(domain) or host == domain[1:]:
                cookies.append(self.jar.get(domain))

        return "; ".join(filter(
            None, sorted(
                ["%s=%s" % (k, v.value) for cookie in filter(None, cookies) for k, v in cookie.items()]
            )))

env_pyrecodes/lib/python3.9/site-packages/websocket/_cookiejar.py
NikolaBlagojevic/pyrecodes
import socket
import struct
import threading
import time

from typing import Optional, Union

# websocket modules
from ._abnf import *
from ._exceptions import *
from ._handshake import *
from ._http import *
from ._logging import *
from ._socket import *
from ._ssl_compat import *
from ._utils import *

"""
_core.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__all__ = ['WebSocket', 'create_connection']


class WebSocket:
    """
    Low level WebSocket interface.

    This class is based on the WebSocket protocol `draft-hixie-thewebsocketprotocol-76 <http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76>`_

    We can connect to the websocket server and send/receive data.
    The following example is an echo client.

    >>> import websocket
    >>> ws = websocket.WebSocket()
    >>> ws.connect("ws://echo.websocket.events")
    >>> ws.recv()
    'echo.websocket.events sponsored by Lob.com'
    >>> ws.send("Hello, Server")
    19
    >>> ws.recv()
    'Hello, Server'
    >>> ws.close()

    Parameters
    ----------
    get_mask_key: func
        A callable function to get new mask keys, see the
        WebSocket.set_mask_key's docstring for more information.
    sockopt: tuple
        Values for socket.setsockopt.
        sockopt must be tuple and each element is argument of sock.setsockopt.
    sslopt: dict
        Optional dict object for ssl socket options. See FAQ for details.
    fire_cont_frame: bool
        Fire recv event for each cont frame. Default is False.
    enable_multithread: bool
        If set to True, lock send method.
    skip_utf8_validation: bool
        Skip utf8 validation.
    """

    def __init__(self, get_mask_key=None, sockopt=None, sslopt=None,
                 fire_cont_frame: bool = False, enable_multithread: bool = True,
                 skip_utf8_validation: bool = False, **_):
        """
        Initialize WebSocket object.

        Parameters
        ----------
        sslopt: dict
            Optional dict object for ssl socket options. See FAQ for details.
        """
        self.sock_opt = sock_opt(sockopt, sslopt)
        self.handshake_response = None
        self.sock = None

        self.connected = False
        self.get_mask_key = get_mask_key
        # These buffer over the build-up of a single frame.
        self.frame_buffer = frame_buffer(self._recv, skip_utf8_validation)
        self.cont_frame = continuous_frame(
            fire_cont_frame, skip_utf8_validation)

        if enable_multithread:
            self.lock = threading.Lock()
            self.readlock = threading.Lock()
        else:
            self.lock = NoLock()
            self.readlock = NoLock()

    def __iter__(self):
        """
        Allow iteration over websocket, implying sequential `recv` executions.
        """
        while True:
            yield self.recv()

    def __next__(self):
        return self.recv()

    def next(self):
        return self.__next__()

    def fileno(self):
        return self.sock.fileno()

    def set_mask_key(self, func):
        """
        Set function to create mask key. You can customize mask key generator.
        Mainly, this is for testing purpose.

        Parameters
        ----------
        func: func
            callable object. the func takes 1 argument as integer.
            The argument means length of mask key.
            This func must return string(byte array),
            which length is argument specified.
        """
        self.get_mask_key = func

    def gettimeout(self) -> float:
        """
        Get the websocket timeout (in seconds) as an int or float

        Returns
        ----------
        timeout: int or float
             returns timeout value (in seconds). This value could be either float/integer.
        """
        return self.sock_opt.timeout

    def settimeout(self, timeout: Optional[float]):
        """
        Set the timeout to the websocket.

        Parameters
        ----------
        timeout: int or float
            timeout time (in seconds). This value could be either float/integer.
        """
        self.sock_opt.timeout = timeout
        if self.sock:
            self.sock.settimeout(timeout)

    timeout = property(gettimeout, settimeout)

    def getsubprotocol(self):
        """
        Get subprotocol
        """
        if self.handshake_response:
            return self.handshake_response.subprotocol
        else:
            return None

    subprotocol = property(getsubprotocol)

    def getstatus(self):
        """
        Get handshake status
        """
        if self.handshake_response:
            return self.handshake_response.status
        else:
            return None

    status = property(getstatus)

    def getheaders(self):
        """
        Get handshake response header
        """
        if self.handshake_response:
            return self.handshake_response.headers
        else:
            return None

    def is_ssl(self):
        try:
            return isinstance(self.sock, ssl.SSLSocket)
        except:
            return False

    headers = property(getheaders)

    def connect(self, url, **options):
        """
        Connect to url. url is websocket url scheme.
        ie. ws://host:port/resource
        You can customize using 'options'.
        If you set "header" list object, you can set your own custom header.

        >>> ws = WebSocket()
        >>> ws.connect("ws://echo.websocket.events",
                ...     header=["User-Agent: MyProgram",
                ...             "x-custom: header"])

        Parameters
        ----------
        header: list or dict
            Custom http header list or dict.
        cookie: str
            Cookie value.
        origin: str
            Custom origin url.
        connection: str
            Custom connection header value.
            Default value "Upgrade" set in _handshake.py
        suppress_origin: bool
            Suppress outputting origin header.
        host: str
            Custom host header string.
        timeout: int or float
            Socket timeout time. This value is an integer or float.
            If you set None for this value, it means "use default_timeout value"
        http_proxy_host: str
            HTTP proxy host name.
        http_proxy_port: str or int
            HTTP proxy port. Default is 80.
        http_no_proxy: list
            Whitelisted host names that don't use the proxy.
        http_proxy_auth: tuple
            HTTP proxy auth information. Tuple of username and password. Default is None.
        http_proxy_timeout: int or float
            HTTP proxy timeout, default is 60 sec as per python-socks.
        redirect_limit: int
            Number of redirects to follow.
        subprotocols: list
            List of available subprotocols. Default is None.
        socket: socket
            Pre-initialized stream socket.
        """
        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
        self.sock, addrs = connect(url, self.sock_opt, proxy_info(**options),
                                   options.pop('socket', None))

        try:
            self.handshake_response = handshake(self.sock, url, *addrs, **options)
            for attempt in range(options.pop('redirect_limit', 3)):
                if self.handshake_response.status in SUPPORTED_REDIRECT_STATUSES:
                    url = self.handshake_response.headers['location']
                    self.sock.close()
                    self.sock, addrs = connect(url, self.sock_opt, proxy_info(**options),
                                               options.pop('socket', None))
                    self.handshake_response = handshake(self.sock, url, *addrs, **options)
            self.connected = True
        except:
            if self.sock:
                self.sock.close()
                self.sock = None
            raise

    def send(self, payload: Union[bytes, str], opcode: int = ABNF.OPCODE_TEXT) -> int:
        """
        Send the data as string.

        Parameters
        ----------
        payload: str
            Payload must be utf-8 string or unicode,
            If the opcode is OPCODE_TEXT.
            Otherwise, it must be string(byte array).
        opcode: int
            Operation code (opcode) to send.
        """

        frame = ABNF.create_frame(payload, opcode)
        return self.send_frame(frame)

    def send_frame(self, frame) -> int:
        """
        Send the data frame.

        >>> ws = create_connection("ws://echo.websocket.events")
        >>> frame = ABNF.create_frame("Hello", ABNF.OPCODE_TEXT)
        >>> ws.send_frame(frame)
        >>> cont_frame = ABNF.create_frame("My name is ", ABNF.OPCODE_CONT, 0)
        >>> ws.send_frame(frame)
        >>> cont_frame = ABNF.create_frame("Foo Bar", ABNF.OPCODE_CONT, 1)
        >>> ws.send_frame(frame)

        Parameters
        ----------
        frame: ABNF frame
            frame data created by ABNF.create_frame
        """
        if self.get_mask_key:
            frame.get_mask_key = self.get_mask_key
        data = frame.format()
        length = len(data)
        if (isEnabledForTrace()):
            trace("++Sent raw: " + repr(data))
            trace("++Sent decoded: " + frame.__str__())
        with self.lock:
            while data:
                l = self._send(data)
                data = data[l:]

        return length

    def send_binary(self, payload: bytes) -> int:
        """
        Send a binary message (OPCODE_BINARY).

        Parameters
        ----------
        payload: bytes
            payload of message to send.
        """
        return self.send(payload, ABNF.OPCODE_BINARY)

    def ping(self, payload: Union[str, bytes] = ""):
        """
        Send ping data.

        Parameters
        ----------
        payload: str
            data payload to send server.
        """
        if isinstance(payload, str):
            payload = payload.encode("utf-8")
        self.send(payload, ABNF.OPCODE_PING)

    def pong(self, payload: Union[str, bytes] = ""):
        """
        Send pong data.

        Parameters
        ----------
        payload: str
            data payload to send server.
        """
        if isinstance(payload, str):
            payload = payload.encode("utf-8")
        self.send(payload, ABNF.OPCODE_PONG)

    def recv(self) -> Union[str, bytes]:
        """
        Receive string data(byte array) from the server.

        Returns
        ----------
        data: string (byte array) value.
        """
        with self.readlock:
            opcode, data = self.recv_data()
        if opcode == ABNF.OPCODE_TEXT:
            return data.decode("utf-8")
        elif opcode == ABNF.OPCODE_TEXT or opcode == ABNF.OPCODE_BINARY:
            return data
        else:
            return ''

    def recv_data(self, control_frame: bool = False) -> tuple:
        """
        Receive data with operation code.

        Parameters
        ----------
        control_frame: bool
            a boolean flag indicating whether to return control frame
            data, defaults to False

        Returns
        -------
        opcode, frame.data: tuple
            tuple of operation code and string(byte array) value.
        """
        opcode, frame = self.recv_data_frame(control_frame)
        return opcode, frame.data

    def recv_data_frame(self, control_frame: bool = False):
        """
        Receive data with operation code.

        If a valid ping message is received, a pong response is sent.

        Parameters
        ----------
        control_frame: bool
            a boolean flag indicating whether to return control frame
            data, defaults to False

        Returns
        -------
        frame.opcode, frame: tuple
            tuple of operation code and string(byte array) value.
        """
        while True:
            frame = self.recv_frame()
            if (isEnabledForTrace()):
                trace("++Rcv raw: " + repr(frame.format()))
                trace("++Rcv decoded: " + frame.__str__())
            if not frame:
                # handle error:
                # 'NoneType' object has no attribute 'opcode'
                raise WebSocketProtocolException(
                    "Not a valid frame {frame}".format(frame=frame))
            elif frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY, ABNF.OPCODE_CONT):
                self.cont_frame.validate(frame)
                self.cont_frame.add(frame)

                if self.cont_frame.is_fire(frame):
                    return self.cont_frame.extract(frame)

            elif frame.opcode == ABNF.OPCODE_CLOSE:
                self.send_close()
                return frame.opcode, frame
            elif frame.opcode == ABNF.OPCODE_PING:
                if len(frame.data) < 126:
                    self.pong(frame.data)
                else:
                    raise WebSocketProtocolException(
                        "Ping message is too long")
                if control_frame:
                    return frame.opcode, frame
            elif frame.opcode == ABNF.OPCODE_PONG:
                if control_frame:
                    return frame.opcode, frame

    def recv_frame(self):
        """
        Receive data as frame from server.

        Returns
        -------
        self.frame_buffer.recv_frame(): ABNF frame object
        """
        return self.frame_buffer.recv_frame()

    def send_close(self, status: int = STATUS_NORMAL, reason: bytes = b""):
        """
        Send close data to the server.

        Parameters
        ----------
        status: int
            Status code to send. See STATUS_XXX.
        reason: str or bytes
            The reason to close. This must be string or UTF-8 bytes.
        """
        if status < 0 or status >= ABNF.LENGTH_16:
            raise ValueError("code is invalid range")
        self.connected = False
        self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE)

    def close(self, status: int = STATUS_NORMAL, reason: bytes = b"", timeout: float = 3):
        """
        Close Websocket object

        Parameters
        ----------
        status: int
            Status code to send. See VALID_CLOSE_STATUS in ABNF.
        reason: bytes
            The reason to close in UTF-8.
        timeout: int or float
            Timeout until receive a close frame.
            If None, it will wait forever until receive a close frame.
        """
        if self.connected:
            if status < 0 or status >= ABNF.LENGTH_16:
                raise ValueError("code is invalid range")

            try:
                self.connected = False
                self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE)
                sock_timeout = self.sock.gettimeout()
                self.sock.settimeout(timeout)
                start_time = time.time()
                while timeout is None or time.time() - start_time < timeout:
                    try:
                        frame = self.recv_frame()
                        if frame.opcode != ABNF.OPCODE_CLOSE:
                            continue
                        if isEnabledForError():
                            recv_status = struct.unpack("!H", frame.data[0:2])[0]
                            if recv_status >= 3000 and recv_status <= 4999:
                                debug("close status: " + repr(recv_status))
                            elif recv_status != STATUS_NORMAL:
                                error("close status: " + repr(recv_status))
                        break
                    except:
                        break
                self.sock.settimeout(sock_timeout)
                self.sock.shutdown(socket.SHUT_RDWR)
            except:
                pass

            self.shutdown()

    def abort(self):
        """
        Low-level asynchronous abort, wakes up other threads that are waiting in recv_*
        """
        if self.connected:
            self.sock.shutdown(socket.SHUT_RDWR)

    def shutdown(self):
        """
        close socket, immediately.
        """
        if self.sock:
            self.sock.close()
            self.sock = None
            self.connected = False

    def _send(self, data: Union[str, bytes]):
        return send(self.sock, data)

    def _recv(self, bufsize):
        try:
            return recv(self.sock, bufsize)
        except WebSocketConnectionClosedException:
            if self.sock:
                self.sock.close()
            self.sock = None
            self.connected = False
            raise


def create_connection(url: str, timeout=None, class_=WebSocket, **options):
    """
    Connect to url and return websocket object.

    Connect to url and return the WebSocket object.
    Passing optional timeout parameter will set the timeout on the socket.
    If no timeout is supplied,
    the global default timeout setting returned by getdefaulttimeout() is used.
    You can customize using 'options'.
    If you set "header" list object, you can set your own custom header.

    >>> conn = create_connection("ws://echo.websocket.events",
         ...     header=["User-Agent: MyProgram",
         ...             "x-custom: header"])

    Parameters
    ----------
    class_: class
        class to instantiate when creating the connection. It has to implement
        settimeout and connect. It's __init__ should be compatible with
        WebSocket.__init__, i.e. accept all of it's kwargs.
    header: list or dict
        custom http header list or dict.
    cookie: str
        Cookie value.
    origin: str
        custom origin url.
    suppress_origin: bool
        suppress outputting origin header.
    host: str
        custom host header string.
    timeout: int or float
        socket timeout time. This value could be either float/integer.
        If set to None, it uses the default_timeout value.
    http_proxy_host: str
        HTTP proxy host name.
    http_proxy_port: str or int
        HTTP proxy port. If not set, set to 80.
    http_no_proxy: list
        Whitelisted host names that don't use the proxy.
    http_proxy_auth: tuple
        HTTP proxy auth information. tuple of username and password. Default is None.
    http_proxy_timeout: int or float
        HTTP proxy timeout, default is 60 sec as per python-socks.
    enable_multithread: bool
        Enable lock for multithread.
    redirect_limit: int
        Number of redirects to follow.
    sockopt: tuple
        Values for socket.setsockopt.
        sockopt must be a tuple and each element is an argument of sock.setsockopt.
    sslopt: dict
        Optional dict object for ssl socket options. See FAQ for details.
    subprotocols: list
        List of available subprotocols. Default is None.
    skip_utf8_validation: bool
        Skip utf8 validation.
    socket: socket
        Pre-initialized stream socket.
    """
    sockopt = options.pop("sockopt", [])
    sslopt = options.pop("sslopt", {})
    fire_cont_frame = options.pop("fire_cont_frame", False)
    enable_multithread = options.pop("enable_multithread", True)
    skip_utf8_validation = options.pop("skip_utf8_validation", False)
    websock = class_(sockopt=sockopt, sslopt=sslopt,
                     fire_cont_frame=fire_cont_frame,
                     enable_multithread=enable_multithread,
                     skip_utf8_validation=skip_utf8_validation, **options)
    websock.settimeout(timeout if timeout is not None else getdefaulttimeout())
    websock.connect(url, **options)
    return websock

env_pyrecodes/lib/python3.9/site-packages/websocket/_core.py
NikolaBlagojevic/pyrecodes
import errno
import selectors
import socket

from typing import Union

from ._exceptions import *
from ._ssl_compat import *
from ._utils import *

"""
_socket.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

DEFAULT_SOCKET_OPTION = [(socket.SOL_TCP, socket.TCP_NODELAY, 1)]
if hasattr(socket, "SO_KEEPALIVE"):
    DEFAULT_SOCKET_OPTION.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
if hasattr(socket, "TCP_KEEPIDLE"):
    DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPIDLE, 30))
if hasattr(socket, "TCP_KEEPINTVL"):
    DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPINTVL, 10))
if hasattr(socket, "TCP_KEEPCNT"):
    DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPCNT, 3))

_default_timeout = None

__all__ = ["DEFAULT_SOCKET_OPTION", "sock_opt", "setdefaulttimeout", "getdefaulttimeout",
           "recv", "recv_line", "send"]


class sock_opt:

    def __init__(self, sockopt: list, sslopt: dict) -> None:
        if sockopt is None:
            sockopt = []
        if sslopt is None:
            sslopt = {}
        self.sockopt = sockopt
        self.sslopt = sslopt
        self.timeout = None


def setdefaulttimeout(timeout: Union[int, float, None]) -> None:
    """
    Set the global timeout setting to connect.

    Parameters
    ----------
    timeout: int or float
        default socket timeout time (in seconds)
    """
    global _default_timeout
    _default_timeout = timeout


def getdefaulttimeout() -> Union[int, float, None]:
    """
    Get default timeout

    Returns
    ----------
    _default_timeout: int or float
        Return the global timeout setting (in seconds) to connect.
    """
    return _default_timeout


def recv(sock: socket.socket, bufsize: int) -> bytes:
    if not sock:
        raise WebSocketConnectionClosedException("socket is already closed.")

    def _recv():
        try:
            return sock.recv(bufsize)
        except SSLWantReadError:
            pass
        except socket.error as exc:
            error_code = extract_error_code(exc)
            if error_code != errno.EAGAIN and error_code != errno.EWOULDBLOCK:
                raise

        sel = selectors.DefaultSelector()
        sel.register(sock, selectors.EVENT_READ)

        r = sel.select(sock.gettimeout())
        sel.close()

        if r:
            return sock.recv(bufsize)

    try:
        if sock.gettimeout() == 0:
            bytes_ = sock.recv(bufsize)
        else:
            bytes_ = _recv()
    except TimeoutError:
        raise WebSocketTimeoutException("Connection timed out")
    except socket.timeout as e:
        message = extract_err_message(e)
        raise WebSocketTimeoutException(message)
    except SSLError as e:
        message = extract_err_message(e)
        if isinstance(message, str) and 'timed out' in message:
            raise WebSocketTimeoutException(message)
        else:
            raise

    if not bytes_:
        raise WebSocketConnectionClosedException(
            "Connection to remote host was lost.")

    return bytes_


def recv_line(sock: socket.socket) -> bytes:
    line = []
    while True:
        c = recv(sock, 1)
        line.append(c)
        if c == b'\n':
            break
    return b''.join(line)


def send(sock: socket.socket, data: Union[bytes, str]) -> int:
    if isinstance(data, str):
        data = data.encode('utf-8')

    if not sock:
        raise WebSocketConnectionClosedException("socket is already closed.")

    def _send():
        try:
            return sock.send(data)
        except SSLWantWriteError:
            pass
        except socket.error as exc:
            error_code = extract_error_code(exc)
            if error_code is None:
                raise
            if error_code != errno.EAGAIN and error_code != errno.EWOULDBLOCK:
                raise

        sel = selectors.DefaultSelector()
        sel.register(sock, selectors.EVENT_WRITE)

        w = sel.select(sock.gettimeout())
        sel.close()

        if w:
            return sock.send(data)

    try:
        if sock.gettimeout() == 0:
            return sock.send(data)
        else:
            return _send()
    except socket.timeout as e:
        message = extract_err_message(e)
        raise WebSocketTimeoutException(message)
    except Exception as e:
        message = extract_err_message(e)
        if isinstance(message, str) and "timed out" in message:
            raise WebSocketTimeoutException(message)
        else:
            raise

env_pyrecodes/lib/python3.9/site-packages/websocket/_socket.py
NikolaBlagojevic/pyrecodes
import os
import socket
import struct

from typing import Optional
from urllib.parse import unquote, urlparse

"""
_url.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

__all__ = ["parse_url", "get_proxy_info"]


def parse_url(url: str) -> tuple:
    """
    parse url and the result is tuple of
    (hostname, port, resource path and the flag of secure mode)

    Parameters
    ----------
    url: str
        url string.
    """
    if ":" not in url:
        raise ValueError("url is invalid")

    scheme, url = url.split(":", 1)

    parsed = urlparse(url, scheme="http")
    if parsed.hostname:
        hostname = parsed.hostname
    else:
        raise ValueError("hostname is invalid")
    port = 0
    if parsed.port:
        port = parsed.port

    is_secure = False
    if scheme == "ws":
        if not port:
            port = 80
    elif scheme == "wss":
        is_secure = True
        if not port:
            port = 443
    else:
        raise ValueError("scheme %s is invalid" % scheme)

    if parsed.path:
        resource = parsed.path
    else:
        resource = "/"

    if parsed.query:
        resource += "?" + parsed.query

    return hostname, port, resource, is_secure


DEFAULT_NO_PROXY_HOST = ["localhost", "127.0.0.1"]


def _is_ip_address(addr: str) -> bool:
    try:
        socket.inet_aton(addr)
    except socket.error:
        return False
    else:
        return True


def _is_subnet_address(hostname: str) -> bool:
    try:
        addr, netmask = hostname.split("/")
        return _is_ip_address(addr) and 0 <= int(netmask) < 32
    except ValueError:
        return False


def _is_address_in_network(ip: str, net: str) -> bool:
    ipaddr = struct.unpack('!I', socket.inet_aton(ip))[0]
    netaddr, netmask = net.split('/')
    netaddr = struct.unpack('!I', socket.inet_aton(netaddr))[0]

    netmask = (0xFFFFFFFF << (32 - int(netmask))) & 0xFFFFFFFF
    return ipaddr & netmask == netaddr


def _is_no_proxy_host(hostname: str, no_proxy: Optional[list]) -> bool:
    if not no_proxy:
        v = os.environ.get("no_proxy", os.environ.get("NO_PROXY", "")).replace(" ", "")
        if v:
            no_proxy = v.split(",")
    if not no_proxy:
        no_proxy = DEFAULT_NO_PROXY_HOST

    if '*' in no_proxy:
        return True
    if hostname in no_proxy:
        return True
    if _is_ip_address(hostname):
        return any([_is_address_in_network(hostname, subnet) for subnet in no_proxy if _is_subnet_address(subnet)])
    for domain in [domain for domain in no_proxy if domain.startswith('.')]:
        if hostname.endswith(domain):
            return True
    return False


def get_proxy_info(
        hostname: str, is_secure: bool, proxy_host: Optional[str] = None, proxy_port: int = 0, proxy_auth: Optional[tuple] = None,
        no_proxy: Optional[list] = None, proxy_type: str = 'http') -> tuple:
    """
    Try to retrieve proxy host and port from environment
    if not provided in options.
    Result is (proxy_host, proxy_port, proxy_auth).
    proxy_auth is tuple of username and password
    of proxy authentication information.

    Parameters
    ----------
    hostname: str
        Websocket server name.
    is_secure: bool
        Is the connection secure? (wss) looks for "https_proxy" in env
        instead of "http_proxy"
    proxy_host: str
        http proxy host name.
    proxy_port: str or int
        http proxy port.
    no_proxy: list
        Whitelisted host names that don't use the proxy.
    proxy_auth: tuple
        HTTP proxy auth information. Tuple of username and password. Default is None.
    proxy_type: str
        Specify the proxy protocol (http, socks4, socks4a, socks5, socks5h). Default is "http".
        Use socks4a or socks5h if you want to send DNS requests through the proxy.
    """
    if _is_no_proxy_host(hostname, no_proxy):
        return None, 0, None

    if proxy_host:
        port = proxy_port
        auth = proxy_auth
        return proxy_host, port, auth

    env_key = "https_proxy" if is_secure else "http_proxy"
    value = os.environ.get(env_key, os.environ.get(env_key.upper(), "")).replace(" ", "")
    if value:
        proxy = urlparse(value)
        auth = (unquote(proxy.username), unquote(proxy.password)) if proxy.username else None
        return proxy.hostname, proxy.port, auth

    return None, 0, None

env_pyrecodes/lib/python3.9/site-packages/websocket/_url.py
NikolaBlagojevic/pyrecodes
from typing import Union

"""
_url.py
websocket - WebSocket client library for Python

Copyright 2023 engn33r

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
__all__ = ["NoLock", "validate_utf8", "extract_err_message", "extract_error_code"]


class NoLock:

    def __enter__(self) -> None:
        pass

    def __exit__(self, exc_type, exc_value, traceback) -> None:
        pass


try:
    # If wsaccel is available we use compiled routines to validate UTF-8
    # strings.
    from wsaccel.utf8validator import Utf8Validator

    def _validate_utf8(utfbytes: bytes) -> bool:
        return Utf8Validator().validate(utfbytes)[0]

except ImportError:
    # UTF-8 validator
    # python implementation of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/

    _UTF8_ACCEPT = 0
    _UTF8_REJECT = 12

    _UTF8D = [
        # The first part of the table maps bytes to character classes that
        # to reduce the size of the transition table and create bitmasks.
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
        0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
        1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
        7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
        8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
        10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,

        # The second part is a transition table that maps a combination
        # of a state of the automaton and a character class to a state.
        0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
        12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
        12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
        12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
        12,36,12,12,12,12,12,12,12,12,12,12, ]

    def _decode(state: int, codep: int, ch: int) -> tuple:
        tp = _UTF8D[ch]

        codep = (ch & 0x3f) | (codep << 6) if (
            state != _UTF8_ACCEPT) else (0xff >> tp) & ch
        state = _UTF8D[256 + state + tp]

        return state, codep

    def _validate_utf8(utfbytes: Union[str, bytes]) -> bool:
        state = _UTF8_ACCEPT
        codep = 0
        for i in utfbytes:
            state, codep = _decode(state, codep, i)
            if state == _UTF8_REJECT:
                return False

        return True


def validate_utf8(utfbytes: Union[str, bytes]) -> bool:
    """
    validate utf8 byte string.
    utfbytes: utf byte string to check.
    return value: if valid utf8 string, return true. Otherwise, return false.
    """
    return _validate_utf8(utfbytes)


def extract_err_message(exception: Exception) -> Union[str, None]:
    if exception.args:
        return exception.args[0]
    else:
        return None


def extract_error_code(exception: Exception) -> Union[int, None]:
    if exception.args and len(exception.args) > 1:
        return exception.args[0] if isinstance(exception.args[0], int) else None

env_pyrecodes/lib/python3.9/site-packages/websocket/_utils.py
NikolaBlagojevic/pyrecodes

# This file was generated by 'versioneer.py' (0.19) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.

import json

version_json = '''
{
 "date": "2023-02-19T15:00:26+0100",
 "dirty": false,
 "error": null,
 "full-revisionid": "6c27ffb07a4176d8283aa0710b6b84dbace4d95f",
 "version": "2023.2.0"
}
'''  # END VERSION_JSON


def get_versions():
    return json.loads(version_json)

env_pyrecodes/lib/python3.9/site-packages/xyzservices/_version.py
NikolaBlagojevic/pyrecodes
"""
Utilities to support XYZservices
"""
from __future__ import annotations

import json
import uuid
import urllib.request
from typing import Optional, Callable, Union
from urllib.parse import quote

QUERY_NAME_TRANSLATION = str.maketrans({x: "" for x in "., -_/"})


class Bunch(dict):
    """A dict with attribute-access

    :class:`Bunch` is used to store :class:`TileProvider` objects.

    Examples
    --------
    >>> black_and_white = TileProvider(
    ...     name="My black and white tiles",
    ...     url="https://myserver.com/bw/{z}/{x}/{y}",
    ...     attribution="(C) xyzservices",
    ... )
    >>> colorful = TileProvider(
    ...     name="My colorful tiles",
    ...     url="https://myserver.com/color/{z}/{x}/{y}",
    ...     attribution="(C) xyzservices",
    ... )
    >>> MyTiles = Bunch(BlackAndWhite=black_and_white, Colorful=colorful)
    >>> MyTiles
    {'BlackAndWhite': {'name': 'My black and white tiles', 'url': \
'https://myserver.com/bw/{z}/{x}/{y}', 'attribution': '(C) xyzservices'}, 'Colorful': \
{'name': 'My colorful tiles', 'url': 'https://myserver.com/color/{z}/{x}/{y}', \
'attribution': '(C) xyzservices'}}
    >>> MyTiles.BlackAndWhite.url
    'https://myserver.com/bw/{z}/{x}/{y}'
    """

    def __getattr__(self, key):
        try:
            return self.__getitem__(key)
        except KeyError:
            raise AttributeError(key)

    def __dir__(self):
        return self.keys()

    def _repr_html_(self, inside=False):

        children = ""
        for key in self.keys():
            if isinstance(self[key], TileProvider):
                obj = "xyzservices.TileProvider"
            else:
                obj = "xyzservices.Bunch"
            uid = str(uuid.uuid4())
            children += f"""
            <li class="xyz-child">
                <input type="checkbox" id="{uid}" class="xyz-checkbox"/>
                <label for="{uid}">{key} <span>{obj}</span></label>
                <div class="xyz-inside">
                    {self[key]._repr_html_(inside=True)}
                </div>
            </li>
            """

        style = "" if inside else f"<style>{CSS_STYLE}</style>"
        html = f"""
        <div>
        {style}
            <div class="xyz-wrap">
                <div class="xyz-header">
                    <div class="xyz-obj">xyzservices.Bunch</div>
                    <div class="xyz-name">{len(self)} items</div>
                </div>
                <div class="xyz-details">
                    <ul class="xyz-collapsible">
                        {children}
                    </ul>
                </div>
            </div>
        </div>
        """

        return html

    def flatten(self) -> dict:
        """Return the nested :class:`Bunch` collapsed into the one level dictionary.

        Dictionary keys are :class:`TileProvider` names (e.g. ``OpenStreetMap.Mapnik``)
        and its values are :class:`TileProvider` objects.

        Returns
        -------
        flattened : dict
            dictionary of :class:`TileProvider` objects

        Examples
        --------
        >>> import xyzservices.providers as xyz
        >>> len(xyz)
        36

        >>> flat = xyz.flatten()
        >>> len(xyz)
        207

        """

        flat = {}

        def _get_providers(provider):
            if isinstance(provider, TileProvider):
                flat[provider.name] = provider
            else:
                for prov in provider.values():
                    _get_providers(prov)

        _get_providers(self)

        return flat

    def filter(
        self,
        keyword: Optional[str] = None,
        name: Optional[str] = None,
        requires_token: Optional[bool] = None,
        function: Callable[[TileProvider], bool] = None,
    ) -> Bunch:
        """Return a subset of the :class:`Bunch` matching the filter conditions

        Each :class:`TileProvider` within a :class:`Bunch` is checked against one or
        more specified conditions and kept if they are satisfied or removed if at least
        one condition is not met.

        Parameters
        ----------
        keyword : str (optional)
            Condition returns ``True`` if ``keyword`` string is present in any string
            value in a :class:`TileProvider` object.
            The comparison is not case sensitive.
        name : str (optional)
            Condition returns ``True`` if ``name`` string is present in
            the name attribute of :class:`TileProvider` object.
            The comparison is not case sensitive.
        requires_token : bool (optional)
            Condition returns ``True`` if :meth:`TileProvider.requires_token` returns
            ``True`` (i.e. if the object requires specification of API token).
        function : callable (optional)
            Custom function taking :class:`TileProvider` as an argument and returns
            bool. If ``function`` is given, other parameters are ignored.

        Returns
        -------
        filtered : Bunch

        Examples
        --------
        >>> import xyzservices.providers as xyz

        You can filter all free providers (not requiring API token):

        >>> free_providers = xyz.filter(requires_token=False)

        Or all providers with ``open`` in the name:

        >>> open_providers = xyz.filter(name="open")

        You can use keyword search to find all providers based on OpenStreetMap data:

        >>> osm_providers = xyz.filter(keyword="openstreetmap")

        You can combine multiple conditions to find providers based on OpenStreetMap
        data that require API token:

        >>> osm_locked = xyz.filter(keyword="openstreetmap", requires_token=True)

        You can also pass custom function that takes :class:`TileProvider` and returns
        boolean value. You can then find all providers with ``max_zoom`` smaller than
        18:

        >>> def zoom18(provider):
        ...    if hasattr(provider, "max_zoom") and provider.max_zoom < 18:
        ...        return True
        ...    return False
        >>> small_zoom = xyz.filter(function=zoom18)
        """

        def _validate(provider, keyword, name, requires_token):

            cond = []

            if keyword is not None:
                keyword_match = False
                for v in provider.values():
                    if isinstance(v, str):
                        if keyword.lower() in v.lower():
                            keyword_match = True
                            break
                cond.append(keyword_match)

            if name is not None:
                name_match = False
                if name.lower() in provider.name.lower():
                    name_match = True
                cond.append(name_match)

            if requires_token is not None:
                token_match = False
                if provider.requires_token() is requires_token:
                    token_match = True
                cond.append(token_match)

            return all(cond)

        def _filter_bunch(bunch, keyword, name, requires_token, function):
            new = Bunch()
            for key, value in bunch.items():
                if isinstance(value, TileProvider):

                    if function is None:
                        if _validate(
                            value,
                            keyword=keyword,
                            name=name,
                            requires_token=requires_token,
                        ):
                            new[key] = value
                    else:
                        if function(value):
                            new[key] = value

                else:
                    filtered = _filter_bunch(
                        value,
                        keyword=keyword,
                        name=name,
                        requires_token=requires_token,
                        function=function,
                    )
                    if filtered:
                        new[key] = filtered

            return new

        return _filter_bunch(
            self,
            keyword=keyword,
            name=name,
            requires_token=requires_token,
            function=function,
        )

    def query_name(self, name: str) -> TileProvider:
        """Return :class:`TileProvider` based on the name query

        Returns a matching :class:`TileProvider` from the :class:`Bunch` if the ``name``
        contains the same letters in the same order as the provider's name irrespective
        of the letter case, spaces, dashes and other characters.
        See examples for details.

        Parameters
        ----------
        name : str
            Name of the tile provider. Formatting does not matter.

        Returns
        -------
        match: TileProvider

        Examples
        --------
        >>> import xyzservices.providers as xyz

        All these queries return the same ``CartoDB.Positron`` TileProvider:

        >>> xyz.query_name("CartoDB Positron")
        >>> xyz.query_name("cartodbpositron")
        >>> xyz.query_name("cartodb-positron")
        >>> xyz.query_name("carto db/positron")
        >>> xyz.query_name("CARTO_DB_POSITRON")
        >>> xyz.query_name("CartoDB.Positron")

        """
        xyz_flat_lower = {
            k.translate(QUERY_NAME_TRANSLATION).lower(): v
            for k, v in self.flatten().items()
        }
        name_clean = name.translate(QUERY_NAME_TRANSLATION).lower()
        if name_clean in xyz_flat_lower:
            return xyz_flat_lower[name_clean]

        raise ValueError(f"No matching provider found for the query '{name}'.")


class TileProvider(Bunch):
    """
    A dict with attribute-access and that
    can be called to update keys


    Examples
    --------

    You can create custom :class:`TileProvider` by passing your attributes to the object
    as it would have been a ``dict()``. It is required to always specify ``name``,
    ``url``, and ``attribution``.

    >>> public_provider = TileProvider(
    ...     name="My public tiles",
    ...     url="https://myserver.com/tiles/{z}/{x}/{y}.png",
    ...     attribution="(C) xyzservices",
    ... )

    Alternatively, you can create it from a dictionary of attributes. When specifying a
    placeholder for the access token, please use the ``"<insert your access token
    here>"`` string to ensure that :meth:`~xyzservices.TileProvider.requires_token`
    method works properly.

    >>> private_provider = TileProvider(
    ...    {
    ...        "url": "https://myserver.com/tiles/{z}/{x}/{y}.png?apikey={accessToken}",
    ...        "attribution": "(C) xyzservices",
    ...        "accessToken": "<insert your access token here>",
    ...        "name": "my_private_provider",
    ...    }
    ... )

    It is customary to include ``html_attribution`` attribute containing HTML string as
    ``'&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>
    contributors'`` alongisde a plain-text ``attribution``.

    You can then fetch all information as attributes:

    >>> public_provider.url
    'https://myserver.com/tiles/{z}/{x}/{y}.png'

    >>> public_provider.attribution
    '(C) xyzservices'

    To ensure you will be able to use the tiles, you can check if the
    :class:`TileProvider` requires a token or API key.

    >>> public_provider.requires_token()
    False
    >>> private_provider.requires_token()
    True

    You can also generate URL in the required format with or without placeholders:

    >>> public_provider.build_url()
    'https://myserver.com/tiles/{z}/{x}/{y}.png'
    >>> private_provider.build_url(x=12, y=21, z=11, accessToken="my_token")
    'https://myserver.com/tiles/11/12/21.png?access_token=my_token'

    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        missing = []
        for el in ["name", "url", "attribution"]:
            if el not in self.keys():
                missing.append(el)
        if len(missing) > 0:
            msg = (
                f"The attributes `name`, `url`, "
                f"and `attribution` are required to initialise "
                f"a `TileProvider`. Please provide values for: "
                f'`{"`, `".join(missing)}`'
            )
            raise AttributeError(msg)

    def __call__(self, **kwargs) -> TileProvider:
        new = TileProvider(self)  # takes a copy preserving the class
        new.update(kwargs)
        return new

    def copy(self, **kwargs) -> TileProvider:
        new = TileProvider(self)  # takes a copy preserving the class
        return new

    def build_url(
        self,
        x: Optional[Union[int, str]] = None,
        y: Optional[Union[int, str]] = None,
        z: Optional[Union[int, str]] = None,
        scale_factor: Optional[str] = None,
        fill_subdomain: Optional[bool] = True,
        **kwargs,
    ) -> str:
        """
        Build the URL of tiles from the :class:`TileProvider` object

        Can return URL with placeholders or the final tile URL.

        Parameters
        ----------

        x, y, z : int (optional)
            tile number
        scale_factor : str (optional)
            Scale factor (where supported). For example, you can get double resolution
            (512 x 512) instead of standard one (256 x 256) with ``"@2x"``. If you want
            to keep a placeholder, pass `"{r}"`.
        fill_subdomain : bool (optional, default True)
            Fill subdomain placeholder with the first available subdomain. If False, the
            URL will contain ``{s}`` placeholder for subdomain.

        **kwargs
            Other potential attributes updating the :class:`TileProvider`.

        Returns
        -------

        url : str
            Formatted URL

        Examples
        --------
        >>> import xyzservices.providers as xyz

        >>> xyz.CartoDB.DarkMatter.build_url()
        'https://a.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png'

        >>> xyz.CartoDB.DarkMatter.build_url(x=9, y=11, z=5)
        'https://a.basemaps.cartocdn.com/dark_all/5/9/11.png'

        >>> xyz.CartoDB.DarkMatter.build_url(x=9, y=11, z=5, scale_factor="@2x")
        'https://a.basemaps.cartocdn.com/dark_all/5/9/11@2x.png'

        >>> xyz.MapBox.build_url(accessToken="my_token")
        'https://api.mapbox.com/styles/v1/mapbox/streets-v11/tiles/{z}/{x}/{y}?access_token=my_token'

        """
        provider = self.copy()

        if x is None:
            x = "{x}"
        if y is None:
            y = "{y}"
        if z is None:
            z = "{z}"

        provider.update(kwargs)

        if provider.requires_token():
            raise ValueError(
                "Token is required for this provider, but not provided. "
                "You can either update TileProvider or pass respective keywords "
                "to build_url()."
            )

        url = provider.pop("url")

        if scale_factor:
            r = scale_factor
            provider.pop("r", None)
        else:
            r = provider.pop("r", "")

        if fill_subdomain:
            subdomains = provider.pop("subdomains", "abc")
            s = subdomains[0]
        else:
            s = "{s}"

        return url.format(x=x, y=y, z=z, s=s, r=r, **provider)

    def requires_token(self) -> bool:
        """
        Returns ``True`` if the TileProvider requires access token to fetch tiles.

        The token attribute name vary and some :class:`TileProvider` objects may require
        more than one token (e.g. ``HERE``). The information is deduced from the
        presence of `'<insert your...'` string in one or more of attributes. When
        specifying a placeholder for the access token, please use the ``"<insert your
        access token here>"`` string to ensure that
        :meth:`~xyzservices.TileProvider.requires_token` method works properly.

        Returns
        -------
        bool

        Examples
        --------
        >>> import xyzservices.providers as xyz
        >>> xyz.MapBox.requires_token()
        True

        >>> xyz.CartoDB.Positron
        False

        We can specify this API key by calling the object or overriding the attribute.
        Overriding the attribute will alter existing object:

        >>> xyz.OpenWeatherMap.Clouds["apiKey"] = "my-private-api-key"

        Calling the object will return a copy:

        >>> xyz.OpenWeatherMap.Clouds(apiKey="my-private-api-key")


        """
        # both attribute and placeholder in url are required to make it work
        for key, val in self.items():
            if isinstance(val, str) and "<insert your" in val:
                if key in self.url:
                    return True
        return False

    @property
    def html_attribution(self):
        if "html_attribution" in self.keys():
            return self["html_attribution"]
        return self["attribution"]

    def _repr_html_(self, inside=False):
        provider_info = ""
        for key, val in self.items():
            if key != "name":
                provider_info += f"<dt><span>{key}</span></dt><dd>{val}</dd>"

        style = "" if inside else f"<style>{CSS_STYLE}</style>"
        html = f"""
        <div>
        {style}
            <div class="xyz-wrap">
                <div class="xyz-header">
                    <div class="xyz-obj">xyzservices.TileProvider</div>
                    <div class="xyz-name">{self.name}</div>
                </div>
                <div class="xyz-details">
                    <dl class="xyz-attrs">
                        {provider_info}
                    </dl>
                </div>
            </div>
        </div>
        """

        return html

    @classmethod
    def from_qms(cls, name: str) -> TileProvider:
        """
        Creates a :class:`TileProvider` object based on the definition from
        the `Quick Map Services <https://qms.nextgis.com/>`__ open catalog.

        Parameters
        ----------
        name : str
            Service name

        Returns
        -------
        :class:`TileProvider`

        Examples
        --------
        >>> from xyzservices.lib import TileProvider
        >>> provider = TileProvider.from_qms("OpenTopoMap")
        """
        QMS_API_URL = "https://qms.nextgis.com/api/v1/geoservices"

        services = json.load(
            urllib.request.urlopen(f"{QMS_API_URL}/?search={quote(name)}&type=tms")
        )

        for service in services:
            if service["name"] == name:
                break
        else:
            raise ValueError(f"Service '{name}' not found.")

        service_id = service["id"]
        service_details = json.load(
            urllib.request.urlopen(f"{QMS_API_URL}/{service_id}")
        )

        return cls(
            name=service_details["name"],
            url=service_details["url"],
            min_zoom=service_details.get("z_min"),
            max_zoom=service_details.get("z_max"),
            attribution=service_details.get("copyright_text"),
        )


def _load_json(f):

    data = json.loads(f)

    providers = Bunch()

    for provider_name in data.keys():
        provider = data[provider_name]

        if "url" in provider.keys():
            providers[provider_name] = TileProvider(provider)

        else:
            providers[provider_name] = Bunch(
                {i: TileProvider(provider[i]) for i in provider}
            )

    return providers


CSS_STYLE = """
/* CSS stylesheet for displaying xyzservices objects in Jupyter.*/
.xyz-wrap {
    --xyz-border-color: var(--jp-border-color2, #ddd);
    --xyz-font-color2: var(--jp-content-font-color2, rgba(128, 128, 128, 1));
    --xyz-background-color-white: var(--jp-layout-color1, white);
    --xyz-background-color: var(--jp-layout-color2, rgba(128, 128, 128, 0.1));
}

html[theme=dark] .xyz-wrap,
body.vscode-dark .xyz-wrap,
body.vscode-high-contrast .xyz-wrap {
    --xyz-border-color: #222;
    --xyz-font-color2: rgba(255, 255, 255, 0.54);
    --xyz-background-color-white: rgba(255, 255, 255, 1);
    --xyz-background-color: rgba(255, 255, 255, 0.05);

}

.xyz-header {
    padding-top: 6px;
    padding-bottom: 6px;
    margin-bottom: 4px;
    border-bottom: solid 1px var(--xyz-border-color);
}

.xyz-header>div {
    display: inline;
    margin-top: 0;
    margin-bottom: 0;
}

.xyz-obj,
.xyz-name {
    margin-left: 2px;
    margin-right: 10px;
}

.xyz-obj {
    color: var(--xyz-font-color2);
}

.xyz-attrs {
    grid-column: 1 / -1;
}

dl.xyz-attrs {
    padding: 0 5px 0 5px;
    margin: 0;
    display: grid;
    grid-template-columns: 135px auto;
    background-color: var(--xyz-background-color);
}

.xyz-attrs dt,
dd {
    padding: 0;
    margin: 0;
    float: left;
    padding-right: 10px;
    width: auto;
}

.xyz-attrs dt {
    font-weight: normal;
    grid-column: 1;
}

.xyz-attrs dd {
    grid-column: 2;
    white-space: pre-wrap;
    word-break: break-all;
}

.xyz-details ul>li>label>span {
    color: var(--xyz-font-color2);
    padding-left: 10px;
}

.xyz-inside {
    display: none;
}

.xyz-checkbox:checked~.xyz-inside {
    display: contents;
}

.xyz-collapsible li>input {
    display: none;
}

.xyz-collapsible>li>label {
    cursor: pointer;
}

.xyz-collapsible>li>label:hover {
    color: var(--xyz-font-color2);
}

ul.xyz-collapsible {
    list-style: none!important;
    padding-left: 20px!important;
}

.xyz-checkbox+label:before {
    content: '►';
    font-size: 11px;
}

.xyz-checkbox:checked+label:before {
    content: '▼';
}

.xyz-wrap {
    margin-bottom: 10px;
}
"""

env_pyrecodes/lib/python3.9/site-packages/xyzservices/lib.py
NikolaBlagojevic/pyrecodes
import os
import pkgutil
import sys

from .lib import _load_json

data_path = os.path.join(sys.prefix, "share", "xyzservices", "providers.json")

if os.path.exists(data_path):
    with open(data_path, "r") as f:
        json = f.read()
else:
    json = pkgutil.get_data("xyzservices", "data/providers.json")

providers = _load_json(json)

env_pyrecodes/lib/python3.9/site-packages/xyzservices/providers.py
NikolaBlagojevic/pyrecodes
import pytest
from urllib.error import URLError

import xyzservices.providers as xyz
from xyzservices import TileProvider, Bunch


@pytest.fixture
def basic_provider():
    return TileProvider(
        url="https://myserver.com/tiles/{z}/{x}/{y}.png",
        attribution="(C) xyzservices",
        name="my_public_provider",
    )


@pytest.fixture
def retina_provider():
    return TileProvider(
        url="https://myserver.com/tiles/{z}/{x}/{y}{r}.png",
        attribution="(C) xyzservices",
        name="my_public_provider2",
        r="@2x",
    )


@pytest.fixture
def silent_retina_provider():
    return TileProvider(
        url="https://myserver.com/tiles/{z}/{x}/{y}{r}.png",
        attribution="(C) xyzservices",
        name="my_public_retina_provider3",
    )


@pytest.fixture
def private_provider():
    return TileProvider(
        url="https://myserver.com/tiles/{z}/{x}/{y}?access_token={accessToken}",
        attribution="(C) xyzservices",
        accessToken="<insert your access token here>",
        name="my_private_provider",
    )


@pytest.fixture
def html_attr_provider():
    return TileProvider(
        url="https://myserver.com/tiles/{z}/{x}/{y}.png",
        attribution="(C) xyzservices",
        html_attribution='&copy; <a href="https://xyzservices.readthedocs.io">xyzservices</a>',  # noqa
        name="my_public_provider_html",
    )


@pytest.fixture
def subdomain_provider():
    return TileProvider(
        url="https://{s}.myserver.com/tiles/{z}/{x}/{y}.png",
        attribution="(C) xyzservices",
        subdomains="abcd",
        name="my_subdomain_provider",
    )


@pytest.fixture
def test_bunch(
    basic_provider,
    retina_provider,
    silent_retina_provider,
    private_provider,
    html_attr_provider,
    subdomain_provider,
):
    return Bunch(
        basic_provider=basic_provider,
        retina_provider=retina_provider,
        silent_retina_provider=silent_retina_provider,
        private_provider=private_provider,
        bunched=Bunch(
            html_attr_provider=html_attr_provider, subdomain_provider=subdomain_provider
        ),
    )


def test_expect_name_url_attribution():
    msg = (
        "The attributes `name`, `url`, and `attribution` are "
        "required to initialise a `TileProvider`. Please provide "
        "values for: "
    )
    with pytest.raises(AttributeError, match=msg + "`name`, `url`, `attribution`"):
        TileProvider({})
    with pytest.raises(AttributeError, match=msg + "`url`, `attribution`"):
        TileProvider({"name": "myname"})
    with pytest.raises(AttributeError, match=msg + "`attribution`"):
        TileProvider({"url": "my_url", "name": "my_name"})
    with pytest.raises(AttributeError, match=msg + "`attribution`"):
        TileProvider(url="my_url", name="my_name")


def test_build_url(
    basic_provider,
    retina_provider,
    silent_retina_provider,
    private_provider,
    subdomain_provider,
):
    expected = "https://myserver.com/tiles/{z}/{x}/{y}.png"
    assert basic_provider.build_url() == expected

    expected = "https://myserver.com/tiles/3/1/2.png"
    assert basic_provider.build_url(1, 2, 3) == expected
    assert basic_provider.build_url(1, 2, 3, scale_factor="@2x") == expected
    assert silent_retina_provider.build_url(1, 2, 3) == expected

    expected = "https://myserver.com/tiles/3/1/2@2x.png"
    assert retina_provider.build_url(1, 2, 3) == expected
    assert silent_retina_provider.build_url(1, 2, 3, scale_factor="@2x") == expected

    expected = "https://myserver.com/tiles/3/1/2@5x.png"
    assert retina_provider.build_url(1, 2, 3, scale_factor="@5x") == expected

    expected = "https://myserver.com/tiles/{z}/{x}/{y}?access_token=my_token"
    assert private_provider.build_url(accessToken="my_token") == expected

    with pytest.raises(ValueError, match="Token is required for this provider"):
        private_provider.build_url()

    expected = "https://{s}.myserver.com/tiles/{z}/{x}/{y}.png"
    assert subdomain_provider.build_url(fill_subdomain=False)

    expected = "https://a.myserver.com/tiles/{z}/{x}/{y}.png"
    assert subdomain_provider.build_url()


def test_requires_token(private_provider, basic_provider):
    assert private_provider.requires_token() is True
    assert basic_provider.requires_token() is False


def test_html_repr(basic_provider, retina_provider):
    provider_strings = [
        '<div class="xyz-wrap">',
        '<div class="xyz-header">',
        '<div class="xyz-obj">xyzservices.TileProvider</div>',
        '<div class="xyz-name">my_public_provider</div>',
        '<div class="xyz-details">',
        '<dl class="xyz-attrs">',
        "<dt><span>url</span></dt><dd>https://myserver.com/tiles/{z}/{x}/{y}.png</dd>",
        "<dt><span>attribution</span></dt><dd>(C) xyzservices</dd>",
    ]

    for html_string in provider_strings:
        assert html_string in basic_provider._repr_html_()

    bunch = Bunch({"first": basic_provider, "second": retina_provider})

    bunch_strings = [
        '<div class="xyz-obj">xyzservices.Bunch</div>',
        '<div class="xyz-name">2 items</div>',
        '<ul class="xyz-collapsible">',
        '<li class="xyz-child">',
        "<span>xyzservices.TileProvider</span>",
        '<div class="xyz-inside">',
    ]

    bunch_repr = bunch._repr_html_()
    for html_string in provider_strings + bunch_strings:
        assert html_string in bunch_repr
    assert bunch_repr.count('<li class="xyz-child">') == 2
    assert bunch_repr.count('<div class="xyz-wrap">') == 3
    assert bunch_repr.count('<div class="xyz-header">') == 3


def test_copy(basic_provider):
    basic2 = basic_provider.copy()
    assert isinstance(basic2, TileProvider)


def test_callable():
    # only testing the callable functionality to override a keyword, as we
    # cannot test the actual providers that need an API key
    updated_provider = xyz.GeoportailFrance.plan(apikey="mykey")
    assert isinstance(updated_provider, TileProvider)
    assert "url" in updated_provider
    assert updated_provider["apikey"] == "mykey"
    # check that original provider dict is not modified
    assert xyz.GeoportailFrance.plan["apikey"] == "essentiels"


def test_html_attribution_fallback(basic_provider, html_attr_provider):
    # TileProvider.html_attribution falls back to .attribution if the former not present
    assert basic_provider.html_attribution == basic_provider.attribution
    assert (
        html_attr_provider.html_attribution
        == '&copy; <a href="https://xyzservices.readthedocs.io">xyzservices</a>'
    )


@pytest.mark.xfail(reason="timeout error", raises=URLError)
def test_from_qms():
    provider = TileProvider.from_qms("OpenStreetMap Standard aka Mapnik")
    assert isinstance(provider, TileProvider)


@pytest.mark.xfail(reason="timeout error", raises=URLError)
def test_from_qms_not_found_error():
    with pytest.raises(ValueError):
        TileProvider.from_qms("LolWut")


def test_flatten(
    basic_provider, retina_provider, silent_retina_provider, private_provider
):
    nested_bunch = Bunch(
        first_bunch=Bunch(first=basic_provider, second=retina_provider),
        second_bunch=Bunch(first=silent_retina_provider, second=private_provider),
    )

    assert len(nested_bunch) == 2
    assert len(nested_bunch.flatten()) == 4


def test_filter(test_bunch):
    assert len(test_bunch.filter(keyword="private").flatten()) == 1
    assert len(test_bunch.filter(keyword="public").flatten()) == 4
    assert len(test_bunch.filter(keyword="{s}").flatten()) == 1
    assert len(test_bunch.filter(name="retina").flatten()) == 1
    assert len(test_bunch.filter(requires_token=True).flatten()) == 1
    assert len(test_bunch.filter(requires_token=False).flatten()) == 5
    assert len(test_bunch.filter(requires_token=False)) == 4  # check nested structure
    assert len(test_bunch.filter(keyword="{s}", requires_token=False).flatten()) == 1
    assert len(test_bunch.filter(name="nonsense").flatten()) == 0

    def custom(provider):
        if hasattr(provider, "subdomains") and provider.subdomains == "abcd":
            return True
        if hasattr(provider, "r"):
            return True
        return False

    assert len(test_bunch.filter(function=custom).flatten()) == 2


def test_query_name():
    options = [
        "CartoDB Positron",
        "cartodbpositron",
        "cartodb-positron",
        "carto db/positron",
        "CARTO_DB_POSITRON",
        "CartoDB.Positron",
        "Carto,db,positron",
    ]

    for option in options:
        queried = xyz.query_name(option)
        assert isinstance(queried, TileProvider)
        assert queried.name == "CartoDB.Positron"

    with pytest.raises(ValueError, match="No matching provider found"):
        xyz.query_name("i don't exist")

    # Name with underscore GH124
    option_with_underscore = "NASAGIBS.ASTER_GDEM_Greyscale_Shaded_Relief"
    queried = xyz.query_name(option_with_underscore)
    assert isinstance(queried, TileProvider)
    assert queried.name == option_with_underscore

env_pyrecodes/lib/python3.9/site-packages/xyzservices/tests/test_lib.py
NikolaBlagojevic/pyrecodes
import os

import pytest
import xyzservices.providers as xyz
import requests
import mercantile

flat_free = xyz.filter(requires_token=False).flatten()


def check_provider(provider):
    for key in ["attribution", "name"]:
        assert key in provider.keys()
    assert provider.url.startswith("http")
    for option in ["{z}", "{y}", "{x}"]:
        assert option in provider.url


def get_tile(provider):
    bounds = provider.get("bounds", [[-180, -90], [180, 90]])
    lat = (bounds[0][0] + bounds[1][0]) / 2
    lon = (bounds[0][1] + bounds[1][1]) / 2
    zoom = (provider.get("min_zoom", 0) + provider.get("max_zoom", 20)) // 2
    tile = mercantile.tile(lon, lat, zoom)
    z = tile.z
    x = tile.x
    y = tile.y
    return (z, x, y)


def get_response(url):
    s = requests.Session()
    a = requests.adapters.HTTPAdapter(max_retries=3)
    s.mount("http://", a)
    s.mount("https://", a)
    try:
        r = s.get(url, timeout=30)
    except requests.ConnectionError:
        pytest.xfail("Timeout.")
    return r.status_code


def get_test_result(provider, allow_403=True):
    if provider.get("status"):
        pytest.xfail("Provider is known to be broken.")

    z, x, y = get_tile(provider)

    try:
        r = get_response(provider.build_url(z=z, x=x, y=y))
        assert r == requests.codes.ok
    except AssertionError:
        if r == 403 and allow_403:
            pytest.xfail("Provider not available due to API restrictions (Error 403).")

        elif r == 503:
            pytest.xfail("Service temporarily unavailable (Error 503).")

        elif r == 502:
            pytest.xfail("Bad Gateway (Error 502).")

        # check another tiles
        elif r == 404:
            # in some cases, the computed tile is not available. trying known tiles.
            options = [
                (12, 2154, 1363),
                (6, 13, 21),
                (16, 33149, 22973),
                (0, 0, 0),
                (2, 6, 7),
                (6, 21, 31),
                (6, 21, 32),
                (6, 21, 33),
                (6, 22, 31),
                (6, 22, 32),
                (6, 22, 33),
                (6, 23, 31),
                (6, 23, 32),
                (6, 23, 33),
                (9, 259, 181),
                (12, 2074, 1410),
            ]
            results = []
            for o in options:
                z, x, y = o
                r = get_response(provider.build_url(z=z, x=x, y=y))
                results.append(r)
            if not any([x == requests.codes.ok for x in results]):
                raise ValueError(f"Response code: {r}")
        else:
            raise ValueError(f"Response code: {r}")


@pytest.mark.parametrize("provider_name", xyz.flatten())
def test_minimal_provider_metadata(provider_name):
    provider = xyz.flatten()[provider_name]
    check_provider(provider)


@pytest.mark.request
@pytest.mark.parametrize("name", flat_free)
def test_free_providers(name):
    provider = flat_free[name]
    get_test_result(provider)


# test providers requiring API keys. Store API keys in GitHub secrets and load them as
# environment variables in CI Action. Note that env variable is loaded as empty on PRs
# from a fork.


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.Thunderforest)
def test_thunderforest(provider_name):
    try:
        token = os.environ["THUNDERFOREST"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.Thunderforest[provider_name](apikey=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.Jawg)
def test_jawg(provider_name):
    try:
        token = os.environ["JAWG"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.Jawg[provider_name](accessToken=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
def test_mapbox():
    try:
        token = os.environ["MAPBOX"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.MapBox(accessToken=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.MapTiler)
def test_maptiler(provider_name):
    try:
        token = os.environ["MAPTILER"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.MapTiler[provider_name](key=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.TomTom)
def test_tomtom(provider_name):
    try:
        token = os.environ["TOMTOM"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.TomTom[provider_name](apikey=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.OpenWeatherMap)
def test_openweathermap(provider_name):
    try:
        token = os.environ["OPENWEATHERMAP"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.OpenWeatherMap[provider_name](apiKey=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.HEREv3)
def test_herev3(provider_name):
    try:
        token = os.environ["HEREV3"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.HEREv3[provider_name](apiKey=token)
    get_test_result(provider, allow_403=False)


@pytest.mark.request
@pytest.mark.parametrize("provider_name", xyz.Stadia)
def test_stadia(provider_name):
    try:
        token = os.environ["STADIA"]
    except KeyError:
        pytest.xfail("Missing API token.")
    if token == "":
        pytest.xfail("Token empty.")

    provider = xyz.Stadia[provider_name](api_key=token)
    provider["url"] = provider["url"] + "?api_key={api_key}"
    get_test_result(provider, allow_403=False)


# NOTE: AzureMaps are not tested as their free account is limited to
# 5000 downloads (total, not per month)

env_pyrecodes/lib/python3.9/site-packages/xyzservices/tests/test_providers.py
NikolaBlagojevic/pyrecodes
"""Test asyncio support"""
# Copyright (c) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.

import asyncio
import json
import os
import sys
from concurrent.futures import CancelledError
from multiprocessing import Process

import pytest
from pytest import mark

import zmq
import zmq.asyncio as zaio


@pytest.fixture
def Context(event_loop):
    return zaio.Context


def test_socket_class(context):
    with context.socket(zmq.PUSH) as s:
        assert isinstance(s, zaio.Socket)


def test_instance_subclass_first(context):
    actx = zmq.asyncio.Context.instance()
    ctx = zmq.Context.instance()
    ctx.term()
    actx.term()
    assert type(ctx) is zmq.Context
    assert type(actx) is zmq.asyncio.Context


def test_instance_subclass_second(context):
    with zmq.Context.instance() as ctx:
        assert type(ctx) is zmq.Context
        with zmq.asyncio.Context.instance() as actx:
            assert type(actx) is zmq.asyncio.Context


async def test_recv_multipart(context, create_bound_pair):
    a, b = create_bound_pair(zmq.PUSH, zmq.PULL)
    f = b.recv_multipart()
    assert not f.done()
    await a.send(b"hi")
    recvd = await f
    assert recvd == [b"hi"]


async def test_recv(create_bound_pair):
    a, b = create_bound_pair(zmq.PUSH, zmq.PULL)
    f1 = b.recv()
    f2 = b.recv()
    assert not f1.done()
    assert not f2.done()
    await a.send_multipart([b"hi", b"there"])
    recvd = await f2
    assert f1.done()
    assert f1.result() == b"hi"
    assert recvd == b"there"


@mark.skipif(not hasattr(zmq, "RCVTIMEO"), reason="requires RCVTIMEO")
async def test_recv_timeout(push_pull):
    a, b = push_pull
    b.rcvtimeo = 100
    f1 = b.recv()
    b.rcvtimeo = 1000
    f2 = b.recv_multipart()
    with pytest.raises(zmq.Again):
        await f1
    await a.send_multipart([b"hi", b"there"])
    recvd = await f2
    assert f2.done()
    assert recvd == [b"hi", b"there"]


@mark.skipif(not hasattr(zmq, "SNDTIMEO"), reason="requires SNDTIMEO")
async def test_send_timeout(socket):
    s = socket(zmq.PUSH)
    s.sndtimeo = 100
    with pytest.raises(zmq.Again):
        await s.send(b"not going anywhere")


async def test_recv_string(push_pull):
    a, b = push_pull
    f = b.recv_string()
    assert not f.done()
    msg = "πøøπ"
    await a.send_string(msg)
    recvd = await f
    assert f.done()
    assert f.result() == msg
    assert recvd == msg


async def test_recv_json(push_pull):
    a, b = push_pull
    f = b.recv_json()
    assert not f.done()
    obj = dict(a=5)
    await a.send_json(obj)
    recvd = await f
    assert f.done()
    assert f.result() == obj
    assert recvd == obj


async def test_recv_json_cancelled(push_pull):
    a, b = push_pull
    f = b.recv_json()
    assert not f.done()
    f.cancel()
    # cycle eventloop to allow cancel events to fire
    await asyncio.sleep(0)
    obj = dict(a=5)
    await a.send_json(obj)
    # CancelledError change in 3.8 https://bugs.python.org/issue32528
    if sys.version_info < (3, 8):
        with pytest.raises(CancelledError):
            recvd = await f
    else:
        with pytest.raises(asyncio.exceptions.CancelledError):
            recvd = await f
    assert f.done()
    # give it a chance to incorrectly consume the event
    events = await b.poll(timeout=5)
    assert events
    await asyncio.sleep(0)
    # make sure cancelled recv didn't eat up event
    f = b.recv_json()
    recvd = await asyncio.wait_for(f, timeout=5)
    assert recvd == obj


async def test_recv_pyobj(push_pull):
    a, b = push_pull
    f = b.recv_pyobj()
    assert not f.done()
    obj = dict(a=5)
    await a.send_pyobj(obj)
    recvd = await f
    assert f.done()
    assert f.result() == obj
    assert recvd == obj


async def test_custom_serialize(create_bound_pair):
    def serialize(msg):
        frames = []
        frames.extend(msg.get("identities", []))
        content = json.dumps(msg["content"]).encode("utf8")
        frames.append(content)
        return frames

    def deserialize(frames):
        identities = frames[:-1]
        content = json.loads(frames[-1].decode("utf8"))
        return {
            "identities": identities,
            "content": content,
        }

    a, b = create_bound_pair(zmq.DEALER, zmq.ROUTER)

    msg = {
        "content": {
            "a": 5,
            "b": "bee",
        }
    }
    await a.send_serialized(msg, serialize)
    recvd = await b.recv_serialized(deserialize)
    assert recvd["content"] == msg["content"]
    assert recvd["identities"]
    # bounce back, tests identities
    await b.send_serialized(recvd, serialize)
    r2 = await a.recv_serialized(deserialize)
    assert r2["content"] == msg["content"]
    assert not r2["identities"]


async def test_custom_serialize_error(dealer_router):
    a, b = dealer_router

    msg = {
        "content": {
            "a": 5,
            "b": "bee",
        }
    }
    with pytest.raises(TypeError):
        await a.send_serialized(json, json.dumps)

    await a.send(b"not json")
    with pytest.raises(TypeError):
        await b.recv_serialized(json.loads)


async def test_recv_dontwait(push_pull):
    push, pull = push_pull
    f = pull.recv(zmq.DONTWAIT)
    with pytest.raises(zmq.Again):
        await f
    await push.send(b"ping")
    await pull.poll()  # ensure message will be waiting
    f = pull.recv(zmq.DONTWAIT)
    assert f.done()
    msg = await f
    assert msg == b"ping"


async def test_recv_cancel(push_pull):
    a, b = push_pull
    f1 = b.recv()
    f2 = b.recv_multipart()
    assert f1.cancel()
    assert f1.done()
    assert not f2.done()
    await a.send_multipart([b"hi", b"there"])
    recvd = await f2
    assert f1.cancelled()
    assert f2.done()
    assert recvd == [b"hi", b"there"]


async def test_poll(push_pull):
    a, b = push_pull
    f = b.poll(timeout=0)
    await asyncio.sleep(0)
    assert f.result() == 0

    f = b.poll(timeout=1)
    assert not f.done()
    evt = await f

    assert evt == 0

    f = b.poll(timeout=1000)
    assert not f.done()
    await a.send_multipart([b"hi", b"there"])
    evt = await f
    assert evt == zmq.POLLIN
    recvd = await b.recv_multipart()
    assert recvd == [b"hi", b"there"]


async def test_poll_base_socket(sockets):
    ctx = zmq.Context()
    url = "inproc://test"
    a = ctx.socket(zmq.PUSH)
    b = ctx.socket(zmq.PULL)
    sockets.extend([a, b])
    a.bind(url)
    b.connect(url)

    poller = zaio.Poller()
    poller.register(b, zmq.POLLIN)

    f = poller.poll(timeout=1000)
    assert not f.done()
    a.send_multipart([b"hi", b"there"])
    evt = await f
    assert evt == [(b, zmq.POLLIN)]
    recvd = b.recv_multipart()
    assert recvd == [b"hi", b"there"]


async def test_poll_on_closed_socket(push_pull):
    a, b = push_pull

    f = b.poll(timeout=1)
    b.close()

    # The test might stall if we try to await f directly so instead just make a few
    # passes through the event loop to schedule and execute all callbacks
    for _ in range(5):
        await asyncio.sleep(0)
        if f.cancelled():
            break
    assert f.cancelled()


@pytest.mark.skipif(
    sys.platform.startswith("win"),
    reason="Windows does not support polling on files",
)
async def test_poll_raw():
    p = zaio.Poller()
    # make a pipe
    r, w = os.pipe()
    r = os.fdopen(r, "rb")
    w = os.fdopen(w, "wb")

    # POLLOUT
    p.register(r, zmq.POLLIN)
    p.register(w, zmq.POLLOUT)
    evts = await p.poll(timeout=1)
    evts = dict(evts)
    assert r.fileno() not in evts
    assert w.fileno() in evts
    assert evts[w.fileno()] == zmq.POLLOUT

    # POLLIN
    p.unregister(w)
    w.write(b"x")
    w.flush()
    evts = await p.poll(timeout=1000)
    evts = dict(evts)
    assert r.fileno() in evts
    assert evts[r.fileno()] == zmq.POLLIN
    assert r.read(1) == b"x"
    r.close()
    w.close()


def test_multiple_loops(push_pull):
    a, b = push_pull

    async def test():
        await a.send(b'buf')
        msg = await b.recv()
        assert msg == b'buf'

    for i in range(3):
        loop = asyncio.new_event_loop()
        loop.run_until_complete(asyncio.wait_for(test(), timeout=10))
        loop.close()


def test_shadow():
    with zmq.Context() as ctx:
        s = ctx.socket(zmq.PULL)
        async_s = zaio.Socket(s)
        assert isinstance(async_s, zaio.Socket)
        assert async_s.underlying == s.underlying
        assert async_s.type == s.type


class ProcessForTeardownTest(Process):
    def run(self):
        """Leave context, socket and event loop upon implicit disposal"""

        actx = zaio.Context.instance()
        socket = actx.socket(zmq.PAIR)
        socket.bind_to_random_port("tcp://127.0.0.1")

        async def never_ending_task(socket):
            await socket.recv()  # never ever receive anything

        loop = asyncio.new_event_loop()
        coro = asyncio.wait_for(never_ending_task(socket), timeout=1)
        try:
            loop.run_until_complete(coro)
        except asyncio.TimeoutError:
            pass  # expected timeout
        else:
            assert False, "never_ending_task was completed unexpectedly"
        finally:
            loop.close()


def test_process_teardown(request):
    proc = ProcessForTeardownTest()
    proc.start()
    request.addfinalizer(proc.terminate)
    proc.join(10)  # starting new Python process may cost a lot
    assert proc.exitcode is not None, "process teardown hangs"
    assert proc.exitcode == 0, f"Python process died with code {proc.exitcode}"

env_pyrecodes/lib/python3.9/site-packages/zmq/tests/test_asyncio.py
NikolaBlagojevic/pyrecodes
import os
import sys

import pytest

import zmq

HERE = os.path.dirname(__file__)
cython_ext = os.path.join(HERE, "cython_ext.pyx")


@pytest.mark.skipif(
    not os.path.exists(cython_ext),
    reason=f"Requires cython test file {cython_ext}",
)
@pytest.mark.skipif(
    'zmq.backend.cython' not in sys.modules, reason="Requires cython backend"
)
@pytest.mark.skipif(
    sys.platform.startswith('win'), reason="Don't try runtime Cython on Windows"
)
@pytest.mark.parametrize('language_level', [3, 2])
def test_cython(language_level, request, tmpdir):
    import pyximport

    assert 'zmq.tests.cython_ext' not in sys.modules

    importers = pyximport.install(
        setup_args=dict(include_dirs=zmq.get_includes()),
        language_level=language_level,
        build_dir=str(tmpdir),
    )

    cython_ext = None

    def unimport():
        pyximport.uninstall(*importers)
        sys.modules.pop('zmq.tests.cython_ext', None)

    request.addfinalizer(unimport)

    # this import tests the compilation
    from . import cython_ext

    assert hasattr(cython_ext, 'send_recv_test')

    # call the compiled function
    # this shouldn't do much
    msg = b'my msg'
    received = cython_ext.send_recv_test(msg)
    assert received == msg

env_pyrecodes/lib/python3.9/site-packages/zmq/tests/test_cython.py
NikolaBlagojevic/pyrecodes
# Copyright (c) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.

import json
import os
import sys
from datetime import timedelta

import pytest

gen = pytest.importorskip('tornado.gen')

from tornado.ioloop import IOLoop

import zmq
from zmq.eventloop import future
from zmq.tests import BaseZMQTestCase


class TestFutureSocket(BaseZMQTestCase):
    Context = future.Context

    def setUp(self):
        self.loop = IOLoop(make_current=False)
        super().setUp()

    def tearDown(self):
        super().tearDown()
        if self.loop:
            self.loop.close(all_fds=True)

    def test_socket_class(self):
        s = self.context.socket(zmq.PUSH)
        assert isinstance(s, future.Socket)
        s.close()

    def test_instance_subclass_first(self):
        actx = self.Context.instance()
        ctx = zmq.Context.instance()
        ctx.term()
        actx.term()
        assert type(ctx) is zmq.Context
        assert type(actx) is self.Context

    def test_instance_subclass_second(self):
        ctx = zmq.Context.instance()
        actx = self.Context.instance()
        ctx.term()
        actx.term()
        assert type(ctx) is zmq.Context
        assert type(actx) is self.Context

    def test_recv_multipart(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_multipart()
            assert not f.done()
            await a.send(b"hi")
            recvd = await f
            assert recvd == [b'hi']

        self.loop.run_sync(test)

    def test_recv(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f1 = b.recv()
            f2 = b.recv()
            assert not f1.done()
            assert not f2.done()
            await a.send_multipart([b"hi", b"there"])
            recvd = await f2
            assert f1.done()
            assert f1.result() == b'hi'
            assert recvd == b'there'

        self.loop.run_sync(test)

    def test_recv_cancel(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f1 = b.recv()
            f2 = b.recv_multipart()
            assert f1.cancel()
            assert f1.done()
            assert not f2.done()
            await a.send_multipart([b"hi", b"there"])
            recvd = await f2
            assert f1.cancelled()
            assert f2.done()
            assert recvd == [b'hi', b'there']

        self.loop.run_sync(test)

    @pytest.mark.skipif(not hasattr(zmq, 'RCVTIMEO'), reason="requires RCVTIMEO")
    def test_recv_timeout(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            b.rcvtimeo = 100
            f1 = b.recv()
            b.rcvtimeo = 1000
            f2 = b.recv_multipart()
            with pytest.raises(zmq.Again):
                await f1
            await a.send_multipart([b"hi", b"there"])
            recvd = await f2
            assert f2.done()
            assert recvd == [b'hi', b'there']

        self.loop.run_sync(test)

    @pytest.mark.skipif(not hasattr(zmq, 'SNDTIMEO'), reason="requires SNDTIMEO")
    def test_send_timeout(self):
        async def test():
            s = self.socket(zmq.PUSH)
            s.sndtimeo = 100
            with pytest.raises(zmq.Again):
                await s.send(b"not going anywhere")

        self.loop.run_sync(test)

    def test_send_noblock(self):
        async def test():
            s = self.socket(zmq.PUSH)
            with pytest.raises(zmq.Again):
                await s.send(b"not going anywhere", flags=zmq.NOBLOCK)

        self.loop.run_sync(test)

    def test_send_multipart_noblock(self):
        async def test():
            s = self.socket(zmq.PUSH)
            with pytest.raises(zmq.Again):
                await s.send_multipart([b"not going anywhere"], flags=zmq.NOBLOCK)

        self.loop.run_sync(test)

    def test_recv_string(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_string()
            assert not f.done()
            msg = 'πøøπ'
            await a.send_string(msg)
            recvd = await f
            assert f.done()
            assert f.result() == msg
            assert recvd == msg

        self.loop.run_sync(test)

    def test_recv_json(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            obj = dict(a=5)
            await a.send_json(obj)
            recvd = await f
            assert f.done()
            assert f.result() == obj
            assert recvd == obj

        self.loop.run_sync(test)

    def test_recv_json_cancelled(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            await gen.sleep(0)
            obj = dict(a=5)
            await a.send_json(obj)
            with pytest.raises(future.CancelledError):
                recvd = await f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = await b.poll(timeout=5)
            assert events
            await gen.sleep(0)
            # make sure cancelled recv didn't eat up event
            recvd = await gen.with_timeout(timedelta(seconds=5), b.recv_json())
            assert recvd == obj

        self.loop.run_sync(test)

    def test_recv_pyobj(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_pyobj()
            assert not f.done()
            obj = dict(a=5)
            await a.send_pyobj(obj)
            recvd = await f
            assert f.done()
            assert f.result() == obj
            assert recvd == obj

        self.loop.run_sync(test)

    def test_custom_serialize(self):
        def serialize(msg):
            frames = []
            frames.extend(msg.get('identities', []))
            content = json.dumps(msg['content']).encode('utf8')
            frames.append(content)
            return frames

        def deserialize(frames):
            identities = frames[:-1]
            content = json.loads(frames[-1].decode('utf8'))
            return {
                'identities': identities,
                'content': content,
            }

        async def test():
            a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

            msg = {
                'content': {
                    'a': 5,
                    'b': 'bee',
                }
            }
            await a.send_serialized(msg, serialize)
            recvd = await b.recv_serialized(deserialize)
            assert recvd['content'] == msg['content']
            assert recvd['identities']
            # bounce back, tests identities
            await b.send_serialized(recvd, serialize)
            r2 = await a.recv_serialized(deserialize)
            assert r2['content'] == msg['content']
            assert not r2['identities']

        self.loop.run_sync(test)

    def test_custom_serialize_error(self):
        async def test():
            a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

            msg = {
                'content': {
                    'a': 5,
                    'b': 'bee',
                }
            }
            with pytest.raises(TypeError):
                await a.send_serialized(json, json.dumps)

            await a.send(b"not json")
            with pytest.raises(TypeError):
                await b.recv_serialized(json.loads)

        self.loop.run_sync(test)

    def test_poll(self):
        async def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.poll(timeout=0)
            assert f.done()
            assert f.result() == 0

            f = b.poll(timeout=1)
            assert not f.done()
            evt = await f
            assert evt == 0

            f = b.poll(timeout=1000)
            assert not f.done()
            await a.send_multipart([b"hi", b"there"])
            evt = await f
            assert evt == zmq.POLLIN
            recvd = await b.recv_multipart()
            assert recvd == [b'hi', b'there']

        self.loop.run_sync(test)

    @pytest.mark.skipif(
        sys.platform.startswith('win'), reason='Windows unsupported socket type'
    )
    def test_poll_base_socket(self):
        async def test():
            ctx = zmq.Context()
            url = 'inproc://test'
            a = ctx.socket(zmq.PUSH)
            b = ctx.socket(zmq.PULL)
            self.sockets.extend([a, b])
            a.bind(url)
            b.connect(url)

            poller = future.Poller()
            poller.register(b, zmq.POLLIN)

            f = poller.poll(timeout=1000)
            assert not f.done()
            a.send_multipart([b'hi', b'there'])
            evt = await f
            assert evt == [(b, zmq.POLLIN)]
            recvd = b.recv_multipart()
            assert recvd == [b'hi', b'there']
            a.close()
            b.close()
            ctx.term()

        self.loop.run_sync(test)

    def test_close_all_fds(self):
        s = self.socket(zmq.PUB)

        async def attach():
            s._get_loop()

        self.loop.run_sync(attach)
        self.loop.close(all_fds=True)
        self.loop = None  # avoid second close later
        assert s.closed

    @pytest.mark.skipif(
        sys.platform.startswith('win'),
        reason='Windows does not support polling on files',
    )
    def test_poll_raw(self):
        async def test():
            p = future.Poller()
            # make a pipe
            r, w = os.pipe()
            r = os.fdopen(r, 'rb')
            w = os.fdopen(w, 'wb')

            # POLLOUT
            p.register(r, zmq.POLLIN)
            p.register(w, zmq.POLLOUT)
            evts = await p.poll(timeout=1)
            evts = dict(evts)
            assert r.fileno() not in evts
            assert w.fileno() in evts
            assert evts[w.fileno()] == zmq.POLLOUT

            # POLLIN
            p.unregister(w)
            w.write(b'x')
            w.flush()
            evts = await p.poll(timeout=1000)
            evts = dict(evts)
            assert r.fileno() in evts
            assert evts[r.fileno()] == zmq.POLLIN
            assert r.read(1) == b'x'
            r.close()
            w.close()

        self.loop.run_sync(test)

env_pyrecodes/lib/python3.9/site-packages/zmq/tests/test_future.py
NikolaBlagojevic/pyrecodes
# Copyright (C) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.

import copy
import errno
import json
import os
import platform
import socket
import sys
import time
import warnings
from unittest import mock

import pytest
from pytest import mark

import zmq
from zmq.tests import BaseZMQTestCase, GreenTest, SkipTest, have_gevent, skip_pypy

pypy = platform.python_implementation().lower() == 'pypy'
windows = platform.platform().lower().startswith('windows')
on_ci = bool(os.environ.get('CI'))

# polling on windows is slow
POLL_TIMEOUT = 1000 if windows else 100


class TestSocket(BaseZMQTestCase):
    def test_create(self):
        ctx = self.Context()
        s = ctx.socket(zmq.PUB)
        # Superluminal protocol not yet implemented
        self.assertRaisesErrno(zmq.EPROTONOSUPPORT, s.bind, 'ftl://a')
        self.assertRaisesErrno(zmq.EPROTONOSUPPORT, s.connect, 'ftl://a')
        self.assertRaisesErrno(zmq.EINVAL, s.bind, 'tcp://')
        s.close()
        ctx.term()

    def test_context_manager(self):
        url = 'inproc://a'
        with self.Context() as ctx:
            with ctx.socket(zmq.PUSH) as a:
                a.bind(url)
                with ctx.socket(zmq.PULL) as b:
                    b.connect(url)
                    msg = b'hi'
                    a.send(msg)
                    rcvd = self.recv(b)
                    assert rcvd == msg
                assert b.closed == True
            assert a.closed == True
        assert ctx.closed == True

    def test_connectbind_context_managers(self):
        url = 'inproc://a'
        msg = b'hi'
        with self.Context() as ctx:
            # Test connect() context manager
            with ctx.socket(zmq.PUSH) as a, ctx.socket(zmq.PULL) as b:
                a.bind(url)
                connect_context = b.connect(url)
                assert f'connect={url!r}' in repr(connect_context)
                with connect_context:
                    a.send(msg)
                    rcvd = self.recv(b)
                    assert rcvd == msg
                # b should now be disconnected, so sending and receiving don't work
                with pytest.raises(zmq.Again):
                    a.send(msg, flags=zmq.DONTWAIT)
                with pytest.raises(zmq.Again):
                    b.recv(flags=zmq.DONTWAIT)
                a.unbind(url)
            # Test bind() context manager
            with ctx.socket(zmq.PUSH) as a, ctx.socket(zmq.PULL) as b:
                # unbind() just stops accepting of new connections, so we have to disconnect to test that
                # unbind happened.
                bind_context = a.bind(url)
                assert f'bind={url!r}' in repr(bind_context)
                with bind_context:
                    b.connect(url)
                    a.send(msg)
                    rcvd = self.recv(b)
                    assert rcvd == msg
                    b.disconnect(url)
                b.connect(url)
                # Since a is unbound from url, b is not connected to anything
                with pytest.raises(zmq.Again):
                    a.send(msg, flags=zmq.DONTWAIT)
                with pytest.raises(zmq.Again):
                    b.recv(flags=zmq.DONTWAIT)

    _repr_cls = "zmq.Socket"

    def test_repr(self):
        with self.context.socket(zmq.PUSH) as s:
            assert f'{self._repr_cls}(zmq.PUSH)' in repr(s)
            assert 'closed' not in repr(s)
        assert f'{self._repr_cls}(zmq.PUSH)' in repr(s)
        assert 'closed' in repr(s)

    def test_dir(self):
        ctx = self.Context()
        s = ctx.socket(zmq.PUB)
        assert 'send' in dir(s)
        assert 'IDENTITY' in dir(s)
        assert 'AFFINITY' in dir(s)
        assert 'FD' in dir(s)
        s.close()
        ctx.term()

    @mark.skipif(mock is None, reason="requires unittest.mock")
    def test_mockable(self):
        s = self.socket(zmq.SUB)
        m = mock.Mock(spec=s)
        s.close()

    def test_bind_unicode(self):
        s = self.socket(zmq.PUB)
        p = s.bind_to_random_port("tcp://*")

    def test_connect_unicode(self):
        s = self.socket(zmq.PUB)
        s.connect("tcp://127.0.0.1:5555")

    def test_bind_to_random_port(self):
        # Check that bind_to_random_port do not hide useful exception
        ctx = self.Context()
        s = ctx.socket(zmq.PUB)
        # Invalid format
        try:
            s.bind_to_random_port('tcp:*')
        except zmq.ZMQError as e:
            assert e.errno == zmq.EINVAL
        # Invalid protocol
        try:
            s.bind_to_random_port('rand://*')
        except zmq.ZMQError as e:
            assert e.errno == zmq.EPROTONOSUPPORT

        s.close()
        ctx.term()

    def test_identity(self):
        s = self.context.socket(zmq.PULL)
        self.sockets.append(s)
        ident = b'identity\0\0'
        s.identity = ident
        assert s.get(zmq.IDENTITY) == ident

    def test_unicode_sockopts(self):
        """test setting/getting sockopts with unicode strings"""
        topic = "tést"
        p, s = self.create_bound_pair(zmq.PUB, zmq.SUB)
        assert s.send_unicode == s.send_unicode
        assert p.recv_unicode == p.recv_unicode
        self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
        self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
        s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
        self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
        s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
        self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
        self.assertRaisesErrno(zmq.EINVAL, s.getsockopt_unicode, zmq.SUBSCRIBE)

        identb = s.getsockopt(zmq.IDENTITY)
        identu = identb.decode('utf16')
        identu2 = s.getsockopt_unicode(zmq.IDENTITY, 'utf16')
        assert identu == identu2
        time.sleep(0.1)  # wait for connection/subscription
        p.send_unicode(topic, zmq.SNDMORE)
        p.send_unicode(topic * 2, encoding='latin-1')
        assert topic == s.recv_unicode()
        assert topic * 2 == s.recv_unicode(encoding='latin-1')

    def test_int_sockopts(self):
        "test integer sockopts"
        v = zmq.zmq_version_info()
        if v < (3, 0):
            default_hwm = 0
        else:
            default_hwm = 1000
        p, s = self.create_bound_pair(zmq.PUB, zmq.SUB)
        p.setsockopt(zmq.LINGER, 0)
        assert p.getsockopt(zmq.LINGER) == 0
        p.setsockopt(zmq.LINGER, -1)
        assert p.getsockopt(zmq.LINGER) == -1
        assert p.hwm == default_hwm
        p.hwm = 11
        assert p.hwm == 11
        # p.setsockopt(zmq.EVENTS, zmq.POLLIN)
        assert p.getsockopt(zmq.EVENTS) == zmq.POLLOUT
        self.assertRaisesErrno(zmq.EINVAL, p.setsockopt, zmq.EVENTS, 2**7 - 1)
        assert p.getsockopt(zmq.TYPE) == p.socket_type
        assert p.getsockopt(zmq.TYPE) == zmq.PUB
        assert s.getsockopt(zmq.TYPE) == s.socket_type
        assert s.getsockopt(zmq.TYPE) == zmq.SUB

        # check for overflow / wrong type:
        errors = []
        backref = {}
        constants = zmq.constants
        for name in constants.__all__:
            value = getattr(constants, name)
            if isinstance(value, int):
                backref[value] = name
        for opt in zmq.constants.SocketOption:
            if opt._opt_type not in {
                zmq.constants._OptType.int,
                zmq.constants._OptType.int64,
            }:
                continue
            if opt.name.startswith(
                (
                    'HWM',
                    'ROUTER',
                    'XPUB',
                    'TCP',
                    'FAIL',
                    'REQ_',
                    'CURVE_',
                    'PROBE_ROUTER',
                    'IPC_FILTER',
                    'GSSAPI',
                    'STREAM_',
                    'VMCI_BUFFER_SIZE',
                    'VMCI_BUFFER_MIN_SIZE',
                    'VMCI_BUFFER_MAX_SIZE',
                    'VMCI_CONNECT_TIMEOUT',
                    'BLOCKY',
                    'IN_BATCH_SIZE',
                    'OUT_BATCH_SIZE',
                    'WSS_TRUST_SYSTEM',
                    'ONLY_FIRST_SUBSCRIBE',
                    'PRIORITY',
                    'RECONNECT_STOP',
                )
            ):
                # some sockopts are write-only
                continue
            try:
                n = p.getsockopt(opt)
            except zmq.ZMQError as e:
                errors.append(f"getsockopt({opt!r}) raised {e}.")
            else:
                if n > 2**31:
                    errors.append(
                        f"getsockopt({opt!r}) returned a ridiculous value."
                        " It is probably the wrong type."
                    )
        if errors:
            self.fail('\n'.join([''] + errors))

    def test_bad_sockopts(self):
        """Test that appropriate errors are raised on bad socket options"""
        s = self.context.socket(zmq.PUB)
        self.sockets.append(s)
        s.setsockopt(zmq.LINGER, 0)
        # unrecognized int sockopts pass through to libzmq, and should raise EINVAL
        self.assertRaisesErrno(zmq.EINVAL, s.setsockopt, 9999, 5)
        self.assertRaisesErrno(zmq.EINVAL, s.getsockopt, 9999)
        # but only int sockopts are allowed through this way, otherwise raise a TypeError
        self.assertRaises(TypeError, s.setsockopt, 9999, b"5")
        # some sockopts are valid in general, but not on every socket:
        self.assertRaisesErrno(zmq.EINVAL, s.setsockopt, zmq.SUBSCRIBE, b'hi')

    def test_sockopt_roundtrip(self):
        "test set/getsockopt roundtrip."
        p = self.context.socket(zmq.PUB)
        self.sockets.append(p)
        p.setsockopt(zmq.LINGER, 11)
        assert p.getsockopt(zmq.LINGER) == 11

    def test_send_unicode(self):
        "test sending unicode objects"
        a, b = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
        self.sockets.extend([a, b])
        u = "çπ§"
        self.assertRaises(TypeError, a.send, u, copy=False)
        self.assertRaises(TypeError, a.send, u, copy=True)
        a.send_unicode(u)
        s = b.recv()
        assert s == u.encode('utf8')
        assert s.decode('utf8') == u
        a.send_unicode(u, encoding='utf16')
        s = b.recv_unicode(encoding='utf16')
        assert s == u

    def test_send_multipart_check_type(self):
        "check type on all frames in send_multipart"
        a, b = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
        self.sockets.extend([a, b])
        self.assertRaises(TypeError, a.send_multipart, [b'a', 5])
        a.send_multipart([b'b'])
        rcvd = self.recv_multipart(b)
        assert rcvd == [b'b']

    @skip_pypy
    def test_tracker(self):
        "test the MessageTracker object for tracking when zmq is done with a buffer"
        addr = 'tcp://127.0.0.1'
        # get a port:
        sock = socket.socket()
        sock.bind(('127.0.0.1', 0))
        port = sock.getsockname()[1]
        iface = "%s:%i" % (addr, port)
        sock.close()
        time.sleep(0.1)

        a = self.context.socket(zmq.PUSH)
        b = self.context.socket(zmq.PULL)
        self.sockets.extend([a, b])
        a.connect(iface)
        time.sleep(0.1)
        p1 = a.send(b'something', copy=False, track=True)
        assert isinstance(p1, zmq.MessageTracker)
        assert p1 is zmq._FINISHED_TRACKER
        # small message, should start done
        assert p1.done

        # disable zero-copy threshold
        a.copy_threshold = 0

        p2 = a.send_multipart([b'something', b'else'], copy=False, track=True)
        assert isinstance(p2, zmq.MessageTracker)
        assert not p2.done

        b.bind(iface)
        msg = self.recv_multipart(b)
        for i in range(10):
            if p1.done:
                break
            time.sleep(0.1)
        assert p1.done == True
        assert msg == [b'something']
        msg = self.recv_multipart(b)
        for i in range(10):
            if p2.done:
                break
            time.sleep(0.1)
        assert p2.done == True
        assert msg == [b'something', b'else']
        m = zmq.Frame(b"again", copy=False, track=True)
        assert m.tracker.done == False
        p1 = a.send(m, copy=False)
        p2 = a.send(m, copy=False)
        assert m.tracker.done == False
        assert p1.done == False
        assert p2.done == False
        msg = self.recv_multipart(b)
        assert m.tracker.done == False
        assert msg == [b'again']
        msg = self.recv_multipart(b)
        assert m.tracker.done == False
        assert msg == [b'again']
        assert p1.done == False
        assert p2.done == False
        m.tracker
        del m
        for i in range(10):
            if p1.done:
                break
            time.sleep(0.1)
        assert p1.done == True
        assert p2.done == True
        m = zmq.Frame(b'something', track=False)
        self.assertRaises(ValueError, a.send, m, copy=False, track=True)

    def test_close(self):
        ctx = self.Context()
        s = ctx.socket(zmq.PUB)
        s.close()
        self.assertRaisesErrno(zmq.ENOTSOCK, s.bind, b'')
        self.assertRaisesErrno(zmq.ENOTSOCK, s.connect, b'')
        self.assertRaisesErrno(zmq.ENOTSOCK, s.setsockopt, zmq.SUBSCRIBE, b'')
        self.assertRaisesErrno(zmq.ENOTSOCK, s.send, b'asdf')
        self.assertRaisesErrno(zmq.ENOTSOCK, s.recv)
        ctx.term()

    def test_attr(self):
        """set setting/getting sockopts as attributes"""
        s = self.context.socket(zmq.DEALER)
        self.sockets.append(s)
        linger = 10
        s.linger = linger
        assert linger == s.linger
        assert linger == s.getsockopt(zmq.LINGER)
        assert s.fd == s.getsockopt(zmq.FD)

    def test_bad_attr(self):
        s = self.context.socket(zmq.DEALER)
        self.sockets.append(s)
        try:
            s.apple = 'foo'
        except AttributeError:
            pass
        else:
            self.fail("bad setattr should have raised AttributeError")
        try:
            s.apple
        except AttributeError:
            pass
        else:
            self.fail("bad getattr should have raised AttributeError")

    def test_subclass(self):
        """subclasses can assign attributes"""

        class S(zmq.Socket):
            a = None

            def __init__(self, *a, **kw):
                self.a = -1
                super().__init__(*a, **kw)

        s = S(self.context, zmq.REP)
        self.sockets.append(s)
        assert s.a == -1
        s.a = 1
        assert s.a == 1
        a = s.a
        assert a == 1

    def test_recv_multipart(self):
        a, b = self.create_bound_pair()
        msg = b'hi'
        for i in range(3):
            a.send(msg)
        time.sleep(0.1)
        for i in range(3):
            assert self.recv_multipart(b) == [msg]

    def test_close_after_destroy(self):
        """s.close() after ctx.destroy() should be fine"""
        ctx = self.Context()
        s = ctx.socket(zmq.REP)
        ctx.destroy()
        # reaper is not instantaneous
        time.sleep(1e-2)
        s.close()
        assert s.closed

    def test_poll(self):
        a, b = self.create_bound_pair()
        time.time()
        evt = a.poll(POLL_TIMEOUT)
        assert evt == 0
        evt = a.poll(POLL_TIMEOUT, zmq.POLLOUT)
        assert evt == zmq.POLLOUT
        msg = b'hi'
        a.send(msg)
        evt = b.poll(POLL_TIMEOUT)
        assert evt == zmq.POLLIN
        msg2 = self.recv(b)
        evt = b.poll(POLL_TIMEOUT)
        assert evt == 0
        assert msg2 == msg

    def test_ipc_path_max_length(self):
        """IPC_PATH_MAX_LEN is a sensible value"""
        if zmq.IPC_PATH_MAX_LEN == 0:
            raise SkipTest("IPC_PATH_MAX_LEN undefined")

        msg = "Surprising value for IPC_PATH_MAX_LEN: %s" % zmq.IPC_PATH_MAX_LEN
        assert zmq.IPC_PATH_MAX_LEN > 30, msg
        assert zmq.IPC_PATH_MAX_LEN < 1025, msg

    def test_ipc_path_max_length_msg(self):
        if zmq.IPC_PATH_MAX_LEN == 0:
            raise SkipTest("IPC_PATH_MAX_LEN undefined")

        s = self.context.socket(zmq.PUB)
        self.sockets.append(s)
        try:
            s.bind('ipc://{}'.format('a' * (zmq.IPC_PATH_MAX_LEN + 1)))
        except zmq.ZMQError as e:
            assert str(zmq.IPC_PATH_MAX_LEN) in e.strerror

    @mark.skipif(windows, reason="ipc not supported on Windows.")
    def test_ipc_path_no_such_file_or_directory_message(self):
        """Display the ipc path in case of an ENOENT exception"""
        s = self.context.socket(zmq.PUB)
        self.sockets.append(s)
        invalid_path = '/foo/bar'
        with pytest.raises(zmq.ZMQError) as error:
            s.bind(f'ipc://{invalid_path}')
        assert error.value.errno == errno.ENOENT
        error_message = str(error.value)
        assert invalid_path in error_message
        assert "no such file or directory" in error_message.lower()

    def test_hwm(self):
        zmq3 = zmq.zmq_version_info()[0] >= 3
        for stype in (zmq.PUB, zmq.ROUTER, zmq.SUB, zmq.REQ, zmq.DEALER):
            s = self.context.socket(stype)
            s.hwm = 100
            assert s.hwm == 100
            if zmq3:
                try:
                    assert s.sndhwm == 100
                except AttributeError:
                    pass
                try:
                    assert s.rcvhwm == 100
                except AttributeError:
                    pass
            s.close()

    def test_copy(self):
        s = self.socket(zmq.PUB)
        scopy = copy.copy(s)
        sdcopy = copy.deepcopy(s)
        assert scopy._shadow
        assert sdcopy._shadow
        assert s.underlying == scopy.underlying
        assert s.underlying == sdcopy.underlying
        s.close()

    def test_send_buffer(self):
        a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
        for buffer_type in (memoryview, bytearray):
            rawbytes = str(buffer_type).encode('ascii')
            msg = buffer_type(rawbytes)
            a.send(msg)
            recvd = b.recv()
            assert recvd == rawbytes

    def test_shadow(self):
        p = self.socket(zmq.PUSH)
        p.bind("tcp://127.0.0.1:5555")
        p2 = zmq.Socket.shadow(p.underlying)
        assert p.underlying == p2.underlying
        s = self.socket(zmq.PULL)
        s2 = zmq.Socket.shadow(s)
        assert s2._shadow_obj is s
        assert s.underlying != p.underlying
        assert s2.underlying == s.underlying
        s3 = zmq.Socket(s)
        assert s3._shadow_obj is s
        assert s3.underlying == s.underlying
        s2.connect("tcp://127.0.0.1:5555")
        sent = b'hi'
        p2.send(sent)
        rcvd = self.recv(s2)
        assert rcvd == sent

    def test_shadow_pyczmq(self):
        try:
            from pyczmq import zctx, zsocket
        except Exception:
            raise SkipTest("Requires pyczmq")

        ctx = zctx.new()
        ca = zsocket.new(ctx, zmq.PUSH)
        cb = zsocket.new(ctx, zmq.PULL)
        a = zmq.Socket.shadow(ca)
        b = zmq.Socket.shadow(cb)
        a.bind("inproc://a")
        b.connect("inproc://a")
        a.send(b'hi')
        rcvd = self.recv(b)
        assert rcvd == b'hi'

    def test_subscribe_method(self):
        pub, sub = self.create_bound_pair(zmq.PUB, zmq.SUB)
        sub.subscribe('prefix')
        sub.subscribe = 'c'
        p = zmq.Poller()
        p.register(sub, zmq.POLLIN)
        # wait for subscription handshake
        for i in range(100):
            pub.send(b'canary')
            events = p.poll(250)
            if events:
                break
        self.recv(sub)
        pub.send(b'prefixmessage')
        msg = self.recv(sub)
        assert msg == b'prefixmessage'
        sub.unsubscribe('prefix')
        pub.send(b'prefixmessage')
        events = p.poll(1000)
        assert events == []

    # CI often can't handle how much memory PyPy uses on this test
    @mark.skipif(
        (pypy and on_ci) or (sys.maxsize < 2**32) or (windows),
        reason="only run on 64b and not on CI.",
    )
    @mark.large
    def test_large_send(self):
        c = os.urandom(1)
        N = 2**31 + 1
        try:
            buf = c * N
        except MemoryError as e:
            raise SkipTest("Not enough memory: %s" % e)
        a, b = self.create_bound_pair()
        try:
            a.send(buf, copy=False)
            rcvd = b.recv(copy=False)
        except MemoryError as e:
            raise SkipTest("Not enough memory: %s" % e)
        # sample the front and back of the received message
        # without checking the whole content
        byte = ord(c)
        view = memoryview(rcvd)
        assert len(view) == N
        assert view[0] == byte
        assert view[-1] == byte

    def test_custom_serialize(self):
        a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

        def serialize(msg):
            frames = []
            frames.extend(msg.get('identities', []))
            content = json.dumps(msg['content']).encode('utf8')
            frames.append(content)
            return frames

        def deserialize(frames):
            identities = frames[:-1]
            content = json.loads(frames[-1].decode('utf8'))
            return {
                'identities': identities,
                'content': content,
            }

        msg = {
            'content': {
                'a': 5,
                'b': 'bee',
            }
        }
        a.send_serialized(msg, serialize)
        recvd = b.recv_serialized(deserialize)
        assert recvd['content'] == msg['content']
        assert recvd['identities']
        # bounce back, tests identities
        b.send_serialized(recvd, serialize)
        r2 = a.recv_serialized(deserialize)
        assert r2['content'] == msg['content']
        assert not r2['identities']


if have_gevent and not windows:
    import gevent

    class TestSocketGreen(GreenTest, TestSocket):
        test_bad_attr = GreenTest.skip_green
        test_close_after_destroy = GreenTest.skip_green
        _repr_cls = "zmq.green.Socket"

        def test_timeout(self):
            a, b = self.create_bound_pair()
            g = gevent.spawn_later(0.5, lambda: a.send(b'hi'))
            timeout = gevent.Timeout(0.1)
            timeout.start()
            self.assertRaises(gevent.Timeout, b.recv)
            g.kill()

        @mark.skipif(not hasattr(zmq, 'RCVTIMEO'), reason="requires RCVTIMEO")
        def test_warn_set_timeo(self):
            s = self.context.socket(zmq.REQ)
            with warnings.catch_warnings(record=True) as w:
                s.rcvtimeo = 5
            s.close()
            assert len(w) == 1
            assert w[0].category == UserWarning

        @mark.skipif(not hasattr(zmq, 'SNDTIMEO'), reason="requires SNDTIMEO")
        def test_warn_get_timeo(self):
            s = self.context.socket(zmq.REQ)
            with warnings.catch_warnings(record=True) as w:
                s.sndtimeo
            s.close()
            assert len(w) == 1
            assert w[0].category == UserWarning

env_pyrecodes/lib/python3.9/site-packages/zmq/tests/test_socket.py
NikolaBlagojevic/pyrecodes
"""
Module used to define the Component class and its subclasses.
"""

import math
from abc import ABC, abstractmethod
from enum import Enum
from pyrecodes import ComponentRecoveryModel
from pyrecodes import Resource
from pyrecodes import Relation
from pyrecodes import ProbabilityDistribution
import numpy as np

class SupplyOrDemand(Enum):
    """
    Enum Class to specify the strings used for component's supply and demand parameters.
    """
    SUPPLY = 'supply'
    DEMAND = 'demand'

class Component(ABC):
    """
    Abstract Class to define the required methods and properties of a component.
    """
    name: str
    functionality_level: float 
    functional: list[int]
    locality: list
    supply: dict
    demand: dict

    @abstractmethod
    def __init__(self) -> None:
        pass

    @abstractmethod
    def construct(self, component_name: str, component_parameters: dict) -> None:
        pass

    @abstractmethod
    def set_initial_damage_level(self, damage_level: float) -> None:
        pass

    @abstractmethod
    def get_damage_level(self) -> float:
        pass

    @abstractmethod
    def update(self, time_step: int) -> None:
        pass

    @abstractmethod
    def set_met_demand_for_recovery_activities(self, resource_name: str, percent_of_met_demand: float) -> None:
        pass

    @abstractmethod
    def recover(self) -> None:
        pass

    @abstractmethod
    def update_supply_based_on_unmet_demand(self, percent_of_met_demand: float) -> None:
        pass

    @abstractmethod
    def set_locality(self, locality_id: list) -> None:
        pass

    @abstractmethod
    def get_locality(self) -> int:
        pass

    @abstractmethod
    def has_operation_demand(self) -> bool:
        pass

    @abstractmethod
    def has_resource_supply(self, resource_name: str) -> bool:
        pass
    
class StandardiReCoDeSComponent(Component):
    """
    Implementation of the Component abstract class to define standard functionality of a component in iRe-CoDeS. 
    """
    class SupplyTypes(Enum):
        """
        | Enum Class to specify the strings used for component's supply types.
        | Only one supply type is currently considered in pyrecodes.
        """
        SUPPLY = 'Supply'

    class DemandTypes(Enum):
        """
        | Enum Class to specify the strings used for component's demand types.
        | Two demand types are currently considered in pyrecodes: operation and recovery demand.
        """
        OPERATION_DEMAND = 'OperationDemand'
        RECOVERY_DEMAND = 'RecoveryDemand'

    def __init__(self) -> None:
        """
        | Initialiaze the StandardiReCoDeSComponent. Set functionality level of a component to 1.
        | Initialize the "functional" time step list and supply and demand of a component as empty dicts.
        """
        self.functionality_level = 1.0
        self.functional = []
        self.recovery_model = None
        self.locality = None
        self.supply = {supply_type.value: dict() for supply_type in self.SupplyTypes}
        self.demand = {demand_type.value: dict() for demand_type in self.DemandTypes}

    def __str__(self) -> str:
        """
        Define a string to describe a component in the print statement.
        """
        return f'{self.name} | Locality: {self.locality}'
    
    def construct(self, component_name: str, component_parameters: dict) -> None:
        self.set_recovery_model(component_parameters['RecoveryModel'])
        self.set_supply(component_parameters.get('Supply', {}))
        self.set_operation_demand(component_parameters.get('OperationDemand', {}))
        self.set_name(component_name)
    
    def set_recovery_model(self, recovery_model_parameters: dict) -> None:
        """
        | Set component recovery model as defined in recovery_model_parameters dict.
        | The data provided in the recovery_model_parameters depends on the recovery model type.
        """
        target_recovery_model = getattr(ComponentRecoveryModel, recovery_model_parameters['Type'])
        self.recovery_model = target_recovery_model(recovery_model_parameters)
    
    def set_supply(self, supply_parameters: dict) -> None:
        """
        | Set component's supply as defined in supply_parameters.
        | Check out the supply_parameters in the examples to get the dict structure.
        | **TODO**: Add the dict structure here. Future work.
        """
        self.add_resources(SupplyOrDemand.SUPPLY.value,
                                self.SupplyTypes.SUPPLY.value,
                                supply_parameters)
    
    def set_operation_demand(self, operation_demand_parameters: dict) -> None:
        """
        | Set component's operation demand as defined in operation_demand_parameters.
        | Check out the operation_demand in the component library files in the examples to get the dict structure.
        | **TODO**: Add the dict structure here. Future work.
        """
        self.add_resources(SupplyOrDemand.DEMAND.value,
                                self.DemandTypes.OPERATION_DEMAND.value,
                                operation_demand_parameters)

    def set_initial_damage_level(self, damage_level: float) -> None:
        """
        Set the initial damage level of the component. Damage level must be between 0 and 1.
        """
        self.recovery_model.set_initial_damage_level(damage_level)

    def get_damage_level(self) -> float:
        """
        Get the current damage level of the component.
        """
        return self.recovery_model.get_damage_level()

    def set_name(self, name: str) -> None:
        """
        Set component name.
        """
        self.name = name

    def set_locality(self, locality_id: list([int])) -> None:
        """
        Set component locality ids. Locality id is a list of integers.
        """
        self.locality = locality_id

    def get_locality(self) -> list:
        """
        Get component locality ids.
        """
        return self.locality

    def has_operation_demand(self) -> bool:
        """
        Check whether the component has any operation demand - if it needs any resource to operate.
        """
        return len(self.demand[self.DemandTypes.OPERATION_DEMAND.value]) > 0

    def has_resource_supply(self, resource_name: str) -> bool:
        """
        Check whether the component has a supply of the resource resource_name.
        """
        return resource_name in list(self.supply[self.SupplyTypes.SUPPLY.value].keys())
    
    def unmet_demand_can_affect_resource_supply(self, resource_name: str) -> bool:
        """
        Check if the component has a supply of the resource resource_name and if the supply can be affected by unmet demand - i.e., if the UnmetDemandToAmountRelation is not constant.
        """
        if resource_name in list(self.supply[self.SupplyTypes.SUPPLY.value].keys()):
            return not(isinstance(self.supply[self.SupplyTypes.SUPPLY.value][resource_name].unmet_demand_to_amount, Relation.Constant))
        else:
            return False

    def add_resources(self, supply_or_demand: str, type: str, resource_dict: dict) -> None:
        """
        | Add resources to the component's supply or demand, depending on the supply_or_demand parameter.
        | The type parameter specifies the type of the supply or demand.
        | The resource_dict parameter is a dict with resource names as keys and resource parameters as values.
        """
        place_to_add = getattr(self, supply_or_demand)[type]
        for resource_name, resource_parameters in resource_dict.items():            
            place_to_add[resource_name] = self.form_resource(resource_name, resource_parameters)
    
    def form_resource(self, resource_name: str, resource_parameters: dict, default_resource_class = 'NonConsumableResource') -> None:
        """
        | Form a resource object based on the resource_name and resource_parameters.
        | Default resource class is NonConsumableResource. Specify a different default resource class if needed.
        | Check out the resource parameters dict in the component library files in the examples to get the dict structure.
        | **TODO**: Add the dict structure here. Future work.
        """
        target_resource_object = getattr(Resource, resource_parameters.get('ResourceClass', default_resource_class))
        return target_resource_object(resource_name, resource_parameters)
    
    def get_current_resource_amount(self, supply_or_demand: str, type: str, resource_name: str) -> float:
        """
        | Get the current amount of a resource resource_name in component's supply or demand and the type of the supply or demand.
        | In case resource is not in component's supply or demand, returns 0.
        """
        resource = getattr(self, supply_or_demand)[type].get(resource_name, None)
        if not (resource is None):
            return resource.current_amount
        else:
            return 0.0

    def update(self, time_step: int) -> None:
        """
        | Update component's state based on its current damage level and damage-to-functionality relation.
        | This includes updating component's functionality_level, checking if the component is functional at time_step and updating its supply, operation and recovery demand.
        """
        self.update_functionality()
        self.check_if_functional(time_step)
        self.update_supply_based_on_component_functionality()
        self.update_operation_demand()
        self.update_recovery_demand()

    def update_functionality(self) -> None:
        """
        Update component's functionality level by calling the recovery model's get_functionality_level method.
        """
        self.functionality_level = self.recovery_model.get_functionality_level()
        
    def check_if_functional(self, time_step: int) -> None:
        """
        | Check if the component is functional at time_step. Component is considered functional if its functionality level is higher than 0.
        | If the component is functional at time_step, add time_step to the component's "functional" list.
        """
        if self.functionality_level > 0:
            self.functional.append(time_step)

    def update_resources_based_on_component_functionality(self, supply_or_demand: str, type: str) -> None:
        """
        Update component's supply or demand based on its current functionality level.
        """
        resources_to_update = getattr(self, supply_or_demand)[type]
        for resource_object in resources_to_update.values():
            resource_object.update_based_on_component_functionality(self.functionality_level)

    def update_supply_based_on_component_functionality(self) -> None:
        """
        Update component's supply based on its current functionality level.
        """
        self.update_resources_based_on_component_functionality(SupplyOrDemand.SUPPLY.value,
                                                               self.SupplyTypes.SUPPLY.value)

    def update_operation_demand(self) -> None:
        """
        Update component's operation demand based on its current functionality level.
        """
        self.update_resources_based_on_component_functionality(SupplyOrDemand.DEMAND.value,
                                                               self.DemandTypes.OPERATION_DEMAND.value)

    def update_recovery_demand(self):
        """
        | Update component's recovery demand based on its currently active recovery activities.
        | Before resources are distributed, it is assumed that all recovery demand is met. 
        | It is important that this method is called before the resource distribution in a time step of the resilience assessment.
        """
        self.set_recovery_model_activities_demand_to_met()
        self.demand[self.DemandTypes.RECOVERY_DEMAND.value] = self.recovery_model.get_demand()

    def set_recovery_model_activities_demand_to_met(self):
        """
        Set the demand of all component's recovery activities as met.
        """
        self.recovery_model.set_activities_demand_to_met()

    def update_supply_based_on_unmet_demand(self, percent_of_met_demand: float) -> None:
        """
        | Update component's supply based on how much of its operation demand is met.
        | This is how component interdependencies are captured.
        | This method is called during resource distribution.
        """
        resources_to_update = getattr(self, SupplyOrDemand.SUPPLY.value)[self.SupplyTypes.SUPPLY.value] 
        for resource_object in resources_to_update.values():
            resource_object.update_based_on_unmet_demand(percent_of_met_demand)   
    
    def update_supply_based_on_consumption(self, resource_name: str, consumed_amount: float) -> None:
        """
        | Update component's supply based on how much of its consumable resource is consumed.
        | If resource is non-consumable, this method does nothing.
        | This method is called during resource distribution.
        """
        resources_to_update = getattr(self, SupplyOrDemand.SUPPLY.value)[self.SupplyTypes.SUPPLY.value] 
        resources_to_update[resource_name].update_based_on_consumption(consumed_amount)
    
    def set_met_demand_for_recovery_activities(self, resource_name: str, percent_of_met_demand: float) -> None:
        """
        | Set the demand_met for the component's recovery activity that require the resource resource_name.
        | This is how recovery resource constraints are captured.
        | This method is called once the resource distribution is performed at a time step.
        | **Note**: At this point, two recovery activities CANNOT ask for the same resource! 
        | **TODO**: Add recovery_activity as input, then it would be possible. Future work. 
        """
        self.recovery_model.set_met_demand_for_recovery_activities(resource_name,
                                                                     percent_of_met_demand)  

    def recover(self, time_step: int):
        """
        Recover the component as defined in its recovery model.
        """
        self.recovery_model.recover(time_step)


class BuildingStockUnitWithEmergencyCalls(StandardiReCoDeSComponent):
    """
    Subclass of the StandardiReCoDeSComponent class that simulates the performance of building stock units 
    with increased post-disaster demand for communication services due to emergency calls.

    Attributes
    ----------
    (only subclass specific)

    COMMUNICATION_RESOURCE_NAME : str
        | The demand for this resource will increase after a disaster.
        | This name should be defined in the component library file by adding the 'PostDisasterIncreaseDueToEmergencyCalls' key with a 'True' value.
    COMMUNICATION_DEMAND_MULTIPLIER : float
        Defines by how much the pre-disaster demand for the communication resource 
        is increased after a disaster.
    COMMUNICATION_DEMAND_INCREASE_TIME_STEP : int
        | Define at which time step of the recovery simulation will the demand for communication resource increase.
        | **Note**: this time step should be after the DISASTER_TIME_STEP defined in the system configuration file.
    COMMUNICATION_DEMAND_EXP_DECREASE_COEFF : float
        Define the parameter of the exponential function that simulates the post-disaster decrease of the
        demand for communication resource following the initial demand surge.   
    """

    COMMUNICATION_RESOURCE_NAME = ''
    COMMUNICATION_DEMAND_MULTIPLIER = 10.0
    COMMUNICATION_DEMAND_INCREASE_TIME_STEP = 1
    COMMUNICATION_DEMAND_EXP_DECREASE_COEFF = -0.3

    def update(self, time_step: int) -> None:
        """
        | *extend parent method*
        | Update the demand for communication services conditioned on the current time step.
        """
        super().update(time_step)
        self.update_communication_demand(time_step)

    def update_communication_demand(self, time_step: int) -> None:
        """
        | Calculate the communication resource demand at the current time step
        | and update in component's operation demand.
        """
        communication_resource_object = \
            getattr(self, SupplyOrDemand.DEMAND.value)[self.DemandTypes.OPERATION_DEMAND.value][
                self.COMMUNICATION_RESOURCE_NAME]
        current_communication_demand = communication_resource_object.current_amount
        initial_communication_demand = communication_resource_object.initial_amount
        if time_step >= self.COMMUNICATION_DEMAND_INCREASE_TIME_STEP:
            current_communication_demand = self.modify_emergency_calls_demand(initial_communication_demand, time_step)
        communication_resource_object.set_current_amount(current_communication_demand)

    def modify_emergency_calls_demand(self, initial_communication_demand: float, time_step: int) -> float:
        """
        | Modify communication demand based on the current time step, the exponential function parameters and initial communication demand.
        | The modified communication demand cannot be smaller than the initial, pre-disaster, demand.
        """
        modified_communication_demand = (initial_communication_demand * self.COMMUNICATION_DEMAND_MULTIPLIER) * math.exp(
            self.COMMUNICATION_DEMAND_EXP_DECREASE_COEFF) ** (time_step - self.COMMUNICATION_DEMAND_INCREASE_TIME_STEP)
        if modified_communication_demand < initial_communication_demand:
            return initial_communication_demand
        else:
            return modified_communication_demand
    
    def add_resources(self, supply_or_demand: str, type: str, resource_dict: dict) -> None:
        """
        | *override parent method*
        | While adding resources check if a resource is a communication resource.
        | If yes, set the COMMUNICATION_RESOURCE_NAME attribute.
        | **Note**: only one resource can be a communication resource that has increased demand following a disaster.
        | **TODO**: Implement a check for multiple communication resources. Future work.
        """
        place_to_add = getattr(self, supply_or_demand)[type]
        for resource_name, resource_parameters in resource_dict.items():
            self.check_if_demand_increase_considered(resource_name, resource_parameters)
            place_to_add[resource_name] = self.form_resource(resource_name, resource_parameters)
    
    def check_if_demand_increase_considered(self, resource_name: str, resource_parameters: dict) -> None:
        """
        | Check if the resource demand should be increased following the disaster for the resource resource_name.
        | Communication resource parameters dict in ComponentLibrary should have the 'PostDisasterIncreaseDueToEmergencyCalls' key with a 'True' value.
        """
        if resource_parameters.get('PostDisasterIncreaseDueToEmergencyCalls', None) == "True":
            self.COMMUNICATION_RESOURCE_NAME = resource_name


class InfrastructureInterface(StandardiReCoDeSComponent):
    """
    | Subclass of the StandardiReCoDeSComponent class used to interface third-party infrastructure simulators into an iRe-CoDeS system-of-systems model.
    | Infrastructure simulator performance is defined in the system configuration file.    
    """   
    def set_supply_dynamics(self, supply_dynamics: dict) -> None:   
        """
        | Set the predefined supply dynamics of the infrastructure interface component by specifying the parameters of the component's recovery model.
        | These are defined in the system configuration file by specifying the amounts of resource supplied by the infrastructure system and the time step at which the supply is restored.
        | **Note 1**: The supply dynamics are defined in the system configuration file, but in the component library file the interface component has to have the resource defined.
        | **Note 2**: It is assumed that last value in the Amount key is the initial supply amount and the highest value that the infrastructure system provides. 
        """
        restoration_times = self.get_restoration_times(supply_dynamics)
        # avoid divison with zero if the last restoration time is zero
        if restoration_times[-1] != 0.0:
            step_limits = list(np.divide(restoration_times, restoration_times[-1]))
        else:
            step_limits = [1.0]
        step_values = list(np.divide(supply_dynamics['Amount'], max(supply_dynamics['Amount'])))   
        step_limits, step_values = self.add_initial_zero_supply(step_limits, step_values)
        
        self.recovery_model.set_parameters({'RestoredIn': supply_dynamics['RestoredIn'],
                                            'StepLimits': step_limits,
                                            'StepValues': step_values})
        self.supply['Supply'][supply_dynamics['Resource']].set_initial_amount(max(supply_dynamics['Amount']))
    
    def add_initial_zero_supply(self, step_limits: list, step_values: list) -> tuple:
        """
        | Add the initial zero supply to the step limits and step values.
        | This is needed to make sure that the supply is zero before the infrastructure system is restored.
        | This is not applied if the interface already defines the initial post-disaster supply amount at time 0.
        """
        if step_limits[0] != 0.0:
            step_limits.insert(0, 0.0)
            step_values.insert(0, 0.0)
        return step_limits, step_values
    
    def get_restoration_times(self, supply_dynamics: dict) -> list:
        """
        | Sample the restoration times from the distribution defined in the system configuration file.
        | These times define the time steps at which the interface restors the supply amounts defined in the Amount key in the system configuration file.
        """
        restoration_times = []
        for restoration_time_dict in supply_dynamics.get('RestoredIn', []):
            distribution_name, distribution_parameters = list(restoration_time_dict.items())[0]
            restoration_time = getattr(ProbabilityDistribution, distribution_name)(distribution_parameters).sample()
            restoration_times.append(restoration_time)
        return restoration_times
    
class Bridge(StandardiReCoDeSComponent):
    """
    Subclass used to identify bridges in a system. For now that's the only purpose.

    **TODO**: Add specific bridge methods and attributes. Future work.
    """
    pass
        
        
pyrecodes/Component.py
NikolaBlagojevic/pyrecodes
"""
Module used to create component library based on which system components are constructed.
"""

import json
from abc import ABC, abstractmethod
from pyrecodes import Component

class ComponentLibraryCreator(ABC):
    """
    Abstract class for ComponentLibraryCreator class used to create component library based on which system components are constructed.
    """

    library: dict

    @abstractmethod
    def form_library(self) -> dict:
        pass


class JSONComponentLibraryCreator(ComponentLibraryCreator):
    """
    Class to create a component library from a single JSON file containing component templates for each component type. 
    """

    library: dict

    def __init__(self, file_name: str) -> None:
        """
        Initiliaze the class. Read the component library JSON file.
        """
        self.file = None
        self.read_library_file(file_name)

    def read_library_file(self, file_name: str) -> None:
        with open(file_name, 'r') as file:
            self.file = json.load(file)

    def form_library(self) -> dict:
        """
        Form component library based on the component template specificed in the component library JSON file. Returns a dict with component templates.
        """
        component_library = dict()
        for component_name, component_params in self.file.items():
            component_library[component_name] = self.form_component(component_name, component_params)
        return component_library
    
    def form_component(self, component_name: str, component_parameters: dict) -> Component.Component:
        """
        Form and return a component object based on the input component_name and component_parameters to be used as component template in system creation.
        """
        target_object = getattr(Component, component_parameters['ComponentClass'])
        component = target_object()
        component.construct(component_name, component_parameters)     
        return component

pyrecodes/ComponentLibraryCreator.py
NikolaBlagojevic/pyrecodes
"""
Module used to define recovery models for components.
"""

import math
from abc import ABC, abstractmethod
from pyrecodes import ProbabilityDistribution
from pyrecodes import Relation
from pyrecodes import Resource

class RecoveryActivity(ABC):
    """
    Abstract class representing a recovery activity.

    Attributes:
        level (float): The current progress level of the activity at a time step (between 0 and 1).
        duration (float): The duration of the activity.
        rate (float): The rate of progress per time unit.
        preceding_activities (list[str]): List of names of preceding activities that have to be completed until this activity can start.
        preceding_activities_finished (bool): True if all preceding activities are finished.
        demand_met (float): The percent of demand met.
        name (str): The name of the recovery activity.
        time_steps (list[int]): List of time steps when progress was recorded.
    """
    level: float
    duration: float
    rate: float
    preceding_activities: list([str])
    preceding_activities_finished: bool
    demand_met: float
    name: str
    time_steps: list([int])

    @abstractmethod
    def __init__(self, name: str) -> None:
        pass

    @abstractmethod
    def set_name(self, name: str) -> None:
        pass

    @abstractmethod
    def set_level(self, level: float) -> None:
        pass

    @abstractmethod
    def set_duration(self, distribution: dict) -> None:
        pass

    @abstractmethod
    def sample_duration(self, distribution: dict) -> float:
        pass

    @abstractmethod
    def set_preceding_activities(self, preceding_activities: list([str])) -> None:
        pass    

    @abstractmethod
    def set_preceding_activities_finished(self, finished: bool) -> None:
        pass

    @abstractmethod
    def set_demand(self, resources: list) -> None:
        pass

    @abstractmethod
    def set_demand_met(self) -> None:
        pass

    @abstractmethod
    def get_demand(self) -> dict:
        pass

    @abstractmethod
    def recover(self, time_step: int) -> None:
        pass

    @abstractmethod
    def record_progress(self, time_step: int) -> None:
        pass

    @abstractmethod
    def activity_finished(self) -> bool:
        pass


class ConcreteRecoveryActivity(RecoveryActivity):
    """
    Concrete implementation of a recovery activity.
    """

    def __init__(self, name: str, initial_level=0.0) -> None:
        """
        Constructor for ConcreteRecoveryActivity.

        Args:
            name (str): The name of the activity.
            initial_level (float, optional): The initial progress level (default is 0.0).
        """
        self.set_name(name)
        self.set_level(initial_level)
        self.time_steps = []
        self.demand_met = {}
        self.demand = {}
        self.preceding_activities_finished = False

    def set_name(self, name: str) -> None:
        """
        Set the name of the activity.

        Args:
            name (str): The name of the activity.
        """
        self.name = name

    def set_level(self, level: float) -> None:
        """
        Set the progress level of the activity.

        Args:
            level (float): The progress level of the activity (between 0 and 1).
        """
        if 0 <= level <= 1:
            self.level = level
        else:
            raise ValueError(f'Level must be between 0 and 1. Recovery activity: {self.name}.')

    def set_duration(self, distribution: dict) -> None:
        """
        Set the duration of the activity based on a distribution. Duration must be a positive number. If duration is zero, the rate is infinite - activity is immediately finished.

        Args:
            distribution (dict): A dictionary describing the duration distribution.
        """
        duration = self.sample_duration(distribution)
        self.duration = duration
        if duration > 0:            
            self.rate = 1 / duration
        elif duration == 0: 
            self.rate = math.inf
        else:
            raise ValueError(f'Duration must be a positive number. Recovery activity: {self.name}.')

    def sample_duration(self, distribution: dict) -> float:
        """
        Sample the duration from a distribution.

        Args:
            distribution (dict): A dictionary describing the duration distribution.

        Returns:
            float: The sampled duration.
        """
        distribution_name, distribution_parameters = list(distribution.items())[0]
        target_distribution = getattr(ProbabilityDistribution, distribution_name)
        distribution = target_distribution(distribution_parameters)
        return distribution.sample()

    def set_preceding_activities(self, preceding_activities: list([str])) -> None:
        """
        Set preceding activities that need to be completed for this activity to start.

        Args:
            preceding_activities (list[str]): List of names of preceding activities.
        """
        self.preceding_activities = preceding_activities

    def set_preceding_activities_finished(self, finished: bool) -> None:
        """
        Set the status of preceding activities.

        Args:
            finished (bool): True if all preceding activities are finished.
        """
        self.preceding_activities_finished = finished

    def set_demand(self, resources: list) -> None:
        """
        Set the resource demand for the activity.

        Args:
            resources (list): List of dictionaries describing resource demands.
        """
        for resource in resources:
            resource_name = resource['Resource']
            resource_parameters = {'Amount': resource['Amount']}          
            self.demand[resource_name] = Resource.ConcreteResource(resource_name, resource_parameters)
            self.demand_met[resource_name] = 1.0

    def set_demand_met(self, resource_name: str, demand_met: float) -> None:
        """
        Set the level of demand met (between 0 and 1) for a resource resource_name.

        Args:
            resource_name (str): The name of the resource.
            demand_met (float): The level of demand met (between 0 and 1).
        """
        if 0 <= demand_met <= 1:
            self.demand_met[resource_name] = demand_met
        else:
            raise ValueError(f'Demand Met must be between 0 and 1. Recovery activity: {self.name}. Resource name: {resource_name}')

    def get_demand(self) -> dict:
        """
        Get the current resource demand of the activity.

        Returns:
            dict: A dictionary of resource demands.
        """
        return self.demand

    def recover(self, time_step: int) -> None:
        """
        Perform recovery for a given time step. Activity recovers (i.e., progresses) if it's not already finished. The rate of progress depends on the percent of met resource demand. 

        Args:
            time_step (int): The time step for which recovery is performed.
        """
        if not self.activity_finished():
            current_level_increase = self.rate * self.effect_of_unmet_demand_on_activity()            
            self.level = min(self.level + current_level_increase, 1.0)
            if current_level_increase > 0:
                self.record_progress(time_step)

    def record_progress(self, time_step: int) -> None:
        """
        Record progress at a specific time step. If the rate at a time step was higher than zero, than the time step is recorded.

        Args:
            time_step (int): The time step when progress is recorded.
        """
        self.time_steps.append(time_step)
    
    def effect_of_unmet_demand_on_activity(self) -> float:
        """
        Determine how demand met for each resource in the recovery demand affects the recovery activity.

        Returns:
            float: The effect of unmet demand on the activity.
        """
        return min(self.demand_met.values(), default=1.0)

    def activity_finished(self) -> bool:
        """
        Check if the activity is finished and return a boolean value.

        Returns:
            bool: True if the activity is finished, False otherwise.
        """
        return math.isclose(self.level, 1.0)

class RecoveryModel(ABC):
    """
    Abstract class representing a recovery model.
    """

    recovery_activities: dict

    @abstractmethod
    def set_parameters(self, parameters: dict) -> None:
        pass

    @abstractmethod
    def set_initial_damage_level(self, damage_level: float) -> None:
        pass

    @abstractmethod
    def set_damage_functionality(self, damage_functionality_relation: dict) -> None:
        pass

    @abstractmethod
    def set_activities_demand_to_met(self) -> None:
        pass

    @abstractmethod
    def recover(self, time_step: int) -> None:
        pass

    @abstractmethod
    def get_functionality_level(self) -> float:
        pass

    @abstractmethod
    def get_demand(self) -> dict:
        pass

    @abstractmethod
    def set_met_demand_for_recovery_activities(self, resource_name: str, percent_of_met_demand: float) -> None:
        pass

class ComponentLevelRecoveryActivitiesModel(RecoveryModel):
    """
    Component recovery model that considers multiple recovery activities defined on a component level.
    
    This model accounts for several impeding factors and a single repair activity.

    Attributes:
        damage_level (float): The damage level of the component (between 0 and 1).
        damage_to_functionality_relation (Relation.Relation): The relation to get the functionality level based on the current damage level.
        recovery_activities (dict): A dictionary of recovery activities parameters.
        REPAIR_ACTIVITY_NAME (str): The name of the repair activity - one that actually reduces the damage level.
    """

    damage_level: float
    damage_to_functionality_relation: Relation.Relation
    recovery_activities: dict

    def __init__(self, recovery_model_parameters: dict, REPAIR_ACTIVITY_NAME = 'Repair') -> None:
        """
        Constructor for ComponentLevelRecoveryActivitiesModel.

        Args:
            recovery_model_parameters (dict): Parameters for the recovery model.
            REPAIR_ACTIVITY_NAME (str, optional): The name of the repair activity (default is 'Repair').
        """
        self.recovery_activities = {}
        self.REPAIR_ACTIVITY_NAME = REPAIR_ACTIVITY_NAME
        self.set_parameters(recovery_model_parameters['Parameters'])
        self.set_damage_functionality(recovery_model_parameters['DamageFunctionalityRelation'])        

    def set_parameters(self, parameters: dict) -> None:
        """
        Set parameters for all recovery activities. Parameters are preceding activities, duration, and demand of activities.

        Args:
            parameters (dict): Parameters for the recovery model.
        """
        for recovery_activity, recovery_activity_parameters in parameters.items():
            self.recovery_activities[recovery_activity] = ConcreteRecoveryActivity(recovery_activity, initial_level=1.0)
            self.recovery_activities[recovery_activity].set_preceding_activities(
                recovery_activity_parameters.get('PrecedingActivities', []))
            self.recovery_activities[recovery_activity].set_duration(recovery_activity_parameters.get('Duration', ''))
            self.recovery_activities[recovery_activity].set_demand(recovery_activity_parameters.get('Demand', ''))

    def set_initial_damage_level(self, damage_level: float) -> None:
        """
        Set the initial damage level. The damage level must be between 0 and 1.

        Sets the progress level of all activities to 0 and the progress level of the repair activity to 1 - damage_level.

        Args:
            damage_level (float): The initial damage level (between 0 and 1).
        """
        if 0 <= damage_level <= 1:
            for recovery_activity_object in self.recovery_activities.values():
                recovery_activity_object.set_level(0.0)
            self.set_initial_repair_activity_state(damage_level)
        else:
            raise ValueError('Damage level must be between 0 and 1.')

    def set_initial_repair_activity_state(self, damage_level: float) -> None:
        """
        Set the initial state of the repair activity. Initial level is 1 - damage_level and the rate is damage_level / duration to ensure that the component is repaired in the specified duration.

        Args:
            damage_level (float): The initial damage level (between 0 and 1).
        """
        self.recovery_activities[self.REPAIR_ACTIVITY_NAME].set_level(1 - damage_level)
        self.recovery_activities[self.REPAIR_ACTIVITY_NAME].rate = damage_level / self.recovery_activities[
            self.REPAIR_ACTIVITY_NAME].duration

    def set_damage_functionality(self, damage_functionality_relation: dict) -> None:
        """
        Set damage functionality relation. This relation is used to get the component's functionality level based on its current damage level.

        Args:
            damage_functionality_relation (dict): Damage functionality relation parameters.
        """
        target_damage_functionality = getattr(Relation, damage_functionality_relation['Type'])
        self.damage_to_functionality_relation = target_damage_functionality()

    def recover(self, time_step: int) -> None:
        """
        Perform recovery for a time step. Increase the level of all activities that are not finished, for which all preceding activities are finished and the resource demand is met.

        Args:
            time_step (int): The time step for which recovery is performed.
        """
        self.check_preceding_activities()
        for recovery_activity_object in self.recovery_activities.values():
            if recovery_activity_object.preceding_activities_finished and self.get_damage_level() > 0:
                recovery_activity_object.recover(time_step)

    def check_preceding_activities(self) -> None:
        """
        Check if preceding activities are finished for all recovery activities and set the preceding_activities_finished attribute accordingly.
        """
        for recovery_activity_object in self.recovery_activities.values():
            if self.preceding_activities_finished(recovery_activity_object):
                recovery_activity_object.set_preceding_activities_finished(True)
            else:
                recovery_activity_object.set_preceding_activities_finished(False)

    def preceding_activities_finished(self, recovery_activity_object: RecoveryActivity) -> bool:
        """
        Check if preceding activities for a single recovery activity are finished.

        Args:
            recovery_activity_object (RecoveryActivity): The recovery activity to check.

        Returns:
            bool: True if all preceding activities are finished, False otherwise.
        """
        for preceding_activity in recovery_activity_object.preceding_activities:
            if not (self.recovery_activities[preceding_activity].activity_finished()):
                return False
        return True

    def get_functionality_level(self) -> float:
        """
        Get the functionality level of the component absed on the current damage level and the damage-functionality relation.

        Returns:
            float: The functionality level (between 0 and 1).
        """
        return self.damage_to_functionality_relation.get_output(self.get_damage_level())

    def get_damage_level(self) -> float:
        """
        Get the current damage level of the component. It's 1 - the progress level of the repair activity.

        Returns:
            float: The damage level (between 0 and 1).
        """
        return 1 - self.recovery_activities[self.REPAIR_ACTIVITY_NAME].level

    def get_demand(self) -> dict:
        """
        Get the resource demand needed to meet the demand of all recovery activities currently in progress.

        Returns:
            dict: A dictionary of resource demands for active recovery activities.
        """
        resource_dict = {}
        for recovery_activity in self.recovery_activities.values():
            if self.preceding_activities_finished(recovery_activity) and self.get_damage_level() > 0 and not (
                    recovery_activity.activity_finished()):
                recovery_activity_demand = recovery_activity.get_demand()
                if len(recovery_activity_demand) > 0:
                    resource_dict = {**resource_dict, **recovery_activity_demand}
        return resource_dict

    def set_activities_demand_to_met(self) -> None:
        """
        Set the demand for recovery activities to met. This is done before resource are distributed at each time step, as resource distribution can only reduce the percent of met demand of a component - this is an iRe-CoDeS assumption.
        """
        for recovery_activity in self.recovery_activities.values():
            for resource_name in recovery_activity.demand.keys():
                recovery_activity.set_demand_met(resource_name, 1.0)

    def set_met_demand_for_recovery_activities(self, resource_name: str, percent_of_met_demand: float) -> None:
        """
        Set the percent of met demand for a recovery activity that requires the resource resource_name. This method is called during resource distribution.

        Args:
            resource_name (str): The name of the resource.
            percent_of_met_demand (float): The percent of met demand (between 0 and 1).:
        
        TODO:
            Add recovery activity name as input parameter. This will allow the same resource to be demanded by multiple recovery activities. Not possible at the moment.
        """
        
        recovery_activity_to_update = self.find_recovery_activity_that_demands_the_resource(resource_name)
        recovery_activity_to_update.set_demand_met(resource_name, percent_of_met_demand)

    def find_recovery_activity_that_demands_the_resource(self, resource_name: str) -> RecoveryActivity:
        """
        Find the recovery activity that demands the resource resource_name.

        Args:
            resource_name (str): The name of the resource.

        Returns:
            RecoveryActivity: The recovery activity that demands the resource.
        
        Raises:
            ValueError: If no recovery activity demands the resource.
        """
        for recovery_activity in self.recovery_activities.values():
            if resource_name in recovery_activity.demand:
                return recovery_activity
        raise ValueError('No recovery activity demands the resource: ' + resource_name)

class NoRecoveryActivityModel(ComponentLevelRecoveryActivitiesModel):
    """
    Recovery model for components that did not experience damage.
    """
    def set_initial_damage_level(self, damage_level: float) -> None:
        """
        Set the initial damage level for undamaged components.

        Args:
            damage_level (float): The initial damage level. For this model, it must be 0.

        Raises:
            ValueError: If the damage level is not 0.
        """
        if damage_level != 0:
            raise ValueError('Initial damage level for NoRecoveryActivity model must be 0.')

    def set_damage_functionality(self, damage_functionality_relation: dict) -> None:
        """
        Set damage functionality relation (not applicable for undamaged components).

        Args:
            damage_functionality_relation (dict): Damage functionality relation parameters.
        """
        pass

    def get_functionality_level(self) -> float:
        """
        Get the functionality level of the undamaged component.

        Returns:
            float: The functionality level (always 1.0 for undamaged components).
        """
        return 1.0

    def get_damage_level(self) -> float:
        """
        Get the damage level of the undamaged component.

        Returns:
            float: The damage level (always 0 for undamaged components).
        """
        return 0

    def get_demand(self) -> dict:
        """
        Get the demand for recovery activities for the undamaged component.

        Returns:
            dict: An empty dictionary, as there is no demand for undamaged components.
        """
        return {}    

    def set_met_demand_for_recovery_activities(self, resource_name: str, percent_of_met_demand: float) -> None:
        pass

class InfrastructureInterfaceRecoveryModel(RecoveryModel):
    """
    Component recovery model for infrastructure interfaces to simulate their pre-defined post-disaster supply/demand dynamics.
    """    

    RECOVERY_ACTIVITY_NAME = 'Recovery'
    
    def __init__(self, recovery_model_parameters: dict) -> None: 
        """
        Constructor for InfrastructureInterfaceRecoveryModel. Model has a single recovery activity, whose name is defined in the RECOVERY_ACTIVITY_NAME attribute.

        Args:
            recovery_model_parameters (dict): Parameters for the recovery model.
        """       
        self.recovery_activities = {self.RECOVERY_ACTIVITY_NAME: ConcreteRecoveryActivity(self.RECOVERY_ACTIVITY_NAME, initial_level=1.0)}
        self.set_damage_functionality()
        self.set_parameters(recovery_model_parameters['Parameters'])        
    
    def set_parameters(self, parameters: dict, default_duration=[{"Deterministic": {"Value": 0}}]) -> None:
        """
        Set model parameters. These include setting the post-disaster functionality levels and the time steps at which they change as multiple steps to achieve the predefined supply/demand dynamics as set in the system configuration file.

        Args:
            parameters (dict): Parameters for the recovery model.
            default_duration (list, optional): Default duration for recovery activities (default is zero).

        Notes:
            - This method sets step durations and demands for the recovery activity.
            - The implementation might be improved in the future.

        TODO:
            - Improve the method. See if StepLimits are redundant as RestoredIn is already defined.
            - The for loop setting the duration is a temporary solution. It should be replaced by a more general solution, just take the max.
        """
        self.damage_to_functionality_relation.set_steps(parameters.get('StepLimits', []), parameters.get('StepValues', []))
        step_durations = parameters.get('RestoredIn', default_duration)
        for step_duration in step_durations:
            self.recovery_activities[self.RECOVERY_ACTIVITY_NAME].set_duration(step_duration)
            self.recovery_activities[self.RECOVERY_ACTIVITY_NAME].set_demand('')  

    def set_initial_damage_level(self, damage_level: float) -> None:
        """
        Set the initial damage level for infrastructure interfaces. The damage level is always 1 to triger the pre-defined supply/demand dynamics.

        Args:
            damage_level (float): The initial damage level. For this model, it must be 1.

        Notes:
            - This method sets both the damage level and the initial state of the recovery activity.
        """
        self.damage_level = 1.0
        self.recovery_activities[self.RECOVERY_ACTIVITY_NAME].level = 0.0

    def set_damage_functionality(self) -> None:
        """
        Set damage functionality relation. For this model it's always a multiple step relation.
        """
        target_damage_functionality = getattr(Relation, 'MultipleStep')
        self.damage_to_functionality_relation = target_damage_functionality()

    def set_activities_demand_to_met(self) -> None:
        pass

    def recover(self, time_step: int) -> None:
        """
        Perform recovery for a time step. Recovery here means simulating the pre-defined post-disaster supply/demand dynamics.

        Args:
            time_step (int): The time step for which recovery is performed.
        """
        self.recovery_activities[self.RECOVERY_ACTIVITY_NAME].recover(time_step)

    def get_functionality_level(self) -> float:
        """
        Get the functionality level of the infrastructure interface.

        Returns:
            float: The functionality level.
        """
        return self.damage_to_functionality_relation.get_output(1 - self.get_damage_level())
    
    def get_damage_level(self) -> float:
        """
        Get the damage level of the infrastructure interface.

        Returns:
            float: The damage level.
        """
        return 1 - self.recovery_activities[self.RECOVERY_ACTIVITY_NAME].level
    
    def get_demand(self) -> dict:
        """
        Get the demand for the infrastructure interface.

        Returns:
            dict: An empty dictionary, as it assumed that there is no recovery demand for infrastructure interfaces.
        """
        return {}
    
    def set_met_demand_for_recovery_activities(self, resource_name: str, percent_of_met_demand: float) -> None:
        pass

pyrecodes/ComponentRecoveryModel.py
NikolaBlagojevic/pyrecodes
"""
Module used to provide initial component damage information to the system class.

More details coming soon.
"""

from abc import ABC, abstractmethod
from pyrecodes import Component

class DamageInput(ABC):
    """
    Different methods for providing initial component damage information to the system class
    """

    @abstractmethod
    def get_initial_damage(self) -> None:
        pass

    @abstractmethod
    def set_initial_damage(self, components: list) -> None:
        pass

class ConcreteDamageInput(DamageInput):

    def __init__(self, parameters):
        self.parameters = parameters

    def get_initial_damage(self):
        pass

    def set_initial_damage(self, components: list([Component.Component])):
        self.get_initial_damage() 
        for damage_level, component in zip(self.damage_levels, components):
            component.set_initial_damage_level(damage_level)

class ListDamageInput(ConcreteDamageInput):

    def get_initial_damage(self):
        """
        self.parameters are set in the system class and contained in the system_creator.system_configuration
        """
        self.damage_levels = self.parameters

class R2DDamageInput(ConcreteDamageInput):
    DAMAGE_STATE_ID_POSITION_IN_COMPONENT_NAME = 2

    def get_initial_damage(self):
        pass

    def set_initial_damage(self, components: list([Component.Component])) -> None:
        for component in components:
            if self.component_is_damaged(component) or self.component_is_interface(component):
                component.set_initial_damage_level(1.0)

    def component_is_damaged(self, component: Component.Component) -> bool:
        damage_state = component.name[self.DAMAGE_STATE_ID_POSITION_IN_COMPONENT_NAME]
        if ('ResidentialBuilding' in component.name) and int(damage_state) > 0:
            return True
        else:
            return False
    
    def component_is_interface(self, component: Component.Component) -> bool:
        if isinstance(component, Component.InfrastructureInterface):
            return True
        else:
            return False

class FileDamageInput(ConcreteDamageInput):

    def get_initial_damage(self):
        with open(self.parameters, 'r') as file:
            damage_input_str = file.read()
        damage_input = list(map(float, damage_input_str.replace('\n', '').split(',')))
        self.damage_levels = damage_input

pyrecodes/DamageInput.py
NikolaBlagojevic/pyrecodes
"""
Module used to define component priority for resource distribution.

More details coming soon.
"""

from abc import ABC, abstractmethod
import random
from pyrecodes import Component
import copy

class DistributionPriority(ABC):
    """
    Abstract class for defining component priority for resource distribution.
    """

    @abstractmethod
    def __init__(self, components: list([Component.Component]), parameters, resource_name: str):
        pass

    @abstractmethod
    def get_component_priorities(self) -> list([int]):
        pass

class ComponentTypeBasedPriority(DistributionPriority):
    """
    Components are prioritized based on their type (i.e., name). Components higher in the system's component list are prioritized among same type components.
    """

    def __init__(self, resource_name: str, parameters, components: list([Component.Component])):
        self.components = components
        self.resource_name = resource_name
        self.set_distribution_priority(parameters)

    def set_distribution_priority(self, parameters):
        categorized_components_dict = self.categorize_components_based_on_type()
        distribution_priority_ids = []
        distribution_priority_demand_types = []
        for component_type_name, component_demand_type in parameters:
            distribution_priority_ids += categorized_components_dict[component_type_name] 
            distribution_priority_demand_types += [component_demand_type for _ in range(len(categorized_components_dict[component_type_name]))]
        self.distribution_priority = distribution_priority_ids, distribution_priority_demand_types
    
    def categorize_components_based_on_type(self) -> dict:
        categorized_components = {}
        for i, component in enumerate(self.components):
            if component.name in categorized_components:
                categorized_components[component.name].append(i)
            else:
               categorized_components[component.name] = [i] 
        return categorized_components     

    def get_component_priorities(self) -> list([int]):  
        return self.distribution_priority

class SupplierOnlyDistributionPriority(DistributionPriority):
    """
    Find all components that have a supply for a specified resource and only include those components when distributing a resource.
    Used for housing service distribution, as it only considers components with housing supply when distributing housing services. Makes housing distribution faster.
    Considers Operation Demand only.
    """

    def __init__(self, resource_name: str, parameters: dict, components: list([Component.Component])):
        self.resource_name = resource_name
        self.components = components
        self.set_distribution_priority(parameters)

    def set_distribution_priority(self, parameters) -> None:
        residential_building_ids = []
        for component_id, component in enumerate(self.components):
            if component.has_resource_supply(self.resource_name) > 0:
                residential_building_ids.append(component_id)
        self.distribution_priority = residential_building_ids, [
            Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value for _ in residential_building_ids]

    def get_component_priorities(self) -> list([int]):
        return self.distribution_priority


class RandomPriority(DistributionPriority):
    """
    Randomly shuffle components for resource distribution. 
    Keeps resource suppliers on top of the priority list to maximize the consumption of components' supply capacity.
    """

    def __init__(self, resource_name: str, parameters: dict, components: list([Component.Component])):
        self.resource_name = resource_name
        self.components = components
        self.set_distribution_priority(parameters)

    def set_distribution_priority(self, parameters: dict) -> None:
        random.seed(parameters['Seed'])
        component_ids = list(range(len(self.components)))
        supplier_ids, nonsupplier_ids = self.get_suppliers_id(component_ids)
        supplier_ids_randomized, supplier_demand_types = self.randomize_ids(supplier_ids,
                                                                                     [Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value])
        nonsupplier_ids_randomized, nonsupplier_demand_types = self.randomize_ids(nonsupplier_ids, parameters['DemandType'])

        self.distribution_priority = supplier_ids_randomized + nonsupplier_ids_randomized, supplier_demand_types + nonsupplier_demand_types

    def get_suppliers_id(self, component_ids: list([int])):
        suppliers_id = []
        remaining_components_id = component_ids
        for component_id, component in enumerate(self.components):       
            if component.has_resource_supply(self.resource_name):
                suppliers_id.append(component_id)
                remaining_components_id.remove(component_id)
        return suppliers_id, remaining_components_id
    
    def randomize_ids(self, component_ids: list, demand_types: list) -> tuple:
        """
        The method randomizes component ids. Demand types are in the same order as in the input variable.
        """
        randomized_priorities = []
        randomized_demand_types = []
        for demand_type in demand_types:
            random.shuffle(component_ids)
            randomized_priorities += copy.deepcopy(component_ids)
            randomized_demand_types += [demand_type for _ in component_ids]
        return randomized_priorities, randomized_demand_types
        
    def get_component_priorities(self) -> list([int]):
        return self.distribution_priority
    
class RandomPriorityWithPrioritizedInterfaces(RandomPriority):
    """
    Randomly shuffle components for resource distribution. 
    Keeps resource suppliers on top of the priority list to maximize the consumption of components' supply capacity. 
    Prioritized infrastructure interfaces over other components to minimize shutdowns due to infrastructure interdependency effects. 
    """

    def set_distribution_priority(self, parameters: dict) -> None:
        component_ids = list(range(len(self.components)))
        suppliers_ids, remaining_components_id = self.get_suppliers_id(component_ids)
        interface_ids, remaining_components_id = self.get_infrastructure_interface_id(remaining_components_id)
        random.seed(parameters['Seed'])
        random.shuffle(suppliers_ids)
        supplier_demand_types = [Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value for _ in
                                 suppliers_ids]
        interface_demand_types = [Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value for _ in
                                 interface_ids]
        random_priorities = []
        random_demand_types = []
        for demand_type in parameters['DemandType']:
            random.shuffle(remaining_components_id)
            random_priorities += copy.deepcopy(remaining_components_id)
            random_demand_types = [demand_type for _ in remaining_components_id]

        self.distribution_priority = suppliers_ids + interface_ids + random_priorities, supplier_demand_types + interface_demand_types + random_demand_types

    def get_infrastructure_interface_id(self, component_ids: list([int])):
        interface_ids = []
        remaining_component_ids = copy.deepcopy(component_ids)
        for component_id in component_ids:  
            if isinstance(self.components[component_id], Component.InfrastructureInterface):
                interface_ids.append(component_id)
                remaining_component_ids.remove(component_id)
        return interface_ids, remaining_component_ids


class ComponentBasedPriority(DistributionPriority):
    """
    Resource distribution priority of each component are explicitly defined.
    """

    def __init__(self, resource_name: str, parameters, components: list([Component.Component])):
        self.resource_name = resource_name
        self.components = components
        self.set_distribution_priority(parameters)

    def set_distribution_priority(self, parameters):
        distribution_priority = []
        component_demand_types = []
        temp_components = copy.deepcopy(self.components)
        for component_name, component_locality, component_demand_type in parameters:
            component_position_in_list, temp_components = self.find_component_position(component_name,
                                                                                       component_locality,
                                                                                       temp_components)
            distribution_priority.append(component_position_in_list)
            component_demand_types.append(component_demand_type)
        self.distribution_priority = distribution_priority, component_demand_types

    def find_component_position(self, component_name: str, component_locality: str, temp_components: list(
        [Component.Component])) -> 'tuple[int, list([Component.Component])]':
        locality_id = self.get_locality_id_from_string(component_locality)
        for i, component in enumerate(temp_components):
            if self.component_not_already_in_priority_list(component):
                if component.name == component_name and component.locality == locality_id:
                    temp_components[i] = None
                    return i, temp_components
        else:
            raise ValueError(
                f'Component {component_name} | Locality: {locality_id} not found in distribution priorities list.')

    @staticmethod
    def component_not_already_in_priority_list(component):
        return not (component is None)

    @staticmethod
    def get_locality_id_from_string(locality_strings: list([str])) -> list:
        return [int(locality_string.split(' ')[-1]) for locality_string in locality_strings]

    def get_component_priorities(self) -> list([int]):
        return self.distribution_priority

pyrecodes/DistributionPriority.py
NikolaBlagojevic/pyrecodes
"""
Module used to visualize the system recovery process using geospatial tools.

More details coming soon.
"""

import time
import contextily as ctx
import geopandas
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.colors as mcolors
import matplotlib.cm as mcm
import imageio
import pandas as pd
import shapely
from pyrecodes import Component
from pyrecodes import Plotter
from pyrecodes import System

COMPONENT_STATE_COLORS = {
    'Waiting': 'black',
    'RapidInspection': 'lightblue',
    'Financing': 'orange',
    'ContractorMobilization': 'springgreen',
    'SitePreparation': 'purple',
    'CleanUp': 'yellow',
    'DetailedInspection': 'tomato',
    'ArchAndEngDesign': 'pink',
    'Permitting': 'darkblue',
    'Demolition': 'gray',
    'Repair': 'red',
    'Functional': 'green'
}

class ConcreteGeoVisualizer():

    def __init__(self, components: list([Component.Component])) -> None:
        self.set_components(components)
        self.set_component_state_dict()
        self.create_buildings_geodataframe()
        self.plotter = Plotter.Plotter()

    def set_components(self, components: list([Component.Component])) -> None:
        self.components = components  
    
    def set_component_state_dict(self) -> None:
        self.state_dict = {}
        states, colors = COMPONENT_STATE_COLORS.keys(), COMPONENT_STATE_COLORS.values()        
        for id, (state, color) in enumerate(zip(states, colors)):
            self.state_dict[state] = {'Color': color, 'ID': id} 
    
    def create_empty_buildings_dataframe(self) -> None:
        columns = ['ID', 'Footprint_WKT', 'State']
        return pd.DataFrame(columns=columns)
    
    def show_recovery_animation(self, max_time_step: int, pause=1):
        for time_step in range(max_time_step):
            self.create_current_state_figure(time_step)
            time.sleep(pause)
            plt.clf()
        
    def create_recovery_gif(self, time_steps: list([int]), file_name='./2D_buildings_with_supply_demand_TIME_STEP.png', savename='./system_recovery.gif') -> None:        
        frames = []
        for time_step in time_steps:           
            frame = imageio.v2.imread(file_name.replace('TIME_STEP', str(time_step)))
            frames.append(frame)
        imageio.mimwrite(savename, # output gif
                frames,          # array of input frames
                duration = 1)         # optional: duration in s
    
    def create_current_state_buildings_and_supply_demand_figure(self, time_step: int, system: System.System, save=True, 
                                                                dpi=300, resources_to_plot=['Shelter', 'RepairCrew'],
                                                                show=False, supply_demand_resilience_calculator_id=0,
                                                                savename=f'2D_buildings_with_supply_demand_TIME_STEP.png') -> None:
        fig, ax_dict = plt.subplot_mosaic([['left', 'upper right'],
                               ['left', 'lower right']],
                              figsize=(24, 12), layout="constrained") 
           
        self.create_current_state_figure(time_step, ax=ax_dict['left'])
        x_axis_label = 'Time step [day]'
        for axis_name, resource_to_plot in zip(['upper right', 'lower right'], resources_to_plot):
            y_axis_label = f'{resource_to_plot} Demand/Supply/Consumption'
            self.plotter.plot_single_resource(list(range(0, time_step+1)), system.resilience_calculators[supply_demand_resilience_calculator_id].system_supply[resource_to_plot][:time_step+1], 
                                        system.resilience_calculators[supply_demand_resilience_calculator_id].system_demand[resource_to_plot][:time_step+1], 
                                        system.resilience_calculators[supply_demand_resilience_calculator_id].system_consumption[resource_to_plot][:time_step+1], ax_dict[axis_name])
            ax_dict[axis_name].set_xlabel(x_axis_label)
            ax_dict[axis_name].set_ylabel(y_axis_label)
            ax_dict[axis_name].grid(True)      

        if show:
            plt.show()

        if save:
            plt.rcParams.update({'font.size': 18})   
            plt.savefig(savename.replace('TIME_STEP', str(time_step)), dpi=dpi, bbox_inches='tight', pad_inches=0)  
            plt.close()       
        
    
    def create_current_state_figure(self, time_step: int, ax=None, save=False, dpi=300):
        state_colors = self.update_buildings_dataframe_with_component_current_state(time_step)     
        if ax == None:   
            fig, ax = plt.subplots(figsize=(8, 6))
        self.buildings_geodataframe.plot(color=state_colors, ax=ax)
        self.create_legend(ax)
        ax.set_title(f'Time Step: {time_step}')
        ctx.add_basemap(ax, zoom="auto", crs=self.buildings_geodataframe.crs, source=ctx.providers.OpenStreetMap.Mapnik)
        ax.axis('tight')   
        ax.axis('off')    
        if save:      
            plt.savefig(f'2D_buildings_{time_step}.png', dpi=dpi, bbox_inches='tight', transparent=True, pad_inches=0)  
    
    def create_current_state_shapefile(self, time_step: int, file_name='./Example 3/shapefiles/NorthEast_SF'):
        self.update_buildings_dataframe_with_component_current_state(time_step)
        self.buildings_geodataframe.to_file(f'{file_name}_Time_Step_{time_step}.shp', driver='ESRI Shapefile')

    def update_buildings_dataframe_with_component_current_state(self, time_step: int) -> list:
        state_colors = []
        for row_id, component_row in self.buildings_geodataframe.iterrows():
            component = self.components[component_row['ID']]
            component_current_state = self.get_component_current_state(component, time_step)
            state_colors.append(self.state_dict[component_current_state[0]]['Color'])
            self.buildings_geodataframe.at[row_id, 'State'] = component_current_state[0]
        return state_colors

    def create_buildings_geodataframe(self) -> None:
        """
        Creates a geodataframe with the buildings of the system. Implemented in subclasses.
        """
        self.buildings_geodataframe = self.create_empty_buildings_dataframe()

    def get_component_current_state(self, component: Component.Component, time_step: int) -> int:        
        component_state = []
        for state in list(self.state_dict.keys())[1:]:
            if self.component_is_in_state(component, state, time_step):                
                component_state.append(state)   
        if self.component_is_waiting(component_state):            
            component_state.append('Waiting')  
        return component_state 

    def component_is_in_state(self, component: Component.Component, state: str, time_step: int) -> bool:
        if state in component.recovery_model.recovery_activities and time_step in \
            component.recovery_model.recovery_activities[state].time_steps:
            return True
        elif state == 'Functional' and self.component_is_functional(component, time_step): 
            return True   
        else:
            return False 
    
    def component_is_functional(self, component: Component.Component, time_step: int) -> bool:        
        return time_step in component.functional
    
    def component_is_waiting(self, component_state: list([int])):
        return len(component_state) == 0
    
    def create_legend(self, axis: plt.axis):
        rectangles = []
        legend_names = []
        for state_name, state_info in self.state_dict.items():
            rectangles.append(mpatches.Rectangle((0, 0), 1, 1, fc=state_info['Color']))
            legend_names.append(state_name)
        axis.legend(rectangles, legend_names, loc='upper right', bbox_to_anchor=(1, 1), frameon=1, facecolor='white')
    
    def get_building_state_color_map(self) -> tuple:
        state_ids = [state['ID'] for state in self.state_dict.values()]   
        state_ids.append(max(state_ids)+1)
        state_colors = [state['Color'] for state in self.state_dict.values()]
        cmap = mcolors.ListedColormap(state_colors, state_ids)
        norm = mcolors.Normalize(vmin=min(state_ids), vmax=max(state_ids))
        return cmap, norm
    
    def create_localities_figure(self, components_to_plot: list, save=False, dpi=300):
        localities_colors = self.get_component_localities_color(components_to_plot)     
        fig, ax = plt.subplots(figsize=(8, 6))        
        self.buildings_geodataframe.plot(color=localities_colors, ax=ax)      
        ax.set_title('Component Localities')
        ctx.add_basemap(ax, zoom="auto", crs=self.buildings_geodataframe.crs, source=ctx.providers.OpenStreetMap.Mapnik)
        ax.axis('tight')   
        ax.axis('off')    
        if save:      
            plt.savefig(f'2D_buildings_Localities.png', dpi=dpi, bbox_inches='tight', transparent=True, pad_inches=0)  
    
    def get_component_localities_color(self, components_to_plot: list) -> list([str]):
        component_localities_color = []
        cmap = mcm.get_cmap('seismic')
        for component in components_to_plot:
                component_localities_color.append(cmap(component.locality[0]/5))
        return component_localities_color
    
    def plot_locality_polygons(self, ax, system_file: dict):
        for locality, locality_content in system_file["Content"].items():
            bounding_box = locality_content['Coordinates']['BoundingBox']
            # Create a method for creating polygons in system creator and use it here - DRY. Future work.
            locality_polygon = shapely.Polygon([(lat, long) for lat, long in 
                                zip(bounding_box['Latitude'], bounding_box['Longitude'])])
            x, y = locality_polygon.exterior.xy
            ax.plot(x, y)

class R2D_GeoVisualizer(ConcreteGeoVisualizer):

    def create_buildings_geodataframe(self) -> None:
        buildings_dataframe = self.create_empty_buildings_dataframe()
        for i, component in enumerate(self.components):
            if self.component_is_a_residential_building(component):
                building_polygon_WKT_format = self.get_building_polygon_in_WKT_format(component)                
                current_row = len(buildings_dataframe) + 1
                component_data_for_df = [i, building_polygon_WKT_format,
                                         'Functional']              
                buildings_dataframe.loc[current_row] = component_data_for_df

        geo_series = geopandas.GeoSeries.from_wkt(buildings_dataframe['Footprint_WKT'])
        geodataframe = geopandas.GeoDataFrame(buildings_dataframe, geometry=geo_series, crs="EPSG:4326")
        self.buildings_geodataframe = geodataframe.to_crs(epsg=3857)           

    def plot_component_localities(self):
        components_to_plot = []
        for component in self.components:
            if self.component_is_a_residential_building(component):
                components_to_plot.append(component)
        self.create_localities_figure(components_to_plot, save=True)

    @staticmethod
    def component_is_a_residential_building(component: Component.Component) -> bool:
        if 'ResidentialBuilding' in component.name:
            return True
        else:
            return False
    
    def get_building_polygon_in_WKT_format(self, component) -> str:
        if self.footprint_in_WKT_format(component.footprint):
            return component.footprint
        else:
            building_polygon_coordinates_as_list = [[str(coordinate_x) + ' ' + str(coordinate_y)][0] for
                                                    coordinate_x, coordinate_y in
                                                    component.footprint['geometry']['coordinates'][0]]
            building_polygons_as_one_string = ','.join(building_polygon_coordinates_as_list)
            building_polygon_WKT_format = 'POLYGON((' + building_polygons_as_one_string + '))'
            return building_polygon_WKT_format
    
    def footprint_in_WKT_format(self, footprint) -> bool:
        if 'POLYGON' in footprint:
            return True
        else:
            return False   
        
pyrecodes/GeoVisualizer.py
NikolaBlagojevic/pyrecodes
"""
Module used to plot the results of the resilience assessment.

More details coming soon.
"""

import matplotlib.pyplot as plt
from pyrecodes import Component

# CONSTANTS
GANTT_BAR_DISTANCE = 10
GANTT_BAR_WIDTH = 5
LOR_ALPHA = 0.2
ALL_RECOVERY_ACTIVITIES_COLORS = {'RapidInspection': 'lightblue',
                                    'Financing': 'orange',
                                    'ContractorMobilization': 'springgreen',
                                    'SitePreparation': 'purple',
                                    'CleanUp': 'yellow',
                                    'DetailedInspection': 'tomato',
                                    'ArchAndEngDesign': 'pink',
                                    'Permitting': 'darkblue',
                                    'Demolition': 'gray',
                                    'Repair': 'red',
                                    'Functional': 'green'}

class Plotter():

    def setup_lor_plot_fig(self, x_axis_label: str, y_axis_label: str) -> plt.axis:
        plt.figure()
        plt.xlabel(x_axis_label)
        plt.ylabel(y_axis_label)
        plt.grid(True)
        return plt.gca()

    def plot_single_resource(self, time_steps: list, supply: list, demand: list, consumption: list,
                             axis_object: plt.axis, warmup=0, show=False, supply_label='Supply', demand_label='Demand',
                             consumption_label='Consumption') -> None:
        [supply, demand, consumption] = self.add_warmup(warmup, [supply, demand, consumption])
        axis_object.plot(time_steps, supply, label=supply_label)
        axis_object.plot(time_steps, demand, label=demand_label)
        axis_object.plot(time_steps, consumption, label=consumption_label)
        axis_object.fill_between(time_steps, demand, consumption,
                         label='LoR', alpha=LOR_ALPHA)
        axis_object.legend(loc='lower right')
        if show:
            plt.show()

    def save_current_figure(self, savename='Figure.png', dpi=300) -> None:        
        plt.gca()
        plt.savefig(savename, dpi=dpi)
    
    def add_warmup(self, warmup: int, lists_to_extend: list([list])) -> list([list]):
        extended_lists = []
        for list_to_extend in lists_to_extend:
            extended_lists.append([list_to_extend[0]] * warmup + list_to_extend)
        return extended_lists       

    def setup_gantt_chart_fig(self, x_axis_label: str, components: list) -> plt.axis:
        plt.figure()
        plt.xlabel(x_axis_label)
        axis_object = plt.gca()
        plt.grid(True)
        return axis_object

    def plot_gantt_chart(self, components: list([Component.Component]), axis_object: plt.axis) -> None:
        recovery_activities_legend = []
        plotted_components = []
        component_row = 0
        for component in components:
            active_recovery_activities = self.plot_component_gantt_bar(component_row, component, axis_object)
            if len(active_recovery_activities) > 0:
                component_row += 1
                recovery_activities_legend += active_recovery_activities
                plotted_components.append(component)        
        self.set_gantt_chart_legend(recovery_activities_legend, axis_object)
        axis_object.set_yticks([(i) * GANTT_BAR_DISTANCE for i in range(len(plotted_components))])   
        axis_object.set_yticklabels([component.__str__() for component in plotted_components])
        plt.show()
    
    def set_gantt_chart_legend(self, recovery_activities_legend: list, axis_object: plt.axis) -> None:
        legend_labels = []
        legend_handles = []
        for recovery_activity_name in set(recovery_activities_legend):
            legend_labels.append(recovery_activity_name)
            legend_handles.append(plt.Rectangle((0, 0), 1, 1, fc=ALL_RECOVERY_ACTIVITIES_COLORS[recovery_activity_name]))
        axis_object.legend(legend_handles, legend_labels, loc='upper left', bbox_to_anchor=(1.02, 1.0))

    def plot_component_gantt_bar(self, component_row: int, component: Component.Component, axis_object: plt.axis) -> list:
        active_recovery_activities = []
        for recovery_activity_name, recovery_activity_object in component.recovery_model.recovery_activities.items():
            duration = len(recovery_activity_object.time_steps)
            if duration > 0:
                Y_position = component_row * GANTT_BAR_DISTANCE - GANTT_BAR_WIDTH/2
                active_recovery_activities.append(recovery_activity_name)
                start = recovery_activity_object.time_steps[0]
                axis_object.broken_barh([(start, duration)], (Y_position, GANTT_BAR_WIDTH),
                                        facecolors=ALL_RECOVERY_ACTIVITIES_COLORS[recovery_activity_name],
                                        edgecolor="none")
        return active_recovery_activities
pyrecodes/Plotter.py
NikolaBlagojevic/pyrecodes
"""
Module used to define probability distributions used in the system.
"""

from abc import ABC, abstractmethod
import math
import numpy as np

class Distribution(ABC):

    @abstractmethod
    def set_parameters(self, parameters: dict) -> None:
        pass

    @abstractmethod
    def sample(self) -> float:
        pass


class Deterministic(Distribution):

    def __init__(self, parameters: dict):
        self.set_parameters(parameters)

    def set_parameters(self, parameters: dict) -> None:
        self.value = parameters['Value']

    def sample(self) -> float:
        return self.value


class Lognormal(Distribution):

    def __init__(self, parameters: dict):
        self.set_parameters(parameters)

    def set_parameters(self, parameters: dict) -> None:
        self.median = parameters['Median']
        self.dispersion = parameters['Dispersion']

    def sample(self) -> float:
        mean_normal = math.log(self.median)
        return np.random.lognormal(mean_normal, self.dispersion)

pyrecodes/ProbabilityDistribution.py
NikolaBlagojevic/pyrecodes
"""
Module used to define relations between damage and functionality of a component and functionality and supply and demand.

More details coming soon.
"""

from abc import ABC, abstractmethod
import math

ABS_TOL = 1e-10

class Relation(ABC):
    """
    Abstract class for all functions used to define relations between damage and functionality of a component and functionality and supply and demand.
    """

    @abstractmethod
    def get_output(self, input: float) -> float():
        pass

class ConcreteRelation(Relation):

    def get_output(self, input: float):
        pass

    def valid_input(self, input: float) -> bool:
        if 0.0 <= input <= 1.0:
            return True
        else:
            raise ValueError('Input must be between 0 and 1.')

class Constant(ConcreteRelation):
    def get_output(self, input: float) -> float:
        if self.valid_input(input):
            return 1.0

class Linear(ConcreteRelation):
    def get_output(self, input: float) -> float:
        if self.valid_input(input):
            return input

class ReverseLinear(ConcreteRelation):
    def get_output(self, input: float) -> float:
        if self.valid_input(input):
            return 1 - input

class Binary(ConcreteRelation):
    def get_output(self, input: float) -> float:
        if self.valid_input(input):
            if math.isclose(input, 1, abs_tol=ABS_TOL):
                return 1
            else:
                return 0

class ReverseBinary(ConcreteRelation):
    def get_output(self, input: float) -> float:
        if self.valid_input(input):
            if math.isclose(input, 0, abs_tol=ABS_TOL):
                return 1
            else:
                return 0

class MultipleStep(ConcreteRelation):
    """
    Multiple step relation between input and output.

    Step limits define the values at which the output changes. Step values define the output values.
    if limit smaller than the lowest limit, output is 0.0
    if limit larger than the largest limit, output is 1.0

    Note: input has to be rounded, otherwise comparison does not work. 
    """
    def get_output(self, input: float) -> float:
        if self.valid_input(input):
            input = round(input, int(math.log10(1 / ABS_TOL)))
            for i in range(len(self.step_limits) - 1):
                if input < self.step_limits[0]:
                    return 0.0
                elif self.step_limits[i] <= input < self.step_limits[i + 1]:
                    return self.step_values[i]
                elif input >= self.step_limits[-1]:
                    return 1.0
        
    def set_steps(self, step_limits: list([float]), step_values: list([float])):   
        if len(step_limits) != len(step_values):
            raise ValueError('Number of step limits and step values should be the same.')
        else:
            self.step_limits = step_limits
            self.step_values = step_values
pyrecodes/Relation.py
NikolaBlagojevic/pyrecodes
"""
Module used to define resilience calculators used in the system.

More details coming soon.
"""

from abc import ABC, abstractmethod
import numpy as np

class ResilienceCalculator(ABC):

    @abstractmethod
    def __str__(self):
        pass

    @abstractmethod
    def calculate_resilience(self):
        pass

    @abstractmethod
    def update(self):
        pass

class FullRecoveryTimeResilienceCalculator(ResilienceCalculator):

    def __str__(self):
        # define what the print() method should return
        return 'Full Recovery Time Resilience Calculator \n' + 'Recovery time: ' + str(self.current_recovery_time) + '\n'

    def calculate_resilience(self):
        return self.current_recovery_time

    def update(self, time_step: int) -> None:
        self.current_time_step = time_step

class ReCoDeSResilienceCalculator(ResilienceCalculator):

    def __init__(self, parameters: dict) -> None:   
        self.system_supply = {}
        self.system_demand = {}
        self.system_consumption = {}
        self.resource_names = parameters["Resources"]
        self.scope = parameters["Scope"]
        for resource_name in self.resource_names:
            self.system_supply[resource_name] = []
            self.system_demand[resource_name] = []
            self.system_consumption[resource_name] = []
    
    def __str__(self):
        # define what the print() method should return
        lack_of_resilience = self.calculate_resilience()
        output = 'Re-CoDeS Resilience Calculator \n'
        output += 'Scope: ' + self.scope + '\n'
        output += '----------------------------- \n'
        output += 'Total unmet demand: \n'
        for resource_name, value in lack_of_resilience.items():
            output += ' ' + resource_name + ': ' + str(value) + '\n'
        return output

    def calculate_resilience(self) -> dict:
        self.lack_of_resilience = dict()
        for resource_name in self.resource_names:
            self.lack_of_resilience[resource_name] = np.sum(
                np.asarray(self.system_demand[resource_name]) - np.asarray(self.system_consumption[resource_name]))
        return self.lack_of_resilience

    def update(self, resources: dict):
        for resource_name, resource_parameters in resources.items():
            if resource_name in self.resource_names:
                self.system_supply[resource_name].append(resource_parameters['DistributionModel'].get_total_supply(scope=self.scope))
                self.system_demand[resource_name].append(resource_parameters['DistributionModel'].get_total_demand(scope=self.scope))
                self.system_consumption[resource_name].append(
                    resource_parameters['DistributionModel'].get_total_consumption(scope=self.scope))

class NISTGoalsResilienceCalculator(ReCoDeSResilienceCalculator):

    def __init__(self, resilience_goals: list) -> None:
        self.set_resilience_goals(resilience_goals)
    
    def __str__(self):
        # define what the print() method should return
        output = 'NIST Resilience Goals Calculator: \n'
        output += '-------------------------------- \n'
        for resilience_goal in self.resilience_goals:
            for key, value in resilience_goal.items():
                output += ' ' + key + ': ' + str(value) + '\n'
            output += '\n'
        return output

    def set_resilience_goals(self, resilience_goals: list):
        self.resilience_goals = []
        for resilience_goal in resilience_goals:
            self.resilience_goals.append(resilience_goal)
            self.resilience_goals[-1]['GoalMet'] = []

    def update(self, resources: dict):
        # compare demand and consumption and check if the goal is met
        # if its met, save the time step in a bool list and then in the end in calculate resilience method get the first time step when the goal is met and maintained
        for resilience_goal in self.resilience_goals:
            resource_parameters = resources.get(resilience_goal['Resource'])            
            total_demand = resource_parameters['DistributionModel'].get_total_demand(scope=resilience_goal['Scope'])
            total_consumption = resource_parameters['DistributionModel'].get_total_consumption(scope=resilience_goal['Scope'])
            resilience_goal['GoalMet'].append(resilience_goal['DesiredFunctionalityLevel'] < total_consumption/total_demand)

    def calculate_resilience(self):
        # find the time step at which the goal is met and MAINTAINED - the last time step at which the goal was not met
        for resilience_goal in self.resilience_goals:
            resilience_goal['MetAtTimeStep'] = np.where(np.asarray(resilience_goal['GoalMet']) == False)[0][-1] + 1
            # remove GoalMet key not important anymore
            del resilience_goal['GoalMet']
        return self.resilience_goals
pyrecodes/ResilienceCalculator.py
NikolaBlagojevic/pyrecodes
"""
Module used to define resources used in the system.

More details coming soon.
"""

from abc import ABC, abstractmethod
from pyrecodes import Relation

class Resource(ABC):
    name: str
    initial_amount: float
    current_amount: float
    component_functionality_to_amount: Relation
    unmet_demand_to_amount: Relation

    @abstractmethod
    def __init__(self, name: str, amount: float, functionality_to_amount_relation: str,
                 unmet_demand_to_amount_relation: str) -> None:
        pass

    @abstractmethod
    def set_current_amount(self, amount: float) -> None:
        pass

    @abstractmethod
    def update_based_on_component_functionality(self, functionality_level: float) -> None:
        pass

    @abstractmethod
    def update_based_on_unmet_demand(self, percent_of_met_demand: float) -> None:
        pass


class ConcreteResource(Resource):

    def __init__(self, name: str, parameters: dict, default_relation='Constant') -> None:
        self.name = name
        self.set_initial_amount(parameters.get('Amount', 0.0))
        self.set_functionality_to_amount_relation(parameters.get('FunctionalityToAmountRelation', default_relation))
        self.set_unmet_demand_to_amount_relation(parameters.get('UnmetDemandToAmountRelation', default_relation)) 

    def set_initial_amount(self, amount: float) -> None:
        if self.amount_is_a_positive_number(amount):
            self.initial_amount = amount
            self.current_amount = amount

    def set_current_amount(self, amount: float) -> None:
        if self.amount_is_a_positive_number(amount):
            self.current_amount = amount

    @staticmethod
    def amount_is_a_positive_number(amount) -> bool:
        if amount >= 0:
            return True
        else:
            raise ValueError('Resource amount must be a positive number.')

    def set_relation(self, relation_class_name: str, attribute_name: str) -> None:
        try:
            target_relation = getattr(Relation, relation_class_name)
            setattr(self, attribute_name, target_relation())
        except:
            raise ValueError(f'Relation {relation_class_name} not defined.')

    def set_functionality_to_amount_relation(self, relation_class_name: str) -> None:
        self.set_relation(relation_class_name, 'component_functionality_to_amount')

    def set_unmet_demand_to_amount_relation(self, relation_class_name: str) -> None:
        self.set_relation(relation_class_name, 'unmet_demand_to_amount')

    def update_based_on_component_functionality(self, component_functionality_level: float) -> None:
        self.current_amount = self.initial_amount * self.component_functionality_to_amount.get_output(
            component_functionality_level)

    def update_based_on_unmet_demand(self, percent_of_met_demand: float) -> None:
        reduced_amount = self.initial_amount * self.unmet_demand_to_amount.get_output(percent_of_met_demand)
        if reduced_amount < self.current_amount:
            self.current_amount = reduced_amount   

class NonConsumableResource(ConcreteResource):
    """
    Class to simulate a resource whose supply does not decrease when it is consumed.
    """

    def update_based_on_consumption(self, consumption: float) -> None:
        pass 

class ConsumableResource(ConcreteResource):
    """
    Class to simulate a resource whose supply decreases when it is consumed.

    Both initial and current amounts are reduced by the consumption amount. 

    The reason is that the initial amount represents the amount that the component provides when 
    functionality level is 1 and this value is also decreased for a consumable resource.
    """

    def update_based_on_consumption(self, consumption: float) -> None:
        self.initial_amount = max(self.initial_amount - consumption, 0)
        self.current_amount = max(self.current_amount - consumption, 0)
pyrecodes/Resource.py
NikolaBlagojevic/pyrecodes
"""
Module used to define resource distribution models used in the system.

More details coming soon.
"""

from abc import ABC, abstractmethod
import numpy as np
import math
import copy
import json
import itertools 
from pyrecodes import Component
from pyrecodes import DistributionPriority

class ResourceDistributionModel(ABC):

    @abstractmethod
    def distribute(self) -> None:
        pass

    @abstractmethod
    def get_total_supply(self, scope: str) -> float:
        pass

    @abstractmethod
    def get_total_demand(self, scope: str) -> float:
        pass

    @abstractmethod
    def get_total_consumption(self, scope: str) -> float:
        pass

class ResourceDistributionModelConstructor(ABC):

    @abstractmethod
    def set_components(self, components: list([Component.Component]), distribution_model: ResourceDistributionModel) -> None:
        pass

    @abstractmethod
    def set_resource_name(self, resource_name: str) -> None:
        pass

class ConcreteResourceDistributionModelConstructor(ResourceDistributionModelConstructor):

    def construct(self, resource_name: str, components: list([Component.Component]), distribution_model: ResourceDistributionModel):
        self.set_resource_name(resource_name, distribution_model)
        self.set_components(components, distribution_model)

    def set_components(self, components: list([Component.Component]), distribution_model: ResourceDistributionModel) -> None:
        distribution_model.components = components

    def set_resource_name(self, resource_name: str, distribution_model: ResourceDistributionModel) -> None:
        distribution_model.resource_name = resource_name

class UtilityDistributionModelConstructor(ConcreteResourceDistributionModelConstructor):

    def construct(self, resource_name: str, resource_parameters: dict, components: list([Component.Component]), distribution_model: ResourceDistributionModel):
        self.set_resource_name(resource_name, distribution_model)
        self.set_components(components, distribution_model)
        self.set_priority_model(resource_name, resource_parameters['DistributionPriority'], components, distribution_model)
        self.set_system_matrix(components, resource_name, distribution_model)

    # def set_components(self, components: list([Component.Component]), distribution_model: ResourceDistributionModel) -> None:
    #     distribution_model.components = components

    # def set_resource_name(self, resource_name: str, distribution_model: ResourceDistributionModel) -> None:
    #     distribution_model.resource_name = resource_name
    
    def set_priority_model(self, resource_name: str, distribution_priority: dict, components: list([Component.Component]), distribution_model: ResourceDistributionModel) -> None:
        target_priority_model = getattr(DistributionPriority, distribution_priority['Type'])
        priority_model = target_priority_model(resource_name, distribution_priority['Parameters'], components)
        distribution_model.priority = priority_model
    
    def set_system_matrix(self, components: list([Component.Component]), resource_name: str, distribution_model: ResourceDistributionModel) -> None:
        distribution_model.system_matrix = SingleResourceSystemMatrixCreator(components, resource_name)
    
    def set_transfer_service_distribution_model(self, transfer_service_distribution_model: ResourceDistributionModel, distribution_model: ResourceDistributionModel) -> None:
        pass

class TransferServiceDistributionModelConstructor(ConcreteResourceDistributionModelConstructor):

    # TODO:Add class specific methods when necessary.
    pass 

class BridgeServiceDistributionModelConstructor(ConcreteResourceDistributionModelConstructor):

    # TODO:Add class specific methods when necessary.
    pass

class SingleResourceSystemMatrixCreator():
    components: list([Component.Component])
    matrix: np.ndarray
    NUM_COLUMN_SETS = 5  # start_locality/end_locality/supply/demand/demand_met
    ROWS_PER_COMPONENT = 2  # operation demand/recovery demand

    def __init__(self, components: list([Component.Component]), resource_name: str):
        self.components = components
        self.resource_name = resource_name
        self.RECOVERY_DEMAND_ROW_OFFSET = len(components)
        self.set_system_matrix_column_ids()
        self.initialize_system_matrix()

    def set_system_matrix_column_ids(self):
        self.START_LOCALITY_COL_ID = 0
        self.END_LOCALITY_COL_ID = 1
        self.SUPPLY_COL_ID = 2
        self.DEMAND_COL_ID = 3
        self.DEMAND_MET_COL_ID = 4

    def initialize_system_matrix(self):
        self.matrix = np.zeros((self.calculate_num_rows_in_system_matrix(),
                                self.calculate_num_columns_in_system_matrix()))

    def calculate_num_rows_in_system_matrix(self):
        return len(self.components) * self.ROWS_PER_COMPONENT

    def calculate_num_columns_in_system_matrix(self):
        return self.NUM_COLUMN_SETS

    def fill_system_matrix(self):
        for row, component in enumerate(self.components):
            self.fill_operation_demand_row(row, component)
            self.fill_recovery_demand_row(row, component)

    def fill_operation_demand_row(self, row: int, component: Component.Component):
        self.matrix[row, :] = np.asarray(self.get_component_properties(component, [
            [Component.SupplyOrDemand.SUPPLY.value, Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value],
            [Component.SupplyOrDemand.DEMAND.value,
             Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]]))

    def fill_recovery_demand_row(self, row: int, component: Component.Component):
        self.matrix[row + self.RECOVERY_DEMAND_ROW_OFFSET, :] = np.asarray(self.get_component_properties(component, [
            [Component.SupplyOrDemand.DEMAND.value,
             Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value]]))

    def get_component_properties(self, component: Component.Component, types: list):
        component_properties = np.zeros(self.NUM_COLUMN_SETS)
        for supply_or_demand, type in types:
            value = self.get_current_resource_amount(component, supply_or_demand, type)
            if supply_or_demand == Component.SupplyOrDemand.SUPPLY.value:
                component_properties[self.SUPPLY_COL_ID] = value
            elif supply_or_demand == Component.SupplyOrDemand.DEMAND.value:
                component_properties[self.DEMAND_COL_ID] = value
        component_properties[self.DEMAND_MET_COL_ID] = self.get_initial_demand_met_indicator()
        component_properties[self.START_LOCALITY_COL_ID] = component.get_locality()[0]
        component_properties[self.END_LOCALITY_COL_ID] = component.get_locality()[-1]
        return component_properties

    def get_current_resource_amount(self, component: Component.Component, supply_or_demand: str, type: str):
        return component.get_current_resource_amount(supply_or_demand, type, self.resource_name)
        # resource = getattr(component, supply_or_demand)[type].get(self.resource_name, None)
        # if not (resource is None):
        #     return resource.current_amount
        # else:
        #     return 0.0        

    def get_initial_demand_met_indicator(self) -> float:
        return 1.0

    def set_demand_met_indicator(self, component_row_id: int, percent_of_met_demand: float):
        self.matrix[component_row_id, self.DEMAND_MET_COL_ID] = percent_of_met_demand

    def get_demand(self, component_row_id: int) -> None:
        return self.matrix[component_row_id, self.DEMAND_COL_ID]

    def update_components(self, components: list([Component.Component])) -> None:
        self.components = components

class UtilityDistributionModel(ResourceDistributionModel):
    components: list([Component.Component])
    resource_name: str
    transfer_service_distribution_model: ResourceDistributionModel
    system_matrix: SingleResourceSystemMatrixCreator

    def __init__(self, resource_name: str, resource_parameters: dict, components: list([Component.Component])):
        self.constructor = UtilityDistributionModelConstructor()
        self.constructor.construct(resource_name, resource_parameters, components, self)
        self.transfer_service_distribution_model = None #consider moving this into the constructor or finding a better solution-the point is to have an initial value for this property

    def distribute(self):
        self.fill_system_matrix()

        component_priorities_by_row, component_demand_types = self.get_component_priorities()    

        suppliers = []
        for component_row_id, component_demand_type in zip(component_priorities_by_row, component_demand_types):
            suppliers, component_is_supplier = self.add_supplier(component_row_id, suppliers)
            suppliers = self.meet_component_demand(suppliers, component_row_id, component_demand_type)

            if component_is_supplier:                
                self.reset_supplier_order(suppliers)
        
        self.update_suppliers_based_on_consumption(suppliers)

    def fill_system_matrix(self):
        self.system_matrix.update_components(self.components)
        self.system_matrix.fill_system_matrix()

    def get_component_priorities(self):
        component_priorities_id, component_demand_types = self.priority.get_component_priorities()
        component_row_in_system_matrix = self.set_component_row_based_on_demand_type(component_priorities_id,
                                                                                     component_demand_types)
        return component_row_in_system_matrix, component_demand_types

    def set_component_row_based_on_demand_type(self, component_priorities_id: list([int]),
                                               component_demand_types: list([str])):
        component_row_ids = []
        for component_priority_id, component_demand_type in zip(component_priorities_id, component_demand_types):
            if component_demand_type == Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value:
                component_row_ids.append(component_priority_id + self.system_matrix.RECOVERY_DEMAND_ROW_OFFSET)
            elif component_demand_type == Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value:
                component_row_ids.append(component_priority_id)
        return component_row_ids

    def add_supplier(self, component_row_id: int, suppliers: list) -> 'tuple[list, bool]':
        current_supply = self.system_matrix.matrix[component_row_id, self.system_matrix.SUPPLY_COL_ID]
        start_locality = self.system_matrix.matrix[component_row_id, self.system_matrix.START_LOCALITY_COL_ID]
        end_locality = self.system_matrix.matrix[component_row_id, self.system_matrix.END_LOCALITY_COL_ID]
        component_is_supplier = False
        if current_supply > 0:
            suppliers.append({'StartLocality': start_locality, 'EndLocality': end_locality, 
                              'CurrentSupply': current_supply, 'ConsumedAmount': 0, 
                              'ComponentRowID': component_row_id})
            self.put_current_supplier_on_top(suppliers)
            component_is_supplier = True
        return suppliers, component_is_supplier

    def put_current_supplier_on_top(self, suppliers):
        suppliers[0], suppliers[1:] = suppliers[-1], suppliers[0:-1]
        return suppliers

    def meet_component_demand(self, suppliers: dict, component_row_id: int, component_demand_type: str):
        component_demand = self.get_demand(component_row_id)  
        component_localities = [self.system_matrix.matrix[component_row_id, self.system_matrix.START_LOCALITY_COL_ID],
                                self.system_matrix.matrix[component_row_id, self.system_matrix.END_LOCALITY_COL_ID]]
        
        if component_demand > 0.0:
            initial_suppliers = copy.deepcopy(suppliers)
            transfer_service_demand = self.get_transfer_service_demand(component_row_id, component_demand_type)
            component_demand_after_distribution, suppliers = self.suppliers_meet_component_demand(suppliers,
                                                                                                  component_demand,  
                                                                                                  component_localities,                                                                                      
                                                                                                  transfer_service_demand)
            self.update_consumed_amounts(suppliers, initial_suppliers)
            
            if component_demand_after_distribution > 0.0:
                percent_of_met_demand = 1 - (component_demand_after_distribution / component_demand)
                self.set_demand_met_indicator(component_row_id, percent_of_met_demand)
                if component_demand_type == Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value:
                    self.reduce_component_supply(component_row_id, percent_of_met_demand)
                elif component_demand_type == Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value:
                    self.set_met_demand_for_recovery_activities(component_row_id, percent_of_met_demand)
                
                # suppliers = self.reset_suppliers(initial_suppliers)     

        return suppliers

    def suppliers_meet_component_demand(self, suppliers, component_demand, component_localities, transfer_service_demand):
        for supplier in suppliers:
            supplier_start_locality, supplier_end_locality, current_supply = supplier['StartLocality'], supplier['EndLocality'], supplier['CurrentSupply']
            supply_needed, path_functionality = self.modify_demand_to_account_for_resource_transfer(supplier_start_locality,
                                                                                   supplier_end_locality,
                                                                                   component_demand,
                                                                                   component_localities,                                                                                   
                                                                                   transfer_service_demand)
            # REFACTOR THIS, CAN BE DONE BETTER - PATH FUNCTIONALITY INTRODUCED TO AVOID SUPPLY_NEEDED = IN SCENARIOS WHEN THERE IS NO PATH BETWEEN TWO LOCALITIES
            if supply_needed < current_supply:
                current_supply -= supply_needed
                component_demand = 0
                supplier = self.update_supplier_resource_amount(supplier, current_supply)
                break
            elif path_functionality > 0:
                component_demand -= current_supply * path_functionality
                current_supply = 0
                supplier = self.update_supplier_resource_amount(supplier, current_supply)

        return component_demand, suppliers

    def update_supplier_resource_amount(self, supplier: dict, current_supply: float) -> None:
        supplier['CurrentSupply'] = current_supply
        return supplier
    
    def update_consumed_amounts(self, suppliers: dict, initial_suppliers: dict) -> None:
        for supplier, initial_supplier in zip(suppliers, initial_suppliers):
            supplier['ConsumedAmount'] += initial_supplier['CurrentSupply'] - supplier['CurrentSupply']
    
    def update_suppliers_based_on_consumption(self, suppliers: list):
        for supplier in suppliers:
            self.components[supplier['ComponentRowID']].update_supply_based_on_consumption(self.resource_name, supplier['ConsumedAmount'])

    def set_demand_met_indicator(self, component_row_id: int, percent_of_met_demand: float):
        self.system_matrix.set_demand_met_indicator(component_row_id, percent_of_met_demand)

    def reduce_component_supply(self, component_row_id: int, percent_of_met_demand: float):
        self.components[component_row_id].update_supply_based_on_unmet_demand(percent_of_met_demand)

    def set_met_demand_for_recovery_activities(self, component_row_id: int, percent_of_met_demand: float):
        self.components[
            component_row_id - self.system_matrix.RECOVERY_DEMAND_ROW_OFFSET].set_met_demand_for_recovery_activities(
            self.resource_name, percent_of_met_demand)

    def reset_supplier_order(self, suppliers: list) -> list:
        suppliers[-1], suppliers[:-1] = suppliers[0], suppliers[1:]
        return suppliers

    def reset_suppliers(self, initial_suppliers: list) -> list:
        return copy.deepcopy(initial_suppliers)

    def get_demand(self, component_row_id: int) -> float:
        return self.system_matrix.get_demand(component_row_id)

    def get_transfer_service_demand(self, component_row_id: int, demand_type: str) -> float:
        """Method assumes that the transfer service demand is the same as the demand for the utility resource."""
        resource_name = self.resource_name
        # consider moving if statement to a separate method
        if demand_type == Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value:
            component_row_id -= self.system_matrix.RECOVERY_DEMAND_ROW_OFFSET
        return self.components[component_row_id].get_current_resource_amount(
                                                                            Component.SupplyOrDemand.DEMAND.value, 
                                                                            demand_type, resource_name)       

    def modify_demand_to_account_for_resource_transfer(self, supplier_start_locality: int, supplier_end_locality: int,
                                                       component_demand: float, component_localities: list([int]),
                                                       transfer_service_demand: float) -> float:
        locality_pairs = self.get_locality_pairs(supplier_start_locality, supplier_end_locality, component_localities)
        
        best_path_functionality = self.get_best_path_functionality(locality_pairs, transfer_service_demand)        

        if best_path_functionality > 0:
            return component_demand / best_path_functionality, best_path_functionality
        else:
            return math.inf, best_path_functionality

    def get_locality_pairs(self, supplier_start_locality: int, supplier_end_locality: int, component_localities: list) -> list:
        """Finds non-duplicate locality pairs between supplier and component."""
        locality_pairs = [[supplier_start_locality, component_localities[0]], [supplier_start_locality, component_localities[1]], 
                            [supplier_end_locality, component_localities[0]], [supplier_end_locality, component_localities[1]]]
        locality_pairs.sort()
        return list(i for i,_ in itertools.groupby(locality_pairs))
               
    def get_best_path_functionality(self, locality_pairs: list, transfer_service_demand: float) -> float:
        best_path_functionality = 0.0
        for start_locality, end_locality in locality_pairs:
            current_path_functionality = self.get_path_functionality(start_locality, end_locality, transfer_service_demand)
            if current_path_functionality > best_path_functionality:
                best_path_functionality = current_path_functionality 
        return best_path_functionality
    
    def get_path_functionality(self, start_locality, end_locality, transfer_service_demand):
        """Method calculates path functionality by comparing the transfer service supply
            of the optimal path with the demand.
            Optimal path not used at the moment. If we need which links are used this is important. Future work."""   
        if start_locality == end_locality:            
            return 1.0
        else:
            optimal_transfer_supply, optimal_path = self.get_optimal_path(start_locality, end_locality)
            return [1.0 if optimal_transfer_supply > transfer_service_demand else 0.0][0]

    def get_optimal_path(self, start_locality: int, end_locality: int) -> float:
        """Method outputs the supply capacity and the localities of the optimal path between two localities.
        If transfer services are not needed to transfer the resource, transfer service distribution model
        should not be assigned (stay None) and the method will return math.inf as the optimal path supply capacity.
        Works only when start and end locality are different. If they are the same, method get_path_functionality returns the appropriate values."""
        if self.transfer_service_distribution_model:
            return self.transfer_service_distribution_model.get_optimal_path(start_locality, end_locality)
        else:
            return math.inf, None
    
    def get_scope(self, scope='All') -> list:
        if scope == 'All':
            return list(range(self.system_matrix.matrix.shape[0]))
        elif "Locality" in scope:
            locality_id = int(scope[-1])
            component_rows_in_locality = np.where(np.logical_or(self.system_matrix.matrix[:, self.system_matrix.START_LOCALITY_COL_ID] == locality_id, 
                                                  self.system_matrix.matrix[:, self.system_matrix.END_LOCALITY_COL_ID] == locality_id))
            return component_rows_in_locality
        else:
            raise ValueError('Scope of the Re-CoDeS Resilience Calculator not well defined: All or Locality X.')

    def get_total_supply(self, scope='All') -> float:
        components_to_include = self.get_scope(scope)
        return np.sum(self.system_matrix.matrix[components_to_include, self.system_matrix.SUPPLY_COL_ID])

    def get_total_demand(self, scope='All') -> float:
        components_to_include = self.get_scope(scope)
        return np.sum(self.system_matrix.matrix[components_to_include, self.system_matrix.DEMAND_COL_ID])

    def get_total_consumption(self, scope='All') -> float:
        components_to_include = self.get_scope(scope)
        return np.sum(np.multiply(self.system_matrix.matrix[components_to_include, self.system_matrix.DEMAND_COL_ID],
                                  self.system_matrix.matrix[components_to_include, self.system_matrix.DEMAND_MET_COL_ID]))


class HousingDistributionModel(ResourceDistributionModel):
    # TODO: Implement if housing distribution is significantly different then utility distribution.
    # this might happen if building housing supply and demand are not the same
    # demand lower than supply, then people from one house move to the next
    # this requries a new constructor class - will take some time, not urgent now

    def distribute(self) -> None:
        pass

    def get_total_supply(self, scope: str) -> float:
        pass

    def get_total_demand(self, scope: str) -> float:
        pass

    def get_total_consumption(self, scope: str) -> float:
        pass
    

class TransferServiceDistributionModelPotentialPathSets(ResourceDistributionModel): 

    components: list([Component.Component])
    resource_name: str
    potential_paths: dict

    def __init__(self, resource_name: str, resource_parameters: dict, components: list([Component.Component])) -> None:
        self.constructor = TransferServiceDistributionModelConstructor()
        self.constructor.construct(resource_name, components, self)
        self.set_potential_paths(resource_parameters["PathSetsFile"])
        self.create_potential_paths() 

    def set_potential_paths(self, filename: str):
        with open(filename, 'r') as file:
            self.potential_paths_list = json.load(file)
    
    def create_potential_paths(self) -> None:
        """ The method finds the links that are used in potential paths. 
        It creates a list of links that is latter queried to find the 
        optimal path from the list of potential paths."""     
        self.potential_paths = {} 
        for path_string, localities_list in self.potential_paths_list.items():       
            self.potential_paths[path_string] = [] 
            start_locality, end_locality = self.get_start_end_locality_from_path_string(path_string)                                  
            for localities in localities_list:
                current_start = start_locality                
                self.initiate_new_link_list(path_string)
                
                for locality in localities[1:]:                
                    self.find_connecting_link(path_string, current_start, locality)

                    if locality == end_locality:                    
                        break
                    else:                    
                        current_start = locality  

    def get_start_end_locality_from_path_string(self, path_string: str) -> tuple:
        return [int(locality) for locality in (path_string.split('from ')[1].split(' to '))]  
    
    def initiate_new_link_list(self, path_string) -> None:
        self.potential_paths[path_string].append([])
    
    def find_connecting_link(self, path_string: str, start_locality: int, next_locality: int):
        for component in self.components:
            if component.locality[0] == start_locality and \
                component.locality[-1] == next_locality and \
                component.has_resource_supply(self.resource_name):
                self.potential_paths[path_string][-1].append(component)
                break

    def distribute(self):
        """Distribute method updates the potential paths between all locality pairs.
        This is done implicitly when using the potential path sets,
        as components' transfer service supply is updated in the system class. """
        pass                                                                                    
                        
    def get_optimal_path(self, start_locality: int, end_locality: int) -> tuple:
        """Method finds the optimal path, from all potential paths between two localities.
            Optimality is defined as maximizing the transfer service supply.
            Transfer service supply of a path is the minimal supply among its constitutive links."""   
        potential_path_links = self.potential_paths.get(f'from {int(start_locality)} to {int(end_locality)}', None)
      
        if potential_path_links is None:
            optimal_transfer_supply = 0.0
            optimal_path = None
            return optimal_transfer_supply, optimal_path
        else:
            optimal_path = None
            optimal_transfer_supply = -math.inf
            
            for i, path in enumerate(potential_path_links):        
                path_supply = self.get_path_supply(path)
          
                if path_supply > optimal_transfer_supply:
                    optimal_transfer_supply = path_supply                
                    optimal_path = i
            
            return optimal_transfer_supply, optimal_path      

    def get_path_supply(self, path: list([Component.Component])) -> float:
        """Get the supply capacity of a path as the minimum of links supply."""    
        return min([link.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value, 
                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value, 
                                                    self.resource_name) for link in path])       

    def get_total_supply(self, scope: str) -> float:
        print(f'System supply for transfer service {self.resource_name} not defined yet.')
        return None

    def get_total_demand(self, scope: str) -> float:
        print(f'System demand for transfer service {self.resource_name} not defined yet.')
        return None

    def get_total_consumption(self, scope: str) -> float:
        print(f'System consumption for transfer service {self.resource_name} not defined yet.')   
        return None

class BridgeServiceDistributionModel(ResourceDistributionModel):   
    """  
    Bridge service is not consumable!
    """    

    def __init__(self, resource_name: str, resource_parameters: dict, components: list([Component.Component])) -> None:
        self.constructor = BridgeServiceDistributionModelConstructor()
        self.constructor.construct(resource_name, components, self)
        self.find_bridges()
        self.map_links_to_bridges()
    
    def find_bridges(self):
        self.bridges = []
        for component in self.components:
            if isinstance(component, Component.Bridge):
                self.bridges.append(component)
    
    def map_links_to_bridges(self):
        """
        Find all links that are on a bridge.
        """
        self.links_on_a_bridge = [[] for _ in range(len(self.bridges))]
        for bridge_id, bridge_component in enumerate(self.bridges):
            for component in self.components:
                if self.component_is_on_the_bridge(component, bridge_component) and not(isinstance(component, Component.Bridge)):
                    self.links_on_a_bridge[bridge_id].append(component)
    
    def component_is_on_the_bridge(self, component, bridge_component):
        # assume bridges go in single direction - that's why use sort
        component_localities = component.get_locality()
        bridges_localities = bridge_component.get_locality()
        return component_localities == bridges_localities 
    
    def distribute(self) -> None:
        for bridge_id, bridge in enumerate(self.bridges):
            bridge_supply = bridge.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value, 
                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value, 
                                                    self.resource_name)
            for link in self.links_on_a_bridge[bridge_id]:
                # TODO: what happens with recovery demand? - if workers need a bridge to cross? but that's the roads then? not sure if needed?
                bridge_demand = link.get_current_resource_amount(Component.SupplyOrDemand.DEMAND.value, 
                                                    Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value, 
                                                    self.resource_name)
                percent_of_met_demand = min(1.0, bridge_supply/bridge_demand)
                link.update_supply_based_on_unmet_demand(percent_of_met_demand)
    
    def get_total_supply(self) -> float:
        print(f'System supply for bridge service {self.resource_name} not defined yet.')
        return None
    
    def get_total_demand(self) -> float:
        print(f'System demand for bridge service {self.resource_name} not defined yet.')
        return None
    
    def get_total_consumption(self) -> float:
        print(f'System consumption for bridge service {self.resource_name} not defined yet.')
        return None

pyrecodes/ResourceDistributionModel.py
NikolaBlagojevic/pyrecodes
"""
Module that defines the **pyrecodes** System class and its two helper classes: DistributionListCreator and RecoveryTargetChecker.
"""

from abc import ABC, abstractmethod
import math
from pyrecodes import SystemCreator
from pyrecodes import DamageInput
from pyrecodes import Component
import pickle
import copy

class System(ABC):
    """
    Abstract base class representing a generic system for resilience assessment.

    Attributes:
        | components (list[Component.Component]): List of system components represented as Component objects.
        | resources (dict): Dictionary containing resource parameters.
        | system_creator (SystemCreator.SystemCreator): The system creator object responsible for system initialization.

    """
    components: list([Component.Component])
    resources: dict
    system_creator: SystemCreator.SystemCreator

    @abstractmethod
    def __init__(self, system_configuration_file: dict, component_library: dict, system_creator: SystemCreator.SystemCreator):
        """
        Constructor method for the System class. Initializes the system with the given configuration.

        Args:
            | configuration_file (dict): The configuration file for the system.
            | component_library (dict): The component library containing system components' configurations.
            | system_creator (SystemCreator.SystemCreator): The system creator object responsible for system initialization.
        """
        pass

    @abstractmethod
    def set_configuration_file(self, system_configuration: dict):
        """
        Sets the configuration file for the system.

        Args:
            system_configuration (dict): System configuration dictionary.
        """
        pass

    @abstractmethod
    def set_component_library(self, component_library: dict):
        """
        Sets the component library for the system.

        Args:
            component_library (dict): The component library to set for the system.
        """
        pass

    @abstractmethod
    def create_system(self, system_creator: SystemCreator.SystemCreator):
        """
        Creates the system based on the provided SystemCreator object.

        Args:
            system_creator (SystemCreator.SystemCreator): The system creator object responsible for system initialization.
        """
        pass

    @abstractmethod
    def set_initial_damage(self):
        """
        Sets the initial damage state of the system components.
        """
        pass

    @abstractmethod
    def start_resilience_assessment(self):
        """
        Initiates the resilience assessment for the system. This is the entry point for running the assessment.
        """
        pass

    @abstractmethod
    def distribute_resources(self):
        """
        Distributes resources among the system component's.
        """
        pass

    @abstractmethod
    def update(self):
        """
        Updates the system state at a time step of the resilience assessment.
        """
        pass

    @abstractmethod
    def recover(self):
        """
        Simulates the recovery process in the system.
        """
        pass

    @abstractmethod
    def calculate_resilience(self):
        """
        Calculates and returns the resilience metric for the system.
        """
        pass

class DistributionListCreator:
    """
    Creates a list specifying the order of resource distribution in the system.

    This class determines the order in which resources should be distributed to capture components' interdependencies and calculate consumption.
    The resource order is fixed: Bridge Services, Transfer Services, Independent Resources, Interdependent Resources.

    Attributes:
        | components (list[Component.Component]): A list of components in the system.
        | resources (dict): A dictionary containing resource parameters for the system.

    """

    def __init__(self, components: list[Component.Component], resources: dict):
        """
        Initialize the DistributionListCreator with components and resources.

        Args:
            | components (list[Component.Component]): A list of components in the system.
            | resources (dict): A dictionary containing resource parameters for the system.

        """
        self.components = components
        self.resources = resources

    def get_resource_distribution_dict(self):
        """
        Generate the resource distribution dict based on the specified order.

        The resource distribution dict is formed by grouping resources into four categories:
        1. Bridge Services
        2. Transfer Services
        3. Other Independent Resources
        4. Interdependent Resources

        The reason is that Bridge Services are here assumed always to be independent resources that only affect Transfer Services.
        Transfer Services are assumed to only be affected by Bridge Services and to affect other resources, 
        both the distribution of Independent and Interdependent Resources.
        Remaining Independent Resources can only be distriuted once per time step, as their supply won't be affected by unmet demand for other resources.
        The supply of Interdependent Resources can be affected by unmet demand and to capture potential cascading effects,
        their distribution is last and in an iterative manner.

        TODO: this method can be improved by making it more generic and flexible. Future work.

        Returns:
            dict: A dict separating independent and interdependent resources and specifying the order of resource distribution.

        
        """
        resource_distribution_dict = {'IndependentResources': [],
                                      'InterdependentResources': []}

        # Group resources that belong to the Bridge Services group
        resource_distribution_dict['IndependentResources'] += self.get_resource_group('BridgeService')

        # Group resources that belong to the Transfer Services group
        resource_distribution_dict['IndependentResources'] += self.get_resource_group('TransferService')

        # Separate Independent and Interdependent Resources excluding Bridge and Transfer Services
        resource_distribution_dict = self.get_independent_interdependent_resources(
            resource_distribution_dict)

        # Form the resource distribution vector using Interdependent Resources
        resource_distribution_dict['InterdependentResources'] = self.form_resource_distribution_vector(resource_distribution_dict['InterdependentResources'])

        return resource_distribution_dict

    def get_resource_group(self, group_name: str):
        """
        Get a list of resource names belonging to the specified group.

        Args:
            group_name (str): The name of the resource group.

        Returns:
            list[str]: A list of resource names in the specified group.

        """
        resource_group = []
        for resource_name, resource_parameters in self.resources.items():
            if resource_parameters['Group'] == group_name:
                resource_group.append(resource_name)
        return resource_group

    def get_independent_interdependent_resources(self, resource_distribution_dict: dict):
        """
        Separate resources that are not Bridge or Transfer services into Independent and Interdependent Resources.

        Args:
            resource_distribution_list (list[str]): The list specifying the order of resource distribution.

        Returns:
            tuple[list[str], list[str]]: Two lists containing Independent and Interdependent Resources, respectively.

        """        
        for resource_name in self.resources.keys():
            if resource_name not in resource_distribution_dict['IndependentResources']:
                for component in self.components:
                    if self.resource_is_interdependent(resource_name, component):
                        resource_distribution_dict['InterdependentResources'].append(resource_name)
                        break
                else:
                    resource_distribution_dict['IndependentResources'].append(resource_name)

        return resource_distribution_dict
    
    def resource_is_interdependent(self, resource_name: str, component: Component.Component) -> bool:
        """
        Check whether the specified resource is interdependent.

        A resource is assumed interdependent if there is a single component that:
        1. Has a supply of the resource
        2. Has an operation demand
        3. Component's supply can be affected by unmet demand of the resource (i.e., the relation is not Constant)

        Args:
            resource_name (str): The name of the resource.
            component (Component.Component): The component to check.

        Returns:
            bool: True if the resource is interdependent, False otherwise.

        """
        return component.has_resource_supply(resource_name) and \
               component.has_operation_demand() and \
               component.unmet_demand_can_affect_resource_supply(resource_name)                 

    def form_resource_distribution_vector(self, interdependent_resources):
        """
        Form the resource distribution vector by repeating Interdependent Resources.

        The number of repetitions is equal to the number of Interdependent Resources.

        Reasoning behind this is provided in:
         Blagojevic et al. (2022) Quantifying disaster resilience of a community with
         interdependent civil infrastructure systems

        Args:
            interdependent_resources (list[str]): List of Interdependent Resources.

        Returns:
            list[str]: The resource distribution vector.

        """
        num_resources = len(interdependent_resources)
        return interdependent_resources * num_resources
    
class RecoveryTargetChecker(ABC):
    """
    Abstract base class for a recovery target checker.

    This class defines the interface for checking whether the system has recovered and the resilience assessment interval
    is finished.
    """

    @abstractmethod
    def recovery_target_met(self, system: System) -> bool:
        """
        Check whether the recovery target has been met for the given system.

        Args:
            system (System): The system object to check for recovery.

        Returns:
            bool: True if the recovery target is met, False otherwise.
        """
        pass


class CompleteDamageRecoveryTargetChecker(RecoveryTargetChecker):
    """
    Concrete subclass of RecoveryTargetChecker.

    This class implements the recovery target checker where the system has recovered once all components are damage-free.

    """

    def recovery_target_met(self, system: System) -> bool:
        """
        Check whether the recovery target has been met for the given system.

        The system is considered to have recovered once all components are damage-free (damage level close to 0).

        Args:
            system (System): The system object to check for recovery.

        Returns:
            bool: True if the recovery target is met, False otherwise.
        """
        if system.time_step <= system.DISASTER_TIME_STEP:
            return False
        else:
            for component in system.components:
                if not (math.isclose(component.get_damage_level(), 0, abs_tol=1e-10)):
                    return False
        return True


class BuiltEnvironmentSystem(System):
    """
    This class represents a built environment system using the iRe-CoDeS framework as an assembly of components that exchange resources. It inherits from the System abstract class
    and implements the necessary methods to run the resilience assessment for the built environment.

    Attributes:
        | components (list[Component.Component]): List of system components represented as Component objects.
        | resources (dict): Dictionary containing resource parameters.
        | system_creator (SystemCreator.SystemCreator): The system creator object responsible for system initialization.
        | FINISH (bool): Flag indicating if the resilience assessment has finished.

    """

    components: list([Component.Component])
    resources: dict
    system_creator: SystemCreator.SystemCreator
    FINISH = False

    def __init__(self, system_configuration: dict, component_library: dict, system_creator: SystemCreator.SystemCreator):
        """
        Constructor method for the BuiltEnvironmentSystem class. Initializes the system with the given configuration.

        Args:
            | configuration_file (str): The configuration file for the system.
            | component_library (dict): The component library containing system components' configurations.
            | system_creator (SystemCreator.SystemCreator): The system creator object responsible for system initialization.
        """
        self.set_configuration_file(system_configuration)
        self.set_component_library(component_library)
        self.set_system_creator(system_creator)
        # self.create_system()

    def __str__(self):
        """
        Returns a string representation of the system state, including the damage level of each component.
        """
        system_state = ''
        for component in self.components:
            system_state += f'{component.__str__()} | Damage Level: {component.get_damage_level()} \n'
        return system_state

    def set_configuration_file(self, system_configuration: dict):
        """
        Sets the configuration file for the system.

        Args:
            system_configuration (dict): Dict with system configuration parameters.
        """
        self.system_configuration = system_configuration

    def set_component_library(self, component_library: dict):
        """
        Sets the component library for the system.

        Args:
            component_library (dict): The component library to set for the system.
        """
        self.component_library = component_library

    def set_system_creator(self, system_creator: SystemCreator.SystemCreator):
        """
        This method allows setting the SystemCreator object responsible for system initialization.

        Args:
            system_creator (SystemCreator.SystemCreator): The SystemCreator object to set for the system.
        """
        self.system_creator = system_creator

    def create_system(self):
        """
        Creates the system based on the provided SystemCreator object.
        """
        self.system_creator.setup(self.component_library, self.system_configuration)
        self.components = self.system_creator.create_components()
        self.resources = self.system_creator.get_resource_parameters(self.components)
        self.resilience_calculators = self.system_creator.get_resilience_calculators()
        self.START_TIME_STEP = self.system_creator.START_TIME_STEP
        self.MAX_TIME_STEP = self.system_creator.MAX_TIME_STEP
        self.DISASTER_TIME_STEP = self.system_creator.DISASTER_TIME_STEP
        self.recovery_target_checker = CompleteDamageRecoveryTargetChecker()
        self.set_resource_distribution_dict()
        self.set_damage_input()

    def set_resource_distribution_dict(self):
        """
        Sets the resource distribution list for the system based on components and resource parameters using the DistributionListCreator object.
        """
        distribution_list_creator = DistributionListCreator(self.components, self.resources)
        self.resource_distribution_dict = distribution_list_creator.get_resource_distribution_dict()
    
    def set_damage_input(self):
        """
        Sets the damage input for the system based on the specified damage input type and parameters.
        """
        target_damage_input_class = getattr(DamageInput, self.system_creator.get_damage_input_type())
        self.damage_input = target_damage_input_class(self.system_creator.get_damage_input_parameters())        

    def start_resilience_assessment(self):
        """
        Initiates the resilience assessment for the built environment.

        The assessment consits of running a for loop where each pass represents the next time step in the system's recovery. 
        
        At each time step, components' state is update, resources are distributed among components to capture their interaction, and components recover following their recovery model. 
        
        Recovery starts after the disaster has occured.
        """
        for self.time_step in range(self.START_TIME_STEP, self.MAX_TIME_STEP):

            if self.recovery_target_met():
                self.FINISH = True

            if self.time_step == self.DISASTER_TIME_STEP:
                self.set_initial_damage()

            if self.time_step == 80:
                print('')

            self.update()

            self.distribute_resources()

            self.update_resilience_calculators()

            if self.time_step > self.DISASTER_TIME_STEP:
                self.recover()

            if self.FINISH:
                print('Resilience assessment finished.')
                break

    def recovery_target_met(self) -> bool:
        """
        Checks whether the recovery target has been met for the built environment system.

        Returns:
            bool: True if the recovery target is met, False otherwise.
        """
        return self.recovery_target_checker.recovery_target_met(self)

    def set_initial_damage(self) -> None: 
        """
        Sets the initial damage for the components based on the damage input.
        """       
        self.damage_input.set_initial_damage(self.components)

    def update(self) -> None:
        """
        Updates the system state during the resilience assessment by updating the state of components.
        """
        for component in self.components:
            component.update(self.time_step)

    def distribute_resources(self) -> None:
        """
        Distributes resources among components following the resource distribution list.
        """
        self.distribute_indepedent_resources()
        self.distribute_interdependent_resources()
    
    def distribute_indepedent_resources(self) -> None:        
        for resource_name in self.resource_distribution_dict['IndependentResources']:
            self.resources[resource_name]['DistributionModel'].distribute()
    
    def distribute_interdependent_resources(self) -> None:
        system_converged = False
        system_supply = self.get_supply_of_interdependent_resources()
        updated_system_supply = self.get_supply_of_interdependent_resources()
        while not(system_converged):
            for resource_name in self.resource_distribution_dict['InterdependentResources']:
                self.resources[resource_name]['DistributionModel'].distribute()
            updated_system_supply = self.get_supply_of_interdependent_resources()
            system_converged = self.check_system_convergence(system_supply, updated_system_supply)
            system_supply = copy.deepcopy(updated_system_supply)
        
    def get_supply_of_interdependent_resources(self) -> dict:
        """
        Gets the supply of interdependent resources in the system.

        Returns:
            dict: A dictionary with the supply of interdependent resources in the system.
        """
        system_supply = {}
        for resource_name in set(self.resource_distribution_dict['InterdependentResources']):            
            system_supply[resource_name] = self.get_system_supply(resource_name)
        return system_supply
    
    def get_system_supply(self, resource_name: str) -> dict:
        """
        Get the system-level supply for a resource resource_name based on current components' state.
        
        This method is already implemented in the ResourceDistributionClass but uses the system matrix. However, using that method requires one to update the system matrix outside of resource distribution which then screws up consumption calculation. This method is a workaround to avoid that.
        """
        system_supply = 0
        for component in self.components:
            if component.has_resource_supply(resource_name):
                system_supply += component.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value,
                                                                       Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value,
                                                                       resource_name)
        return system_supply
        
    def check_system_convergence(self, system_supply: dict, updated_system_supply: dict) -> bool:
        """
        Checks whether the system has converged to a stable state.

        This is checked by comparing the system-level resource supply before and after the resource distribution.

        Returns:
            bool: True if the system has converged, False otherwise.
        """
        system_converged = True
        for resource_name in set(self.resource_distribution_dict['InterdependentResources']):
            if not(math.isclose(system_supply[resource_name], updated_system_supply[resource_name])):
                system_converged = False
                break            
        return system_converged         

    def recover(self) -> None:
        """
        Simulates the recovery process in the system at a single time step.
        """
        for component in self.components:
            component.recover(self.time_step)

    def update_resilience_calculators(self) -> None:
        """
        Updates the resilience calculators with the current resource allocations.
        """
        for resilience_calculator in self.resilience_calculators:
            resilience_calculator.update(self.resources)

    def calculate_resilience(self, print_output=True, return_output=False) -> dict:
        """
        Calculates and returns the resilience metrics for the system.

        Returns:
            list: A list containing resilience metrics specific to each of the resilience calculators.
        """
        resilience_metrics = []
        for resilience_calculator in self.resilience_calculators:
            resilience_metrics.append(resilience_calculator.calculate_resilience())
            if print_output:
                print(resilience_calculator)
        if return_output:
            return resilience_metrics
    
    def save_as_pickle(self, savename='./system_object.pickle') -> None:
        """
        Saves the system object as a pickle file.

        Args:
            savename (str, optional): The name of the pickle file to save the system object. Defaults to './system_object.pickle'.
        """
        with open(savename, 'wb') as file:
            pickle.dump(self, file) 
    
    def load_as_pickle(self, loadname='./system_object.pickle') -> None:
        """
        Loads the system object from a pickle file and returns it.

        Args:
            loadname (str, optional): The name of the pickle file to load the system object. Defaults to './system_object.pickle'.

        Returns:
            BuiltEnvironmentSystem: The loaded system object.
        """
        with open(loadname, 'rb') as file:
            system = pickle.load(file) 
        return system


pyrecodes/System.py
NikolaBlagojevic/pyrecodes
"""
Module used to create a system based on a system configuration file.

More details coming soon.
"""

from abc import ABC, abstractmethod
from pyrecodes import Component
from pyrecodes import ComponentLibraryCreator
from pyrecodes import ResourceDistributionModel
from pyrecodes import ResilienceCalculator
import json
import copy
import math
import pandas as pd
import geopandas as gpd
import shapely

class SystemCreator(ABC):
    """
    Abstract class for creating a System from a file
    """

    @abstractmethod
    def setup(self, component_library: ComponentLibraryCreator.ComponentLibraryCreator, file_name: str) -> None:
        pass

    @abstractmethod
    def create_components(self) -> list([Component.Component]):
        pass

    @abstractmethod
    def get_damage_input_type(self) -> str:
        pass

    @abstractmethod
    def get_damage_input_parameters(self) -> dict:
        pass

    @abstractmethod
    def get_resource_distribution_parameters(self) -> dict:
        pass

    @abstractmethod
    def get_resilience_calculators(self) -> list([dict]):
        pass

class ConcreteSystemCreator(SystemCreator):
    """
    Class that implements methods common to all SystemCreator subclasses.
    """

    def __init__(self):
        self.component_library = {}
        self.system_configuration = {}
        self.components = []

    def setup(self, component_library: dict, system_configuration: dict) -> None:
        self.component_library = component_library
        self.system_configuration = system_configuration
        self.set_constants()
       
    def set_constants(self) -> None:
        constants = self.system_configuration.get('Constants', {})
        for constant_label, constant_value in constants.items():
            setattr(self, constant_label, constant_value)
    
    def get_damage_input_type(self) -> str:
        return self.system_configuration['DamageInput']['Type']

    def get_damage_input_parameters(self) -> dict:
        return self.system_configuration['DamageInput']['Parameters']

    def get_resource_distribution_parameters(self) -> dict:
        return self.system_configuration['Resources']
    
    def get_resource_parameters(self, components) -> dict:
        all_resources_parameters = self.get_resource_distribution_parameters()
        transfer_services = self.get_transfer_services(components, all_resources_parameters)
        non_transfer_services = self.get_non_transfer_services(components, all_resources_parameters, transfer_services)
        return {**transfer_services, **non_transfer_services}
    
    def get_transfer_services(self, components: list([Component.Component]), all_resources_parameters: dict) -> dict:
        resources = dict()
        for resource_name, resource_parameters in all_resources_parameters.items():
            if resource_parameters['Group'] == 'TransferService':
                resources[resource_name] = dict()        
                resources[resource_name]['Group'] = resource_parameters['Group']
                resources[resource_name]['DistributionModel'] = self.get_resource_distribution_model(resource_name, resource_parameters, components)                
        return resources
    
    def get_non_transfer_services(self, components: list([Component.Component]), all_resources_parameters: dict, transfer_services: dict) -> dict:
        resources = {}
        for resource_name, resource_parameters in all_resources_parameters.items():
            if resource_parameters['Group'] != 'TransferService':
                resources[resource_name] = dict()        
                resources[resource_name]['Group'] = resource_parameters['Group']
                resources[resource_name]['DistributionModel'] = self.get_resource_distribution_model(resource_name, resource_parameters, components)
                required_transfer_service = resource_parameters['DistributionModel']['Parameters'].get('TransferService', None)
                if required_transfer_service is not None: 
                    resources[resource_name]['DistributionModel'].transfer_service_distribution_model = transfer_services[required_transfer_service]['DistributionModel']
        return resources
    
    def get_resource_distribution_model(self, resource_name: str, resource_parameters: dict, components: list([Component.Component])) -> ResourceDistributionModel:
        target_distribution_model = getattr(ResourceDistributionModel, resource_parameters['DistributionModel']['Type'])
        return target_distribution_model(resource_name, resource_parameters['DistributionModel']['Parameters'], components)
    
    def get_resilience_calculators(self) -> list([ResilienceCalculator.ResilienceCalculator]):
        resilience_calculators = []
        for resilience_calculator_parameters in self.system_configuration.get('ResilienceCalculator', []):
            target_resilience_calculator = getattr(ResilienceCalculator, resilience_calculator_parameters['Type'])
            resilience_calculators.append(target_resilience_calculator(resilience_calculator_parameters['Parameters']))
        return resilience_calculators
    
    def create_components(self) -> list([Component.Component]):
        self.components = []
        for locality, content in self.system_configuration.get("Content", {}).items():
            self.create_components_in_localities(locality, content.get('ComponentsInLocality', {}))
            self.create_components_between_localities(locality, content.get('LinkTo', ()))
        return self.components
    
    def create_components_in_localities(self, locality: str, content: dict) -> None:
        """
        Implemented by subclasses.
        """
        pass

    def create_components_between_localities(self, locality: str, content: dict) -> None:
        """
        Implemented by subclasses.
        """
        pass

    def get_component_object(self, component_type: str) -> Component.Component:
        return copy.deepcopy(self.component_library[component_type])

    def add_component(self, component_object: Component.Component) -> None:
        self.components.append(copy.deepcopy(component_object))
    
    @staticmethod
    def format_locality_id(locality_string) -> int:
        return int(locality_string.split(' ')[-1])   

class JSONSystemCreator(ConcreteSystemCreator):
    """
    Create a System from a JSON file.
    """

    def create_components_in_localities(self, locality: str, content: dict) -> None:
        for component_type, amount in content.items():
            component_object = self.get_component_object(component_type)
            component_object.set_locality([self.format_locality_id(locality)])
            for _ in range(amount):
                self.add_component(component_object)

    def create_components_between_localities(self, locality: str, content: dict) -> None:
        for link_to, link_type_list in content.items():
            for link_type in link_type_list:
                component_object = self.get_component_object(link_type)
                component_object.set_locality([self.format_locality_id(locality), self.format_locality_id(link_to)])
                self.add_component(component_object)

class R2DSystemCreator(ConcreteSystemCreator):
    """
    Create a System based on SimCenter R2D Tool's output files for Example 1.
    """

    def __init__(self) -> None:
        super().__init__()
        self.building_area_per_person = {}
        self.scenario_id = None
        self.component_parameters_setter = None

    def setup(self, component_library: dict, system_configuration: dict) -> None:
        super().setup(component_library, system_configuration)
        self.set_building_area_per_person()
        self.scenario_id = self.system_configuration["DamageInput"]["Parameters"]["ScenarioID"]
        self.set_component_parameters_setter()
        
    def set_building_area_per_person(self) -> None:
        for locality, locality_content in self.system_configuration["Content"].items():
            self.building_area_per_person[locality] = locality_content['ComponentsInLocality']['AreaPerPerson']
    
    def set_component_parameters_setter(self) -> None:
        system_level_data = {key: self.__dict__.get(key, None) for key in R2DResidentialBuildingParametersSetter.SYSTEM_LEVEL_DATA_REQUIRED_FOR_BUILDINGS}
        self.component_parameters_setter = R2DResidentialBuildingParametersSetter(system_level_data)        

    def create_components_in_localities(self, locality: str, content: dict) -> list([Component.Component]):
        self.components += self.create_recovery_resource_suppliers(locality, content)
        self.components += self.create_residential_building_components(locality, content)     

    def create_recovery_resource_suppliers(self, locality: str, content: dict) -> list([Component.Component]):
        suppliers = []
        for component_name in content.get('RecoveryResourceSuppliers', []):
            component = copy.deepcopy(self.component_library[component_name])
            component.set_locality([self.format_locality_id(locality)])
            suppliers.append(component)
        return suppliers

    def create_residential_building_components(self, locality: str, content: dict) -> list([Component.Component]):
        building_info_folder = content['BuildingsInfoFolder']
        building_id_range = range(content['BuildingIDsRange'][0],
                                  content['BuildingIDsRange'][1])
        bounding_box = content['Coordinates']['BoundingBox']
        max_num_buildings = content['MaxNumBuildings']
        num_buildings = 0
        components = []
        for building_id in building_id_range:
            building_data = self.load_building_data(building_id, building_info_folder)
            building_location = self.get_building_location(building_data)
            if self.building_location_inside_bounding_box(building_location,
                                                          bounding_box) and self.building_is_residential(building_data):
                components.append(self.create_residential_building(locality, building_data))
                num_buildings += 1
                if num_buildings >= max_num_buildings:
                    break
        return components

    def load_building_data(self, building_id: int, building_info_folder: str) -> dict:
        building_json_name = str(building_id) + '-BIM.json'
        building_data = {}
        with open(building_info_folder + str(building_id) + '/' + building_json_name, 'r') as file:
            building_data['Information'] = json.load(file)
        with open(building_info_folder + str(building_id) + '/' + 'DL_summary.csv', 'r') as file:
            building_data['Damage'] = pd.read_csv(file)
        return building_data

    @staticmethod
    def get_building_location(building_data: dict) -> list:
        return [building_data['Information']['GeneralInformation']['Latitude'],
                building_data['Information']['GeneralInformation']['Longitude']]

    @staticmethod
    def building_location_inside_bounding_box(building_location: list, bounding_box: dict) -> bool:
        # TODO: this can be optimized if needed. polygon created only once per locality, building locations in a list for vectorization.
        polygon = shapely.Polygon([(lat, long) for lat, long in zip(bounding_box['Latitude'], bounding_box['Longitude'])])
        building_centroid = shapely.Point(building_location[0], building_location[1])
        return building_centroid.within(polygon)

    @staticmethod
    def building_is_residential(building_data: dict) -> bool:
        return 'RES' in building_data['Information']['GeneralInformation']['OccupancyClass']

    def create_residential_building(self, locality, building_data) -> Component.Component:
        building_DS = self.get_building_damage_state(building_data)        
        component_template = copy.deepcopy(self.component_library[f'DS{building_DS}_ResidentialBuilding'])
        residential_building = self.component_parameters_setter.set_parameters(component_template, locality, building_data, building_DS)
        return residential_building

    def get_building_damage_state(self, building_data: dict) -> int:
        return building_data['Damage']['highest_damage_state/S'][self.scenario_id]

class R2DSystemWithInterfacesCreator(R2DSystemCreator):
    """
    Create a System based on SimCenter R2D Tool's output files for Example 1 with infrastructure interfaces.
    """

    def create_components_in_localities(self, locality: str, content: dict) -> list([Component.Component]):
        super().create_components_in_localities(locality, content)        
        self.components += self.create_infrastructure_service_suppliers(locality, content)
        
    def create_infrastructure_service_suppliers(self, locality: str, content: dict) -> list([Component.Component]):
        suppliers = []
        for interface_dict in content['Infrastructure']:
            component = copy.deepcopy(self.component_library[list(interface_dict.keys())[0]])
            self.set_infrastructure_system_parameters(component, list(interface_dict.values())[0])
            component.set_locality([self.format_locality_id(locality)])
            suppliers.append(component)
        return suppliers
    
    def set_infrastructure_system_parameters(self, component: Component.Component, interface_parameters: dict) -> None:
        component.set_supply_dynamics(interface_parameters)   
        if 'Demand' in interface_parameters:
            self.component_parameters_setter.set_component_operation_demand(component, interface_parameters['Demand']['Resource'], interface_parameters['Demand']['Amount'])
    
class ComponentParametersSetter:
    """
    Class that sets component parameters unique to each component and defined during system creation.
    """

    def set_component_supply(self, component: Component.Component, resource_name: str, 
                             resource_supply_amount: float, 
                             resource_supply_type=Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value) -> None:        
        component_supply = getattr(component, Component.SupplyOrDemand.SUPPLY.value)[resource_supply_type]
        component_supply[resource_name].set_initial_amount(resource_supply_amount)

    def set_component_operation_demand(self, component: Component.Component, resource_name: str, 
                             resource_demand_amount: float) -> None: 
        component_demand = getattr(component, Component.SupplyOrDemand.DEMAND.value)[
                            Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]
        component_demand[resource_name].set_initial_amount(resource_demand_amount)
    
    def set_component_recovery_demand(self, component: Component.Component, recovery_activity_name: str, 
                                      resource_name: str, resource_amount: float):
        if recovery_activity_name in component.recovery_model.recovery_activities.keys():
            component.recovery_model.recovery_activities[recovery_activity_name].demand[resource_name].set_initial_amount(
                resource_amount)

class R2DResidentialBuildingParametersSetter(ComponentParametersSetter):
    """
    Class that sets parameters of residential building components as provided in the R2DTool output files
    """

    SYSTEM_LEVEL_DATA_REQUIRED_FOR_BUILDINGS = ['DS4_REPAIR_DURATION', 'MAX_REPAIR_CREW_DEMAND_PER_BUILDING',
                                                'MAX_RESIDENTS_PER_BUILDING', 'REPAIR_CREW_DEMAND_PER_SQFT',
                                                'DEFAULT_REPAIR_DURATION_DICT', 'DEMAND_PER_PERSON', 'HOUSING_RESOURCES',
                                                'building_area_per_person', 'scenario_id']
    
    def __init__(self, system_level_data: dict):
        self.system_level_data = system_level_data

    def set_parameters(self, component: Component.Component, locality: str, building_data: dict, building_DS: int):  
        self.set_building_geometry(component, building_data)      
        self.set_housing_parameters(component, locality)
        self.set_operation_demand_parameters(component, locality, building_data.get('OperationDemandResources', []))
        self.set_repair_parameters(component, building_data, building_DS)       
        component.set_locality([R2DSystemCreator.format_locality_id(locality)])
        return component

    def set_building_geometry(self, component, building_data: dict):
        self.set_building_area(component, building_data)
        self.set_building_footprint(component, building_data)
        self.set_building_num_stories(component, building_data)
    
    def set_building_area(self, component: Component.Component, building_data: dict) -> None:
        component.area = self.get_total_building_area(building_data)

    def set_building_footprint(self, component: Component.Component, building_data: dict) -> None:
        component.footprint = json.loads(building_data['Information']['GeneralInformation']['Footprint'])

    def set_building_num_stories(self, component: Component.Component, building_data: dict) -> None:
        component.num_stories = building_data['Information']['GeneralInformation']['NumberOfStories']

    def get_total_building_area(self, building_data: dict) -> float:
        return building_data['Information']['GeneralInformation']['PlanArea'] * \
            building_data['Information']['GeneralInformation']['NumberOfStories']
    
    def set_housing_parameters(self, component: Component.Component, locality: str):
        building_housing_capacity = self.get_building_housing_capacity(component.area, locality)
        for housing_resource_name in self.system_level_data['HOUSING_RESOURCES']:
            self.set_component_supply(component, housing_resource_name, building_housing_capacity)
            self.set_component_operation_demand(component, housing_resource_name, building_housing_capacity)

    def get_building_housing_capacity(self, building_area: float, locality: str) -> int:        
        building_housing_capacity = min(int(building_area / self.system_level_data['building_area_per_person'][locality]), self.system_level_data['MAX_RESIDENTS_PER_BUILDING'])
        return building_housing_capacity  
    
    def set_operation_demand_parameters(self, component: Component.Component, locality: str, operation_demand_resources: list):
        if self.system_level_data['DEMAND_PER_PERSON'] is not None:
            building_housing_capacity = self.get_building_housing_capacity(component.area, locality)
            # consider only those resources defined in the "DEMAND_PER_PERSON" attribute in the system configuration file
            operation_demand_resources = [resource_name for resource_name in self.system_level_data.get('DEMAND_PER_PERSON', {}).keys()]
            for operation_demand_resource in operation_demand_resources:
                amount = building_housing_capacity*self.system_level_data['DEMAND_PER_PERSON'][operation_demand_resource]
                self.set_component_operation_demand(component, operation_demand_resource, amount)
     
    def set_repair_parameters(self, component: Component.Component, building_data: dict, building_DS: int):
        self.set_repair_demand(component, building_data, building_DS)
        self.set_repair_duration(component, building_data)
        self.set_building_repair_cost(component, building_data)

    def set_repair_demand(self, component: Component.Component, building_data: dict, building_DS: int) -> None:
        if 'Repair' in component.recovery_model.recovery_activities.keys():
            repair_demand = self.get_repair_crew_demand(building_DS, component.area)
            self.set_component_recovery_demand(component, 'Repair', 'RepairCrew', repair_demand)

    def get_repair_crew_demand(self, building_DS: int, building_area: float) -> int:
        repair_crew_demand = math.ceil(building_area / self.system_level_data['REPAIR_CREW_DEMAND_PER_SQFT'].get(f'DS{building_DS}', math.inf))
        return min(self.system_level_data['MAX_REPAIR_CREW_DEMAND_PER_BUILDING'], repair_crew_demand)

    def set_repair_duration(self, component: Component.Component, building_data: dict) -> None:
        if 'Repair' in component.recovery_model.recovery_activities.keys():
            median_repair_duration = self.get_median_repair_duration(building_data)
            repair_duration_dict = copy.deepcopy(self.system_level_data['DEFAULT_REPAIR_DURATION_DICT'])
            repair_duration_dict['Lognormal']['Median'] = median_repair_duration
            component.recovery_model.recovery_activities['Repair'].set_duration(repair_duration_dict)

    def get_median_repair_duration(self, building_data: dict) -> int:
        """
        Get median repair duration from the R2D .csv building file. 
        Note that if the building collapsed, a dummy repair duration of 1 is assigned in R2D.
        In that case, set the repair duration to DS4 duration.
        """
        median_repair_duration = building_data['Damage']['reconstruction/time'][self.system_level_data['scenario_id']]
        if median_repair_duration == 1 and building_data['Damage']['collapses/collapsed'][self.system_level_data['scenario_id']] == 1:
            return self.system_level_data['DS4_REPAIR_DURATION']
        else:
            return median_repair_duration
  
    def set_building_repair_cost(self, component: Component.Component, building_data: dict) -> None:
        repair_cost = building_data['Damage']['reconstruction/cost'][self.system_level_data['scenario_id']]
        financing_activity = component.recovery_model.recovery_activities.get('Financing', None)
        if financing_activity is not None:
            repair_cost_distributed_over_duration = repair_cost / component.recovery_model.recovery_activities['Financing'].duration
        else:
            repair_cost_distributed_over_duration = 0
        self.set_component_recovery_demand(component, 'Financing', 'Money', repair_cost_distributed_over_duration)
        # financing = component.recovery_model.recovery_activities.get('Financing', None)
        # if financing:
        #     repair_cost = building_data['Damage']['reconstruction/cost'][system_level_data['scenario_id']]            
        #     financing.set_demand([{'Resource': 'Money', 'Amount': repair_cost}])

    
pyrecodes/SystemCreator.py
NikolaBlagojevic/pyrecodes
"""
Module used to run the **pyrecodes** resilience assessment as defined in the component library and system configuration files.
"""

from pyrecodes import ComponentLibraryCreator
from pyrecodes import SystemCreator
from pyrecodes import System
import json

def read_file(file_name: str) -> dict:
    """Reads a JSON file and returns its contents as a dictionary.

    Args:
        file_name (str): The name of the JSON file to be read.

    Returns:
        dict: The dictionary representation of the JSON data.
    """
    with open(file_name, 'r') as file:
        file = json.load(file)
    return file

def form_component_library(input_dict: dict) -> dict:
    """Forms a component library based on the input dictionary.

    Args:
        input_dict (dict): The input dictionary containing component library configuration.

    Returns:
        dict: The component library dict.
    """
    # Extract the target ComponentLibraryCreator class from the input dictionary and instantiate it.
    component_library_target_object = getattr(ComponentLibraryCreator, input_dict['ComponentLibrary']['ComponentLibraryCreatorClass'])  
    component_library_object = component_library_target_object(input_dict['ComponentLibrary']['ComponentLibraryFile'])
    
    # Call the 'form_library()' method on the component_library_object and return the component library as a dict.
    return component_library_object.form_library()

def create_system(input_dict: dict) -> System.System:
    """Creates a system object using the input dictionary.

    Args:
        input_dict (dict): The input dictionary containing System configuration.

    Returns:
        System.System: The created system object.
    """
    # Form the component library dict using the input dictionary.
    component_library = form_component_library(input_dict)

    # Read the system configuration file.
    system_configuration = read_file(input_dict['System']['SystemConfigurationFile'])
    
    # Extract the target SystemCreator class from the input dictionary and instantiate it.
    system_creator_target_object = getattr(SystemCreator, input_dict['System']['SystemCreatorClass'])  
    system_creator = system_creator_target_object()
    
    # Extract the target System class from the input dictionary and instantiate it with the necessary arguments.
    system_target_object = getattr(System, input_dict['System']['SystemClass']) 
    system = system_target_object(system_configuration, component_library, system_creator)
    system.create_system()
    return system

def run(main_file: str) -> System.System:
    """Runs the resilience assessment for the system.

    Args:
        main_file (str): The name of the main JSON configuration file.

    Returns:
        System.System: The system object after running the resilience assessment.
    """
    # Read the main JSON configuration file and obtain the input dictionary.
    input_dict = read_file(main_file)    
    # Create the system object using the input dictionary.
    system = create_system(input_dict)
    # Start the resilience assessment for the system.
    system.start_resilience_assessment()
    # Return the system object.
    return system

def load_system(loadname, system_class_name='BuiltEnvironmentSystem') -> System.System:
    target_system_class = getattr(System, system_class_name)
    system_object = target_system_class({}, {}, None) 
    return system_object.load_as_pickle(loadname)
pyrecodes/main.py
NikolaBlagojevic/pyrecodes
from setuptools import setup, find_packages

setup(
    name='pyrecodes',
    version='0.1.0',
    packages=find_packages(),
    install_requires=[
        'numpy>=1.24.2',
        'pandas>=2.0.0',
        'geopandas>=0.12.2',
        'contextily>=1.3.0',
        'matplotlib>=3.7.1',
        'imageio>=2.27.0'
    ],
)
setup.py
NikolaBlagojevic/pyrecodes

import pytest
import math
from pyrecodes import ComponentRecoveryModel
from pyrecodes import Relation
from pyrecodes import Component
from pyrecodes import Resource

component_name = 'TestComponent'
component_parameters = {
        "ComponentClass": "StandardiReCoDeSComponent",
        "RecoveryModel": {
            "Type": "ComponentLevelRecoveryActivitiesModel",
            "Parameters": {
                "Repair": {
                    "Duration": {
                        "Deterministic": {
                            "Value": 100
                        }
                    },
                    "Demand": [
                        {
                            "Resource": "RecoveryResource1",
                            "Amount": 0.1
                        }],
                    }
                },
            "DamageFunctionalityRelation": {
                "Type": "ReverseLinear"
            }
        },
        "Supply": {
            "SupplyResource1": {
                "Amount": 10,
                "FunctionalityToAmountRelation": "Constant",
                "UnmetDemandToAmountRelation": "Binary"
            },
            "SupplyResource2": {
                "Amount": 100,
                "FunctionalityToAmountRelation": "Linear",
                "UnmetDemandToAmountRelation": "Constant"
            }
        },
        "OperationDemand": {
            "DemandResource1": {
                "Amount": 1,
                "FunctionalityToAmountRelation": "Constant"
            },
            "DemandResource2": {
                "Amount": 5,
                "FunctionalityToAmountRelation": "Binary"
            }
        }
    }

class TestStandardiReCoDeSComponent():

    @pytest.fixture
    def component(self) -> Component.StandardiReCoDeSComponent:
        return Component.StandardiReCoDeSComponent()

    def test_init(self, component):
        assert component.functionality_level == 1.0
        assert component.functional == []
        assert component.recovery_model == None
        assert component.locality == None
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value] == {}
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value] == {}
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}

    def test_str(self, component):
        component.set_name(component_name)
        component.set_locality(1)
        assert component.__str__() == f'{component_name} | Locality: {1}'
    
    def test_construct(self, component):
        component.construct(component_name, component_parameters)
        assert component.name == component_name
        assert isinstance(component.recovery_model, ComponentRecoveryModel.ComponentLevelRecoveryActivitiesModel)
        assert component.recovery_model.recovery_activities['Repair'].rate == 0.01
        assert component.recovery_model.recovery_activities['Repair'].duration == 100
        assert isinstance(component.recovery_model.damage_to_functionality_relation, Relation.ReverseLinear)

        for resource_name, resource_parameters in component_parameters['Supply'].items():
            assert component.supply['Supply'][resource_name].initial_amount == resource_parameters['Amount']
            assert component.supply['Supply'][resource_name].current_amount == resource_parameters['Amount']
            assert isinstance(component.supply['Supply'][resource_name].component_functionality_to_amount, getattr(Relation, resource_parameters['FunctionalityToAmountRelation']))
            assert isinstance(component.supply['Supply'][resource_name].unmet_demand_to_amount, getattr(Relation, resource_parameters['UnmetDemandToAmountRelation']))

        for resource_name, resource_parameters in component_parameters['OperationDemand'].items():
            assert component.demand['OperationDemand'][resource_name].initial_amount == resource_parameters['Amount']
            assert component.demand['OperationDemand'][resource_name].current_amount == resource_parameters['Amount']
            assert isinstance(component.demand['OperationDemand'][resource_name].component_functionality_to_amount, getattr(Relation, resource_parameters['FunctionalityToAmountRelation']))
    
    def test_set_recovery_model(self, component):
        component.set_recovery_model(component_parameters['RecoveryModel'])
        assert isinstance(component.recovery_model, ComponentRecoveryModel.ComponentLevelRecoveryActivitiesModel)
        assert component.recovery_model.recovery_activities['Repair'].rate == 0.01
        assert component.recovery_model.recovery_activities['Repair'].duration == 100
        assert isinstance(component.recovery_model.damage_to_functionality_relation, Relation.ReverseLinear)
  
    def test_set_supply(self, component):
        assert component.demand['OperationDemand'] == {}
        assert component.supply['Supply'] == {}
        component.set_supply(component_parameters['Supply'])        
        for resource_name, resource_parameters in component_parameters['Supply'].items():
            assert component.supply['Supply'][resource_name].initial_amount == resource_parameters['Amount']
            assert component.supply['Supply'][resource_name].current_amount == resource_parameters['Amount']
            assert isinstance(component.supply['Supply'][resource_name].component_functionality_to_amount, getattr(Relation, resource_parameters['FunctionalityToAmountRelation']))
            assert isinstance(component.supply['Supply'][resource_name].unmet_demand_to_amount, getattr(Relation, resource_parameters['UnmetDemandToAmountRelation']))

    def test_operation_demand(self, component): 
        component.set_operation_demand(component_parameters['OperationDemand'])
        for resource_name, resource_parameters in component_parameters['OperationDemand'].items():
            assert component.demand['OperationDemand'][resource_name].initial_amount == resource_parameters['Amount']
            assert component.demand['OperationDemand'][resource_name].current_amount == resource_parameters['Amount']
            assert isinstance(component.demand['OperationDemand'][resource_name].component_functionality_to_amount, getattr(Relation, resource_parameters['FunctionalityToAmountRelation']))
    
    def test_set_initial_damage_level(self, component):
        component.set_recovery_model(component_parameters['RecoveryModel'])
        component.set_initial_damage_level(0.5)
        assert component.get_damage_level() == 0.5
        component.set_initial_damage_level(1.0)
        assert component.get_damage_level() == 1.0
        component.set_initial_damage_level(0.0)
        assert component.get_damage_level() == 0.0
        with pytest.raises(ValueError):
            component.set_initial_damage_level(-0.5)

    def test_set_name(self, component):
        component.set_name(component_name)
        assert component.name == component_name
        another_name = 'TestComponent2'
        component.set_name(another_name)
        assert component.name == another_name
    
    def test_set_locality(self, component):
        component.set_locality([0])
        assert component.get_locality() == [0]
        component.set_locality([0, 1])
        assert component.get_locality() == [0, 1]
    
    def test_has_operation_demand(self, component):
        assert component.has_operation_demand() == False
        component.set_operation_demand(component_parameters['OperationDemand']) 
        assert component.has_operation_demand() == True
    
    def test_has_resource_supply(self, component):
        assert component.has_resource_supply('SupplyResource1') == False
        component.set_supply(component_parameters['Supply'])
        assert component.has_resource_supply('SupplyResource1') == True
        assert component.has_resource_supply('SupplyResource2') == True
        assert component.has_resource_supply('SupplyResource3') == False

    def test_unmet_demand_can_affect_resource_supply(self, component):
        assert component.unmet_demand_can_affect_resource_supply('SupplyResource1') == False
        component.set_supply(component_parameters['Supply'])
        assert component.unmet_demand_can_affect_resource_supply('SupplyResource1') == True
        assert component.unmet_demand_can_affect_resource_supply('SupplyResource2') == False
        assert component.unmet_demand_can_affect_resource_supply('SupplyResource3') == False

    def test_add_resources_supply(self, component):
        component = Component.StandardiReCoDeSComponent()
        component.add_resources(Component.SupplyOrDemand.SUPPLY.value,
                                Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value, 
                                component_parameters['Supply'])
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value] == {}
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}
        assert len(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]) == 2
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                        'SupplyResource1'].name == 'SupplyResource1'
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                        'SupplyResource1'].initial_amount == 10
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                        'SupplyResource1'].current_amount == 10
        assert isinstance(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                           'SupplyResource1'].component_functionality_to_amount, Relation.Constant)
        assert isinstance(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                            'SupplyResource1'].unmet_demand_to_amount, Relation.Binary)
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                        'SupplyResource2'].name == 'SupplyResource2'
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                        'SupplyResource2'].initial_amount == 100
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                        'SupplyResource2'].current_amount == 100
        assert isinstance(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                           'SupplyResource2'].component_functionality_to_amount, Relation.Linear)
        assert isinstance(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                            'SupplyResource2'].unmet_demand_to_amount, Relation.Constant)
    
    def test_add_resources_demand(self, component):
        component.add_resources(Component.SupplyOrDemand.DEMAND.value,
                                Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value, 
                                component_parameters['OperationDemand'])
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}
        assert len(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]) == 2
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                        'DemandResource1'].name == 'DemandResource1'
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                        'DemandResource1'].initial_amount == 1
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                        'DemandResource1'].current_amount == 1
        assert isinstance(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                           'DemandResource1'].component_functionality_to_amount, Relation.Constant)
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                        'DemandResource2'].name == 'DemandResource2'
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                        'DemandResource2'].initial_amount == 5
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                        'DemandResource2'].current_amount == 5
        assert isinstance(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                           'DemandResource2'].component_functionality_to_amount, Relation.Binary)
        
    def test_add_one_resource_wrong_supply_key_string(self, component):
        with pytest.raises(KeyError):
            component.add_resources(Component.SupplyOrDemand.SUPPLY.value, 
                                    'supply', 
                                    component_parameters['Supply'])

    def test_add_one_resource_wrong_demand_key_string(self, component):
        with pytest.raises(KeyError):
            component.add_resources(Component.SupplyOrDemand.DEMAND.value, 
                                    'operation_demand', 
                                    component_parameters['OperationDemand'])

    def test_add_multiple_resources(self, component):
        resource_dict = {"SupplyResource1": {
                "Amount": 10,
                "FunctionalityToAmountRelation": "Constant",
                "UnmetDemandToAmountRelation": "Binary"
                }
            }
        for _ in range(5):
            component.add_resources(Component.SupplyOrDemand.SUPPLY.value,
                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value, 
                                    resource_dict)

        for resource in component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value].values():
            assert resource.name == 'SupplyResource1'
            assert resource.initial_amount == 10
            assert resource.current_amount == 10
        
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value] == {}
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}

    def test_form_resource_non_consumable(self, component):
        resource_dict = {"SupplyResource1": {
                "Amount": 10,
                "FunctionalityToAmountRelation": "Constant",
                "UnmetDemandToAmountRelation": "Binary"
                }
            }
        resource_name, resource_parameters = list(resource_dict.items())[0]
        resource = component.form_resource(resource_name, resource_parameters)
        assert isinstance(resource, Resource.NonConsumableResource)
        assert resource.name == 'SupplyResource1'
        assert resource.initial_amount == 10
        assert resource.current_amount == 10
        assert isinstance(resource.component_functionality_to_amount, Relation.Constant)
        assert isinstance(resource.unmet_demand_to_amount, Relation.Binary)
    
    def test_form_resource_consumable(self, component):
        resource_dict = {"SupplyResource1": {
                "ResourceClass": "ConsumableResource",
                "Amount": 10,
                "FunctionalityToAmountRelation": "Constant",
                "UnmetDemandToAmountRelation": "Binary"
                }
            }
        resource_name, resource_parameters = list(resource_dict.items())[0]
        resource = component.form_resource(resource_name, resource_parameters)
        assert isinstance(resource, Resource.ConsumableResource)
        assert resource.name == 'SupplyResource1'
        assert resource.initial_amount == 10
        assert resource.current_amount == 10
        assert isinstance(resource.component_functionality_to_amount, Relation.Constant)
        assert isinstance(resource.unmet_demand_to_amount, Relation.Binary)
    
    def test_get_current_resource_amount_no_damage(self, component):
        component.construct(component_name, component_parameters)
        assert component.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value, 
                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value,
                                                    'SupplyResource1') == 10
        assert component.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value, 
                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value,
                                                    'SupplyResource2') == 100
        assert component.get_current_resource_amount(Component.SupplyOrDemand.DEMAND.value,
                                                    Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value,
                                                    'DemandResource1') == 1
        assert component.get_current_resource_amount(Component.SupplyOrDemand.DEMAND.value,
                                                    Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value,
                                                    'DemandResource2') == 5
    
    def test_get_current_resource_amount_with_damage(self, component):
        component.construct(component_name, component_parameters)
        component.set_initial_damage_level(0.5)
        component.update(1)
        assert component.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value,
                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value,
                                                    'SupplyResource1') == 10
        assert component.get_current_resource_amount(Component.SupplyOrDemand.SUPPLY.value,
                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value,
                                                    'SupplyResource2') == 50
        assert component.get_current_resource_amount(Component.SupplyOrDemand.DEMAND.value,
                                                    Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value,
                                                    'DemandResource1') == 1
        assert component.get_current_resource_amount(Component.SupplyOrDemand.DEMAND.value,
                                                    Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value,
                                                    'DemandResource2') == 0
        
    def test_update_functionality(self, component):
        component.set_recovery_model(component_parameters['RecoveryModel'])
        damage_level = 0.23
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        assert math.isclose(component.functionality_level, 0.77)

        damage_level = 0.56
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        assert math.isclose(component.functionality_level, 0.44)

        damage_level = 0.1
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        assert math.isclose(component.functionality_level, 0.9)
    
    def test_check_if_functional_partial_damage(self, component):
        component.set_recovery_model(component_parameters['RecoveryModel'])
        damage_level = 0.23
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        component.check_if_functional(time_step=1)
        assert component.functional == [1]

    def test_check_if_functional_full_damage(self, component):
        component.set_recovery_model(component_parameters['RecoveryModel'])
        damage_level = 1.0
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        component.check_if_functional(time_step=5)   
        assert component.functional == []
    
    def test_check_if_functional_during_recovery(self, component):
        component.set_recovery_model(component_parameters['RecoveryModel'])
        damage_level = 1.0
        component.set_initial_damage_level(damage_level)
        for time_step in range(150):
            component.update_functionality()
            component.check_if_functional(time_step=time_step)
            component.recover(time_step=time_step)
        assert component.functional == list(range(1, 150))
    
    def test_update_resources_based_on_component_functionality(self, component):
        component.construct(component_name, component_parameters)
        damage_level = 0.23
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        component.update_resources_based_on_component_functionality(Component.SupplyOrDemand.SUPPLY.value,
                                                                    Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].current_amount,
                            10)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource2'].current_amount,
                            100*0.77)
        
        component.update_resources_based_on_component_functionality(Component.SupplyOrDemand.DEMAND.value,
                                                                    Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value)
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                                'DemandResource1'].current_amount,
                            1)
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                                'DemandResource2'].current_amount,
                            0)
        
    def test_update_supply_based_on_component_functionality(self, component):
        component.construct(component_name, component_parameters)
        damage_level = 0.4
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        component.update_supply_based_on_component_functionality()
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].current_amount,
                            10)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource2'].current_amount,
                            100*0.6)
    
    def test_update_operation_demand(self, component):
        component.construct(component_name, component_parameters)
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                                'DemandResource1'].current_amount,
                            1)
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                                'DemandResource2'].current_amount,
                            5)    
        damage_level = 1.0
        component.set_initial_damage_level(damage_level)
        component.update_functionality()
        component.update_operation_demand()
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                                'DemandResource1'].current_amount,
                            1)
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value][
                                'DemandResource2'].current_amount,
                            0)
    
    def test_update_recovery_demand(self, component):
        component.construct(component_name, component_parameters)
        component.update_recovery_demand()
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}
        component.set_initial_damage_level(0.5)
        component.update_recovery_demand()
        assert math.isclose(component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value][
                                'RecoveryResource1'].current_amount,
                            0.1)
        component.set_initial_damage_level(0)
        component.update_recovery_demand()
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}

    def test_set_recovery_model_activities_demand_to_met(self, component):
        component.construct(component_name, component_parameters)
        assert component.recovery_model.recovery_activities['Repair'].demand_met['RecoveryResource1'] == 1.0
        component.recovery_model.recovery_activities['Repair'].demand_met['RecoveryResource1'] = 0.5
        component.set_recovery_model_activities_demand_to_met()
        assert component.recovery_model.recovery_activities['Repair'].demand_met['RecoveryResource1'] == 1.0
    
    def test_update_supply_based_on_unmet_demand(self, component):
        component.construct(component_name, component_parameters)
        percent_of_met_demand = 0.6
        component.update_supply_based_on_unmet_demand(percent_of_met_demand)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].current_amount,
                            0)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource2'].current_amount,
                            100)
    
    def test_update_supply_based_on_consumption(self, component):
        component_parameters['Supply']['SupplyResource1']['ResourceClass'] = 'ConsumableResource'
        component.construct(component_name, component_parameters)
        component.update_supply_based_on_consumption('SupplyResource1', 1)
        component.update_supply_based_on_consumption('SupplyResource2', 1)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].current_amount,
                            9)  
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].initial_amount,
                            9) 
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource2'].current_amount,
                            100)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource2'].initial_amount,
                            100)
        
        component.update_supply_based_on_consumption('SupplyResource1', 10)
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].current_amount,
                            0)  
        assert math.isclose(component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value][
                                'SupplyResource1'].initial_amount,
                            0) 
    
    def test_set_met_demand_for_recovery_activities(self, component):
        component.construct(component_name, component_parameters)
        component.set_met_demand_for_recovery_activities('RecoveryResource1', 0.3)
        assert component.recovery_model.recovery_activities['Repair'].demand_met['RecoveryResource1'] == 0.3

        component.set_met_demand_for_recovery_activities('RecoveryResource1', 1.0)
        assert component.recovery_model.recovery_activities['Repair'].demand_met['RecoveryResource1'] == 1.0

    def test_recover(self, component):
        component.construct(component_name, component_parameters)
        component.set_initial_damage_level(0.4)
        component.update(0)
        assert math.isclose(component.get_damage_level(), 0.4)
        component.recover(0)
        assert math.isclose(component.get_damage_level(), 0.396)

        component.set_met_demand_for_recovery_activities('RecoveryResource1', 0.0)
        component.recover(1)
        assert math.isclose(component.get_damage_level(), 0.396)

        component.set_met_demand_for_recovery_activities('RecoveryResource1', 1.0)
        component.recover(2)
        assert math.isclose(component.get_damage_level(), 0.392)
        assert component.recovery_model.recovery_activities['Repair'].time_steps == [0, 2]

        for time_step in range (3, 101):
            component.recover(time_step)
        assert component.get_damage_level() == 0.0
        assert component.recovery_model.recovery_activities['Repair'].time_steps == [0, 2] + list(range(3, 101))

    def test_update(self, component):
        component.construct(component_name, component_parameters)
        component.update(0)
        assert component.functionality_level == 1
        assert component.functional == [0]
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource1'].current_amount == 10
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource2'].current_amount == 100
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource1'].current_amount == 1
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource2'].current_amount == 5
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}

        component.set_initial_damage_level(0.4)
        component.update(1)
        assert component.functionality_level == 0.6
        assert component.functional == [0, 1]
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource1'].current_amount == 10
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource2'].current_amount == 60
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource1'].current_amount == 1
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource2'].current_amount == 0
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value]['RecoveryResource1'].current_amount == 0.1

        component.recover(2)
        component.update(2)
        assert component.functionality_level == 0.604
        assert component.functional == [0, 1, 2]
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource1'].current_amount == 10
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource2'].current_amount == 60.4
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource1'].current_amount == 1
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource2'].current_amount == 0
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value]['RecoveryResource1'].current_amount == 0.1

        for time_step in range (3, 102):
            component.recover(time_step)
            component.update(time_step)
        assert component.get_damage_level() == 0.0
        assert component.recovery_model.recovery_activities['Repair'].time_steps == list(range(2, 102))
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource1'].current_amount == 10
        assert component.supply[Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value]['SupplyResource2'].current_amount == 100
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource1'].current_amount == 1
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value]['DemandResource2'].current_amount == 5
        assert component.demand[Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value] == {}
    

class TestBuildingStockUnitWithEmergencyCalls():

    @pytest.fixture
    def component(self) -> Component.StandardiReCoDeSComponent:
        return Component.BuildingStockUnitWithEmergencyCalls()
    
    @pytest.fixture
    def component_parameters(self):
        component_parameters['ComponentClass'] = 'BuidlingStockUnitWithEmergencyCalls'
        component_parameters['OperationDemand']['Communication'] = {'Amount': 1.0, 
                                                                    'FunctionalityToAmountRelation': 'Linear',
                                                                    'PostDisasterIncreaseDueToEmergencyCalls': 'True'}
        return component_parameters

    def test_update_communication_demand(self, component, component_parameters):
        component.construct(component_name, component_parameters)
        initial_communication_demand = component.demand['OperationDemand']['Communication'].current_amount
        time_steps = [0, 1, 5, 10, 50]
        target_communication_demands = [initial_communication_demand,
                                        initial_communication_demand * component.COMMUNICATION_DEMAND_MULTIPLIER,
                                        initial_communication_demand * component.COMMUNICATION_DEMAND_MULTIPLIER * math.exp(
                                            component.COMMUNICATION_DEMAND_EXP_DECREASE_COEFF * 4),
                                        initial_communication_demand,
                                        initial_communication_demand]
        for time_step, target_communication_demand in zip(time_steps, target_communication_demands):
            component.update_communication_demand(time_step)
            assert math.isclose(target_communication_demand,
                                component.demand['OperationDemand']['Communication'].current_amount)

    def test_modify_emergency_calls(self, component, component_parameters):
        component.construct(component_name, component_parameters)
        initial_communication_demand = component.demand['OperationDemand']['Communication'].current_amount
        component.COMMUNICATION_RESOURCE_NAME = 'Communication'
        time_steps = [1, 3, 50]
        target_communication_demands = [initial_communication_demand * component.COMMUNICATION_DEMAND_MULTIPLIER,
                                        initial_communication_demand * component.COMMUNICATION_DEMAND_MULTIPLIER * math.exp(
                                            component.COMMUNICATION_DEMAND_EXP_DECREASE_COEFF * 2),
                                        initial_communication_demand]
        for time_step, target_communication_demand in zip(time_steps, target_communication_demands):
            communication_demand = component.modify_emergency_calls_demand(initial_communication_demand, time_step)
            assert math.isclose(communication_demand, target_communication_demand)
    
    def test_check_if_demand_increase_considered(self, component, component_parameters):
        component.check_if_demand_increase_considered('DemandResource1', component_parameters['OperationDemand']['DemandResource1'])
        assert component.COMMUNICATION_RESOURCE_NAME == ''

        component.check_if_demand_increase_considered('SupplyResource1', component_parameters['Supply']['SupplyResource1'])
        assert component.COMMUNICATION_RESOURCE_NAME == ''

        component.check_if_demand_increase_considered('Communication', component_parameters['OperationDemand']['Communication'])
        assert component.COMMUNICATION_RESOURCE_NAME == 'Communication'

class TestInfrastructureInterface():

    supply_dict_simple = {
        "TestInterface": {
                            "Resource": "SupplyInterfaceResource1",
                            "Amount": [100, 200],
                            "RestoredIn": [
                                {
                                    "Lognormal": {
                                        "Median": 20,
                                        "Dispersion": 0
                                    }
                                },
                                {
                                    "Deterministic": {
                                        "Value": 50
                                    }
                                }
                            ],
                            "Demand": {
                                "Resource": "DemandInterfaceResource1",
                                "Amount": 5
                            }
                        }
    }

    supply_dict_complicated = {
        "TestInterface": {
                            "Resource": "SupplyInterfaceResource1",
                            "Amount": [100, 50, 0, 200, 500, 600],
                            "RestoredIn": [
                                {
                                    "Deterministic": {
                                        "Value": 0
                                    }
                                },
                                {
                                    "Lognormal": {
                                        "Median": 20,
                                        "Dispersion": 0
                                    }
                                },
                                {
                                    "Deterministic": {
                                        "Value": 50
                                    }
                                },
                                {
                                    "Deterministic": {
                                        "Value": 80
                                    }
                                },
                                {
                                    "Deterministic": {
                                        "Value": 150
                                    }
                                },
                                {
                                    "Deterministic": {
                                        "Value": 200
                                    }
                                }
                            ],
                            "Demand": {
                                "Resource": "DemandInterfaceResource1",
                                "Amount": 5
                            }
                        }
    }

    @pytest.fixture
    def component(self) -> Component.StandardiReCoDeSComponent:
        return Component.InfrastructureInterface()
    
    @pytest.fixture
    def component_parameters(self) -> dict:
        component_parameters['ComponentClass'] = 'InfrastructureInterface'
        component_parameters['RecoveryModel']['Type'] = 'InfrastructureInterfaceRecoveryModel'
        component_parameters['Supply'] = {'SupplyInterfaceResource1': {'Amount': 0}}                                          
        return component_parameters

    def test_set_supply_dynamics_simple(self, component, component_parameters):
        component.construct(component_name, component_parameters)
        component.set_supply_dynamics(self.supply_dict_simple['TestInterface'])
        assert component.supply['Supply']['SupplyInterfaceResource1'].initial_amount == 200
        assert component.recovery_model.recovery_activities[component.recovery_model.RECOVERY_ACTIVITY_NAME].duration == 50
        assert isinstance(component.recovery_model.damage_to_functionality_relation, Relation.MultipleStep)
        assert component.recovery_model.damage_to_functionality_relation.step_values == [0.0, 0.5, 1]
        assert math.isclose(component.recovery_model.damage_to_functionality_relation.step_limits[1], 0.4)
        assert component.recovery_model.damage_to_functionality_relation.step_limits[2] == 1
    
    def test_set_supply_dynamics_complicated(self, component, component_parameters):
        component.construct(component_name, component_parameters)
        component.set_supply_dynamics(self.supply_dict_complicated['TestInterface'])
        assert component.supply['Supply']['SupplyInterfaceResource1'].initial_amount == 600
        assert component.recovery_model.recovery_activities[component.recovery_model.RECOVERY_ACTIVITY_NAME].duration == 200
        assert isinstance(component.recovery_model.damage_to_functionality_relation, Relation.MultipleStep)
        assert component.recovery_model.damage_to_functionality_relation.step_values == [1/6, 1/12, 0, 1/3, 5/6, 1.0]
        assert math.isclose(component.recovery_model.damage_to_functionality_relation.step_limits[0], 0)
        assert math.isclose(component.recovery_model.damage_to_functionality_relation.step_limits[1], 0.1)
        assert math.isclose(component.recovery_model.damage_to_functionality_relation.step_limits[2], 0.25)
        assert math.isclose(component.recovery_model.damage_to_functionality_relation.step_limits[3], 0.4)
        assert math.isclose(component.recovery_model.damage_to_functionality_relation.step_limits[4], 0.75)
        assert component.recovery_model.damage_to_functionality_relation.step_limits[-1] == 1
    
    def test_supply_dynamics_values(self, component, component_parameters):
        component.construct(component_name, component_parameters)
        component.set_supply_dynamics(self.supply_dict_complicated['TestInterface'])
        component.supply['Supply']['SupplyInterfaceResource1'].component_functionality_to_amount = Relation.Linear()
        assert component.supply['Supply']['SupplyInterfaceResource1'].initial_amount == 600
        component.set_initial_damage_level(1.0)
        component.update(0)
        assert component.supply['Supply']['SupplyInterfaceResource1'].current_amount == 100
        recovery_time_step_ranges = [1, 5, 20, 21, 35, 50, 60, 85, 130, 180, 200, 500]
        target_supply_amounts = [100, 100, 50, 50, 50, 0, 200, 200, 500, 500, 600]
        for i, recovery_time_step_upper_bound in enumerate(recovery_time_step_ranges[1:]):
            recovery_time_step_lower_bound = recovery_time_step_ranges[i]
            for time_step in range(recovery_time_step_lower_bound, recovery_time_step_upper_bound):
                component.recover(time_step)
                component.update(time_step)
            assert math.isclose(component.supply['Supply']['SupplyInterfaceResource1'].current_amount,
                                target_supply_amounts[i])

tests/test_Component.py
NikolaBlagojevic/pyrecodes
import pytest
from pyrecodes import ComponentLibraryCreator


class TestJSONComponentLibraryCreator_ThreeLocalitiesCommunity():
    COMPONENT_LIBRARY_FILE = './Example 1/ThreeLocalitiesCommunity_ComponentLibrary.json'

    @pytest.fixture
    def component_library_creator(self):
        return ComponentLibraryCreator.JSONComponentLibraryCreator(self.COMPONENT_LIBRARY_FILE)

    def test_init(self, component_library_creator: ComponentLibraryCreator.ComponentLibraryCreator):
        for component_name, component_parameters in component_library_creator.file.items():
            assert 'ComponentClass' in component_parameters
            assert 'RecoveryModel' in component_parameters
            assert 'Type' in component_parameters['RecoveryModel']
            assert 'Parameters' in component_parameters['RecoveryModel']
            assert'DamageFunctionalityRelation' in component_parameters['RecoveryModel']
            if component_name != 'BuildingStockUnit' and component_name != 'SuperLink':
                assert 'Supply' in component_parameters
                assert 'OperationDemand' in component_parameters

    def test_form_component_BTS1(self, component_library_creator: ComponentLibraryCreator.ComponentLibraryCreator):
        component_name = 'BaseTransceiverStation_1'
        component_parameters = component_library_creator.file[component_name]
        component = component_library_creator.form_component(component_name, component_parameters)
        assert type(component).__name__ == 'StandardiReCoDeSComponent'
        assert type(component.recovery_model).__name__ == 'ComponentLevelRecoveryActivitiesModel'
        assert component.recovery_model.recovery_activities['Repair'].duration == 10
        assert type(component.recovery_model.damage_to_functionality_relation).__name__ == 'ReverseBinary'
        assert component.supply['Supply']['Communication'].initial_amount == 1
        assert component.supply['Supply']['Communication'].current_amount == 1
        assert type(component.supply['Supply']['Communication'].component_functionality_to_amount).__name__ == 'Linear'
        assert type(component.supply['Supply']['Communication'].unmet_demand_to_amount).__name__ == 'Binary'
        assert component.demand['OperationDemand']['ElectricPower'].initial_amount == 1
        assert component.demand['OperationDemand']['ElectricPower'].current_amount == 1
        assert type(component.demand['OperationDemand'][
                                  'ElectricPower'].component_functionality_to_amount).__name__ == 'Linear'
    
    def test_form_component_BSU(self, component_library_creator: ComponentLibraryCreator.ComponentLibraryCreator):
        component_name = 'BuildingStockUnit'
        component_parameters = component_library_creator.file[component_name]
        component = component_library_creator.form_component(component_name, component_parameters)
        assert type(component).__name__ == 'BuildingStockUnitWithEmergencyCalls'
        assert type(component.recovery_model).__name__ == 'ComponentLevelRecoveryActivitiesModel'
        assert component.recovery_model.recovery_activities['Repair'].duration == 10
        assert type(component.recovery_model.damage_to_functionality_relation).__name__ == 'ReverseLinear'
        assert component.demand['OperationDemand']['Communication'].initial_amount == 1
        assert component.demand['OperationDemand']['Communication'].current_amount == 1
        assert type(component.demand['OperationDemand'][
                                  'Communication'].component_functionality_to_amount).__name__ == 'Constant'
        assert component.demand['OperationDemand']['ElectricPower'].initial_amount == 1
        assert component.demand['OperationDemand']['ElectricPower'].current_amount == 1
        assert type(component.demand['OperationDemand'][
                                  'ElectricPower'].component_functionality_to_amount).__name__ == 'Linear'

    def test_form_library(self, component_library_creator: ComponentLibraryCreator.ComponentLibraryCreator):
        component_library = component_library_creator.form_library()
        component_type_names = ['BaseTransceiverStation_1', 'BaseTransceiverStation_2', 'ElectricPowerPlant',
                                'CoolingWaterFacility', 'BuildingStockUnit', 'SuperLink']
        components_in_library = [component_type_name in component_library for component_type_name in
                                 component_type_names]
        assert len(component_library) == 6 and all(components_in_library)

tests/test_ComponentLibraryCreator.py
NikolaBlagojevic/pyrecodes

import pytest
import math
import copy
import numpy as np
from pyrecodes import Relation
from pyrecodes import ComponentRecoveryModel
from pyrecodes import Resource

class TestComponentRecoveryActivity:

    @pytest.fixture
    def recovery_activity(self):
        return ComponentRecoveryModel.ConcreteRecoveryActivity('DummyActivity')

    def test_init(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        assert recovery_activity.name == 'DummyActivity'
        assert recovery_activity.level == 0.0
        assert recovery_activity.time_steps == []
        assert recovery_activity.demand_met == {}
        assert recovery_activity.demand == {}
        assert recovery_activity.preceding_activities_finished == False

    def test_set_name(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        assert recovery_activity.name == 'DummyActivity'
        recovery_activity.set_name('AnotherDummyActivity')
        assert recovery_activity.name == 'AnotherDummyActivity'

    def test_set_level(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        assert recovery_activity.level == 0.0
        recovery_activity.set_level(0.4)
        assert recovery_activity.level == 0.4

    def test_set_too_high_level(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        with pytest.raises(ValueError):
            recovery_activity.set_level(1.5)

    def test_set_negative_level(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        with pytest.raises(ValueError):
            recovery_activity.set_level(-1.5)

    def test_deterministic_sample_duration(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        distribution = {'Deterministic': {'Value': 5}}
        deterministic_sample = recovery_activity.sample_duration(distribution)
        assert math.isclose(deterministic_sample, 5)

    def test_lognormal_sample_duration(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        distribution = {'Lognormal': {'Median': 2, 'Dispersion': 1}}
        lognormal_samples = [recovery_activity.sample_duration(distribution) for _ in range(100000)]
        assert math.isclose(np.median(lognormal_samples), 2, abs_tol=0.05) and math.isclose(
            np.std(np.log(lognormal_samples)), 1, abs_tol=0.01)

    def test_set_deterministic_duration(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        distribution = {'Deterministic': {'Value': 5}}
        recovery_activity.set_duration(distribution)
        assert math.isclose(recovery_activity.duration, 5) and math.isclose(recovery_activity.rate, 1 / 5)

    def test_set_negative_duration(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        distribution = {'Deterministic': {'Value': -5}}
        with pytest.raises(ValueError):
            recovery_activity.set_duration(distribution)

    def test_set_empty_preceding_activities(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        preceding_activities = []
        recovery_activity.set_preceding_activities(preceding_activities)
        assert recovery_activity.preceding_activities == []

    def test_set_two_preceding_activities(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        preceding_activities = ['Activity 1', 'Acitivity 2']
        recovery_activity.set_preceding_activities(preceding_activities)
        assert recovery_activity.preceding_activities == ['Activity 1', 'Acitivity 2']

    def test_set_preceding_activities_twice(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        preceding_activities = ['Activity 1', 'Activity 2']
        recovery_activity.set_preceding_activities(preceding_activities)
        recovery_activity.set_preceding_activities(preceding_activities)
        assert recovery_activity.preceding_activities == ['Activity 1', 'Activity 2']

    def test_initial_preceding_activities_finished(self,
                                                   recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        assert recovery_activity.preceding_activities_finished == False

    def test_set_preceding_activities_finished(self,
                                               recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        recovery_activity.set_preceding_activities_finished(True)
        assert recovery_activity.preceding_activities_finished == True

    def test_set_demand(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        demand = [{'Resource': 'Resource 1', 'Amount': 5}, {'Resource': 'Resource 2', 'Amount': 3}]
        recovery_activity.set_demand(demand)
        assert all([recovery_activity.demand['Resource 1'].initial_amount == 5,
                    recovery_activity.demand['Resource 1'].current_amount == 5,
                    recovery_activity.demand['Resource 2'].initial_amount == 3,
                    recovery_activity.demand['Resource 2'].current_amount == 3])

    def test_set_demand_met(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        demand = [{'Resource': 'Resource 1', 'Amount': 5}, {'Resource': 'Resource 2', 'Amount': 3}]
        recovery_activity.set_demand(demand)
        recovery_activity.set_demand_met('Resource 1', 0.5)
        assert recovery_activity.demand_met['Resource 1'] == 0.5
        recovery_activity.set_demand_met('Resource 2', 0.9)
        assert recovery_activity.demand_met['Resource 2'] == 0.9

    def test_set_invalid_demand_met(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        demand = [{'Resource': 'Resource 1', 'Amount': 5}]
        recovery_activity.set_demand(demand)
        with pytest.raises(ValueError):
            recovery_activity.set_demand_met('Resource 1', -0.5)

    def test_get_demand(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        demand = [{'Resource': 'Resource 1', 'Amount': 5}, {'Resource': 'Resource 2', 'Amount': 3}]
        recovery_activity.set_demand(demand)
        demand = recovery_activity.get_demand()
        assert all([demand['Resource 1'].initial_amount == 5,
                    demand['Resource 1'].current_amount == 5,
                    demand['Resource 2'].initial_amount == 3,
                    demand['Resource 2'].current_amount == 3])

    def test_recover_without_set_duration(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        with pytest.raises(AttributeError):
            recovery_activity.recover(0)

    def test_recover_demand_not_met(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        recovery_activity.set_duration({'Deterministic': {'Value': 5}})
        demand = [{'Resource': 'Resource 1', 'Amount': 5}]
        recovery_activity.set_demand(demand)
        recovery_activity.set_demand_met('Resource', 0.0)
        recovery_activity.recover(0)
        assert recovery_activity.level == 0 and recovery_activity.time_steps == []

    def test_recover_full_demand_met(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        recovery_activity.set_duration({'Deterministic': {'Value': 5}})
        demand = [{'Resource': 'Resource 1', 'Amount': 5}]
        recovery_activity.set_demand(demand)
        recovery_activity.set_demand_met('Resource 1', 1.0)
        for time_step in range(3):
            recovery_activity.recover(time_step)
        assert math.isclose(recovery_activity.level, 0.6)
        assert recovery_activity.time_steps == [0, 1, 2]
        for time_step in range(3, 5):
            recovery_activity.recover(time_step)
        assert recovery_activity.level == 1.0
        assert recovery_activity.time_steps == [0, 1, 2, 3, 4]
        for time_step in range(5, 10):
            recovery_activity.recover(time_step)
        assert recovery_activity.level == 1.0
        assert recovery_activity.time_steps == [0, 1, 2, 3, 4]

    def test_recover_partial_demand_met(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        recovery_activity.set_duration({'Deterministic': {'Value': 5}})
        demand = [{'Resource': 'Resource 1', 'Amount': 5}]
        recovery_activity.set_demand(demand)
        recovery_activity.set_demand_met('Resource 1', 0.5)
        for time_step in range(3):
            recovery_activity.recover(time_step)
        assert math.isclose(recovery_activity.level, 0.3)
        assert recovery_activity.time_steps == [0, 1, 2]
        for time_step in range(3, 5):
            recovery_activity.recover(time_step)
        assert math.isclose(recovery_activity.level, 0.5)
        assert recovery_activity.time_steps == [0, 1, 2, 3, 4]
        for time_step in range(5, 10):
            recovery_activity.recover(time_step)
        assert math.isclose(recovery_activity.level, 1.0)
        assert recovery_activity.time_steps == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

    def test_effect_of_unmet_demand_on_activity(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        assert recovery_activity.effect_of_unmet_demand_on_activity() == 1.0
        demand = [{'Resource': 'Resource 1', 'Amount': 5}, {'Resource': 'Resource 2', 'Amount': 3}]
        recovery_activity.set_demand(demand)
        recovery_activity.set_demand_met('Resource 1', 0.5)
        recovery_activity.set_demand_met('Resource 2', 0.9)
        assert recovery_activity.effect_of_unmet_demand_on_activity() == 0.5
        recovery_activity.set_demand_met('Resource 1', 0.2)
        recovery_activity.set_demand_met('Resource 2', 0.1)
        assert recovery_activity.effect_of_unmet_demand_on_activity() == 0.1

    def test_activity_finished_false(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        recovery_activity.set_duration({'Deterministic': {'Value': 5}})
        for time_step in range(4):
            recovery_activity.recover(time_step)
        assert recovery_activity.activity_finished() == False

    def test_activity_finished_true(self, recovery_activity: ComponentRecoveryModel.ConcreteRecoveryActivity):
        recovery_activity.set_duration({'Deterministic': {'Value': 5}})
        for time_step in range(5):
            recovery_activity.recover(time_step)
        assert recovery_activity.activity_finished() == True

class TestComponentLevelRecoveryActivitiesModel_SingleActivity:

    RECOVERY_MODEL_PARAMETERS = {"Type": "ComponentLevelRecoveryActivitiesModel",
                                 "Parameters": {"Repair": {"Duration": {"Deterministic": {"Value": 10}},
                                                           "Demand": [{"Resource": "RepairCrew", "Amount": 10}], }},
                                 "DamageFunctionalityRelation": {"Type": "ReverseLinear"}}

    @pytest.fixture()
    def recovery_model(self):
        target_recovery_model = getattr(ComponentRecoveryModel,
                                        self.RECOVERY_MODEL_PARAMETERS['Type'])
        return target_recovery_model(self.RECOVERY_MODEL_PARAMETERS)
    
    def test_init(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.REPAIR_ACTIVITY_NAME == 'Repair'
        assert len(recovery_model.recovery_activities) == 1
        assert isinstance(recovery_model.recovery_activities['Repair'], ComponentRecoveryModel.ConcreteRecoveryActivity)
        assert recovery_model.recovery_activities['Repair'].name == 'Repair'
        assert recovery_model.recovery_activities['Repair'].duration == 10
        assert recovery_model.recovery_activities['Repair'].rate == 1/10
        assert isinstance(recovery_model.recovery_activities['Repair'].demand, dict)
        assert isinstance(recovery_model.recovery_activities['Repair'].demand['RepairCrew'], Resource.Resource)
        assert recovery_model.recovery_activities['Repair'].demand['RepairCrew'].initial_amount == 10
        assert recovery_model.recovery_activities['Repair'].demand['RepairCrew'].current_amount == 10

    def test_set_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.3
        recovery_model.set_initial_damage_level(damage_level)
        assert math.isclose(recovery_model.get_damage_level(), damage_level)

    def test_set_damage_level_error_negative(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = -0.3
        with pytest.raises(ValueError):
            recovery_model.set_initial_damage_level(damage_level)

    def test_set_damage_level_error_above_one(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 1.3
        with pytest.raises(ValueError):
            recovery_model.set_initial_damage_level(damage_level)
    
    def test_set_initial_repair_activity_state(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.recovery_activities['Repair'].level == 1.0
        recovery_model.set_initial_repair_activity_state(0.3)
        assert recovery_model.recovery_activities['Repair'].level == 0.7
        assert recovery_model.recovery_activities['Repair'].rate == 0.3/10      

    def test_set_damage_functionality(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_damage_functionality(self.RECOVERY_MODEL_PARAMETERS['DamageFunctionalityRelation'])
        damage_functionality_type = self.RECOVERY_MODEL_PARAMETERS['DamageFunctionalityRelation']['Type']
        assert isinstance(recovery_model.damage_to_functionality_relation, getattr(Relation, damage_functionality_type))

    def test_recover_once(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.6
        recovery_model.set_initial_damage_level(damage_level)
        recovery_model.recover(0)
        assert math.isclose(recovery_model.get_damage_level(),
                            0.6 - 0.6 / 10) and recovery_model.recovery_activities['Repair'].time_steps == [0]

    def test_recover_partially(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.6
        recovery_model.set_initial_damage_level(damage_level)
        for time_step in range(6):
            recovery_model.recover(time_step)
        assert math.isclose(recovery_model.get_damage_level(), 0.6 - 6 * 0.6 / 10,
                            abs_tol=1e-10) and recovery_model.recovery_activities['Repair'].time_steps == list(range(6))

    def test_recover_fully(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.6
        recovery_model.set_initial_damage_level(damage_level)
        for time_step in range(10):
            recovery_model.recover(time_step)
        assert math.isclose(recovery_model.get_damage_level(), 0,
                            abs_tol=1e-10) and recovery_model.recovery_activities['Repair'].time_steps == list(range(10))

    def test_recover_fully_partial_demand_met(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.6
        partial_demand_met = 0.1
        recovery_model.set_initial_damage_level(damage_level)
        recovery_model.recovery_activities['Repair'].demand_met['RepairCrew'] = partial_demand_met
        for time_step in range(100):
            recovery_model.recover(time_step)
        assert math.isclose(recovery_model.get_damage_level(), 0,
                            abs_tol=1e-10) and recovery_model.recovery_activities['Repair'].time_steps == list(range(100))

    def test_recover_partially_partial_demand_met(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.6
        partial_demand_met = 0.1
        recovery_model.set_initial_damage_level(damage_level)
        recovery_model.recovery_activities['Repair'].demand_met['RepairCrew'] = partial_demand_met
        for time_step in range(10):
            recovery_model.recover(time_step)
        assert math.isclose(recovery_model.get_damage_level(), 0.6 * 0.9,
                            abs_tol=1e-10) and recovery_model.recovery_activities['Repair'].time_steps == list(range(10))

    def test_recover_fully_full_damage(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 1.0
        recovery_model.set_initial_damage_level(damage_level)
        for time_step in range(10):
            recovery_model.recover(time_step)
        assert math.isclose(recovery_model.get_damage_level(), 0,
                            abs_tol=1e-10) and recovery_model.recovery_activities['Repair'].time_steps == list(range(10))

    def test_overrecover(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.6
        recovery_model.set_initial_damage_level(damage_level)
        for time_step in range(20):
            recovery_model.recover(time_step)
        assert math.isclose(recovery_model.get_damage_level(), 0,
                            abs_tol=1e-10) and recovery_model.recovery_activities['Repair'].time_steps == list(range(10))

    def test_check_preceding_activities(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.check_preceding_activities()
        assert recovery_model.recovery_activities['Repair'].preceding_activities_finished == True

    def test_preceding_activities_finished(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.preceding_activities_finished(recovery_model.recovery_activities['Repair']) == True

    def test_get_functionality_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.4
        recovery_model.set_initial_damage_level(damage_level)
        assert math.isclose(recovery_model.get_functionality_level(), 0.6)

    def test_get_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 0.4
        recovery_model.set_initial_damage_level(damage_level)
        assert math.isclose(recovery_model.get_damage_level(), 0.4)
        
    def test_get_demand_no_damage(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        demand = recovery_model.get_demand()
        assert demand == {}
    
    def test_get_demand_with_damage(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(0.1)
        demand = recovery_model.get_demand()
        assert demand['RepairCrew'].current_amount == 10 and demand['RepairCrew'].initial_amount == 10

    def test_set_activities_demand_to_met(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_activities_demand_to_met()
        assert recovery_model.recovery_activities['Repair'].demand_met['RepairCrew'] == 1.0
    
    def test_set_met_demand_for_recovery_activities(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_met_demand_for_recovery_activities('RepairCrew', 0.5)
        assert recovery_model.recovery_activities['Repair'].demand_met['RepairCrew'] == 0.5

    def test_set_wrong_unmet_demand_for_recovery_activities(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        with pytest.raises(ValueError):
            recovery_model.set_met_demand_for_recovery_activities('Inspectors', 0.5)
    
    def test_find_recovery_activity_that_demands_the_resource(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_activity = recovery_model.find_recovery_activity_that_demands_the_resource('RepairCrew')
        assert recovery_activity.name == 'Repair'
        with pytest.raises(ValueError):
            recovery_model.find_recovery_activity_that_demands_the_resource('SomeResource')

class TestComponentLevelRecoveryActivitiesModel_MultipleActivities:
    RECOVERY_MODEL_PARAMETERS = {"Type": "ComponentLevelRecoveryActivitiesModel",
                                 "Parameters": {
                                     "RapidInspection": {
                                         "Duration": {"Lognormal": {"Median": 1, "Dispersion": 0.0}},
                                         "Demand": [{"Resource": "FirstResponderEngineer", "Amount": 0.1}],
                                         "PrecedingActivities": []
                                     },
                                     "ContractorMobilization": {
                                         "Duration": {"Lognormal": {"Median": 1, "Dispersion": 0.0}},
                                         "Demand": [{"Resource": "Contractor", "Amount": 1}],
                                         "PrecedingActivities": ["RapidInspection"]
                                     },
                                     "Repair": {
                                         "Duration": {"Lognormal": {"Median": 10, "Dispersion": 0.0}},
                                         "Demand": [{"Resource": "RepairCrew", "Amount": 10}],
                                         "PrecedingActivities": ["RapidInspection", "ContractorMobilization"]
                                     }
                                 },
                                 "DamageFunctionalityRelation": {
                                     "Type": "ReverseLinear"
                                 }
                                 }

    @pytest.fixture
    def recovery_model(self):
        target_recovery_model = getattr(ComponentRecoveryModel, self.RECOVERY_MODEL_PARAMETERS['Type'])
        return target_recovery_model(self.RECOVERY_MODEL_PARAMETERS)

    def test_set_init(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.REPAIR_ACTIVITY_NAME == 'Repair'
        assert len(recovery_model.recovery_activities) == 3
        assert isinstance(recovery_model.recovery_activities['RapidInspection'], ComponentRecoveryModel.ConcreteRecoveryActivity)
        assert isinstance(recovery_model.recovery_activities['ContractorMobilization'], ComponentRecoveryModel.ConcreteRecoveryActivity)
        assert isinstance(recovery_model.recovery_activities['Repair'], ComponentRecoveryModel.ConcreteRecoveryActivity)
        assert recovery_model.recovery_activities['RapidInspection'].name == 'RapidInspection'
        assert recovery_model.recovery_activities['RapidInspection'].duration == 1
        assert recovery_model.recovery_activities['RapidInspection'].rate == 1
        assert recovery_model.recovery_activities['ContractorMobilization'].name == 'ContractorMobilization'
        assert recovery_model.recovery_activities['ContractorMobilization'].duration == 1
        assert recovery_model.recovery_activities['ContractorMobilization'].rate == 1
        assert recovery_model.recovery_activities['Repair'].name == 'Repair'
        assert math.isclose(recovery_model.recovery_activities['Repair'].duration, 10)
        assert math.isclose(recovery_model.recovery_activities['Repair'].rate, 1/10)
        assert isinstance(recovery_model.recovery_activities['RapidInspection'].demand, dict)
        assert len(recovery_model.recovery_activities['RapidInspection'].demand) == 1
        assert isinstance(recovery_model.recovery_activities['RapidInspection'].demand['FirstResponderEngineer'], Resource.Resource)
        assert recovery_model.recovery_activities['RapidInspection'].demand['FirstResponderEngineer'].initial_amount == 0.1
        assert recovery_model.recovery_activities['RapidInspection'].demand['FirstResponderEngineer'].current_amount == 0.1
        assert isinstance(recovery_model.recovery_activities['ContractorMobilization'].demand, dict)
        assert isinstance(recovery_model.recovery_activities['ContractorMobilization'].demand['Contractor'], Resource.Resource)
        assert recovery_model.recovery_activities['ContractorMobilization'].demand['Contractor'].initial_amount == 1
        assert recovery_model.recovery_activities['ContractorMobilization'].demand['Contractor'].current_amount == 1
        assert len(recovery_model.recovery_activities['ContractorMobilization'].demand) == 1
        assert isinstance(recovery_model.recovery_activities['Repair'].demand, dict)
        assert isinstance(recovery_model.recovery_activities['Repair'].demand['RepairCrew'], Resource.Resource)
        assert recovery_model.recovery_activities['Repair'].demand['RepairCrew'].initial_amount == 10
        assert recovery_model.recovery_activities['Repair'].demand['RepairCrew'].current_amount == 10
        assert len(recovery_model.recovery_activities['Repair'].demand) == 1

    def test_set_initial_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        damage_level = 1.0
        recovery_model.set_parameters(self.RECOVERY_MODEL_PARAMETERS['Parameters'])
        recovery_model.set_initial_damage_level(damage_level)
        for recovery_activity_name, recovery_activity in recovery_model.recovery_activities.items():
            if recovery_activity_name == recovery_model.REPAIR_ACTIVITY_NAME:
                assert recovery_activity.level == 1 - damage_level
            else:
                assert recovery_activity.level == 0.0

    def test_set_wrong_initial_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        with pytest.raises(ValueError):
            recovery_model.set_initial_damage_level(1.5)

    def test_check_preceding_activities(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.check_preceding_activities()
        target_bools = [True, True, True]
        for target_bool, recovery_activity in zip(target_bools, recovery_model.recovery_activities.values()):
            assert recovery_activity.preceding_activities_finished == target_bool

        recovery_model.set_initial_damage_level(0.5)
        recovery_model.check_preceding_activities()
        target_bools = [True, False, False]
        for target_bool, recovery_activity in zip(target_bools, recovery_model.recovery_activities.values()):
            assert recovery_activity.preceding_activities_finished == target_bool

        recovery_model.recovery_activities['RapidInspection'].level = 1.0
        recovery_model.check_preceding_activities()
        target_bools = [True, True, False]
        for target_bool, recovery_activity in zip(target_bools, recovery_model.recovery_activities.values()):
            assert recovery_activity.preceding_activities_finished == target_bool
    
    def test_preceding_activities_finished(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        for recovery_activity in recovery_model.recovery_activities.values():
            assert recovery_model.preceding_activities_finished(recovery_activity) == True

        recovery_model.set_initial_damage_level(1.0)
        target_bools = [True, False, False]
        for target_bool, recovery_activity in zip(target_bools, recovery_model.recovery_activities.values()):
            assert recovery_model.preceding_activities_finished(recovery_activity) == target_bool

        recovery_model.recovery_activities['RapidInspection'].level = 1.0
        target_bools = [True, True, False]
        for target_bool, recovery_activity in zip(target_bools, recovery_model.recovery_activities.values()):
            assert recovery_model.preceding_activities_finished(recovery_activity) == target_bool
    
    def test_get_functionality_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.get_functionality_level() == 1.0
        recovery_model.set_initial_damage_level(1.0)
        assert recovery_model.get_functionality_level() == 0.0
        recovery_model.set_initial_damage_level(0.6)
        assert recovery_model.get_functionality_level() == 0.4

    def test_get_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.get_damage_level() == 0.0
        recovery_model.set_initial_damage_level(1.0)
        assert recovery_model.get_damage_level() == 1.0
        recovery_model.set_initial_damage_level(0.6)
        assert recovery_model.get_damage_level() == 0.6   

    def test_get_demand(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.get_demand() == {}

        recovery_model.set_initial_damage_level(0.1)
        current_demand = recovery_model.get_demand()
        assert len(current_demand) == 1
        assert current_demand['FirstResponderEngineer'].initial_amount == 0.1
        assert current_demand['FirstResponderEngineer'].current_amount == 0.1

        recovery_model.recovery_activities['RapidInspection'].level = 1.0
        current_demand = recovery_model.get_demand()
        assert len(current_demand) == 1
        assert current_demand['Contractor'].initial_amount == 1
        assert current_demand['Contractor'].current_amount == 1

        recovery_model.recovery_activities['ContractorMobilization'].level = 1.0
        current_demand = recovery_model.get_demand()
        assert len(current_demand) == 1
        assert current_demand['RepairCrew'].initial_amount == 10
        assert current_demand['RepairCrew'].current_amount == 10

        recovery_model.recovery_activities['Repair'].level = 1.0
        assert recovery_model.get_demand() == {}

    def test_set_activities_demand_to_met(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert all([list(recovery_activity.demand_met.values())[0] == 1.0 for recovery_activity in
                              recovery_model.recovery_activities.values()])
        recovery_model.recovery_activities['RapidInspection'].demand_met['FirstResponderEngineer'] = 0.3
        recovery_model.set_activities_demand_to_met()
        assert all([list(recovery_activity.demand_met.values())[0] == 1.0 for recovery_activity in
                              recovery_model.recovery_activities.values()])

    def test_set_met_demand_for_recovery_activities(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert all([list(recovery_activity.demand_met.values())[0] == 1.0 for recovery_activity in
                              recovery_model.recovery_activities.values()])
        recovery_model.set_met_demand_for_recovery_activities('FirstResponderEngineer', 0.5)
        assert recovery_model.recovery_activities['RapidInspection'].demand_met['FirstResponderEngineer'] == 0.5
        assert recovery_model.recovery_activities['ContractorMobilization'].demand_met['Contractor'] == 1.0
        assert recovery_model.recovery_activities['Repair'].demand_met['RepairCrew'] == 1.0
        recovery_model.set_met_demand_for_recovery_activities('Contractor', 0.6)
        assert recovery_model.recovery_activities['ContractorMobilization'].demand_met['Contractor'] == 0.6
        recovery_model.set_met_demand_for_recovery_activities('RepairCrew', 0.4)
        assert recovery_model.recovery_activities['Repair'].demand_met['RepairCrew'] == 0.4

    def test_find_recovery_activity_that_demands_the_resource(self,
                                                              recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_activity = recovery_model.find_recovery_activity_that_demands_the_resource('Contractor')
        assert recovery_activity.name == 'ContractorMobilization'
        recovery_activity = recovery_model.find_recovery_activity_that_demands_the_resource('RepairCrew')
        assert recovery_activity.name == 'Repair'

    def test_recover_no_damage(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        for time_step in range(10):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['RapidInspection'].time_steps == []
        assert recovery_model.recovery_activities['ContractorMobilization'].time_steps == []
        assert recovery_model.recovery_activities['Repair'].time_steps == []
        assert recovery_model.recovery_activities['RapidInspection'].level == 1.0
        assert recovery_model.recovery_activities['ContractorMobilization'].level == 1.0
        assert recovery_model.recovery_activities['Repair'].level == 1.0

    def test_recover_once(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(0.01)
        recovery_model.recover(time_step=1)
        assert recovery_model.recovery_activities['RapidInspection'].time_steps == [1]
        assert recovery_model.recovery_activities['ContractorMobilization'].time_steps == []
        assert recovery_model.recovery_activities['Repair'].time_steps == []
        assert recovery_model.recovery_activities['RapidInspection'].level == 1.0
        assert recovery_model.recovery_activities['ContractorMobilization'].level == 0.0
        assert recovery_model.recovery_activities['Repair'].level == 0.99

    def test_recover_partially(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(1.0)
        for time_step in range(5):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['RapidInspection'].time_steps == [0]
        assert recovery_model.recovery_activities['ContractorMobilization'].time_steps == [1]
        assert recovery_model.recovery_activities['Repair'].time_steps == [2, 3, 4]
        assert recovery_model.recovery_activities['RapidInspection'].level == 1.0
        assert recovery_model.recovery_activities['ContractorMobilization'].level == 1.0
        assert math.isclose(recovery_model.recovery_activities['Repair'].level, 0.3)

    def test_recover_full(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(1.0)
        for time_step in range(12):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['RapidInspection'].time_steps == [0]
        assert recovery_model.recovery_activities['ContractorMobilization'].time_steps == [1]
        assert recovery_model.recovery_activities['Repair'].time_steps == [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
        assert recovery_model.recovery_activities['RapidInspection'].level == 1.0
        assert recovery_model.recovery_activities['ContractorMobilization'].level == 1.0
        assert math.isclose(recovery_model.recovery_activities['Repair'].level, 1.0)

    def test_recover_partial_demand_met(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(1.0)
        recovery_model.recovery_activities['ContractorMobilization'].demand_met['Contractor'] = 0.5
        for time_step in range(13):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['RapidInspection'].time_steps == [0]
        assert recovery_model.recovery_activities['ContractorMobilization'].time_steps == [1, 2]
        assert recovery_model.recovery_activities['Repair'].time_steps == [3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
        assert recovery_model.recovery_activities['RapidInspection'].level == 1.0
        assert recovery_model.recovery_activities['ContractorMobilization'].level == 1.0
        assert math.isclose(recovery_model.recovery_activities['Repair'].level, 1.0)

class TestNoRecoveryActivityComponentRecoveryModel:

    RECOVERY_MODEL_PARAMETERS = {"Type": "NoRecoveryActivityModel",
                                 "Parameters": {},
                                 "DamageFunctionalityRelation": {
                                     "Type": "Constant"
                                 }
                                 }

    @pytest.fixture()
    def recovery_model(self):
        return ComponentRecoveryModel.NoRecoveryActivityModel(self.RECOVERY_MODEL_PARAMETERS)  
    
    def test_init(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert len(recovery_model.recovery_activities) == 0        

    def test_set_initial_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(0.0)
        assert recovery_model.get_damage_level() == 0.0

    def test_set_nonzero_initial_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        with pytest.raises(ValueError):
            recovery_model.set_initial_damage_level(0.5)

    def test_get_functionality_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.get_functionality_level() == 1.0

    def test_get_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.get_damage_level() == 0.0

    def test_get_demand(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert recovery_model.get_demand() == {}

class TestInfrastructureInterfaceRecoveryModel:

    RECOVERY_MODEL_PARAMETERS = {"Type": "InfrastructureInterfaceRecoveryModel",
                                "Parameters": {},
                                "DamageFunctionalityRelation": ""}
    
    STEP_DICT = {'StepLimits': [0.2, 0.4, 1.0],
                 'StepValues': [0.1, 0.6, 1.0],
                 'RestoredIn': [{'Deterministic': {'Value': 10}},
                                {'Deterministic': {'Value': 20}},
                                {'Deterministic': {'Value': 50}}]}

    @pytest.fixture()
    def recovery_model(self):
        return ComponentRecoveryModel.InfrastructureInterfaceRecoveryModel(self.RECOVERY_MODEL_PARAMETERS)
    
    def test_init(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        assert len(recovery_model.recovery_activities) == 1
        assert recovery_model.recovery_activities['Recovery'].time_steps == []
        assert recovery_model.recovery_activities['Recovery'].level == 1.0
        assert recovery_model.recovery_activities['Recovery'].duration == 0.0
        assert recovery_model.recovery_activities['Recovery'].rate == math.inf
        assert isinstance(recovery_model.damage_to_functionality_relation, Relation.MultipleStep)

    def test_set_initial_damage_level(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_initial_damage_level(0.0)
        assert recovery_model.get_damage_level() == 1.0
        assert recovery_model.recovery_activities['Recovery'].level == 0.0
    
    def test_set_parameters(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_parameters(self.STEP_DICT)
        assert recovery_model.damage_to_functionality_relation.step_limits == self.STEP_DICT['StepLimits']
        assert recovery_model.damage_to_functionality_relation.step_values == self.STEP_DICT['StepValues']
        assert recovery_model.recovery_activities['Recovery'].duration == 50
        assert math.isclose(recovery_model.recovery_activities['Recovery'].rate, 1/50)
    
    def test_recover_no_steps_defined(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.recover(0)
        assert recovery_model.recovery_activities['Recovery'].time_steps == []
        assert recovery_model.recovery_activities['Recovery'].level == 1.0
        recovery_model.set_initial_damage_level(1.0)
        recovery_model.recover(1)
        recovery_model.recover(2)
        assert recovery_model.recovery_activities['Recovery'].time_steps == [1]
    
    def test_recover_steps_defined(self, recovery_model: ComponentRecoveryModel.RecoveryModel):
        recovery_model.set_parameters(self.STEP_DICT)
        recovery_model.set_initial_damage_level(1.0)
        recovery_model.recover(1)
        recovery_model.recover(2)
        assert recovery_model.recovery_activities['Recovery'].time_steps == [1, 2]
        assert math.isclose(recovery_model.recovery_activities['Recovery'].level, 0.04)
        assert recovery_model.get_functionality_level() == 0.0
        
        for time_step in range(3, 12):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['Recovery'].time_steps == list(range(1, 12))
        assert math.isclose(recovery_model.recovery_activities['Recovery'].level, 0.22)
        assert recovery_model.get_functionality_level() == 0.1

        for time_step in range(12, 22):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['Recovery'].time_steps == list(range(1, 22))
        assert math.isclose(recovery_model.recovery_activities['Recovery'].level, 0.42)
        assert recovery_model.get_functionality_level() == 0.6

        for time_step in range(22, 52):
            recovery_model.recover(time_step)
        assert recovery_model.recovery_activities['Recovery'].time_steps == list(range(1, 51))
        assert math.isclose(recovery_model.recovery_activities['Recovery'].level, 1.0)
        assert recovery_model.get_functionality_level() == 1.0



tests/test_ComponentRecoveryModel.py
NikolaBlagojevic/pyrecodes
import pytest
import math
from pyrecodes import DamageInput
from pyrecodes import Component
from pyrecodes import main

class TestListDamageInput():

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'    

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture
    def damage_input(self, system):
        return DamageInput.ListDamageInput(system.system_creator.get_damage_input_parameters())

    def test_get_initial_damage(self, damage_input):
        damage_input.get_initial_damage()
        assert damage_input.damage_levels == [0.0, 0.4, 0.0, 0.0, 0.4, 0.0, 0.0, 0.4, 0.4, 0.0, 0.0]

    def test_set_initial_damage(self, system):
        system.set_initial_damage()
        target_damage_levels = [0.0, 0.4, 0.0, 0.0, 0.4, 0.0, 0.0, 0.4, 0.4, 0.0, 0.0]
        for target_damage_level, component in zip(target_damage_levels, system.components):
            assert component.get_damage_level() == target_damage_level
    
    def test_set_initial_damage_fewer_levels(self, system):
        system.damage_input.parameters = [0.1, 0.4, 0.3, 0.5, 0.4]
        system.set_initial_damage()
        target_damage_levels = [0.1, 0.4, 0.3, 0.5, 0.4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
        for target_damage_level, component in zip(target_damage_levels, system.components):
            assert math.isclose(component.get_damage_level(), target_damage_level)

class TestR2DDamageInput:

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Interface_Infrastructure_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture
    def damage_input(self, system):
        return DamageInput.R2DDamageInput(system.system_creator.get_damage_input_parameters())

    def test_set_initial_damage_buildings(self, system):
        system.set_initial_damage()
        target_damage_states = [1, 0, 3, 1, 0, 3, 1, 0, 0]
        target_damage_levels = [1, 0, 1, 1, 0, 1, 1, 0, 0]
        for i, component in enumerate(system.components[1:10]):
            assert int(component.name[2]) == target_damage_states[i]
            assert component.get_damage_level() == target_damage_levels[i]
    
    def test_set_initial_damage_interfaces(self, system):
        system.set_initial_damage()
        interface_ids = [433, 434, 435, 553, 554, 555, 606, 607, 608, 749, 750, 751]
        for interface_id in interface_ids:
            assert system.components[interface_id].get_damage_level() == 1.0
    
    def test_component_is_damaged(self, damage_input):
        component = Component.StandardiReCoDeSComponent()
        component.name = 'DS0_ResidentialBuilding'
        assert damage_input.component_is_damaged(component) == False
        component.name = 'DS1_ResidentialBuilding'
        assert damage_input.component_is_damaged(component) == True
    
    def test_component_is_interface(self, damage_input):
        component = Component.StandardiReCoDeSComponent()
        assert damage_input.component_is_interface(component) == False
        component = Component.InfrastructureInterface()
        assert damage_input.component_is_interface(component) == True

class TestFileDamageInput:

    DAMAGE_FILE = './tests/test_inputs/dummy_damage_input_file.txt'

    @pytest.fixture
    def damage_input(self):
        return DamageInput.FileDamageInput(self.DAMAGE_FILE)
    
    def test_get_initial_damage(self, damage_input):
        damage_input.get_initial_damage()
        assert damage_input.damage_levels == [0.0, 0.1, 0.2, 0.3, 0.4]

    



tests/test_DamageInput.py
NikolaBlagojevic/pyrecodes
import pytest
from pyrecodes import main
from pyrecodes import ComponentLibraryCreator
from pyrecodes import DistributionPriority
from pyrecodes import System
from pyrecodes import SystemCreator
import copy
import numpy as np


class TestDistributionPriority():

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        system = main.create_system(input_dict)
        return system

class TestComponentTypeBasedPriority(TestDistributionPriority):

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def distribution_priority(self, system: System.System):
        distribution_priority = DistributionPriority.ComponentTypeBasedPriority(resource_name='Resource 1', 
                                                                                parameters=[['ElectricPowerPlant', 'OperationDemand'], 
                                                                                            ['CoolingWaterFacility', 'OperationDemand'], 
                                                                                            ['BuildingStockUnit', 'OperationDemand']],
                                                                                components=system.components)
        return distribution_priority

    def test_categorize_components_based_on_type(self, distribution_priority: DistributionPriority.DistributionPriority):
        categorized_components_dict = distribution_priority.categorize_components_based_on_type()
        assert categorized_components_dict['BaseTransceiverStation_1'] == [0]
        assert categorized_components_dict['ElectricPowerPlant'] == [1]
        assert categorized_components_dict['SuperLink'] == [2, 3, 5, 6, 9, 10]
        assert categorized_components_dict['CoolingWaterFacility'] == [4]
        assert categorized_components_dict['BuildingStockUnit'] == [7]
        assert categorized_components_dict['BaseTransceiverStation_2'] == [8]

    def test_set_distribution_priority(self, distribution_priority: DistributionPriority.DistributionPriority):
        assert distribution_priority.get_component_priorities() == ([1, 4, 7], ['OperationDemand', 'OperationDemand', 'OperationDemand'])

class TestSupplierOnlyDistributionPriority_NorthEastSF(TestDistributionPriority):
    
    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Main.json'

    def test_set_distribution_priority(self, system: System.System):
        target_ids = [component_id for component_id in range(len(system.components)) if
                      'ResidentialBuilding' in system.components[component_id].name]
        print(target_ids)
        housing_priorities_ids, demand_types = system.resources['Shelter'][
            'DistributionModel'].priority.get_component_priorities()
        print(housing_priorities_ids)
        print(demand_types)
        assert target_ids == housing_priorities_ids and all(
            [demand_type == 'OperationDemand' for demand_type in demand_types])

class TestRandomDistributionPriority_NorthEastSF(TestDistributionPriority):

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Main.json'

    def test_set_distribution_priority(self, system: System.System):
        target_remaining_component_ids = list(range(1, len(system.components)))
        target_remaining_demand_types = ['RecoveryDemand' for _ in target_remaining_component_ids]
        target_supplier_ids = 0
        target_supplier_dummy_demand_types = 'OperationDemand'
        recovery_resource_names = ['FirstResponderEngineer', 'SeniorEngineer', 'PlanCheckEngineeringTeam', 'Money',
                                   'Contractor', 'RepairCrew']
        for recovery_resource in recovery_resource_names:
            component_ids, component_demand_types = system.resources[recovery_resource][
                'DistributionModel'].priority.get_component_priorities()
            print(component_ids)
            assert target_supplier_ids == component_ids[0]
            assert target_supplier_dummy_demand_types == component_demand_types[0]
            sorted_user_component_ids = sorted(component_ids[1:])
            assert target_remaining_component_ids == sorted_user_component_ids
            assert target_remaining_demand_types == component_demand_types[1:]
            newly_generated_component_ids, newly_generated_component_demand_types = system.resources[recovery_resource][
                'DistributionModel'].priority.get_component_priorities()
            assert component_ids == newly_generated_component_ids
            assert component_demand_types == newly_generated_component_demand_types
            parameters = {"Seed": 40.0, "DemandType": ["RecoveryDemand"]}
            system.resources[recovery_resource]['DistributionModel'].priority.set_distribution_priority(parameters)
            different_seed_component_ids, different_seed_component_demand_types = system.resources[recovery_resource][
                'DistributionModel'].priority.get_component_priorities()
            assert different_seed_component_ids != newly_generated_component_ids
            assert different_seed_component_demand_types == newly_generated_component_demand_types

    def test_get_suppliers_id(self, system: System.System):
        target_remaining_component_ids = list(range(1, len(system.components)))
        target_supplier_ids = [0]
        recovery_resource_names = ['FirstResponderEngineer', 'PlanCheckEngineeringTeam', 'Money']
        for recovery_resource in recovery_resource_names:
            remaining_component_ids = list(range(len(system.components)))
            supplier_ids, remaining_component_ids = system.resources[recovery_resource][
                'DistributionModel'].priority.get_suppliers_id(remaining_component_ids)
            assert target_supplier_ids == supplier_ids
            assert target_remaining_component_ids == remaining_component_ids
   
    def test_randomize_ids_multiple_components_one_demand_types(self, system: System.System):
        component_ids = list(range(100))
        demand_types = ['OperationDemand']
        randomized_ids, randomized_demand_types = system.resources['RepairCrew'][
                'DistributionModel'].priority.randomize_ids(component_ids, demand_types)
        target_id_count = [1] * 100
        assert [randomized_ids.count(id) for id in component_ids] == target_id_count
        assert randomized_demand_types.count('OperationDemand') == 100

    def test_randomize_ids_one_component_two_demand_types(self, system: System.System):
        component_ids = [0]
        demand_types = ['RecoveryDemand', 'OperationDemand']
        randomized_ids, randomized_demand_types = system.resources['RepairCrew'][
                'DistributionModel'].priority.randomize_ids(component_ids, demand_types)
        bool_list = []
        assert randomized_ids == [0, 0]
        assert 'RecoveryDemand' in randomized_demand_types and 'OperationDemand' in randomized_demand_types
    
    def test_randomize_ids_multiple_components_two_demand_types(self, system: System.System):
        component_ids = [0, 1, 2]
        demand_types = ['RecoveryDemand', 'OperationDemand']
        randomized_ids, randomized_demand_types = system.resources['RepairCrew'][
                'DistributionModel'].priority.randomize_ids(component_ids, demand_types)
        bool_list = []
        assert [randomized_ids.count(id) for id in component_ids] == [2, 2, 2]
        assert randomized_demand_types.count('RecoveryDemand') == 3
        assert randomized_demand_types.count('OperationDemand') == 3

class TestRandomDistributionPriorityWithPrioritizedInterfaces_NorthEastSFWithInterfaces(TestDistributionPriority):

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Interface_Infrastructure_Main.json'
    INTERFACE_IDS = {'ElectricPower': [433, 553, 606, 749, 756],
                     'PotableWater': [434, 554, 607, 750, 757],
                     'CellularCommunication': [435, 555, 608, 751, 758]}

    def test_set_distribution_priority(self, system: System.System):
        all_resources = ['ElectricPower', 'PotableWater', 'CellularCommunication']
        for resource in all_resources:
            component_priorities = system.resources[resource][
                'DistributionModel'].priority.get_component_priorities()
            other_resources = [some_resource for some_resource in all_resources if some_resource != resource]            
            assert all(interface_id in component_priorities[0][:len(self.INTERFACE_IDS[resource])] for interface_id in self.INTERFACE_IDS[resource])
            number_of_other_resource_interface_components = sum([len(self.INTERFACE_IDS[other_resource]) for other_resource in other_resources])
            for other_resource in other_resources:                 
                start_index = len(self.INTERFACE_IDS[resource])
                end_index = start_index + number_of_other_resource_interface_components
                assert all(interface_id in component_priorities[0][start_index:end_index]
                                    for interface_id in self.INTERFACE_IDS[other_resource])

    def test_get_infrastructure_interface_id(self, system: System.System):
        all_interface_ids = system.resources['ElectricPower'][
                'DistributionModel'].priority.get_infrastructure_interface_id(list(range(len(system.components))))
        for resource, interface_ids in self.INTERFACE_IDS.items():
            assert all(interface_id for interface_id in interface_ids if interface_id in all_interface_ids)
        
class TestComponentBasedDistributionPriority_ThreeLocalitiesCommunity(TestDistributionPriority):

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    def test_get_locality_id_from_string(self):
        example_strings = [['Locality 1'], ['Locality 123'], ['Locality 435'], ['Locality 1', 'Locality 100']]
        target_ids = [[1], [123], [435], [1, 100]]
        for example_string, target_id in zip(example_strings, target_ids):
            locality_id = DistributionPriority.ComponentBasedPriority.get_locality_id_from_string(example_string)
            assert locality_id == target_id

    def test_component_not_already_in_priority_list(self, system: System.System):
        component = system.components[0]
        assert not (DistributionPriority.ComponentBasedPriority.component_not_already_in_priority_list(None))
        assert DistributionPriority.ComponentBasedPriority.component_not_already_in_priority_list(component)        

    def test_find_component_position(self, system: System.System):
        components = system.components
        distribution_priority = system.resources['ElectricPower']['DistributionModel'].priority
        target_components = [['BaseTransceiverStation_1', ['Locality 1']], ['ElectricPowerPlant', ['Locality 1']],
                             ['BuildingStockUnit', ['Locality 3']], ['SuperLink', ['Locality 1', 'Locality 2']]]
        target_positions = [0, 1, 7, 2]
        for target_component, target_position in zip(target_components, target_positions):
            component_position, temp_components = distribution_priority.find_component_position(target_component[0],
                                                                                                target_component[1],
                                                                                                components)
            assert component_position == target_position

    def test_find_component_position_error(self, system: System.System):
        distribution_priority = system.resources['ElectricPower']['DistributionModel'].priority
        with pytest.raises(ValueError):
            distribution_priority.find_component_position('ElectricPowerPlant', ['Locality 3'], system.components)

    def test_set_distribution_priority(self, system: System.System):
        target_priorities = [[1, 0, 4, 8, 7], [4, 1, 0, 8, 7], [0, 8, 1, 4, 7]]
        for target_priority, resource_parameters in zip(target_priorities, list(system.resources.values())[1:]):
            component_positions, demand_types = resource_parameters[
                'DistributionModel'].priority.get_component_priorities()
            assert target_priority == component_positions
            assert all([demand_type == 'OperationDemand' for demand_type in demand_types])

        
tests/test_DistributionPriority.py
NikolaBlagojevic/pyrecodes
import pytest
import pandas as pd
from pyrecodes import SystemCreator
from pyrecodes import ComponentLibraryCreator
from pyrecodes import System
from pyrecodes import GeoVisualizer
from pyrecodes import Plotter
from pyrecodes import main

class TestConcreteGeoVisualizer:

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        system = main.create_system(input_dict)
        return system
    
    @pytest.fixture
    def geo_visualizer(self, system: System.System):
        return GeoVisualizer.ConcreteGeoVisualizer(system.components)
    
    def test_init(self, geo_visualizer: GeoVisualizer.ConcreteGeoVisualizer, system: System.System):
        assert geo_visualizer.components == system.components
        assert isinstance(geo_visualizer.buildings_geodataframe, pd.DataFrame)
        assert isinstance(geo_visualizer.state_dict, dict)
        assert isinstance(geo_visualizer.plotter, Plotter.Plotter)        

    def test_set_component_state_dict(self, geo_visualizer: GeoVisualizer.ConcreteGeoVisualizer):
        assert geo_visualizer.state_dict['Waiting'] == {'Color': 'black', 'ID': 0}
        assert geo_visualizer.state_dict['CleanUp'] == {'Color': 'yellow', 'ID': 5}
        assert geo_visualizer.state_dict['Functional'] == {'Color': 'green', 'ID': 11}

    def test_get_component_current_state(self, geo_visualizer, system):
        system.components[1].functional = [0, 20, 21, 22, 23, 24, 25]
        system.components[1].recovery_model.recovery_activities['RapidInspection'].time_steps = [1]
        system.components[1].recovery_model.recovery_activities['CleanUp'].time_steps = [2, 3, 4]
        system.components[1].recovery_model.recovery_activities['Permitting'].time_steps = [4, 5, 6]
        system.components[1].recovery_model.recovery_activities['Repair'].time_steps = list(range(15, 20))
        assert geo_visualizer.get_component_current_state(system.components[1], 0) == ['Functional']
        assert geo_visualizer.get_component_current_state(system.components[1], 20) == ['Functional']
        assert geo_visualizer.get_component_current_state(system.components[1], 19) == ['Repair']
        assert geo_visualizer.get_component_current_state(system.components[1], 1) == ['RapidInspection']
        assert geo_visualizer.get_component_current_state(system.components[1], 2) == ['CleanUp']
        assert geo_visualizer.get_component_current_state(system.components[1], 4) == ['CleanUp', 'Permitting']
        assert geo_visualizer.get_component_current_state(system.components[1], 10) == ['Waiting']

    def test_component_is_in_state(self, geo_visualizer, system: System.System):
        system.components[1].functional = [0, 20, 21, 22, 23, 24, 25]
        system.components[1].recovery_model.recovery_activities['RapidInspection'].time_steps = [1]
        system.components[1].recovery_model.recovery_activities['Repair'].time_steps = list(range(15, 20))
        assert geo_visualizer.component_is_in_state(system.components[1], 'Functional', 0) == True
        assert geo_visualizer.component_is_in_state(system.components[1], 'Functional', 20) == True
        assert geo_visualizer.component_is_in_state(system.components[1], 'Functional', 19) == False
        assert geo_visualizer.component_is_in_state(system.components[1], 'RapidInspection', 1) == True
        assert geo_visualizer.component_is_in_state(system.components[1], 'RapidInspection', 2) == False
        assert geo_visualizer.component_is_in_state(system.components[1], 'Repair', 2) == False
        assert geo_visualizer.component_is_in_state(system.components[1], 'Repair', 15) == True






tests/test_GeoVisualizer.py
NikolaBlagojevic/pyrecodes
import math
import numpy as np
import pytest
from pyrecodes import ProbabilityDistribution

class TestProbabilityDistribution:

    def test_deterministic_sampling(self):
        parameters = {'Value': 5}
        dist = ProbabilityDistribution.Deterministic(parameters)
        assert dist.sample() == 5

    def test_lognormal_median(self):
        num_samples = 100000
        parameters_list = [{'Median': 1, 'Dispersion': 0}, {'Median': 1, 'Dispersion': 1},
                           {'Median': 0.1, 'Dispersion': 1}, {'Median': 5, 'Dispersion': 0.5}]
        target_medians = [1, 1, 0.1, 5]

        for target_median, parameters in zip(target_medians, parameters_list):
            dist = ProbabilityDistribution.Lognormal(parameters)
            samples = [dist.sample() for _ in range(num_samples)]
            assert math.isclose(target_median, np.median(samples), abs_tol=0.03)

    def test_lognormal_dispersion(self):
        num_samples = 100000
        parameters_list = [{'Median': 1, 'Dispersion': 0}, {'Median': 1, 'Dispersion': 1},
                           {'Median': 0.1, 'Dispersion': 1}, {'Median': 5, 'Dispersion': 0.5}]
        target_dispersions = [0, 1, 1, 0.5]

        for target_dispersion, parameters in zip(target_dispersions, parameters_list):
            dist = ProbabilityDistribution.Lognormal(parameters)
            samples = [dist.sample() for _ in range(num_samples)]
            assert math.isclose(target_dispersion, np.std(np.log(samples)), abs_tol=0.03)


tests/test_ProbabilityDistribution.py
NikolaBlagojevic/pyrecodes
import pytest
from pyrecodes import Relation
import math


class TestRelation():

    test_values = [0.0, 0.3, 0.5, 0.8, 1.0]

    def construct_relation_object(self, relation_name: str):
        target_relation = getattr(Relation, relation_name)
        self.relation = target_relation()

    def test_invalid_negative_input(self):
        self.construct_relation_object('Linear')
        with pytest.raises(ValueError):
            self.relation.get_output(-0.5)

    def test_invalid_high_input(self):
        self.construct_relation_object('Linear')
        with pytest.raises(ValueError):
            self.relation.get_output(50)
    
    def test_constant_relation(self):
        self.construct_relation_object('Constant')
        output = []
        correct_output = [1.0, 1.0, 1.0, 1.0, 1.0]
        for value in self.test_values:
            output.append(self.relation.get_output(value))
        assert all([math.isclose(a, b) for a, b in zip(output, correct_output)])

    def test_linear_relation(self):
        self.construct_relation_object('Linear')
        output = []
        correct_output = [0.0, 0.3, 0.5, 0.8, 1.0]
        for value in self.test_values:
            output.append(self.relation.get_output(value))
        assert output == correct_output

    def test_reverse_linear_relation(self):
        self.construct_relation_object('ReverseLinear')
        output = []
        correct_output = [1.0, 0.7, 0.5, 0.2, 0.0]
        for value in self.test_values:
            output.append(self.relation.get_output(value))
        assert all([math.isclose(a, b) for a, b in zip(output, correct_output)])

    def test_binary_relation(self):
        self.construct_relation_object('Binary')
        output = []
        correct_output = [0.0, 0.0, 0.0, 0.0, 1.0]
        for value in self.test_values:
            output.append(self.relation.get_output(value))
        assert all([math.isclose(a, b) for a, b in zip(output, correct_output)])

    def test_reverse_binary_relation(self):
        self.construct_relation_object('ReverseBinary')
        output = []
        correct_output = [1.0, 0.0, 0.0, 0.0, 0.0]
        for value in self.test_values:
            output.append(self.relation.get_output(value))
        assert all([math.isclose(a, b) for a, b in zip(output, correct_output)])
    
    def test_multiple_step_relation(self):
        self.construct_relation_object('MultipleStep')
        self.relation.set_steps([0.0, 0.2, 0.4, 0.6, 0.8], [0.15, 0.25, 0.55, 0.75, 0.95])
        output = []
        correct_output = [0.15, 0.25, 0.55, 1.0, 1.0]
        for value in self.test_values:
            output.append(self.relation.get_output(value))
        assert all([math.isclose(a, b) for a, b in zip(output, correct_output)])
    
    def test_multiple_step_relation_wrong_steps(self):
        self.construct_relation_object('MultipleStep')
        with pytest.raises(ValueError):
            self.relation.set_steps([0.0, 0.2, 0.4, 0.6, 0.8], [0.15, 0.25, 0.55, 0.75, 0.95, 1.0])   

    

tests/test_Relation.py
NikolaBlagojevic/pyrecodes
import pytest
import math
from pyrecodes import main
from pyrecodes import ResilienceCalculator
from pyrecodes import System

class TestReCoDeSResilienceCalculator():

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture()
    def resilience_calculator(self, system: System.System) -> ResilienceCalculator.ResilienceCalculator:
        return ResilienceCalculator.ReCoDeSResilienceCalculator(system.system_configuration['ResilienceCalculator'][0]['Parameters'])

    def test_init_(self, system: System.System, resilience_calculator):
        resource_names = ['ElectricPower', 'CoolingWater', 'Communication']
        assert resilience_calculator.resource_names == resource_names

    def test_update_no_damage(self, system: System.System,
                              resilience_calculator: ResilienceCalculator.ResilienceCalculator):
        system.distribute_resources()
        resilience_calculator.update(system.resources)
        target_values = [[5.0, 4.0, 4.0], [3.0, 1.0, 1.0], [3.0, 3.0, 3.0]]
        for target_value, resource_name in zip(target_values, resilience_calculator.resource_names):
            assert math.isclose(target_value[0], resilience_calculator.system_supply[resource_name][0])
            assert math.isclose(target_value[1], resilience_calculator.system_demand[resource_name][0])
            assert math.isclose(target_value[2], resilience_calculator.system_consumption[resource_name][0])

    def test_update_with_damage(self, system: System.System,
                                resilience_calculator: ResilienceCalculator.ResilienceCalculator):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        system.resources['ElectricPower']['DistributionModel'].distribute()
        system.resources['CoolingWater']['DistributionModel'].distribute()
        system.resources['Communication']['DistributionModel'].distribute()
        resilience_calculator.update(system.resources)
        target_values = [[3.0, 2.6, 2.6], [1.8, 1.0, 1.0], [1.0, 12.0, 1.0]]
        for target_value, resource_name in zip(target_values, resilience_calculator.resource_names):
            assert math.isclose(target_value[0], resilience_calculator.system_supply[resource_name][0])
            print(resilience_calculator.system_supply[resource_name])
            assert math.isclose(target_value[1], resilience_calculator.system_demand[resource_name][0])
            assert math.isclose(target_value[2], resilience_calculator.system_consumption[resource_name][0])

    def test_update_with_damage_and_interdependency(self, system: System.System,
                                                    resilience_calculator: ResilienceCalculator.ResilienceCalculator):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        system.distribute_resources()
        resilience_calculator.update(system.resources)
        target_values = [[0.0, 2.6, 0.0], [0.0, 1.0, 0.0], [0.0, 12.0, 0.0]]
        for target_value, resource_name in zip(target_values, resilience_calculator.resource_names):
            assert math.isclose(target_value[0], resilience_calculator.system_supply[resource_name][0])
            assert math.isclose(target_value[1], resilience_calculator.system_demand[resource_name][0])
            assert math.isclose(target_value[2], resilience_calculator.system_consumption[resource_name][0])

    def test_calculate_resilience(self, system: System.System,
                                  resilience_calculator: ResilienceCalculator.ResilienceCalculator):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        system.distribute_resources()
        resilience_calculator.update(system.resources)
        lack_of_resilience = resilience_calculator.calculate_resilience()
        target_values = [2.6, 1.0, 12.0]
        for target_value, resource_name in zip(target_values, resilience_calculator.resource_names):
            assert math.isclose(lack_of_resilience[resource_name], target_value)

class TestNISTGoalsResilienceCalculator:

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture()
    def resilience_calculator(self, system: System.System) -> ResilienceCalculator.ResilienceCalculator:
        return ResilienceCalculator.NISTGoalsResilienceCalculator(system.system_configuration['ResilienceCalculator'][1]['Parameters'])
    
    def test_init_(self, resilience_calculator):
        assert len(resilience_calculator.resilience_goals) == 3

    def test_update(self, system, resilience_calculator):
        system.distribute_resources()
        resilience_calculator.update(system.resources)
        assert resilience_calculator.resilience_goals[0]['GoalMet'][0] == True
        assert resilience_calculator.resilience_goals[1]['GoalMet'][0] == True
        assert resilience_calculator.resilience_goals[2]['GoalMet'][0] == True

        system.set_initial_damage()
        system.time_step = 1
        system.update()
        system.distribute_resources()
        resilience_calculator.update(system.resources)
        assert resilience_calculator.resilience_goals[0]['GoalMet'][1] == False
        assert resilience_calculator.resilience_goals[1]['GoalMet'][1] == False
        assert resilience_calculator.resilience_goals[2]['GoalMet'][1] == False

        for time_step in range(1, 12):
            system.time_step = time_step
            system.recover()
            system.update()
            system.distribute_resources()
            resilience_calculator.update(system.resources)
        assert resilience_calculator.resilience_goals[0]['GoalMet'][-1] == True
        assert resilience_calculator.resilience_goals[1]['GoalMet'][-1] == True
        assert resilience_calculator.resilience_goals[2]['GoalMet'][-1] == True
    
    def test_calculate_resilience(self, system):
        system.start_resilience_assessment()
        system.calculate_resilience()
        assert system.resilience_calculators[1].resilience_goals[0]['MetAtTimeStep'] == [12]
        assert system.resilience_calculators[1].resilience_goals[1]['MetAtTimeStep'] == [12]
        assert system.resilience_calculators[1].resilience_goals[2]['MetAtTimeStep'] == [12]

tests/test_ResilienceCalculator.py
NikolaBlagojevic/pyrecodes
import pytest
from pyrecodes import Resource
from pyrecodes import Relation


class TestResource():
    test_resource_name = 'Resource 1'
    test_initial_amount = 5.0

    def test_concrete_resource_initialization(self):
        parameters = {'Amount': 100, 'FunctionalityToAmountRelation': 'Linear', 'UnmetDemandToAmountRelation': 'Constant'}
        resource = Resource.ConcreteResource('TestResource', parameters)
        assert resource.name == 'TestResource'
        assert resource.initial_amount == 100
        assert resource.current_amount == 100
        assert isinstance(resource.component_functionality_to_amount, Relation.Linear)
        assert isinstance(resource.unmet_demand_to_amount, Relation.Constant)
    
    def test_concrete_resource_initialization_with_no_parameters(self):
        resource = Resource.ConcreteResource('TestResource', {})
        assert resource.name == 'TestResource'
        assert resource.initial_amount == 0
        assert resource.current_amount == 0
        assert isinstance(resource.component_functionality_to_amount, Relation.Constant)
        assert isinstance(resource.unmet_demand_to_amount, Relation.Constant)
    
    def test_set_initial_amount_positive(self):
        resource = Resource.ConcreteResource('TestResource', {})
        resource.set_initial_amount(50)
        assert resource.current_amount == 50 and resource.initial_amount == 50

    def test_set_initial_amount_negative(self):
        resource = Resource.ConcreteResource('TestResource', {})
        with pytest.raises(ValueError):
            resource.set_initial_amount(-50)
    
    def test_set_current_amount_positive(self):
        resource = Resource.ConcreteResource('TestResource', {})
        resource.set_current_amount(50)
        assert resource.current_amount == 50 and resource.initial_amount == 0

    def test_set_current_amount_negative(self):
        resource = Resource.ConcreteResource('TestResource', {})
        with pytest.raises(ValueError):
            resource.set_current_amount(-50)

    def test_set_relation(self):
        resource = Resource.ConcreteResource('TestResource', {})
        resource.set_relation('Linear', 'component_functionality_to_amount')
        assert isinstance(resource.component_functionality_to_amount, Relation.Linear)
        resource.set_relation('Constant', 'unmet_demand_to_amount')
        assert isinstance(resource.unmet_demand_to_amount, Relation.Constant)
        with pytest.raises(ValueError):
            resource.set_relation('NotARelation', 'component_functionality_to_amount')
    
    def test_update_based_on_component_functionality_linear_relation(self):
        parameters = {'Amount': 100, 'FunctionalityToAmountRelation': 'Linear'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the component functionality level to 0.5
        resource.update_based_on_component_functionality(0.5)
        # The current amount should be 0.5 * 100 = 50
        assert resource.current_amount == 50
        assert resource.initial_amount == 100

        # Set the component functionality level to 0
        resource.update_based_on_component_functionality(0)
        # The current amount should be 0 * 100 = 0
        assert resource.current_amount == 0
        assert resource.initial_amount == 100

        # Set the component functionality level to 1
        resource.update_based_on_component_functionality(1)
        # The current amount should be 1 * 100 = 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100
    
    def test_update_based_on_component_functionality_binary_relation(self):
        parameters = {'Amount': 100, 'FunctionalityToAmountRelation': 'Binary'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the component functionality level to 0.5
        resource.update_based_on_component_functionality(0.5)
        # The current amount should be 0.0 * 100 = 0.0
        assert resource.current_amount == 0.0
        assert resource.initial_amount == 100

        # Set the component functionality level to 0
        resource.update_based_on_component_functionality(0)
        # The current amount should be 0 * 100 = 0
        assert resource.current_amount == 0
        assert resource.initial_amount == 100

        # Set the component functionality level to 1
        resource.update_based_on_component_functionality(1)
        # The current amount should be 1 * 100 = 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100

    def test_update_based_on_component_functionality_linear_relation(self):
        parameters = {'Amount': 100, 'FunctionalityToAmountRelation': 'Linear'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the component functionality level to 0.5
        resource.update_based_on_component_functionality(0.5)
        # The current amount should be 0.5 * 100 = 50
        assert resource.current_amount == 50
        assert resource.initial_amount == 100

        # Set the component functionality level to 0
        resource.update_based_on_component_functionality(0)
        # The current amount should be 0 * 100 = 0
        assert resource.current_amount == 0
        assert resource.initial_amount == 100

        # Set the component functionality level to 1
        resource.update_based_on_component_functionality(1)
        # The current amount should be 1 * 100 = 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100
    
    def test_update_based_on_component_functionality_constant_relation(self):
        parameters = {'Amount': 100, 'FunctionalityToAmountRelation': 'Constant'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the component functionality level to 0.5
        resource.update_based_on_component_functionality(0.5)
        # The current amount should be 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100

        # Set the component functionality level to 0
        resource.update_based_on_component_functionality(0)
        # The current amount should be 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100

        # Set the component functionality level to 1
        resource.update_based_on_component_functionality(1)
        # The current amount should be 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100
    
    def test_update_based_on_component_functionality_wrong_input(self):
        parameters = {'Amount': 100, 'FunctionalityToAmountRelation': 'Linear'}
        resource = Resource.ConcreteResource('TestResource', parameters)
        with pytest.raises(ValueError):
            resource.update_based_on_component_functionality(-5)

    def test_update_based_on_unmet_demand_constant_relation(self):
        parameters = {'Amount': 100, 'UnmetDemandToAmountRelation': 'Constant'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the unmet demand to 0.5
        resource.update_based_on_unmet_demand(0.5)
        # The current amount should be 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100

        # Set the unmet demand to 0
        resource.update_based_on_unmet_demand(0)
        # The current amount should be 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100

        # Set the unmet demand to 1
        resource.update_based_on_unmet_demand(1)
        # The current amount should be 100
        assert resource.current_amount == 100
        assert resource.initial_amount == 100
    
    def test_update_based_on_unmet_demand_linear_relation(self):
        parameters = {'Amount': 100, 'UnmetDemandToAmountRelation': 'Linear'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the unmet demand to 0.5
        resource.update_based_on_unmet_demand(0.5)
        # The current amount should be 50
        assert resource.current_amount == 50
        assert resource.initial_amount == 100

        # Set the unmet demand to 0
        resource.update_based_on_unmet_demand(0)
        # The current amount should be 100
        assert resource.current_amount == 0
        assert resource.initial_amount == 100

        # Set the unmet demand to 1
        resource.update_based_on_unmet_demand(1)
        # The current amount should still be 0
        assert resource.current_amount == 0
        assert resource.initial_amount == 100
    
    def test_update_based_on_unmet_demand_wrong_input(self):
        parameters = {'Amount': 100, 'UnmetDemandToAmountRelation': 'Linear'}
        resource = Resource.ConcreteResource('TestResource', parameters)
        with pytest.raises(ValueError):
            resource.update_based_on_unmet_demand(-5)

    def test_update_based_on_unmet_demand_binary_relation(self):
        parameters = {'Amount': 100, 'UnmetDemandToAmountRelation': 'Binary'}
        resource = Resource.ConcreteResource('TestResource', parameters)

        # Set the unmet demand to 0.5
        resource.update_based_on_unmet_demand(0.5)
        # The current amount should be 100
        assert resource.current_amount == 0
        assert resource.initial_amount == 100

        # Set the unmet demand to 0
        resource.update_based_on_unmet_demand(0)
        # The current amount should be 100
        assert resource.current_amount == 0
        assert resource.initial_amount == 100

        # Set the unmet demand to 1
        resource.update_based_on_unmet_demand(1)
        # The current amount should still be 0
        assert resource.current_amount == 0
        assert resource.initial_amount == 100
    
    def test_non_consumable_resource_update_based_on_consumption(self):
        resource = Resource.NonConsumableResource('TestResource', {'Amount': 100})
        # Non-consumable resources should not be affected by consumption
        resource.update_based_on_consumption(50)
        assert resource.initial_amount == 100
        assert resource.current_amount == 100
    
    def test_consumable_resource_update_based_on_consumption(self):
        resource = Resource.ConsumableResource('TestResource', {'Amount': 100})
        # Consumable resource with initial amount 100
        resource.update_based_on_consumption(30)
        # After consumption of 30, the initial and current amounts should be reduced by 30
        assert resource.initial_amount == 70
        assert resource.current_amount == 70

    def test_consumable_resource_update_based_on_consumption_exhausted(self):
        resource = Resource.ConsumableResource('TestResource', {'Amount': 30})
        # Consumable resource with initial amount 30
        resource.update_based_on_consumption(50)
        # If consumption exceeds the initial amount, the current amount should be 0
        assert resource.initial_amount == 0
        assert resource.current_amount == 0
     




tests/test_Resource.py
NikolaBlagojevic/pyrecodes
import math
import numpy as np
import pytest
from pyrecodes import Component
from pyrecodes import ComponentLibraryCreator
from pyrecodes import DistributionPriority
from pyrecodes import System
from pyrecodes import SystemCreator
from pyrecodes import ResourceDistributionModel
from pyrecodes import main

class TestUtilityDistributionModelConstructor:

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    DUMMY_PRIORITY = {"Type": "ComponentTypeBasedPriority",
                    "Parameters": [
                        [
                            "BaseTransceiverStation_1",
                            "OperationDemand"
                        ],
                        [
                            "ElectricPowerPlant",
                            "OperationDemand"
                        ],
                        [
                            "CoolingWaterFacility",
                            "OperationDemand"
                        ],
                        [
                            "BuildingStockUnit",
                            "OperationDemand"
                        ]
                    ]}
    
    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture
    def distribution_model_constructor(self):
        return ResourceDistributionModel.UtilityDistributionModelConstructor()
    
    @pytest.fixture
    def distribution_model(self, system):
        distribution_model = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if resource_name == 'ElectricPower'}
        return distribution_model  
       
    def test_set_components(self, distribution_model, distribution_model_constructor, system: System.System):
        component_subset = system.components[:10]
        distribution_model_constructor.set_components(component_subset, distribution_model['ElectricPower'])
        assert distribution_model['ElectricPower'].components == component_subset

    def test_set_resource_name(self, distribution_model, distribution_model_constructor):
        distribution_model_constructor.set_resource_name('DummyResource', distribution_model['ElectricPower'])
        assert distribution_model['ElectricPower'].resource_name == 'DummyResource'

    def test_set_priority_model(self, distribution_model, distribution_model_constructor, system):
        distribution_model_constructor.set_priority_model('ElectricPower', self.DUMMY_PRIORITY, system.components, distribution_model['ElectricPower'])
        assert isinstance(distribution_model['ElectricPower'].priority, DistributionPriority.ComponentTypeBasedPriority)
    
    def test_set_system_matrix(self, distribution_model, distribution_model_constructor, system):
        distribution_model_constructor.set_system_matrix(system.components, 'ElectricPower', distribution_model['ElectricPower'])
        assert isinstance(distribution_model['ElectricPower'].system_matrix, ResourceDistributionModel.SingleResourceSystemMatrixCreator)

class TestTransferServiceDistributionModelConstructor:

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture
    def distribution_model_constructor(self):
        return ResourceDistributionModel.TransferServiceDistributionModelConstructor()
    
    @pytest.fixture
    def distribution_model(self, system):
        distribution_model = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if resource_name == 'SuperTransferService'}
        return distribution_model
    
    def test_set_components(self, distribution_model, distribution_model_constructor, system: System.System):
        component_subset = system.components[:10]
        distribution_model_constructor.set_components(component_subset, distribution_model['SuperTransferService'])
        assert distribution_model['SuperTransferService'].components == component_subset
    
    def test_set_resource_name(self, distribution_model, distribution_model_constructor):
        distribution_model_constructor.set_resource_name('DummyResource', distribution_model['SuperTransferService'])
        assert distribution_model['SuperTransferService'].resource_name == 'DummyResource'
    
class TestSingleResourceSystemMatrixCreator():

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture
    def distribution_models(self, system):
        distribution_models = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if 'TransferService' not in resource_name}
        return distribution_models    

    def test_initialize_system_matrix(self, distribution_models: dict, system: System.System):
        for distribution_model in distribution_models.values():
            distribution_model.system_matrix.initialize_system_matrix()
            assert np.all(distribution_model.system_matrix.matrix == np.zeros(
                (2 * len(system.components), distribution_models['ElectricPower'].system_matrix.NUM_COLUMN_SETS)))

    def test_calculate_num_rows_in_system_matrix(self, distribution_models: dict, system: System.System):
        assert distribution_models['ElectricPower'].system_matrix.calculate_num_rows_in_system_matrix() == len(
            system.components) * distribution_models['ElectricPower'].system_matrix.ROWS_PER_COMPONENT

    def test_calculate_num_columns_in_system_matrix(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].system_matrix.calculate_num_columns_in_system_matrix() == \
               distribution_models['ElectricPower'].system_matrix.NUM_COLUMN_SETS

    def test_fill_operation_demand_row(self, distribution_models: dict):
        for resource_name, distribution_model in distribution_models.items():
            for i, component in enumerate(distribution_model.components):
                supply = distribution_model.system_matrix.get_current_resource_amount(component, 'supply',
                                                                                      Component.StandardiReCoDeSComponent.SupplyTypes.SUPPLY.value)
                demand = distribution_model.system_matrix.get_current_resource_amount(component, 'demand',
                                                                                      Component.StandardiReCoDeSComponent.DemandTypes.OPERATION_DEMAND.value)
                target_row = [component.get_locality()[0], component.get_locality()[-1], supply, demand, 1.0]
                distribution_model.system_matrix.fill_operation_demand_row(i, component)
                assert np.all(target_row == distribution_model.system_matrix.matrix[i, :])

    def test_fill_recovery_demand_row(self, distribution_models: dict):
        for resource_name, distribution_model in distribution_models.items():
            for i, component in enumerate(distribution_model.components):
                recovery_demand = distribution_model.system_matrix.get_current_resource_amount(component, 'demand',
                                                                                               Component.StandardiReCoDeSComponent.DemandTypes.RECOVERY_DEMAND.value)
                target_row = [component.get_locality()[0], component.get_locality()[-1], 0.0, recovery_demand, 1.0]
                distribution_model.system_matrix.fill_recovery_demand_row(i, component)
                assert np.all(target_row == distribution_model.system_matrix.matrix[
                                                      i + distribution_model.system_matrix.RECOVERY_DEMAND_ROW_OFFSET,
                                                      :])

    def test_fill_system_matrix_no_damage(self, distribution_models: dict):
        for distribution_model in distribution_models.values():
            distribution_model.system_matrix.fill_system_matrix()

        locality_matrix = []
        for component in distribution_model.components:
            locality_matrix.append([component.get_locality()[0], component.get_locality()[-1]])

        recovery_demand_matrix_part = np.concatenate(
            (locality_matrix, np.asarray([[0.0, 0.0, 1.0] for _ in distribution_models['ElectricPower'].components])),
            axis=1)
        target_power_matrix = np.concatenate(
            (np.asarray([[1, 1, 0.0, 1.0, 1.0], [1, 1, 5.0, 0.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 0.0, 1.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 1.0, 1.0],
                         [3, 3, 0.0, 1.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)

        assert np.all(target_power_matrix == distribution_models['ElectricPower'].system_matrix.matrix)
        target_communication_matrix = np.concatenate(
            (np.asarray([[1, 1, 1.0, 0.0, 1.0], [1, 1, 0.0, 1.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 0.0, 1.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 1.0, 1.0],
                         [3, 3, 2.0, 0.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        assert np.all(target_communication_matrix == distribution_models['Communication'].system_matrix.matrix)

        target_cooling_water_matrix = np.concatenate(
            (np.asarray([[1, 1, 0.0, 0.0, 1.0], [1, 1, 0.0, 1.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 3.0, 0.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 0.0, 1.0],
                         [3, 3, 0.0, 0.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        assert np.all(target_cooling_water_matrix == distribution_models['CoolingWater'].system_matrix.matrix)

    def test_fill_system_matrix_with_damage(self, distribution_models: dict, system: System.System):
        system.set_initial_damage()
        system.time_step = 0
        system.update()
        for distribution_model in distribution_models.values():
            distribution_model.components = system.components
            distribution_model.fill_system_matrix()
        locality_matrix = []
        for component in distribution_model.components:
            locality_matrix.append([component.get_locality()[0], component.get_locality()[-1]])

        recovery_demand_matrix_part = np.concatenate(
            (locality_matrix, np.asarray([[0.0, 0.0, 1.0] for _ in distribution_models['ElectricPower'].components])),
            axis=1)
        target_power_matrix = np.concatenate(
            (np.asarray([[1, 1, 0.0, 1.0, 1.0], [1, 1, 3.0, 0.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 0.0, 1.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 0.6, 1.0],
                         [3, 3, 0.0, 0.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        assert np.all(np.isclose(target_power_matrix, distribution_models['ElectricPower'].system_matrix.matrix))

        target_communication_matrix = np.concatenate(
            (np.asarray([[1, 1, 1.0, 0.0, 1.0], [1, 1, 0.0, 1.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 0.0, 1.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 1.0, 1.0],
                         [3, 3, 0.0, 0.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        assert np.all(np.isclose(target_communication_matrix, distribution_models['Communication'].system_matrix.matrix))

        target_cooling_water_matrix = np.concatenate(
            (np.asarray([[1, 1, 0.0, 0.0, 1.0], [1, 1, 0.0, 1.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 1.8, 0.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 0.0, 1.0],
                         [3, 3, 0.0, 0.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        assert np.all(np.isclose(target_cooling_water_matrix, distribution_models['CoolingWater'].system_matrix.matrix))

    def test_get_component_properties(self, distribution_models: dict):
        
        electric_power_plant = distribution_models['ElectricPower'].components[1]
        component_properties = distribution_models['ElectricPower'].system_matrix.get_component_properties(
            electric_power_plant, [['supply', 'Supply'], ['demand', 'OperationDemand']])
        assert all(component_properties == [1, 1, 5, 0, 1])
        component_properties = distribution_models['ElectricPower'].system_matrix.get_component_properties(
            electric_power_plant, [['supply', 'Supply'], ['demand', 'RecoveryDemand']])
        assert all(component_properties == [1, 1, 5, 0, 1])
        component_properties = distribution_models['CoolingWater'].system_matrix.get_component_properties(
            electric_power_plant, [['supply', 'Supply'], ['demand', 'OperationDemand']])
        assert all(component_properties == [1, 1, 0, 1, 1])
        component_properties = distribution_models['Communication'].system_matrix.get_component_properties(
            electric_power_plant, [['supply', 'Supply'], ['demand', 'OperationDemand']])
        assert all(component_properties == [1, 1, 0, 1, 1])

    def test_get_current_resource_amount(self, distribution_models: dict):
        building_stock_unit = distribution_models['ElectricPower'].components[7]
        assert distribution_models['ElectricPower'].system_matrix.get_current_resource_amount(building_stock_unit,
                                                                                           'supply', 'Supply') == 0.0
        assert distribution_models['ElectricPower'].system_matrix.get_current_resource_amount(building_stock_unit,
                                                                                           'demand',
                                                                                           'OperationDemand') == 1.0
        assert distribution_models['CoolingWater'].system_matrix.get_current_resource_amount(building_stock_unit, 'demand',
                                                                                          'OperationDemand') == 0.0
        assert distribution_models['Communication'].system_matrix.get_current_resource_amount(building_stock_unit,
                                                                                           'demand',
                                                                                           'OperationDemand') == 1.0

    def test_set_demand_met_indicator(self, distribution_models: dict):
        distribution_models['ElectricPower'].system_matrix.set_demand_met_indicator(0, 0.5)
        distribution_models['CoolingWater'].system_matrix.set_demand_met_indicator(0, 0.5)
        distribution_models['Communication'].system_matrix.set_demand_met_indicator(0, 0.5)
        assert all([distribution_models['ElectricPower'].system_matrix.matrix[
                        0, distribution_models['ElectricPower'].system_matrix.DEMAND_MET_COL_ID] == 0.5,
                    distribution_models['CoolingWater'].system_matrix.matrix[
                        0, distribution_models['CoolingWater'].system_matrix.DEMAND_MET_COL_ID] == 0.5,
                    distribution_models['Communication'].system_matrix.matrix[
                        0, distribution_models['Communication'].system_matrix.DEMAND_MET_COL_ID] == 0.5])

    def test_get_demand(self, distribution_models: dict):
        assert all([distribution_models['ElectricPower'].system_matrix.get_demand(0) ==
                    distribution_models['ElectricPower'].system_matrix.matrix[
                        0, distribution_models['ElectricPower'].system_matrix.DEMAND_COL_ID],
                    distribution_models['Communication'].system_matrix.get_demand(0) ==
                    distribution_models['Communication'].system_matrix.matrix[
                        0, distribution_models['Communication'].system_matrix.DEMAND_COL_ID],
                    distribution_models['CoolingWater'].system_matrix.get_demand(0) ==
                    distribution_models['CoolingWater'].system_matrix.matrix[
                        0, distribution_models['CoolingWater'].system_matrix.DEMAND_COL_ID]])

    def test_get_initial_demand_met_indicators(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].system_matrix.get_initial_demand_met_indicator() == 1.0


class TestUtilityDistributionModel_ThreeLocalitiesCommunity():

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        return main.create_system(input_dict)

    @pytest.fixture
    def distribution_models(self, system):
        distribution_models = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if 'TransferService' not in resource_name}
        return distribution_models 

    def test_init_(self, distribution_models: dict, system: System.System):
        assert distribution_models['ElectricPower'].system_matrix.RECOVERY_DEMAND_ROW_OFFSET == len(system.components)
        assert isinstance(distribution_models['ElectricPower'].priority, DistributionPriority.ComponentBasedPriority)
        assert isinstance(distribution_models['ElectricPower'].transfer_service_distribution_model, ResourceDistributionModel.TransferServiceDistributionModelPotentialPathSets)

    def test_distribute(self, distribution_models: dict, system: System.System):
        distribution_models['Communication'].distribute()
        assert np.all(distribution_models['Communication'].system_matrix.matrix[:, -1] == 1.0)
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        distribution_models['Communication'].components = system.components
        distribution_models['Communication'].distribute()
        assert np.all(distribution_models['Communication'].system_matrix.matrix[7, -1] == 0.0)

    def test_fill_system_matrix(self, distribution_models: dict, system: System.System):
        locality_matrix = []
        for component in distribution_models['ElectricPower'].components:
            locality_matrix.append([component.get_locality()[0], component.get_locality()[-1]])
        recovery_demand_matrix_part = np.concatenate(
            (locality_matrix, np.asarray([[0.0, 0.0, 1.0] for _ in distribution_models['ElectricPower'].components])),
            axis=1)
        target_initial_matrix_no_damage = np.concatenate(
            (np.asarray([[1, 1, 0.0, 1.0, 1.0], [1, 1, 5.0, 0.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 0.0, 1.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 1.0, 1.0],
                         [3, 3, 0.0, 1.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        distribution_models['ElectricPower'].fill_system_matrix()
        assert np.all(distribution_models['ElectricPower'].system_matrix.matrix == target_initial_matrix_no_damage)

        target_initial_matrix_with_damage = np.concatenate(
            (np.asarray([[1, 1, 0.0, 1.0, 1.0], [1, 1, 3.0, 0.0, 1.0], [1, 2, 0.0, 0.0, 1.0], [1, 3, 0.0, 0.0, 1.0],
                         [2, 2, 0.0, 1.0, 1.0], [2, 1, 0.0, 0.0, 1.0], [2, 3, 0.0, 0.0, 1.0], [3, 3, 0.0, 0.6, 1.0],
                         [3, 3, 0.0, 0.0, 1.0], [3, 1, 0.0, 0.0, 1.0], [3, 2, 0.0, 0.0, 1.0]]),
             recovery_demand_matrix_part), axis=0)
        
        system.set_initial_damage()
        system.time_step = 0
        system.update()
        distribution_models['ElectricPower'].components = system.components
        distribution_models['ElectricPower'].fill_system_matrix()
        assert np.all(distribution_models['ElectricPower'].system_matrix.matrix == target_initial_matrix_with_damage)
        
    def test_get_component_priorities(self, distribution_models: dict):
        target_priorities = [1, 0, 4, 8, 7]
        target_demand_types = ['OperationDemand' for _ in range(len(target_priorities))]
        component_rows, component_demand_types = distribution_models['ElectricPower'].get_component_priorities()
        assert target_priorities == component_rows
        assert target_demand_types == component_demand_types
    
    def test_set_component_row_based_on_demand_type(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].set_component_row_based_on_demand_type([1], ['OperationDemand']) == [1]
        assert distribution_models['ElectricPower'].set_component_row_based_on_demand_type([1], ['RecoveryDemand']) == [1 + len(distribution_models['ElectricPower'].components)]                          

    def test_add_supplier(self, distribution_models: dict):
        power_plant_row_id = 1
        target_component_is_supplier = [True, False, False]
        target_suppliers_length = [1, 0, 0]
        for target_bool, target_length, distribution_model in zip(target_component_is_supplier, target_suppliers_length,
                                                                  distribution_models.values()):
            distribution_model.system_matrix.fill_system_matrix()
            suppliers = []
            suppliers, component_is_supplier = distribution_model.add_supplier(power_plant_row_id, suppliers)
            assert len(suppliers) == target_length
            assert component_is_supplier == target_bool

    def test_add_supplier_with_damage(self, distribution_models: dict, system: System.System):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        target_damaged_supply_values = [3.0, 1.8, 1.0]
        for target_value, distribution_model in zip(target_damaged_supply_values, distribution_models.values()):
            distribution_model.components = system.components
            distribution_model.fill_system_matrix()
            suppliers = []
            for component_row_id in range(len(distribution_model.components)):
                suppliers, component_is_supplier = distribution_model.add_supplier(component_row_id, suppliers)
            assert math.isclose(suppliers[0]['CurrentSupply'], target_value)
    
    def test_meet_component_demand_single_supplier(self, distribution_models: dict):
        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 5.0, 'ConsumedAmount': 0,
                      'ComponentRowID': 0}]
        componet_row_id = 7
        component_demand_type = 'OperationDemand'
        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, componet_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 4.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 1.0)

        component_row_id = 4
        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, component_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 3.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 2.0)

        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 0.0, 'ConsumedAmount': 0,
                      'ComponentRowID': 0}]
        
        component_row_id = 4
        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, componet_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 0.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 0.0)
    
    def test_meet_component_demand_single_supplier(self, distribution_models: dict):
        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 5.0, 'ConsumedAmount': 0,
                      'ComponentRowID': 0}]
        componet_row_id = 7
        component_demand_type = 'OperationDemand'
        distribution_models['ElectricPower'].fill_system_matrix()

        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, componet_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 4.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 1.0)

        component_row_id = 4
        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, component_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 3.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 2.0)

        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 0.0, 'ConsumedAmount': 0,
                      'ComponentRowID': 0}]
        
        component_row_id = 4
        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, componet_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 0.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 0.0)
    
    def test_meet_component_demand_multiple_suppliers(self, distribution_models: dict):
        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 0.5, 'ConsumedAmount': 0,
                      'ComponentRowID': 0},
                      {'StartLocality': 1, 'EndLocality': 1,
                       'CurrentSupply': 1.0, 'ConsumedAmount': 0,
                       'ComponentRowID': 8}]
        component_row_id = 7
        component_demand_type = 'OperationDemand'
        distribution_models['ElectricPower'].fill_system_matrix()

        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, component_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 0.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 0.5)
        assert math.isclose(suppliers[1]['CurrentSupply'], 0.5)
        assert math.isclose(suppliers[1]['ConsumedAmount'], 0.5)

        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 0, 'ConsumedAmount': 0,
                      'ComponentRowID': 0},
                      {'StartLocality': 1, 'EndLocality': 1,
                       'CurrentSupply': 1.0, 'ConsumedAmount': 0,
                       'ComponentRowID': 8}]

        suppliers = distribution_models['ElectricPower'].meet_component_demand(suppliers, component_row_id, component_demand_type)
        assert math.isclose(suppliers[0]['CurrentSupply'], 0.0)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 0.0)
        assert math.isclose(suppliers[1]['CurrentSupply'], 0.0)
        assert math.isclose(suppliers[1]['ConsumedAmount'], 1.0)        

    def test_suppliers_meet_component_demand_no_damage(self, distribution_models: dict):
        building_stock_id = 7
        component_demand_type = 'OperationDemand'
        for distribution_model in distribution_models.values():
            distribution_model.fill_system_matrix()
            suppliers = []
            component_priorities, component_demand_types = distribution_model.get_component_priorities()
            for component_row_id, component_demand_type in zip(component_priorities, component_demand_types):                
                suppliers, component_is_supplier = distribution_model.add_supplier(component_row_id, suppliers) 
            component_demand = distribution_model.get_demand(building_stock_id)
            component_localities = [distribution_model.system_matrix.matrix[building_stock_id, distribution_model.system_matrix.START_LOCALITY_COL_ID],
                                distribution_model.system_matrix.matrix[building_stock_id, distribution_model.system_matrix.END_LOCALITY_COL_ID]]
            transfer_service_demand = distribution_model.get_transfer_service_demand(building_stock_id, component_demand_type)
            demand_after_distribution, suppliers= distribution_model.suppliers_meet_component_demand(suppliers, component_demand, component_localities, transfer_service_demand)
            assert math.isclose(demand_after_distribution, 0, abs_tol=1e-5)

    def test_suppliers_meet_component_demand_with_damage(self, distribution_models: dict, system: System.System):
        system.set_initial_damage()
        system.time_step = 0
        system.update()
        building_stock_id = 7
        component_demand_type = 'OperationDemand'
        target_demand_values = [0.0, 0.0, 1.0]
        set_supplier_amount = [2.0, 0.0, 0.0]
        for target_value, supplier_amount, distribution_model in zip(target_demand_values, set_supplier_amount,
                                                                     distribution_models.values()):
            distribution_model.components = system.components
            distribution_model.fill_system_matrix()
            suppliers = [{'StartLocality': 1, 'EndLocality': 1, 
                              'CurrentSupply': supplier_amount, 'ConsumedAmount': 0, 
                              'ComponentRowID': 0}]
            component_demand = distribution_model.get_demand(building_stock_id)
            component_localities = [distribution_model.system_matrix.matrix[building_stock_id, distribution_model.system_matrix.START_LOCALITY_COL_ID],
                                distribution_model.system_matrix.matrix[building_stock_id, distribution_model.system_matrix.END_LOCALITY_COL_ID]]
            transfer_service_demand = distribution_model.get_transfer_service_demand(building_stock_id, component_demand_type)
            demand_after_distribution, suppliers= distribution_model.suppliers_meet_component_demand(suppliers, 
                                                                                                    component_demand, 
                                                                                                    component_localities, 
                                                                                                    transfer_service_demand)
            assert math.isclose(demand_after_distribution, target_value, abs_tol=1e-10)

    def test_update_consumed_amounts(self, distribution_models: dict):
        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 0.5, 'ConsumedAmount': 0,
                      'ComponentRowID': 0}]
        initial_suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                              'CurrentSupply': 5.0, 'ConsumedAmount': 0,
                              'ComponentRowID': 0}]
        distribution_models['ElectricPower'].update_consumed_amounts(suppliers, initial_suppliers)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 4.5)
        distribution_models['ElectricPower'].update_consumed_amounts(suppliers, initial_suppliers)
        assert math.isclose(suppliers[0]['ConsumedAmount'], 9.0)
    
    def test_update_suppliers_based_on_consumption(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].components[1].supply['Supply']['ElectricPower'].current_amount == 5.0
        suppliers = [{'StartLocality': 1, 'EndLocality': 1,
                      'CurrentSupply': 0.5, 'ConsumedAmount': 1.0,
                      'ComponentRowID': 1}]
        distribution_models['ElectricPower'].update_suppliers_based_on_consumption(suppliers)
        assert distribution_models['ElectricPower'].components[1].supply['Supply']['ElectricPower'].current_amount == 5.0
        
    def test_reduce_component_supply(self, distribution_models: dict):
        power_plant_row_id = 1
        base_station_row_id = 0
        percent_of_met_demand = 0.4
        reduced_supply_of_the_power_plant = [0.0, 0.0, 0.0]
        reduced_supply_of_the_base_station = [0.0, 0.0, 0.0]
        for power_plant_supply, base_station_supply, distribution_model in zip(reduced_supply_of_the_power_plant,
                                                                               reduced_supply_of_the_base_station,
                                                                               distribution_models.values()):
            distribution_model.reduce_component_supply(power_plant_row_id, percent_of_met_demand)
            distribution_model.reduce_component_supply(base_station_row_id, percent_of_met_demand)
            assert math.isclose(distribution_model.system_matrix.get_current_resource_amount(
                distribution_model.components[power_plant_row_id], Component.SupplyOrDemand.SUPPLY.value, 'Supply'),
                power_plant_supply)
            assert math.isclose(distribution_model.system_matrix.get_current_resource_amount(
                distribution_model.components[base_station_row_id], Component.SupplyOrDemand.SUPPLY.value, 'Supply'),
                base_station_supply)

    def test_get_demand(self, distribution_models: dict):
        demand_of_the_power_plant = [0.0, 1.0, 1.0]
        power_plant_row_id = 1
        for target_demand, distribution_model in zip(demand_of_the_power_plant, distribution_models.values()):
            distribution_model.fill_system_matrix()
            assert distribution_model.get_demand(power_plant_row_id) == target_demand

    def test_get_transfer_service_demand(self, distribution_models: dict):
        component_row_BSU = 7
        component_row_EPP = 1
        component_demand_type = 'OperationDemand'
        assert distribution_models['ElectricPower'].get_transfer_service_demand(component_row_BSU, component_demand_type) == 1.0
        assert distribution_models['Communication'].get_transfer_service_demand(component_row_BSU, component_demand_type) == 1.0
        assert distribution_models['CoolingWater'].get_transfer_service_demand(component_row_EPP, component_demand_type) == 1.0

    def test_modify_demand_to_account_for_resource_transfer(self, distribution_models: dict):
        # TODO: Improve the test. Test more cases. Use Virtual Community.
        supplier_start_locality = 1
        supplier_end_locality = 1
        component_demand = 10
        component_localities = [3, 3]
        transfer_service_demand = 5
        assert distribution_models['ElectricPower'].modify_demand_to_account_for_resource_transfer(supplier_start_locality,                                                                                                   
                                                                                                    supplier_end_locality,
                                                                                                    component_demand,
                                                                                                    component_localities,
                                                                                                    transfer_service_demand)[0] == component_demand
        for component in distribution_models['ElectricPower'].components:
            if component.name == 'SuperLink':
                component.set_initial_damage_level(1.0)
                component.update(1)
        assert distribution_models['ElectricPower'].modify_demand_to_account_for_resource_transfer(supplier_start_locality,
                                                                                                    supplier_end_locality,
                                                                                                    component_demand,
                                                                                                    component_localities,
                                                                                                    transfer_service_demand)[0] == math.inf

    def test_get_locality_pairs(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].get_locality_pairs(1, 1, [1, 1]) == [[1, 1]]
        assert distribution_models['ElectricPower'].get_locality_pairs(1, 2, [1, 1]) == [[1, 1], [2, 1]]
        assert distribution_models['ElectricPower'].get_locality_pairs(1, 3, [1, 2]) == [[1, 1], [1, 2], [3, 1], [3, 2]]
        assert distribution_models['ElectricPower'].get_locality_pairs(1, 3, [1, 3]) == [[1, 1], [1, 3], [3, 1], [3, 3]]
        assert distribution_models['ElectricPower'].get_locality_pairs(2, 1, [2, 2]) == [[1, 2], [2, 2]]
    
    def test_get_best_path_functionality(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].get_best_path_functionality([[1, 1]], 10) == 1.0
        assert distribution_models['ElectricPower'].get_best_path_functionality([[1, 2], [1, 3]], 10) == 1.0

        for component in distribution_models['ElectricPower'].components:
            if component.name == 'SuperLink':
                component.set_initial_damage_level(1.0)
                component.update(1)
            
        assert distribution_models['ElectricPower'].get_best_path_functionality([[1, 1], [1, 2]], 10) == 1.0
        assert distribution_models['ElectricPower'].get_best_path_functionality([[1, 2], [1, 3]], 10) == 0.0
        assert distribution_models['ElectricPower'].get_best_path_functionality([[1, 3], [2, 1]], 10) == 0.0
    
    def test_get_path_functionality(self, distribution_models: dict):
        assert distribution_models['ElectricPower'].get_path_functionality(1, 2, 10) == 1.0
        assert distribution_models['ElectricPower'].get_path_functionality(1, 1, 10) == 1.0
        assert distribution_models['ElectricPower'].get_path_functionality(1, 3, 1001) == 0.0

        for component in distribution_models['ElectricPower'].components:
            if component.name == 'SuperLink':
                component.set_initial_damage_level(1.0)
                component.update(1)
            
        assert distribution_models['ElectricPower'].get_path_functionality(1, 1, 1000) == 1.0
        assert distribution_models['ElectricPower'].get_path_functionality(1, 2, 1) == 0.0

    def test_get_optimal_path(self, distribution_models: dict):
        distribution_models['ElectricPower'].transfer_service_distribution_model.set_potential_paths('./tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_potential_path_sets.json')
        assert distribution_models['ElectricPower'].get_optimal_path(2, 1) == (1000, 0)
        assert distribution_models['Communication'].get_optimal_path(2, 1) == (math.inf, None)

        for component in distribution_models['ElectricPower'].components:
            if component.name == 'SuperLink':
                component.set_initial_damage_level(1.0)
                component.update(1)
        
        assert distribution_models['ElectricPower'].get_optimal_path(2, 1) == (0, 0)

    def test_get_scope(self, distribution_models: dict):
        distribution_models['ElectricPower'].fill_system_matrix()
        
        target_all_components_id = list(range(2 * len(distribution_models['ElectricPower'].components)))
        assert distribution_models['ElectricPower'].get_scope() == target_all_components_id

        target_locality_1_components_id = [0, 1, 2, 3, 5, 9, 11, 12, 13, 14, 16, 20]
        assert distribution_models['ElectricPower'].get_scope('Locality 1')[0].tolist() == target_locality_1_components_id

        with pytest.raises(ValueError):
            distribution_models['ElectricPower'].get_scope('1')
    
    def test_get_total_supply_no_damage(self, distribution_models: dict):
        target_supplies = [5.0, 3.0, 3.0]
        for target_supply, distribution_model in zip(target_supplies, distribution_models.values()):
            distribution_model.fill_system_matrix()
            current_supply = distribution_model.get_total_supply()
            assert math.isclose(target_supply, current_supply)

    def test_get_total_supply_with_damage(self, distribution_models: dict, system: System.System):
        target_supplies = [3.0, 1.8, 1.0]
        system.set_initial_damage()
        system.time_step = 0
        system.update()
        for target_supply, distribution_model in zip(target_supplies, distribution_models.values()):
            distribution_model.components = system.components
            distribution_model.fill_system_matrix()
            current_supply = distribution_model.get_total_supply()
            assert math.isclose(target_supply, current_supply)

    def test_get_total_demand_no_damage(self, distribution_models: dict):
        target_demands = [4.0, 1.0, 3.0]
        for target_demand, distribution_model in zip(target_demands, distribution_models.values()):
            distribution_model.fill_system_matrix()
            current_demand = distribution_model.get_total_demand()
            assert math.isclose(target_demand, current_demand)

    def test_get_total_demand_with_damage(self, distribution_models: dict, system: System.System):
        target_demands = [2.6, 1.0, 12.0]
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        for target_demand, distribution_model in zip(target_demands, distribution_models.values()):
            distribution_model.components = system.components
            distribution_model.fill_system_matrix()
            current_demand = distribution_model.get_total_demand()
            assert math.isclose(target_demand, current_demand)

    def test_get_system_consumption_no_damage(self, distribution_models: dict):
        target_consumptions = [4.0, 1.0, 3.0]
        for target_consumption, distribution_model in zip(target_consumptions, distribution_models.values()):
            distribution_model.fill_system_matrix()
            current_consumption = distribution_model.get_total_consumption()
            assert math.isclose(target_consumption, current_consumption)

    def test_get_total_consumption_with_damage(self, distribution_models: dict, system: System.System):
        target_demands = [2.6, 1.0, 1.0]
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        for target_demand, distribution_model in zip(target_demands, distribution_models.values()):
            distribution_model.components = system.components
            distribution_model.distribute()
            current_consumption = distribution_model.get_total_consumption()
            assert math.isclose(target_demand, current_consumption)

class TestUtilityDistributionModel_NorthEast_SF():

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Main.json'
    RECOVERY_RESOURCE_SUPPLY = {"FirstResponderEngineer": 5,
                                "SeniorEngineer": 4,
                                "Contractor": 10,
                                "Money": 5500000000,
                                "PlanCheckEngineeringTeam": 10,
                                "SitePreparationCrew": 10,
                                "CleanUpCrew": 10,
                                "EngineeringDesignTeam": 10,
                                "DemolitionCrew": 10,
                                "RepairCrew": 500}
   
    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        system =  main.create_system(input_dict)
        self.set_recovery_resource_supply(system.components)
        emergency_response_center = system.components[0]
        return system

    @pytest.fixture
    def distribution_models(self, system):
        distribution_models = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if 'TransferService' not in resource_name}
        return distribution_models 

    def set_recovery_resource_supply(self, components: list([Component.Component])):
        emergency_response_center = components[0]
        for resource_name, resource_amount in self.RECOVERY_RESOURCE_SUPPLY.items():
            emergency_response_center.supply['Supply'][resource_name].initial_amount = resource_amount
            emergency_response_center.supply['Supply'][resource_name].current_amount = resource_amount

    def test_housing_distribution_no_damage(self, distribution_models: dict):
        distribution_models['Shelter'].distribute()
        target_total_supply = 0
        for component in distribution_models['Shelter'].components[1:]:
            assert component.supply['Supply']['Shelter'].current_amount == component.demand['OperationDemand'][
                    'Shelter'].current_amount
            target_total_supply += component.supply['Supply']['Shelter'].current_amount
        assert target_total_supply == distribution_models['Shelter'].get_total_consumption()
        assert target_total_supply == distribution_models['Shelter'].get_total_supply()

    def test_housing_distribution_with_damage(self, system: System.System, distribution_models: dict):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        distribution_models['Shelter'].components = system.components
        distribution_models['Shelter'].distribute()
        target_total_supply = 0
        target_total_demand = 0
        for component in distribution_models['Shelter'].components[1:]:      
            if component.name in ['DS0_ResidentialBuilding', 'DS1_ResidentialBuilding']:
                assert component.supply['Supply']['Shelter'].current_amount == component.demand['OperationDemand']['Shelter'].current_amount
                target_total_supply += component.supply['Supply']['Shelter'].current_amount
            else:
                assert math.isclose(component.supply['Supply']['Shelter'].current_amount, 0)
            assert component.supply['Supply']['Shelter'].initial_amount == component.demand['OperationDemand']['Shelter'].current_amount
            target_total_demand += component.demand['OperationDemand']['Shelter'].current_amount

        assert math.isclose(target_total_supply, distribution_models['Shelter'].get_total_consumption())
        assert math.isclose(target_total_supply, distribution_models['Shelter'].get_total_supply())
        assert target_total_demand == distribution_models['Shelter'].get_total_demand()

    def test_recovery_resource_distribution_no_damage(self, distribution_models: dict):
        for resource_name, distribution_model in distribution_models.items():
            if not (resource_name == 'Shelter'):
                distribution_model.distribute()
                assert distribution_model.get_total_consumption() == 0
                assert distribution_model.get_total_demand() == 0
                assert distribution_model.get_total_supply() == self.RECOVERY_RESOURCE_SUPPLY[resource_name]

        for component in distribution_models['Shelter'].components:
            for recovery_activity in component.recovery_model.recovery_activities.values():
                assert list(recovery_activity.demand_met.values())[0] == 1.0

    def test_rapid_inspection_distribution_with_damage(self, system: System.System, distribution_models: dict):
        system.set_initial_damage()
        system.time_step = 0
        system.update()  
        buildings_need_inspection = 14
        distribution_models['FirstResponderEngineer'].components = system.components
        distribution_models['FirstResponderEngineer'].distribute()
        assert math.isclose(distribution_models['FirstResponderEngineer'].get_total_consumption(), buildings_need_inspection*0.1)
        assert math.isclose(distribution_models['FirstResponderEngineer'].get_total_demand(), buildings_need_inspection*0.1)
        assert distribution_models['FirstResponderEngineer'].get_total_supply() == self.RECOVERY_RESOURCE_SUPPLY['FirstResponderEngineer']
        
        for component in distribution_models['FirstResponderEngineer'].components:
            for recovery_activity in component.recovery_model.recovery_activities.values():
                assert list(recovery_activity.demand_met.values())[0] == 1.0

    def test_detailed_inspection_distribution_with_damage(self, system: System.System, distribution_models: dict):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        system.recover()   
        system.time_step = 2
        system.update()                         
        demand_per_component = 2
        components_with_demand_id = [component_id for component_id, component in enumerate(system.components) if 'SeniorEngineer' in component.demand['RecoveryDemand']]         
        distribution_models['SeniorEngineer'].components = system.components
        distribution_models['SeniorEngineer'].distribute()
        component_priorities, demand_type = distribution_models['SeniorEngineer'].get_component_priorities()              
        assert math.isclose(distribution_models['SeniorEngineer'].get_total_consumption(), min(self.RECOVERY_RESOURCE_SUPPLY['SeniorEngineer'], len(components_with_demand_id)*demand_per_component))
        assert math.isclose(distribution_models['SeniorEngineer'].get_total_demand(), len(components_with_demand_id)*demand_per_component)
        assert distribution_models['SeniorEngineer'].get_total_supply() == self.RECOVERY_RESOURCE_SUPPLY['SeniorEngineer']
        num_components_with_met_demand = math.floor(self.RECOVERY_RESOURCE_SUPPLY['SeniorEngineer'] / demand_per_component)
        counter = 0
        for component_offset_id in component_priorities[1:]:  
            component_id = component_offset_id - distribution_models['SeniorEngineer'].system_matrix.RECOVERY_DEMAND_ROW_OFFSET
            component = system.components[component_id]                        
            if component.name in ['DS2_ResidentialBuilding', 'DS3_ResidentialBuilding', 'DS4_ResidentialBuilding'] and component_id in components_with_demand_id:
                if counter < num_components_with_met_demand:
                    assert list(component.recovery_model.recovery_activities['DetailedInspection'].demand_met.values())[0] == 1.0
                    counter += 1
                else:
                    assert list(component.recovery_model.recovery_activities['DetailedInspection'].demand_met.values())[0] == 0.0
        
    def test_consumable_resource_with_money(self, system: System.System, distribution_models: dict):
        system.set_initial_damage()
        system.time_step = 1
        system.update()
        total_money_demand = 0
        for component in system.components:
            for recovery_activity in component.recovery_model.recovery_activities.values():
                if recovery_activity.name != 'Financing' or recovery_activity.name != 'Repair':
                    recovery_activity.level = 1.0
                elif recovery_activity.name == 'Financing':
                    total_money_demand += recovery_activity.demand['Money'].current_amount                   
        system.update()
        distribution_models['Money'].components = system.components
        distribution_models['Money'].distribute()
        target_consumed_supply = self.RECOVERY_RESOURCE_SUPPLY['Money'] - total_money_demand
        assert math.isclose(distribution_models['Money'].components[0].supply['Supply']['Money'].current_amount, target_consumed_supply)

class TestTransferServiceDistributionModelPotentialPathSet:

    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        system =  main.create_system(input_dict)
        return system
    
    @pytest.fixture
    def distribution_models(self, system):
        distribution_models = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if 'TransferService' in resource_name}
        return distribution_models

    def test_find_connecting_link(self, distribution_models: dict):
        path_string = 'from 1 to 3'
        start_locality = 1
        next_locality = 3
        distribution_models['SuperTransferService'].find_connecting_link(path_string, start_locality, next_locality)
        assert distribution_models['SuperTransferService'].potential_paths[path_string][-1][-1].locality[0] == 1
        assert distribution_models['SuperTransferService'].potential_paths[path_string][-1][-1].locality[1] == 3

        path_string = 'from 2 to 3'
        start_locality = 2
        next_locality = 3
        distribution_models['SuperTransferService'].find_connecting_link(path_string, start_locality, next_locality)
        assert distribution_models['SuperTransferService'].potential_paths[path_string][-1][-1].locality[0] == 2
        assert distribution_models['SuperTransferService'].potential_paths[path_string][-1][-1].locality[1] == 3

    def test_get_start_end_locality_from_path_string(self, distribution_models: dict):
        path_string = 'from 10 to 20'
        start_locality, end_locality = distribution_models['SuperTransferService'].get_start_end_locality_from_path_string(path_string)
        assert start_locality == 10
        assert end_locality == 20
    
    def test_get_path_supply(self, distribution_models: dict):
        path = [distribution_models['SuperTransferService'].components[2], distribution_models['SuperTransferService'].components[3]]
        assert distribution_models['SuperTransferService'].get_path_supply(path) == 1000.0
        distribution_models['SuperTransferService'].components[2].set_initial_damage_level(0.5)
        distribution_models['SuperTransferService'].components[2].update(1)
        assert distribution_models['SuperTransferService'].get_path_supply(path) == 500.0

    def test_get_optimal_path(self, distribution_models: dict):
        optimal_transfer_supply, optimal_path = distribution_models['SuperTransferService'].get_optimal_path(1, 3)
        assert optimal_transfer_supply == 1000.0
        assert optimal_path == 0

        distribution_models['SuperTransferService'].components[3].set_initial_damage_level(0.5)
        distribution_models['SuperTransferService'].components[3].update(1)
        optimal_transfer_supply, optimal_path = distribution_models['SuperTransferService'].get_optimal_path(1, 3)
        assert optimal_transfer_supply == 1000.0
        assert optimal_path == 1

        distribution_models['SuperTransferService'].components[2].set_initial_damage_level(0.5)
        distribution_models['SuperTransferService'].components[2].update(1)
        optimal_transfer_supply, optimal_path = distribution_models['SuperTransferService'].get_optimal_path(1, 3)
        assert optimal_transfer_supply == 500.0
        assert optimal_path == 0

class TestBridgeServiceDistributionModel:

    MAIN_FILE = './tests/test_inputs/test_inputs_VirtualCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.MAIN_FILE)
        system =  main.create_system(input_dict)
        return system
    
    @pytest.fixture
    def distribution_models(self, system):
        distribution_models = {resource_name: resource_parameters['DistributionModel'] for
                               resource_name, resource_parameters in system.resources.items() if 'Bridge' in resource_name}
        return distribution_models
    
    def test_find_bridges(self, distribution_models):
        distribution_models['BridgeService'].bridges = []
        distribution_models['BridgeService'].find_bridges()
        assert len(distribution_models['BridgeService'].bridges) == 4
        target_localities = [[201, 301], [301, 201], [301, 302], [302, 301]]
        for target_locality_pair, bridge in zip(target_localities, distribution_models['BridgeService'].bridges):
            assert isinstance(bridge, Component.Bridge)
            assert bridge.locality == target_locality_pair
    
    def test_map_links_to_bridges(self, distribution_models):
        distribution_models['BridgeService'].links_on_a_bridge = []
        distribution_models['BridgeService'].map_links_to_bridges()
        assert len(distribution_models['BridgeService'].links_on_a_bridge) == 4
        target_localities = [[201, 301], [301, 201], [301, 302], [302, 301]]
        for i, target_locality_pair in enumerate(target_localities):
            for link in distribution_models['BridgeService'].links_on_a_bridge[i]:
                assert link.locality == target_locality_pair
                assert not isinstance(link, Component.Bridge)
    
    def test_component_is_on_the_bridge(self, distribution_models):
        bridge = distribution_models['BridgeService'].bridges[0]
        component_on_bridge = distribution_models['BridgeService'].components[77]
        component_not_on_bridge = distribution_models['BridgeService'].components[0]
        assert distribution_models['BridgeService'].component_is_on_the_bridge(bridge, component_on_bridge) == True
        assert distribution_models['BridgeService'].component_is_on_the_bridge(bridge, component_not_on_bridge) == False
    
    def test_distribute(self, distribution_models):
        distribution_models['BridgeService'].distribute()
        target_EPTS = distribution_models['BridgeService'].components[75]
        target_PWP = distribution_models['BridgeService'].components[76]
        target_CWP = distribution_models['BridgeService'].components[77]       
        target_bridge = distribution_models['BridgeService'].bridges[0]
        assert target_CWP.supply['Supply']['CoolingWaterTransferService'].current_amount == 1000.0
        assert target_PWP.supply['Supply']['PotableWaterTransferService'].current_amount == 1000.0
        assert target_EPTS.supply['Supply']['ElectricPowerTransferService'].current_amount == 1000.0
        target_bridge.set_initial_damage_level(0.5)
        target_bridge.update(1)
        distribution_models['BridgeService'].distribute()
        assert target_CWP.supply['Supply']['CoolingWaterTransferService'].current_amount == 0
        assert target_PWP.supply['Supply']['PotableWaterTransferService'].current_amount == 0
        assert target_EPTS.supply['Supply']['ElectricPowerTransferService'].current_amount == 0
        
  
        

tests/test_ResourceDistributionModel.py
NikolaBlagojevic/pyrecodes
import pytest
import numpy as np
import math
from pyrecodes import System
from pyrecodes import SystemCreator
from pyrecodes import ComponentLibraryCreator
from pyrecodes import DamageInput
from pyrecodes import main

class TestDistributionListCreator:
    main_file = './tests/test_inputs/test_inputs_VirtualCommunity_Main.json'

    @pytest.fixture
    def distribution_list_creator(self):
        input_dict = main.read_file(self.main_file)
        system = main.create_system(input_dict)
        return System.DistributionListCreator(system.components, system.resources)
    
    def test_get_resource_distribution_dict(self, distribution_list_creator: System.DistributionListCreator):
        target_dict = {'IndependentResources': [
                        'BridgeService', 'ElectricPowerTransferService', 
                        'PotableWaterTransferService',
                        'CoolingWaterTransferService', 'Shelter'],

                       'InterdependentResources': [                       
                       'ElectricPower', 'HighLevelCommunication',
                       'LowLevelCommunication', 'PotableWater', 
                       'CoolingWater', 'FunctionalHousing',

                       'ElectricPower', 'HighLevelCommunication',
                       'LowLevelCommunication', 'PotableWater',
                       'CoolingWater', 'FunctionalHousing',

                       'ElectricPower', 'HighLevelCommunication',
                       'LowLevelCommunication', 'PotableWater',
                       'CoolingWater', 'FunctionalHousing',

                       'ElectricPower', 'HighLevelCommunication',
                       'LowLevelCommunication', 'PotableWater',
                       'CoolingWater', 'FunctionalHousing',

                       'ElectricPower', 'HighLevelCommunication',
                       'LowLevelCommunication', 'PotableWater', 
                       'CoolingWater', 'FunctionalHousing',
                       
                       'ElectricPower', 'HighLevelCommunication',
                       'LowLevelCommunication', 'PotableWater', 
                       'CoolingWater', 'FunctionalHousing']}
        
        assert distribution_list_creator.get_resource_distribution_dict() == target_dict
    
    def test_get_resource_group(self, distribution_list_creator):
        assert distribution_list_creator.get_resource_group('Utilities') == ['ElectricPower', 
                                                                             'HighLevelCommunication',
                                                                             'LowLevelCommunication',
                                                                             'PotableWater', 
                                                                             'CoolingWater', 
                                                                             'Shelter', 
                                                                             'FunctionalHousing']
        assert distribution_list_creator.get_resource_group('TransferService') == ['ElectricPowerTransferService',                                                                             
                                                                                   'PotableWaterTransferService', 
                                                                                   'CoolingWaterTransferService']
        assert distribution_list_creator.get_resource_group('BridgeService') == ['BridgeService']

    def test_get_independent_interdependent_resources(self, distribution_list_creator):
        target_interdependent_resources = ['ElectricPower', 'HighLevelCommunication', 'LowLevelCommunication', 'PotableWater', 'CoolingWater', 'FunctionalHousing']
        target_independent_resources = ['BridgeService', 'ElectricPowerTransferService', 'PotableWaterTransferService', 'CoolingWaterTransferService', 'Shelter']
        target_resource_distribution_dict = {'IndependentResources': target_independent_resources, 'InterdependentResources': target_interdependent_resources}
        resource_distribution_dict = distribution_list_creator.get_independent_interdependent_resources({'IndependentResources': ['BridgeService', 'ElectricPowerTransferService', 'PotableWaterTransferService', 'CoolingWaterTransferService'], 
                                                                                                         'InterdependentResources': []})
        assert resource_distribution_dict == target_resource_distribution_dict
    
    def test_resource_is_interdependent(self, distribution_list_creator):
        for component in distribution_list_creator.components:
            if component.name == 'ElectricPowerPlant':
                assert distribution_list_creator.resource_is_interdependent('ElectricPower', component) == True
            elif component.name == 'CoolingWaterFacility':
                assert distribution_list_creator.resource_is_interdependent('CoolingWater', component) == True
            else:
                assert distribution_list_creator.resource_is_interdependent('ElectricPower', component) == False
                assert distribution_list_creator.resource_is_interdependent('CoolingWater', component) == False
                assert distribution_list_creator.resource_is_interdependent('BridgeService', component) == False
                assert distribution_list_creator.resource_is_interdependent('Shelter', component) == False

    def test_form_resource_distribution_vector(self, distribution_list_creator):
        interdependent_resources = ['ElectricPower', 'Communication', 'CoolingWater']
        vector = distribution_list_creator.form_resource_distribution_vector(interdependent_resources)
        assert vector == ['ElectricPower', 'Communication', 'CoolingWater', 'ElectricPower', 'Communication',
                          'CoolingWater', 'ElectricPower', 'Communication', 'CoolingWater']

class TestCompleteDamageRecoveryTargetChecker:

    main_file = './tests/test_inputs/test_inputs_VirtualCommunity_Main.json'

    @pytest.fixture
    def system(self):
        input_dict = main.read_file(self.main_file)
        return main.create_system(input_dict)
    
    def test_recovery_target_met(self, system):
        system.time_step = system.START_TIME_STEP
        recovery_target_checker = System.CompleteDamageRecoveryTargetChecker()
        assert recovery_target_checker.recovery_target_met(system) == False

        not_enough_time_steps_to_recover = 10
        system.set_initial_damage()
        for time_step in range(1, not_enough_time_steps_to_recover):
            system.time_step = time_step
            system.recover()
        assert recovery_target_checker.recovery_target_met(system) == False

        enough_time_steps_to_recovery = 200
        for time_step in range(not_enough_time_steps_to_recover, enough_time_steps_to_recovery):
            system.time_step = time_step
            system.recover()
        assert recovery_target_checker.recovery_target_met(system) == True      

class TestBuiltEnvironmentSystem():

    main_file = ''

    @pytest.fixture()
    def system_creator(self) -> SystemCreator.SystemCreator:
        return SystemCreator.JSONSystemCreator()

    @pytest.fixture()
    def component_library_creator(self) -> ComponentLibraryCreator.ComponentLibraryCreator:
        return ComponentLibraryCreator.JSONComponentLibraryCreator(self.component_library_file)

    @pytest.fixture()
    def system(self) -> System.System:
        input_dict = main.read_file(self.main_file)
        system = main.create_system(input_dict)
        return system

class TestThreeLocalitiesSystem(TestBuiltEnvironmentSystem):

    main_file = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    def test_init(self, system: System.System):
        assert isinstance(system.system_configuration, dict)
        assert isinstance(system.component_library, dict)
        assert isinstance(system.system_creator, SystemCreator.SystemCreator)
    
    def test_create_system(self, system: System.System):
        assert len(system.components) == 11 # number of components in the ThreeLocalitiesCommunity 
        assert len(system.resources) == 4 # number of resources in the ThreeLocalitiesCommunity
        assert len(system.resilience_calculators) == 2 # number of resilience calculators in the ThreeLocalitiesCommunity
        assert isinstance(system.recovery_target_checker, System.CompleteDamageRecoveryTargetChecker)
        assert isinstance(system.resource_distribution_dict, dict)
        assert isinstance(system.damage_input, DamageInput.ListDamageInput)
        assert system.START_TIME_STEP == 0
        assert system.MAX_TIME_STEP == 500
        assert system.DISASTER_TIME_STEP == 1
    
    def test_start_resilience_assessment(self, system: System.System):
        system.start_resilience_assessment()
        assert system.FINISH == True
    
    def test_set_initial_damage(self, system: System.System):
        target_damage_levels = [0.0, 0.4, 0.0, 0.0, 0.4, 0.0, 0.0, 0.4, 0.4, 0.0, 0.0]
        system.set_initial_damage()
        bool_list = []
        for target_damage_level, component in zip(target_damage_levels, system.components):
            bool_list.append(component.get_damage_level() == target_damage_level)
        assert all(bool_list)
    
    def test_update(self, system: System.System):
        for component in system.components:
            assert component.functionality_level == 1

        system.time_step = system.DISASTER_TIME_STEP
        system.set_initial_damage()
        system.update()
        target_functionality_levels = [1.0, 0.6, 1.0, 1.0, 0.6, 1.0, 1.0, 0.6, 0.0, 1.0, 1.0]
        for target_functionality_level, component in zip(target_functionality_levels, system.components):
            assert component.functionality_level == target_functionality_level

    def test_distribute_resources(self, system: System.System):
        system.distribute_resources()
        target_consumptions = [4, 1, 3]
        demand_col = system.resources['ElectricPower']['DistributionModel'].system_matrix.DEMAND_COL_ID
        demand_met_col = system.resources['ElectricPower']['DistributionModel'].system_matrix.DEMAND_MET_COL_ID
        for target_consumption, resource in zip(target_consumptions, list(system.resources.values())[1:]): # start from second resource, first resource is a Transfer Service
                consumption = np.sum(np.multiply(resource['DistributionModel'].system_matrix.matrix[:, demand_col],
                                                resource['DistributionModel'].system_matrix.matrix[:, demand_met_col]))
                assert math.isclose(consumption, target_consumption, abs_tol=1e-10)

        system.time_step = system.DISASTER_TIME_STEP
        system.set_initial_damage()
        system.update()
        system.distribute_resources()
        target_consumptions = [0, 0, 0]
        for target_consumption, resource in zip(target_consumptions, list(system.resources.values())[1:]): # start from second resource, first resource is a Transfer Service
            if resource['Group'] != 'TransferService':
                consumption = np.sum(np.multiply(resource['DistributionModel'].system_matrix.matrix[:, demand_col],
                                                resource['DistributionModel'].system_matrix.matrix[:, demand_met_col]))
                print(consumption)
                assert math.isclose(consumption, target_consumption, abs_tol=1e-10)
    
    def test_get_supply_of_interdependent_resources(self, system: System.System):
        target_supply = {'ElectricPower': 5, 'Communication': 3, 'CoolingWater': 3}
        system.distribute_resources()
        supply = system.get_supply_of_interdependent_resources()
        assert supply == target_supply
    
    def test_check_system_convergence(self, system: System.System):
        system.distribute_resources()
        supply = system.get_supply_of_interdependent_resources()
        assert system.check_system_convergence(supply, supply) == True

        system.time_step = system.DISASTER_TIME_STEP
        system.set_initial_damage()
        system.update()
        system.distribute_resources()
        damaged_supply = system.get_supply_of_interdependent_resources()
        assert system.check_system_convergence(supply, damaged_supply) == False

    def test_recover(self, system: System.System):
        system.time_step = system.DISASTER_TIME_STEP
        system.set_initial_damage()
        system.recover()
        target_damage_levels = [0.0, 0.36, 0.0, 0.0, 0.36, 0.0, 0.0, 0.36, 0.36, 0.0, 0.0]        
        for target_damage_level, component in zip(target_damage_levels, system.components):
            assert component.get_damage_level() == target_damage_level
        
class TestVirtualCommunity(TestBuiltEnvironmentSystem):

    main_file = './tests/test_inputs/test_inputs_VirtualCommunity_Main.json'

    def test_distribute_interdependent_resources(self, system: System.System):
        target_supplies = [80.0, 600.0, 495.0, 0.8, 0.24, 3600.0]
        system.distribute_interdependent_resources()
        for target_supply, resource_name in zip(target_supplies, system.resource_distribution_dict['InterdependentResources']):
            assert target_supply == system.resources[resource_name]['DistributionModel'].get_total_supply()
        
        system.time_step = system.DISASTER_TIME_STEP
        system.set_initial_damage()
        system.update()
        system.distribute_interdependent_resources()
        target_supplies = [0, 0, 0, 0, 0, 0]
        for target_supply, resource_name in zip(target_supplies, system.resource_distribution_dict['InterdependentResources']):
            assert target_supply == system.resources[resource_name]['DistributionModel'].get_total_supply()

   

#         # test when only the bridge is damaged
#         damage_vector = [0.0, 0.1, 0.0, 0.0, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.distribute_bridge_transfer_service()
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 40*0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 40*0.0)


#         # test with a three localities system and multiple potential path between two localities
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         # define the dict key as a string from i-th locality to j-th locality, as a list of 
#         # localities that the path will go through
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2], [1, 3, 2]], 'from 2 to 1': [[2, 1], [2, 3, 1]],
#                                                                         'from 2 to 3': [[2, 1, 3], [2, 3]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                                           'from 2 to 3': [[2, 3], [2, 1, 3]], 'from 3 to 2': [[3, 2], [3, 1, 2]]}
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                                           'from 2 to 3': [[2, 3], [2, 1, 3]], 'from 3 to 2': [[3, 2], [3, 1, 2]]}
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         # test create potential paths for undamaged two localities system
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][0][0], Component.CWP))
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][1][0], Component.CWP))
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][1][1], Component.CWP))
#         self.assertEqual(len(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][1]), 2)
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][0][0], Component.CWP))
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][0][1], Component.CWP))
#         self.assertEqual(len(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][1]), 1)
#         self.assertEqual(len(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][0]), 2)
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 1'][0][0], Component.EPTL))
#         self.assertTrue(isinstance(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 1 to 2'][0][0], Component.EPTL))
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 40)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 40)
#         self.assertEqual(self.sys_test.potential_paths_links['PotableWaterTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['PotableWaterTransferService'], 0.8)
#         self.assertEqual(self.sys_test.potential_paths_links['PotableWaterTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['PotableWaterTransferService'], 0.8)
#         # create the damage vector to test the damaged potential paths
#         damage_vector = [0.0, 0.0, 0.0, 0.3, 0.0, 0.4, 0.01, 0.01, 0.0, 0.0, 0.0, 0.0, 0.35, 0.0, 0.0, 0.0, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         # test the damaged potential paths   - Cooling Water      
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.7)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][1][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.6)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 1 to 2'][1][1].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.65)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.7)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 1'][1][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.65)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 1'][1][1].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.6)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][0][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.7)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][0][1].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.6)
#         self.assertEqual(self.sys_test.potential_paths_links['CoolingWaterTransferService']['from 2 to 3'][1][0].supply['transfer_services'].current['CoolingWaterTransferService'], 0.8*0.65)
#         # test the damaged paths - Electric Power
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 1 to 2'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 1'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 3'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 40.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 3'][1][0].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 2 to 3'][1][1].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 3 to 2'][0][0].supply['transfer_services'].current['ElectricPowerTransferService'], 40.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 3 to 2'][1][0].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 3 to 2'][1][1].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)
#         self.assertEqual(self.sys_test.potential_paths_links['ElectricPowerTransferService']['from 3 to 2'][1][0].supply['transfer_services'].current['ElectricPowerTransferService'], 0.0)

#     def test_get_optimal_path(self):
#         # test the algorithm that finds the optimal path from the list of potential paths
#         self.sys_test.create_system(self.two_comp_sys_wlinks)
#         potential_paths = dict()
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]]} 
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]]} 
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         # test the two localities undamaged system
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8)
#         self.assertEqual(opt_path, 0)        
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8)
#         self.assertEqual(opt_path, 0)        
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0)                
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0)        
#         # test damaged system        
#         damage_vector = [0.4, 0.0, 0.4, 0.4, 0.4, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8*0.6)
#         self.assertEqual(opt_path, 0)        
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8*0.6)
#         self.assertEqual(opt_path, 0)        
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, 0)                
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, 0)

#         # test the system with three localities
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         # define the dict key as a string from i-th locality to j-th locality, as a list of 
#         # localities that the path will go through
#         self.sys_test.potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2], [1, 3, 2]], 'from 2 to 1': [[2, 1], [2, 3, 1]],
#                                                                         'from 2 to 3': [[2, 1, 3], [2, 3]]}
#         self.sys_test.potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                                           'from 2 to 3': [[2, 3], [2, 1, 3]], 'from 3 to 2': [[3, 2], [3, 1, 2]]}
#         self.sys_test.create_potential_paths()
#         # test undamaged system
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8)
#         self.assertEqual(opt_path, 0)        
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8)
#         self.assertEqual(opt_path, 0)           
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 3, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, None) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 3, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8)
#         self.assertEqual(opt_path, 0) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0)                
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0)
#         opt_cost, opt_path = self.sys_test.get_optimal_path(3, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 3, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, None) 
#         # create the damage vector to test the damaged potential paths
#         damage_vector = [0.0, 0.0, 0.0, 0.3, 0.0, 0.4, 0.01, 0.01, 0.0, 0.0, 0.0, 0.0, 0.35, 0.0, 0.0, 0.0, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         # test the damaged potential paths - Cooling Water       
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8*0.7)
#         self.assertEqual(opt_path, 0)        
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8*0.7)
#         self.assertEqual(opt_path, 0)           
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 3, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, None) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 3, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.65*0.8)
#         self.assertEqual(opt_path, 1) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, 0)                
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 1, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, 0)
#         opt_cost, opt_path = self.sys_test.get_optimal_path(3, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 3, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, None)

#         # test when the bridge is damaged
#         damage_vector = np.zeros(17)
#         damage_vector[2] = 0.01
#         damage_vector[4] = 0.01
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.distribute_bridge_transfer_service()
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, 0) 
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 2, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, 0)
#         opt_cost, opt_path = self.sys_test.get_optimal_path(1, 3, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 0.0)
#         self.assertEqual(opt_path, None)
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 3, 'CoolingWaterTransferService')
#         self.assertEqual(opt_cost, 0.8)
#         self.assertEqual(opt_path, 1)
#         opt_cost, opt_path = self.sys_test.get_optimal_path(2, 3, 'ElectricPowerTransferService')
#         self.assertEqual(opt_cost, 40)
#         self.assertEqual(opt_path, 0)


#     def test_resource_distribution_two_localities(self):            
#         # test when system is undamaged
#         self.sys_test.create_system(self.two_comp_sys_wlinks)
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.priorities = Priorities.PrioritiesTwoCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         # set the potential paths
#         potential_paths = dict()
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]]}
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]]}
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         ep_id = 0
#         self.sys_test.resource_distribution(ep_id)
#         # nothing should change in the system matrix
#         epp_vector = [1, 1, 0.0, 1.0, 1]
#         epp_supply = [40, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, :5] == epp_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 10:15] == epp_supply))
#         bsu_vector = [2, 2, 0, 1, 6]
#         bsu_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu_demand = [7.7, 0.0, 33.3, 0.086, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, :5] == bsu_vector))      
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 5:10] == bsu_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 10:15] == bsu_supply))   
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))

#         # test when two comp system is damage - the damage is low enough so that the demand is still met        
#         damage_vector = [0.4, 0.0, 0.0, 0.0, 0.4]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()            
#         self.sys_test.resource_distribution(ep_id)
#         epp_vector = [1, 1, 0.4, 0.6, 1]
#         epp_supply = [40*0.6, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, :5] == epp_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 10:15] == epp_supply))
#         bsu_vector = [2, 2, 0.4, 0.0, 6]
#         bsu_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu_demand = [7.7*0.0, 0.0, 33.3, 0.086*0.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, :5] == bsu_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 5:10] == bsu_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 10:15] == bsu_supply))     
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))

#         # test when two comp system is damage - the damage is high enough so that the demand is not met
#         damage_vector = [0.9, 0.0, 0.0, 0.0, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(ep_id)
#         epp_vector = [1, 1, 0.9, 0.1, 1]
#         epp_supply = [40*0.1, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, :5] == epp_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 10:15] == epp_supply))
#         bsu_vector = [2, 2, 0, 1, 6]
#         bsu_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu_demand = [7.7, 0.0, 33.3, 0.086, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, :5] == bsu_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 5:10] == bsu_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 10:15] == bsu_supply))     
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))
#         # demand met indicator for ep for bsu should be zero
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))

#         # test with multiple users and one supplier
#         # add one more user to the input dict
#         self.two_comp_sys_wlinks[1] = {'LocalityID': 2,
#                                 'Coord. X': 5,
#                                 'Coord. Y': 5,
#                                 'Content': {'BSU': 2},
#                                 # LinkTo field determines which link types is connected to which locality_id
#                                 # here locality 1 is connected to locality 2 with one CWP and one EPTL
#                                 'LinkTo': {'CWP': [1], 'EPTL': [1]},
#                                 'BridgeTo': {'CWP': [1],
#                                            'EPTL': [1],
#                                            }} 
#         self.sys_test.create_system(self.two_comp_sys_wlinks)
#         # recreate potential paths - because the components have changed
#         self.sys_test.create_potential_paths()
#         # first test with low damage - there should be enough supply
#         damage_vector = [0.1, 0.0, 0.0, 0.0, 0.4, 0.3]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.priorities = Priorities.PrioritiesTwoCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         self.sys_test.resource_distribution(ep_id)
#         epp_vector = [1, 1, 0.1, 0.9, 1]
#         epp_supply = [40*0.9, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, :5] == epp_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 10:15] == epp_supply))
#         bsu1_vector = [2, 2, 0.4, 0.0, 6]
#         bsu1_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu1_demand = [7.7*0.0, 0.0, 33.3, 0.086*0.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, :5] == bsu1_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 5:10] == bsu1_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 10:15] == bsu1_supply))  
#         bsu2_vector = [2, 2, 0.3, 1.0, 6]
#         bsu2_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu2_demand = [7.7*1.0, 0.0, 33.3, 0.086*1.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][5, :5] == bsu2_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][5, 5:10] == bsu2_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][5, 10:15] == bsu2_supply))
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))
#         # now test with high damage - no user gets enough ep
#         damage_vector = [0.9, 0.0, 0.0, 0.0, 0.1, 0.1]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(ep_id)
#         epp_vector = [1, 1, 0.9, 0.1, 1]
#         epp_supply = [40*0.1, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, :5] == epp_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 10:15] == epp_supply))
#         bsu1_vector = [2, 2, 0.1, 1.0, 6]
#         bsu1_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu1_demand = [7.7*1.0, 0.0, 33.3, 0.086*1.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, :5] == bsu1_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 5:10] == bsu1_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4, 10:15] == bsu1_supply))  
#         bsu2_vector = [2, 2, 0.1, 1.0, 6]
#         bsu2_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu2_demand = [7.7*1.0, 0.0, 33.3, 0.086*1.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][5, :5] == bsu2_vector))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][5, 5:10] == bsu2_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][5, 10:15] == bsu2_supply))
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][4:, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))

#         # test with damaged links - damaged CWP
#         damage_vector = [0.0, 0.0, 1.0, 0.0, 0.0, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix() 
#         # distribute power - everything should still be working since only the CWP is damaged
#         self.sys_test.resource_distribution(ep_id)
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))
#         # set the damage to the EPTL - minimal damage should still cause a loss of transfer supply
#         damage_vector = [0.0, 0.0, 0.0, 0.01, 0.0, 0.0]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(ep_id)
#         # the demand of BSUs for power should not be met
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][-2:, self.sys_test.system_matrix_demand_met_col_ids[0]] == 0.0))
#         # repair the system once and check again - now the EPTL should be functional
#         self.sys_test.repair()
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(ep_id)
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][-2:, self.sys_test.system_matrix_demand_met_col_ids[0]] == 1.0))       


#     def test_resource_distribution_five_components(self):             
#         # test with multiple suppliers and multiple users - 5-comp system
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.priorities = Priorities.PrioritiesFiveCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         potential_paths = dict()
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2], [1, 3, 2]], 'from 2 to 1': [[2, 1], [2, 3, 1]],
#                                                           'from 1 to 3': [[1, 3], [1, 2, 3]], 'from 3 to 1': [[3, 1], [3, 2, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2], [1, 3, 2]], 'from 2 to 1': [[2, 1], [2, 3, 1]],
#                                                           'from 1 to 3': [[1, 3], [1, 2, 3]], 'from 3 to 1': [[3, 1], [3, 2, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2], [1, 3, 2]], 'from 2 to 1': [[2, 1], [2, 3, 1]],
#                                                           'from 1 to 3': [[1, 3], [1, 2, 3]], 'from 3 to 1': [[3, 1], [3, 2, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         # distribute all rss 
#         ep_id = 0
#         self.sys_test.resource_distribution(ep_id)
#         llc_id = 2
#         self.sys_test.resource_distribution(llc_id)
#         cw_id = 4
#         self.sys_test.resource_distribution(cw_id)
#         # nothing should change in the system matrix - no damage
#         # only testing EPP and BSU
#         epp_vector = [1, 1, 0.0, 1.0, 1]
#         epp_supply = [40, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, :5] == epp_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, 10:15] == epp_supply))
#         bsu_vector = [3, 3, 0, 1, 6]
#         bsu_supply = [0, 0.0, 0.0, 0.0, 0.0]
#         bsu_demand = [7.7, 0.0, 33.3, 0.086, 0.0]        
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][16, :5] == bsu_vector))      
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][16, 5:10] == bsu_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][16, 10:15] == bsu_supply))   
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))        

#         # test with damaged community - EPP is damaged
#         damage_vector = np.zeros(len(self.sys_test.components))
#         damage_vector[0] = 0.999
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.time_step = 2
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         # test the system matrix
#         epp_vector = [1, 1, 0.999, 0.001, 1]
#         epp_supply = [0.001 * 40, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.2, 0.0, 0.001, 0.0, 0.05]  
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, :5] == epp_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, 10:15] == epp_supply))
#         bts_1_vector = [1, 1, 0.0, 1.0, 3]
#         bts_1_supply = [0.0, 0.0, 45.0, 0.0, 0.0]
#         bts_1_demand = [0.1, 50.0, 0.0, 0.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][1, :5] == bts_1_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][1, 5:10] == bts_1_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][1, 10:15] == bts_1_supply))   
#         # test rs distribution
#         # distribute EP
#         ep_id = 0
#         self.sys_test.resource_distribution(ep_id, consider_interdep = True)     
#         # check the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][[0, 1, 10, 15, 16], self.sys_test.system_matrix_demand_met_col_ids[0]] == 0.0))

#         # test damaged links - damage the bridge from 1 to 2
#         damage_vector = np.zeros(len(self.sys_test.components))
#         damage_vector[2] = 0.01
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         # distribute EP
#         ep_id = 0
#         self.sys_test.resource_distribution(ep_id, consider_interdep = True)  
#         # because the paths are redundant, even though the bridge is damaged, the demand of all components should be met
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))      

#         # damage the CWP from 1 to 2 - nothing should change
#         damage_vector = np.zeros(len(self.sys_test.components))
#         damage_vector[3] = 1.0
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()       
#         # distribute CW
#         cw_id = 4
#         self.sys_test.resource_distribution(cw_id, consider_interdep = True)
#         # test the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))      

#         # change potential paths - reduce redundancy
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         # damage the CWP from 1 to 2
#         damage_vector = np.zeros(len(self.sys_test.components))
#         damage_vector[3] = 1.0
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()  
#         # EPP should not get CW - CWP is too damaged
#         self.sys_test.resource_distribution(cw_id, consider_interdep = True)
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, self.sys_test.system_matrix_demand_met_col_ids[cw_id]] == 0.0))   

#         damage_vector = np.zeros(len(self.sys_test.components))
#         damage_vector[3] = 0.5
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         # self.sys_test.create_potential_paths()
#         self.sys_test.assemble_system_matrix()  
#         # EPP should get CW - CWP is damaged but its transfer supply is still laarger than the demand
#         self.sys_test.resource_distribution(cw_id, consider_interdep = True)
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, self.sys_test.system_matrix_demand_met_col_ids[cw_id]] == 1.0)) 

#         # damage the bridge from 1 to 2
#         damage_vector = np.zeros(len(self.sys_test.components))
#         damage_vector[2] = 1.0
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.distribute_bridge_transfer_service()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(cw_id, consider_interdep = True)
#         # EPP should not get CW
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, self.sys_test.system_matrix_demand_met_col_ids[cw_id]] == 0.0))  

#         # continue writing test for situations you find out are sketchy


#     def test_get_supply_capacities(self):
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         self.sys_test.assemble_system_matrix()
#         expected_supply_caps = [40, 0, 45, 0.2, 0.06]
#         supply_capacities = self.sys_test.get_supply_capacities()
#         self.assertTrue(np.all(expected_supply_caps == supply_capacities))

#         # test when system is damaged
#         damage_vector = np.zeros(len(self.sys_test.components))
#         # damage only the facilities
#         damage_vector[0] = 0.5
#         damage_vector[1] = 0.1
#         damage_vector[10] = 0.5
#         damage_vector[15] = 0.1
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()        
#         expected_supply_caps = [20, 0.0, 0.0, 0.0, 0.03]
#         supply_capacities = np.round(self.sys_test.get_supply_capacities(), 5)
#         self.assertTrue(np.all(expected_supply_caps == supply_capacities))

#     def test_get_total_demand(self):
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         self.sys_test.assemble_system_matrix()
#         expected_demands = [8.3, 50, 33.302, 0.086, 0.05]
#         total_demand = np.round(self.sys_test.get_total_demand(), 5)        
#         self.assertTrue(np.all(expected_demands == total_demand))

#         # test when system is damaged
#         damage_vector = np.zeros(len(self.sys_test.components))
#         # damage only the facilities
#         damage_vector[0] = 0.5
#         damage_vector[1] = 0.1
#         damage_vector[10] = 0.5
#         damage_vector[15] = 0.1
#         damage_vector[16] = 0.1
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         expected_demands = [8.1, 0.0, 33.302, 0.086, 0.05]        
#         total_demand = np.round(self.sys_test.get_total_demand(), 5)   
#         self.assertTrue(np.all(expected_demands == total_demand))

#         # test when llc surges
#         self.sys_test.time_step = 2
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         expected_demands = [8.1, 0.0, 333.002, 0.086, 0.05]        
#         total_demand = np.round(self.sys_test.get_total_demand(), 5)   
#         self.assertTrue(np.all(expected_demands == total_demand))

#     def test_get_total_consumption(self):
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         self.sys_test.assemble_system_matrix()
#         expected_cons = [8.3, 50, 33.302, 0.086, 0.05]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))

#         # test when system is damaged
#         damage_vector = np.zeros(len(self.sys_test.components))
#         # damage only the facilities
#         damage_vector[0] = 0.5
#         damage_vector[1] = 0.1
#         damage_vector[10] = 0.5
#         damage_vector[15] = 0.1
#         damage_vector[16] = 0.4
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.priorities = Priorities.PrioritiesFiveCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         # set up potential paths
#         potential_paths = dict()
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         ep_id = 0
#         self.sys_test.resource_distribution(ep_id, consider_interdep = True)  
#         # test consumption after distributing electricity - for this level of damage, all demands should be met
#         expected_cons = [0.4, 0.0, 33.302, 0.0, 0.05]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))        
#         cw_id = 4
#         self.sys_test.resource_distribution(cw_id, consider_interdep = True)
#         # test consumption after distributing CW - EPP should not get enough
#         expected_cons = [0.4, 0.0, 33.302, 0.0, 0.0]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))       
#         # test the consumption after distributing LLC - 
#         llc_id = 2
#         self.sys_test.resource_distribution(llc_id, consider_interdep = True) 
#         expected_cons = [0.4, 0.0, 0.0, 0.0, 0.0]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))       

#         # test when links are damaged

#         # damage EPTLs
#         damage_vector = np.zeros(len(self.sys_test.components))
#         # damage EPTL from 1 to 2
#         damage_vector[6] = 0.99
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         # should have no effect on CWF and LLC
#         self.sys_test.resource_distribution(cw_id, consider_interdep = True)
#         self.sys_test.resource_distribution(llc_id, consider_interdep = True) 
#         expected_cons = [8.3, 50, 33.302, 0.086, 0.05]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))
#         # should prevent EP from reaching CWF at locality 2
#         self.sys_test.resource_distribution(ep_id, consider_interdep = True)
#         expected_cons = [8.1, 50, 33.302, 0.086, 0.05]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))

#         # damage also the EPTL from 2 to 3 - should have no effect
#         damage_vector[13] = 0.01
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(ep_id, consider_interdep = True)
#         expected_cons = [8.1, 50, 33.302, 0.086, 0.05]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))

#         # damage the EPTL from 1 to 3 - should prevent EP from reaching PWF and BSU
#         damage_vector[7] = 0.01
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.assemble_system_matrix()
#         self.sys_test.resource_distribution(ep_id, consider_interdep = True)
#         expected_cons = [0.3, 50, 33.302, 0.086, 0.05]
#         total_cons = np.round(self.sys_test.get_total_consumption(), 5)        
#         self.assertTrue(np.all(expected_cons == total_cons))       


#     def test_interdependency_modelling(self):
#         # add a BSC to the five comp system - to enable the functioning of BTS
#         # add one more CWF - to meet the demand of the BSC
#         self.five_comp_sys_wlinks[1] = {'LocalityID': 2,
#                                 'Coord. X': 0,
#                                 'Coord. Y': 0,
#                                 'Content': {'CWF': 2,
#                                             'BSC': 1},
#                                 'LinkTo': {'CWP': [1, 3],
#                                            'EPTL': [1, 3],
#                                            'PWP': [1, 3]},
#                                 'BridgeTo': {'CWP': [1, 3],
#                                            'EPTL': [1, 3]}}
#         # test interdependency modelling with the five_comp system
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         self.sys_test.priorities = Priorities.PrioritiesFiveCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         # set up potential paths
#         potential_paths = dict()
#         potential_paths['CoolingWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['ElectricPowerTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         potential_paths['PotableWaterTransferService'] = {'from 1 to 2': [[1, 2]], 'from 2 to 1': [[2, 1]],
#                                                           'from 1 to 3': [[1, 3]], 'from 3 to 1': [[3, 1]],
#                                                           'from 2 to 3': [[2, 3]], 'from 3 to 2': [[3, 2]]}
#         self.sys_test.set_potential_paths(potential_paths)
#         self.sys_test.create_potential_paths()
#         self.sys_test.interdependency_modelling(consider_interdep = True)
#         # there should be enough supply for all components - check the demand met columns
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][:, self.sys_test.system_matrix_demand_met_col_ids] == 1.0))

#         # damage the bridge from 1 to 3 - should prevent the functionality of PWF and BSU
#         damage_vector = np.zeros(len(self.sys_test.components))
#         # damage EPTL from 1 to 2
#         damage_vector[4] = 0.1
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.interdependency_modelling(consider_interdep = True)
#         # BSU should have no power and potable water
#         ep_id = 0
#         pw_id = 3        
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][18, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][18, self.sys_test.system_matrix_demand_met_col_ids[pw_id]] == 0.0))
#         # PWF should have no power
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][17, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))

#         # damage the CWP from 1 to 2 - should prevent the EPP from getting CWF and cause chaos (components do not have power)
#         damage_vector = np.zeros(len(self.sys_test.components))   
#         damage_vector[3] = 0.99 # CWP will have a transfer capacity of 0.008
#         self.sys_test.set_damage(damage_vector)      
#         self.sys_test.update()
#         self.sys_test.interdependency_modelling(consider_interdep = True)
#         # test whether the demands are met
#         cw_id = 4
#         llc_id = 2
#         hlc_id = 1
#         # EPP should not get CW, and therefore the BSC won't get power and EPP won't also get LLC
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, self.sys_test.system_matrix_demand_met_col_ids[cw_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][0, self.sys_test.system_matrix_demand_met_col_ids[llc_id]] == 0.0))
#         # BTS should not get power and HLC
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][1, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][1, self.sys_test.system_matrix_demand_met_col_ids[hlc_id]] == 0.0))
#         # BSC should not get EP and CW
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][12, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][12, self.sys_test.system_matrix_demand_met_col_ids[cw_id]] == 0.0))
#         # CWFs should not get EP and LLC
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][10, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][10, self.sys_test.system_matrix_demand_met_col_ids[llc_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][11, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][11, self.sys_test.system_matrix_demand_met_col_ids[llc_id]] == 0.0))
#         # PWF should not get EP 
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][17, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         # BSU should not get EP, LLC and PW
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][18, self.sys_test.system_matrix_demand_met_col_ids[ep_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][18, self.sys_test.system_matrix_demand_met_col_ids[llc_id]] == 0.0))
#         self.assertTrue(np.all(self.sys_test.system_matrix[0][18, self.sys_test.system_matrix_demand_met_col_ids[pw_id]] == 0.0))

#     def test_check_recovery_target(self):
#         self.sys_test.create_system(self.five_comp_sys)
#         # when initialized the recovery traget should be met - all components are initialized with 0 damage level
#         self.assertTrue(self.sys_test.check_recovery_target())

#         # test when system is damaged
#         damage_vector = np.zeros(len(self.sys_test.components))
#         # damage one facility
#         damage_vector[0] = 0.5

#         self.sys_test.set_damage(damage_vector)     
#         self.sys_test.update() 
#         self.assertTrue(not(self.sys_test.check_recovery_target()))

#         # repair the system and then test the condition - assumed repair rate is 0.1, so it recovers in 4 time steps
#         for i in range(50):
#             self.sys_test.repair()

#         self.assertTrue(self.sys_test.check_recovery_target())

#     def test_recovery_simulation(self):
#         self.sys_test.create_system(self.five_comp_sys)
#         self.sys_test.priorities = Priorities.PrioritiesFiveCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         damage_vector = [0.4, 0.0, 0.4, 0.4, 0.4]
#         self.sys_test.recovery_simulation(damage_vector, plot_LoR=False)
#         self.assertEqual(len(self.sys_test.system_matrix), 7)        
#         # TODO: Implement more tests - test the system matrix at each time_step!


#     def test_calculate_LoR(self):
#         self.sys_test.create_system(self.five_comp_sys)
#         self.sys_test.priorities = Priorities.PrioritiesFiveCompSys()
#         self.sys_test.priorities.set_all_priorities(self.sys_test)
#         damage_vector = [0.4, 0.0, 0.4, 0.4, 0.4]
#         self.sys_test.recovery_simulation(damage_vector, plot_LoR = False)

#         # TODO: test the LoR values for the five component community without links

#     def test_create_links(self):
#         self.sys_test.create_system(self.two_comp_sys_wlinks)        
#         cwp = [self.sys_test.components[i] for i,component in enumerate(self.sys_test.components) if type(component)==Component.CWP]
#         eptl = [self.sys_test.components[i] for i,component in enumerate(self.sys_test.components) if type(component)==Component.EPTL]
#         # check if there is only one CWP - there is no duplication of links
#         self.assertEqual(len(cwp), 1)
#         self.assertEqual(len(eptl), 1)
#         # check link type  
#         # get the variables from the list
#         cwp = cwp[0]
#         eptl = eptl[0]
#         self.assertTrue(Component.CWP == type(cwp))
#         self.assertTrue(Component.EPTL == type(eptl))
#         self.assertTrue(cwp.start_locality == 1)
#         self.assertTrue(cwp.end_locality == 2)
#         self.assertTrue(eptl.start_locality == 1)
#         self.assertTrue(eptl.end_locality == 2)
#         # TODO: Test with five comp system

#     def test_assemble_transfer_supply_matrix(self):
#         self.sys_test.create_system(self.two_comp_sys_wlinks)
#         self.sys_test.assemble_transfer_supply_matrix()
#         transfer_supply_matrix = self.sys_test.transfer_supply_matrix[0]
#         # test whether the matrix picked up all the considered transfer services
#         self.assertTrue(np.all([transfer_service in transfer_supply_matrix for transfer_service in self.sys_test.considered_transfer_services]))
#         self.assertTrue(np.shape(transfer_supply_matrix['ElectricPowerTransferService']) == (2, 2))
#         # test whether diagonal values are infinite
#         self.assertTrue(np.all([transfer_supply_matrix['ElectricPowerTransferService'][0, 0] == math.inf,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 1] == math.inf]))
#         # test offdiagonal values
#         self.assertTrue(np.all([transfer_supply_matrix['ElectricPowerTransferService'][0, 1] == 5,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 0] == 5]))
#         # test the offdiagonal values
#         self.assertTrue(np.all([transfer_supply_matrix['CoolingWaterTransferService'][0, 1] == 3,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][1, 0] == 3]))
#         # test five comp sys with links
#         self.sys_test.create_system(self.five_comp_sys_wlinks)
#         self.sys_test.assemble_transfer_supply_matrix()
#         transfer_supply_matrix = self.sys_test.transfer_supply_matrix[0]
#         self.assertTrue(np.shape(transfer_supply_matrix['ElectricPowerTransferService']) == (3, 3))
#         # test offdiagonal values
#         self.assertTrue(np.all([transfer_supply_matrix['ElectricPowerTransferService'][0, 1] == 5,                              
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 0] == 5,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][2, 0] == 5,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][0, 2] == 5,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 2] == 5,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][2, 1] == 5,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][0, 0] == math.inf,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 1] == math.inf,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][2, 2] == math.inf]))
#         # test the offdiagonal values
#         self.assertTrue(np.all([transfer_supply_matrix['CoolingWaterTransferService'][0, 1] == 3,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][1, 0] == 3]))

#         # test when system is damaged
#         # set all components to 0.4 damage level
#         damage_vector = [0.4 for _ in self.sys_test.components]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.update()
#         self.sys_test.assemble_transfer_supply_matrix()
#         transfer_supply_matrix = self.sys_test.transfer_supply_matrix[0]
#         # test EPTLs - binary links
#         self.assertTrue(np.all([transfer_supply_matrix['ElectricPowerTransferService'][0, 1] == 0,                              
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 0] == 0,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][2, 0] == 0,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][0, 2] == 0,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 2] == 0,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][2, 1] == 0,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][0, 0] == math.inf,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][1, 1] == math.inf,
#                                 transfer_supply_matrix['ElectricPowerTransferService'][2, 2] == math.inf]))

#         # test CWPs - linear links
#         self.assertTrue(np.all([transfer_supply_matrix['CoolingWaterTransferService'][0, 1] == 3*0.6,                              
#                                 transfer_supply_matrix['CoolingWaterTransferService'][1, 0] == 3*0.6,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][2, 0] == 3*0.6,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][0, 2] == 3*0.6,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][1, 2] == 3*0.6,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][2, 1] == 3*0.6,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][0, 0] == math.inf,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][1, 1] == math.inf,
#                                 transfer_supply_matrix['CoolingWaterTransferService'][2, 2] == math.inf]))

# =============================================================================
#     def test_interdependency_modelling_irecodes_small_example(self):
#         # TODO: Needs to be properly initialized - component properties need to be set
#         # test damaged system
#         damage_vector = [0.4, 0.0, 0.4, 0.4, 0.4]
#         self.sys_test.set_damage(damage_vector)
#         self.sys_test.time_step = 2
#         self.sys_test.update()        
#         self.sys_test.interdependency_modelling()
#         # test the system matrix - following the example from the iReCoDeS paper
#         epp_vector = [1, 1, 0.4, 0.6, 1]
#         epp_supply = [0.0, 0.0, 0.0, 0.0, 0.0]
#         epp_demand = [0.0, 0.0, 1.0, 0.0, 1.0]  
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, :5] == epp_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, 5:10] == epp_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][0, 10:15] == epp_supply))
#         bts_1_vector = [1, 1, 0.0, 1.0, 3]
#         bts_1_supply = [0.0, 0.0, 0.0, 0.0, 0.0]
#         bts_1_demand = [1.0, 0.0, 0.0, 0.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][1, :5] == bts_1_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][1, 5:10] == bts_1_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][1, 10:15] == bts_1_supply))
#         cwf_vector = [2, 2, 0.4, 0.6, 5]
#         cwf_supply = [0.0, 0.0, 0.0, 0.0, 0.0]
#         cwf_demand = [1.0, 0.0, 1.0, 0.0, 0.0]        
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][2, :5] == cwf_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][2, 5:10] == cwf_demand))
#         self.assertTrue(np.all(np.round(self.sys_test.system_matrix[2][2, 10:15], 10) == cwf_supply))
#         bts_2_vector = [3, 3, 0.4, 0.0, 4]
#         bts_2_supply = [0.0, 0.0, 0.0, 0.0, 0.0]
#         bts_2_demand = [0.0, 0.0, 0.0, 0.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][3, :5] == bts_2_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][3, 5:10] == bts_2_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][3, 10:15] == bts_2_supply))
#         bsu_vector = [3, 3, 0.4, 0.6, 6]
#         bsu_supply = [0.0, 0.0, 0.0, 0.0, 0.0]
#         bsu_demand = [0.6, 0.0, 10.0, 0.0, 0.0]
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][4, :5] == bsu_vector))   
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][4, 5:10] == bsu_demand))
#         self.assertTrue(np.all(self.sys_test.system_matrix[2][4, 10:15] == bsu_supply))       
# =============================================================================


tests/test_System.py
NikolaBlagojevic/pyrecodes
import pytest
import math
import copy
from pyrecodes import SystemCreator
from pyrecodes import ComponentLibraryCreator
from pyrecodes import Component
from pyrecodes import ComponentRecoveryModel
from pyrecodes import ResourceDistributionModel
from pyrecodes import ResilienceCalculator
from pyrecodes import main


class TestThreeLocalitiesJSONSystemCreator():
    MAIN_FILE = './tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_Main.json'

    @pytest.fixture()
    def json_system_creator(self) -> SystemCreator.SystemCreator:
        return SystemCreator.JSONSystemCreator()

    @pytest.fixture()
    def component_library(self) -> ComponentLibraryCreator.JSONComponentLibraryCreator:
        input_dict = main.read_file(self.MAIN_FILE)
        return main.form_component_library(input_dict)
    
    @pytest.fixture()
    def system_configuration(self) -> dict:
        input_dict = main.read_file(self.MAIN_FILE)
        system_configuration = main.read_file(input_dict['System']['SystemConfigurationFile'])
        return system_configuration
    
    def test_init(self, json_system_creator: SystemCreator.SystemCreator):
        assert json_system_creator.component_library == {}
        assert json_system_creator.system_configuration == {}
        assert json_system_creator.components == []
    
    def test_setup(self, json_system_creator: SystemCreator.SystemCreator,
                   component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        assert isinstance(json_system_creator.component_library, dict)
        assert isinstance(json_system_creator.system_configuration, dict)
        assert json_system_creator.START_TIME_STEP == 0
        assert json_system_creator.DISASTER_TIME_STEP == 1
        assert json_system_creator.MAX_TIME_STEP == 500

    def test_get_damage_input_type(self, json_system_creator: SystemCreator.SystemCreator,
                                   component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        assert json_system_creator.get_damage_input_type() == "ListDamageInput"

    def test_get_damage_input_parameters(self, json_system_creator: SystemCreator.SystemCreator,
                                     component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        assert json_system_creator.get_damage_input_parameters() == [0.0, 0.4, 0.0, 0.0, 0.4, 0.0, 0.0, 0.4, 0.4, 0.0,
                                                                     0.0]
    
    def test_get_resource_distribution_parameters(self, json_system_creator: SystemCreator.SystemCreator,
                                                  component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        assert len(json_system_creator.get_resource_distribution_parameters()) == 4
        assert isinstance(json_system_creator.get_resource_distribution_parameters(), dict)

    def test_get_resource_parameters(self, json_system_creator: SystemCreator.SystemCreator,
                                     component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        components = json_system_creator.create_components()
        resource_parameters = json_system_creator.get_resource_parameters(components)
        assert len(resource_parameters) == 4
        assert isinstance(resource_parameters, dict)
        assert list(resource_parameters.keys()) == ['SuperTransferService', 'ElectricPower', 'CoolingWater', 'Communication']
    
    def test_get_transfer_services(self, json_system_creator: SystemCreator.SystemCreator,
                                   component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        components = json_system_creator.create_components()
        all_resources_parameters = json_system_creator.get_resource_distribution_parameters()
        transfer_services = json_system_creator.get_transfer_services(components, all_resources_parameters)
        assert len(transfer_services) == 1
        assert isinstance(transfer_services, dict)
        assert list(transfer_services.keys()) == ['SuperTransferService']
        assert isinstance(transfer_services['SuperTransferService']['DistributionModel'], ResourceDistributionModel.TransferServiceDistributionModelPotentialPathSets)
    
    def test_get_nontransfer_services(self, json_system_creator: SystemCreator.SystemCreator,
                                   component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        components = json_system_creator.create_components()
        all_resources_parameters = json_system_creator.get_resource_distribution_parameters()
        transfer_services = json_system_creator.get_transfer_services(components, all_resources_parameters)
        nontransfer_services = json_system_creator.get_non_transfer_services(components, all_resources_parameters, transfer_services)        
        assert len(nontransfer_services) == 3
        assert isinstance(nontransfer_services, dict)
        assert list(nontransfer_services.keys()) == ['ElectricPower', 'CoolingWater', 'Communication']
        assert isinstance(nontransfer_services['ElectricPower']['DistributionModel'], ResourceDistributionModel.UtilityDistributionModel)
        assert isinstance(nontransfer_services['ElectricPower']['DistributionModel'].transfer_service_distribution_model, ResourceDistributionModel.TransferServiceDistributionModelPotentialPathSets)
        assert isinstance(nontransfer_services['CoolingWater']['DistributionModel'], ResourceDistributionModel.UtilityDistributionModel)
        assert isinstance(nontransfer_services['CoolingWater']['DistributionModel'].transfer_service_distribution_model, ResourceDistributionModel.TransferServiceDistributionModelPotentialPathSets)
        assert isinstance(nontransfer_services['Communication']['DistributionModel'], ResourceDistributionModel.UtilityDistributionModel)
    
    def test_get_resilience_calculators(self, json_system_creator: SystemCreator.SystemCreator,
                                        component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        resilience_calculators = json_system_creator.get_resilience_calculators()
        assert len(resilience_calculators) == 2
        assert isinstance(resilience_calculators, list)
        assert isinstance(resilience_calculators[0], ResilienceCalculator.ReCoDeSResilienceCalculator)
        assert isinstance(resilience_calculators[1], ResilienceCalculator.NISTGoalsResilienceCalculator)

    def test_get_component_object(self, json_system_creator: SystemCreator.SystemCreator,
                                  component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        component_types = ['ElectricPowerPlant', 'BuildingStockUnit', 'BaseTransceiverStation_1',
                           'BaseTransceiverStation_2', 'CoolingWaterFacility', 'SuperLink']
        for component_type in component_types:
            assert isinstance(json_system_creator.get_component_object(component_type),
                               Component.StandardiReCoDeSComponent)
        
    def test_add_components_empty(self, json_system_creator: SystemCreator.SystemCreator,
                                  component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        assert json_system_creator.components == []
    
    def test_add_multiple_components(self, json_system_creator: SystemCreator.SystemCreator,
                                  component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        num_components = 5
        for _ in range(num_components):
            component = json_system_creator.get_component_object('ElectricPowerPlant')
            json_system_creator.add_component(component)
        assert len(json_system_creator.components) == num_components

    def test_format_locality_id(self, json_system_creator: SystemCreator.SystemCreator):
        locality_strings = ['Locality 12', 'Locality 1', 'Locality 312', 'Locality 3124']
        locality_ids = [12, 1, 312, 3124]
        boolean_list = []
        for locality_id, locality_string in zip(locality_ids, locality_strings):
            boolean_list.append(locality_id == json_system_creator.format_locality_id(locality_string))
        assert all(boolean_list) == True
    
    def test_create_components_between_localities(self, json_system_creator: SystemCreator.SystemCreator,
                                                            component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)
        localities = ['Locality 1', 'Locality 2', 'Locality 3']
        target_start_end_localities = [[[1, 2], [1, 3]], [[2, 1], [2, 3]], [[3, 1], [3, 2]]]
        for locality, target_start_end_locality in zip(localities, target_start_end_localities):
            content = system_configuration['Content'][locality]['LinkTo']
            json_system_creator.components = []
            json_system_creator.create_components_between_localities(locality, content)
            assert len(json_system_creator.components) == 2
            assert isinstance(json_system_creator.components[0], Component.StandardiReCoDeSComponent)
            assert isinstance(json_system_creator.components[1], Component.StandardiReCoDeSComponent)
            assert json_system_creator.components[0].locality == target_start_end_locality[0]
            assert json_system_creator.components[1].locality == target_start_end_locality[1]

    def test_create_components_in_localities_locality_1(self, json_system_creator: SystemCreator.SystemCreator,
                                                        component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)                                                        
        locality = 'Locality 1'
        content = system_configuration['Content'][locality]['ComponentsInLocality']
        json_system_creator.create_components_in_localities(locality, content)       
        
        assert len(json_system_creator.components) == 2
        assert isinstance(json_system_creator.components[0], Component.StandardiReCoDeSComponent)
        assert isinstance(json_system_creator.components[1], Component.StandardiReCoDeSComponent)
        assert json_system_creator.components[0].name == 'BaseTransceiverStation_1'
        assert json_system_creator.components[0].locality == [1]
        assert json_system_creator.components[1].name == 'ElectricPowerPlant'
        assert json_system_creator.components[1].locality == [1]
    
    def test_create_components_in_localities_locality_2(self, json_system_creator: SystemCreator.SystemCreator,
                                                        component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)                                                        
        locality = 'Locality 2'
        content = system_configuration['Content'][locality]['ComponentsInLocality']
        json_system_creator.create_components_in_localities(locality, content)       
        
        assert len(json_system_creator.components) == 1
        assert isinstance(json_system_creator.components[0], Component.StandardiReCoDeSComponent)
        assert json_system_creator.components[0].name == 'CoolingWaterFacility'
        assert json_system_creator.components[0].locality == [2]

    def test_create_components_in_localities_locality_3(self, json_system_creator: SystemCreator.SystemCreator,
                                                        component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)                                                        
        locality = 'Locality 3'
        content = system_configuration['Content'][locality]['ComponentsInLocality']
        json_system_creator.create_components_in_localities(locality, content)       
        
        assert len(json_system_creator.components) == 2
        assert isinstance(json_system_creator.components[0], Component.BuildingStockUnitWithEmergencyCalls)
        assert json_system_creator.components[0].name == 'BuildingStockUnit'
        assert json_system_creator.components[0].locality == [3]
        assert json_system_creator.components[1].name == 'BaseTransceiverStation_2'
        assert json_system_creator.components[1].locality == [3]
    
    def test_create_components_num_components_created(self, json_system_creator: SystemCreator.SystemCreator,
                                                        component_library: dict, system_configuration: dict):
        json_system_creator.setup(component_library, system_configuration)                                                        
        json_system_creator.create_components()
        assert len(json_system_creator.components) == 11

class TestR2DSystemCreator_NorthEast_SF_Housing:

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Main.json'

    @pytest.fixture()
    def r2d_system_creator(self) -> SystemCreator.SystemCreator:
        return SystemCreator.R2DSystemCreator()

    @pytest.fixture()
    def component_library(self) -> ComponentLibraryCreator.JSONComponentLibraryCreator:
        input_dict = main.read_file(self.MAIN_FILE)
        return main.form_component_library(input_dict)
    
    @pytest.fixture()
    def system_configuration(self) -> dict:
        input_dict = main.read_file(self.MAIN_FILE)
        system_configuration = main.read_file(input_dict['System']['SystemConfigurationFile'])
        return system_configuration

    def test_setup(self, r2d_system_creator: SystemCreator.SystemCreator, 
                   component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)

        assert isinstance(r2d_system_creator.component_library, dict)
        assert isinstance(r2d_system_creator.system_configuration, dict)
        assert r2d_system_creator.scenario_id == 1
        assert r2d_system_creator.DS4_REPAIR_DURATION == 240
        assert r2d_system_creator.MAX_REPAIR_CREW_DEMAND_PER_BUILDING == 50
        assert r2d_system_creator.REPAIR_CREW_DEMAND_PER_SQFT['DS1'] == 5400
        assert r2d_system_creator.REPAIR_CREW_DEMAND_PER_SQFT['DS3'] == 2700
        assert r2d_system_creator.building_area_per_person['Locality 1'] == 541
        assert isinstance(r2d_system_creator.component_parameters_setter, SystemCreator.R2DResidentialBuildingParametersSetter)

    def test_create_components(self, r2d_system_creator: SystemCreator.SystemCreator, 
                               component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        r2d_system_creator.create_components()
        target_num_components = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
                                    'MaxNumBuildings'] + 1
        assert len(r2d_system_creator.components) == target_num_components

    def test_create_recovery_resource_suppliers(self, r2d_system_creator: SystemCreator.SystemCreator,
                                                component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        suppliers = r2d_system_creator.create_recovery_resource_suppliers('Locality 1',
                                                                          r2d_system_creator.system_configuration['Content'][
                                                                              'Locality 1']['ComponentsInLocality'])
        assert len(suppliers) == 1
        assert suppliers[0].name == 'EmergencyResponseCenter'
        assert suppliers[0].locality == [1]
        assert isinstance(suppliers[0], Component.StandardiReCoDeSComponent)

    def test_create_residential_building_components(self, r2d_system_creator: SystemCreator.SystemCreator,
                                                    component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)        
        r2d_system_creator.set_building_area_per_person()
        components = r2d_system_creator.create_residential_building_components('Locality 1',
                                                                               r2d_system_creator.system_configuration['Content'][
                                                                                   'Locality 1']['ComponentsInLocality'])
        target_num_components = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'MaxNumBuildings']
        assert len(components) == target_num_components
        for component in components:
            assert 'ResidentialBuilding' in component.name
            assert isinstance(component, Component.StandardiReCoDeSComponent)

    def test_load_building_data_information(self, r2d_system_creator: SystemCreator.SystemCreator,
                                            component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        building_info_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'BuildingsInfoFolder']
        building_id = '8000'
        building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
        assert building_data['Information']['GeneralInformation']['BIM_id'] == building_id
        target_coords = [37.803317, -122.4429]
        assert building_data['Information']['GeneralInformation']['Latitude'] == target_coords[0]
        assert building_data['Information']['GeneralInformation']['Longitude'] == target_coords[1]
        target_plan_area = 57242.0
        assert building_data['Information']['GeneralInformation']['PlanArea'] == target_plan_area
        target_num_stories = 1
        assert building_data['Information']['GeneralInformation']['NumberOfStories'] == target_num_stories

    def test_load_building_data_damage(self, r2d_system_creator: SystemCreator.SystemCreator, 
                                       component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        building_info_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'BuildingsInfoFolder']
        building_id = 8000
        building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
        scenario_id = 0
        target_damage_state = 2
        assert building_data['Damage']['highest_damage_state/S'][scenario_id] == target_damage_state
        target_reconstruction_cost = 813408.82
        assert math.isclose(building_data['Damage']['reconstruction/cost'][scenario_id], target_reconstruction_cost)
        target_reconstruction_time = 30
        assert building_data['Damage']['reconstruction/time'][scenario_id] == target_reconstruction_time

    def test_get_building_location(self, r2d_system_creator: SystemCreator.SystemCreator, 
                                   component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        building_info_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'BuildingsInfoFolder']
        building_ids = [8000, 8001, 8002, 8003]
        target_locations = [[37.803317, -122.4429], [37.799682, -122.434602], [37.797494, -122.434566],
                            [37.796672, -122.43739]]
        for target_location, building_id in zip(target_locations, building_ids):
            building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
            building_location = r2d_system_creator.get_building_location(building_data)
            assert all([math.isclose(i, j) for i, j in zip(target_location, building_location)])

    def test_building_location_inside_building_box(self, r2d_system_creator: SystemCreator.SystemCreator,
                                                   component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        building_info_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'BuildingsInfoFolder']
        building_ids = [8000, 8001, 8002, 8003]
        bounding_box = {'Latitude': [37.797, 37.797, 37.8, 37.8], 'Longitude': [-122.44, -122.43, -122.43, -122.44]}
        target_bools = [False, True, True, False]
        for target_bool, building_id in zip(target_bools, building_ids):
            building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
            building_location = r2d_system_creator.get_building_location(building_data)
            building_in_bounding_box = r2d_system_creator.building_location_inside_bounding_box(building_location,
                                                                                                bounding_box)
            assert target_bool == building_in_bounding_box

    def test_building_is_residential(self, r2d_system_creator: SystemCreator.SystemCreator, 
                                     component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        building_info_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'BuildingsInfoFolder']
        building_ids = [8000, 8001, 8002, 8003]
        target_bools = [False, True, False, True]
        for target_bool, building_id in zip(target_bools, building_ids):
            building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
            building_is_residential = r2d_system_creator.building_is_residential(building_data)
            assert target_bool == building_is_residential

    def test_create_residential_building(self, r2d_system_creator: SystemCreator.SystemCreator,
                                            component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        r2d_system_creator.set_building_area_per_person()
        locality = 'Locality 1'
        building_info_folder = r2d_system_creator.system_configuration['Content'][locality]['ComponentsInLocality'][
            'BuildingsInfoFolder']
        r2d_system_creator.system_configuration['Constants']['DEFAULT_REPAIR_DURATION_DICT']['Lognormal']['Dispersion']  = 0.0
        building_ids = [8001, 8005, 8020]
        scenario_ids = [2, 4]
        target_repair_durations = [[30, 120, 240], [0, 5, 5]]
        target_repair_demands = [[2, 22, 11], [0, 11, 6]]
        target_housing_capacities = [[11, 106, 53], [11, 106, 53]]
        target_DSs = [[2, 3, 4], [0, 1, 1]]
        target_repair_costs = [[40300, 3276525.175, 2014177.5], [0, 158669.5, 40283.55]]

        for i, scenario_id in enumerate(scenario_ids):
            r2d_system_creator.scenario_id = scenario_id
            r2d_system_creator.set_component_parameters_setter()
            for ii, building_id in enumerate(building_ids):
                building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
                component = r2d_system_creator.create_residential_building(locality, building_data)
                assert isinstance(component, Component.StandardiReCoDeSComponent)
                assert component.name == f'DS{target_DSs[i][ii]}_ResidentialBuilding'
                assert component.supply['Supply']['Shelter'].initial_amount == target_housing_capacities[i][ii]
                assert component.demand['OperationDemand']['Shelter'].initial_amount == target_housing_capacities[i][ii]
                if target_DSs[i][ii] != 0:
                    assert math.isclose(component.recovery_model.recovery_activities['Repair'].duration,
                                                  target_repair_durations[i][ii])
                    assert component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].initial_amount == \
                        target_repair_demands[i][ii]
                else:
                    assert isinstance(component.recovery_model, ComponentRecoveryModel.NoRecoveryActivityModel)
                if target_DSs[i][ii] > 1:
                    target_financing_demand = target_repair_costs[i][ii] / component.recovery_model.recovery_activities['Financing'].duration
                    assert math.isclose(
                        component.recovery_model.recovery_activities['Financing'].demand['Money'].initial_amount,
                        target_financing_demand)
                    
    def test_get_building_damage_state(self, r2d_system_creator: SystemCreator.SystemCreator, 
                                       component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        building_info_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality'][
            'BuildingsInfoFolder']
        building_id = 8000
        building_data = r2d_system_creator.load_building_data(building_id, building_info_folder)
        scenario_ids = list(range(5))
        target_DSs = [2, 3, 2, 0, 4]
        for target_DS, scenario_id in zip(target_DSs, scenario_ids):
            r2d_system_creator.scenario_id = scenario_id
            damage_state = r2d_system_creator.get_building_damage_state(building_data)
            assert damage_state == target_DS

class TestR2DSystemWithInterfacesCreator:

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Interface_Infrastructure_Main.json'

    @pytest.fixture()
    def r2d_system_creator(self) -> SystemCreator.SystemCreator:
        return SystemCreator.R2DSystemWithInterfacesCreator()

    @pytest.fixture()
    def component_library(self) -> ComponentLibraryCreator.JSONComponentLibraryCreator:
        input_dict = main.read_file(self.MAIN_FILE)
        return main.form_component_library(input_dict)
    
    @pytest.fixture()
    def system_configuration(self) -> dict:
        input_dict = main.read_file(self.MAIN_FILE)
        system_configuration = main.read_file(input_dict['System']['SystemConfigurationFile'])
        return system_configuration
    
    def test_create_components_in_localities(self, r2d_system_creator: SystemCreator.R2DSystemWithInterfacesCreator,
                                             component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        locality = 'Locality 1'
        content = r2d_system_creator.system_configuration['Content'][locality]['ComponentsInLocality']
        r2d_system_creator.create_components_in_localities(locality, content)
        assert isinstance(r2d_system_creator.components[-4], Component.StandardiReCoDeSComponent)
        assert 'ResidentialBuilding' in r2d_system_creator.components[-4].name
        assert isinstance(r2d_system_creator.components[-3], Component.InfrastructureInterface)
        assert r2d_system_creator.components[-3].name == 'ElectricPowerSupplySystem'
        assert isinstance(r2d_system_creator.components[-2], Component.InfrastructureInterface)
        assert r2d_system_creator.components[-2].name == 'WaterSupplySystem'
        assert isinstance(r2d_system_creator.components[-1], Component.InfrastructureInterface)
        assert r2d_system_creator.components[-1].name == 'CellularCommunicationSystem'

    def test_create_infrastructure_service_suppliers(self, r2d_system_creator: SystemCreator.R2DSystemWithInterfacesCreator,
                                                     component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        locality = 'Locality 1'
        content = r2d_system_creator.system_configuration['Content'][locality]['ComponentsInLocality']
        suppliers = r2d_system_creator.create_infrastructure_service_suppliers(locality, content)
        assert isinstance(suppliers[0], Component.InfrastructureInterface)
        assert suppliers[0].name == 'ElectricPowerSupplySystem'
        assert suppliers[0].locality == [1]
        assert isinstance(suppliers[1], Component.InfrastructureInterface)
        assert suppliers[1].name == 'WaterSupplySystem'
        assert suppliers[1].locality == [1]
        assert isinstance(suppliers[2], Component.InfrastructureInterface)
        assert suppliers[2].name == 'CellularCommunicationSystem'
        assert suppliers[2].locality == [1]
    
    def test_set_infrastructure_system_parameters(self, r2d_system_creator: SystemCreator.R2DSystemWithInterfacesCreator,
                                                     component_library: dict, system_configuration: dict):
        r2d_system_creator.setup(component_library, system_configuration)
        locality = 'Locality 2'
        content = r2d_system_creator.system_configuration['Content'][locality]['ComponentsInLocality']
        component = copy.deepcopy(component_library['ElectricPowerSupplySystem'])
        assert component.supply['Supply']['ElectricPower'].initial_amount == 0
        assert component.supply['Supply']['ElectricPower'].current_amount == 0
        assert component.recovery_model.recovery_activities['Recovery'].duration == 0

        r2d_system_creator.set_infrastructure_system_parameters(component, content['Infrastructure'][0]['ElectricPowerSupplySystem'])
        assert component.supply['Supply']['ElectricPower'].initial_amount == 80
        assert component.recovery_model.recovery_activities['Recovery'].duration == 30
        assert component.recovery_model.damage_to_functionality_relation.step_values == [0.0, 0.5, 1.0]
        assert component.recovery_model.damage_to_functionality_relation.step_limits == [0.0, 0.5, 1.0]
    
class TestComponentParametersSetter:

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Interface_Infrastructure_Main.json'

    @pytest.fixture()
    def r2d_system_creator(self) -> SystemCreator.SystemCreator:
        return SystemCreator.R2DSystemWithInterfacesCreator()

    @pytest.fixture()
    def component_library(self) -> ComponentLibraryCreator.JSONComponentLibraryCreator:
        input_dict = main.read_file(self.MAIN_FILE)
        return main.form_component_library(input_dict)
    
    @pytest.fixture()
    def building_component(self, component_library: dict) -> Component.StandardiReCoDeSComponent:
        return copy.deepcopy(component_library['DS3_ResidentialBuilding'])

    @pytest.fixture()
    def component_parameters_setter(self) -> SystemCreator.ComponentParametersSetter:
        return SystemCreator.ComponentParametersSetter()

    def test_set_component_supply(self, 
                                  building_component: Component.StandardiReCoDeSComponent,
                                  component_parameters_setter: SystemCreator.ComponentParametersSetter):
        component_parameters_setter.set_component_supply(building_component,
                                                         'Shelter', 100)
        assert building_component.supply['Supply']['Shelter'].initial_amount == 100
        assert building_component.supply['Supply']['Shelter'].current_amount == 100
    
    def test_set_component_supply_wrong_resource(self, 
                                        building_component: Component.StandardiReCoDeSComponent,
                                        component_parameters_setter: SystemCreator.ComponentParametersSetter):
        with pytest.raises(KeyError):
            component_parameters_setter.set_component_supply(building_component,
                                                             'ElectricPower', 100)
    
    def test_set_component_supply_negative_amount(self,
                                        building_component: Component.StandardiReCoDeSComponent,
                                        component_parameters_setter: SystemCreator.ComponentParametersSetter):
        with pytest.raises(ValueError):
            component_parameters_setter.set_component_supply(building_component,
                                                             'Shelter', -100) 

    def test_set_component_operation_demand(self,
                                            building_component: Component.StandardiReCoDeSComponent,
                                            component_parameters_setter: SystemCreator.ComponentParametersSetter):
        component_parameters_setter.set_component_operation_demand(building_component,
                                                                    'Shelter', 100)
        assert building_component.demand['OperationDemand']['Shelter'].initial_amount == 100
        assert building_component.demand['OperationDemand']['Shelter'].current_amount == 100

    def test_set_component_operation_demand_wrong_resource(self,
                                            building_component: Component.StandardiReCoDeSComponent,
                                            component_parameters_setter: SystemCreator.ComponentParametersSetter):
        with pytest.raises(KeyError):
            component_parameters_setter.set_component_operation_demand(building_component,
                                                                        'DummyResource', 100)
            
    def test_set_component_operation_demand_negative_amount(self,
                                            building_component: Component.StandardiReCoDeSComponent,
                                            component_parameters_setter: SystemCreator.ComponentParametersSetter):
        with pytest.raises(ValueError):
            component_parameters_setter.set_component_operation_demand(building_component,
                                                                        'Shelter', -100)
    
    def test_set_recovery_demand(self,
                                 building_component: Component.StandardiReCoDeSComponent,
                                 component_parameters_setter: SystemCreator.ComponentParametersSetter):
        component_parameters_setter.set_component_recovery_demand(building_component,
                                                                  'Repair', 'RepairCrew', 100)
        assert building_component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].initial_amount == 100
        assert building_component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].current_amount == 100

    def test_set_recovery_demand_wrong_resource(self,
                                                building_component: Component.StandardiReCoDeSComponent,
                                                component_parameters_setter: SystemCreator.ComponentParametersSetter):
        with pytest.raises(KeyError):
            component_parameters_setter.set_component_recovery_demand(building_component,
                                                                      'Repair', 'DummyResource', 100)
    
    def test_set_recovery_demand_negative_amount(self,
                                                    building_component: Component.StandardiReCoDeSComponent,
                                                    component_parameters_setter: SystemCreator.ComponentParametersSetter):
        with pytest.raises(ValueError):
            component_parameters_setter.set_component_recovery_demand(building_component,
                                                                      'Repair', 'RepairCrew', -100)     

    def test_set_recovery_demand_wrong_activity(self,
                                                    building_component: Component.StandardiReCoDeSComponent,
                                                    component_parameters_setter: SystemCreator.ComponentParametersSetter):
        component_parameters_setter.set_component_recovery_demand(building_component,
                                                                      'DummyActivity', 'RepairCrew', 100) 
        assert 'DummyActivity' not in building_component.recovery_model.recovery_activities.keys()

class TestR2DComponentParametersSetter:

    MAIN_FILE = './tests/test_inputs/test_inputs_NorthEast_SF_Housing_Interface_Infrastructure_Main.json'
    
    @pytest.fixture()
    def component_library(self) -> ComponentLibraryCreator.JSONComponentLibraryCreator:
        input_dict = main.read_file(self.MAIN_FILE)
        return main.form_component_library(input_dict)

    @pytest.fixture()
    def system_configuration(self) -> dict:
        input_dict = main.read_file(self.MAIN_FILE)
        system_configuration = main.read_file(input_dict['System']['SystemConfigurationFile'])
        return system_configuration

    @pytest.fixture()
    def r2d_system_creator(self, component_library: dict, system_configuration: dict) -> SystemCreator.SystemCreator:
        r2d_system_creator = SystemCreator.R2DSystemWithInterfacesCreator()
        r2d_system_creator.setup(component_library, system_configuration)
        return r2d_system_creator
    
    @pytest.fixture()
    def building_component(self, component_library: dict) -> Component.StandardiReCoDeSComponent:
        return copy.deepcopy(component_library['DS3_ResidentialBuilding'])
    
    @pytest.fixture()
    def component_parameters_setter(self, r2d_system_creator) -> SystemCreator.ComponentParametersSetter:
        system_level_data = {key: r2d_system_creator.__dict__.get(key, None) for key in SystemCreator.R2DResidentialBuildingParametersSetter.SYSTEM_LEVEL_DATA_REQUIRED_FOR_BUILDINGS}
        return SystemCreator.R2DResidentialBuildingParametersSetter(system_level_data)
    
    def test_init(self, component_parameters_setter):
        assert isinstance(component_parameters_setter.system_level_data, dict)
        assert len(component_parameters_setter.system_level_data) == len(SystemCreator.R2DResidentialBuildingParametersSetter.SYSTEM_LEVEL_DATA_REQUIRED_FOR_BUILDINGS)
    
    def test_set_parameters(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_parameters(building_component, 'Locality 1', building_data, 3)
        assert building_component.area == 57242.0
        assert building_component.footprint == {"type": "Feature", "geometry": {"type": "Polygon", "coordinates":[[[-122.442624,37.803560],[-122.442634,37.803620],[-122.442965,37.803585],[-122.442954,37.803519],[-122.442784,37.803536],[-122.442785,37.803544],[-122.442624,37.803560]]]}, "properties":{}}
        assert building_component.num_stories == 1
        assert building_component.supply['Supply']['Shelter'].current_amount == 105
        assert building_component.supply['Supply']['Shelter'].initial_amount == 105
        assert building_component.demand['OperationDemand']['Shelter'].current_amount == 105
        assert building_component.demand['OperationDemand']['Shelter'].initial_amount == 105
        assert building_component.demand['OperationDemand']['ElectricPower'].current_amount == 105 * 0.02
        assert building_component.demand['OperationDemand']['ElectricPower'].initial_amount == 105 * 0.02
        assert building_component.demand['OperationDemand']['PotableWater'].current_amount == 105 * 150
        assert building_component.demand['OperationDemand']['PotableWater'].initial_amount == 105 * 150
        assert building_component.demand['OperationDemand']['CellularCommunication'].initial_amount == 105 * 0.033
        assert building_component.demand['OperationDemand']['CellularCommunication'].current_amount == 105 * 0.033
        assert building_component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].current_amount == 22
        assert building_component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].initial_amount == 22
        assert building_component.locality == [1]

    def test_set_building_area(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_area(building_component, building_data)
        assert building_component.area == 57242.0
    
    def test_set_building_footprint(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_footprint(building_component, building_data)
        target_building_footprint = {"type": "Feature", "geometry": {"type": "Polygon", "coordinates":[[[-122.442624,37.803560],[-122.442634,37.803620],[-122.442965,37.803585],[-122.442954,37.803519],[-122.442784,37.803536],[-122.442785,37.803544],[-122.442624,37.803560]]]}, "properties":{}}
        assert building_component.footprint == target_building_footprint

    def test_set_building_num_stories(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_num_stories(building_component, building_data)
        assert building_component.num_stories == 1

    def test_get_total_building_area(self,
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_ids = [8000, 8001, 8002, 8004]
        target_areas = [57242.0, 6200.0, 6240.0, 13404.0]
        for target_area, building_id in zip(target_areas, building_ids):
            building_data = r2d_system_creator.load_building_data(building_id, building_folder)
            building_area = component_parameters_setter.get_total_building_area(building_data)
            assert building_area == target_area

    def test_get_building_housing_capacity(self, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_ids = [8000, 8001, 8002, 8004]
        target_housing_capacities = [105, 11, 11, 24]
        r2d_system_creator.set_building_area_per_person()
        for target_housing_capacity, building_id in zip(target_housing_capacities, building_ids):
            building_data = r2d_system_creator.load_building_data(building_id, building_folder)
            building_area = component_parameters_setter.get_total_building_area(building_data)
            building_housing_capacity = component_parameters_setter.get_building_housing_capacity(building_area, 'Locality 1')
            assert building_housing_capacity == target_housing_capacity

    def test_set_housing_parameters(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_area(building_component, building_data)
        component_parameters_setter.set_housing_parameters(building_component, 'Locality 1')
        assert building_component.supply['Supply']['Shelter'].current_amount == 105
        assert building_component.supply['Supply']['Shelter'].initial_amount == 105
        assert building_component.demand['OperationDemand']['Shelter'].current_amount == 105
        assert building_component.demand['OperationDemand']['Shelter'].initial_amount == 105
    
    def test_set_operation_demand_parameters(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_area(building_component, building_data)
        component_parameters_setter.set_housing_parameters(building_component, 'Locality 1')
        component_parameters_setter.set_operation_demand_parameters(building_component, 'Locality 1', building_data.get('OperationDemandResources', []))
        assert building_component.demand['OperationDemand']['ElectricPower'].current_amount == 105 * 0.02
        assert building_component.demand['OperationDemand']['ElectricPower'].initial_amount == 105 * 0.02
        assert building_component.demand['OperationDemand']['PotableWater'].current_amount == 105 * 150
        assert building_component.demand['OperationDemand']['PotableWater'].initial_amount == 105 * 150
        assert building_component.demand['OperationDemand']['CellularCommunication'].current_amount == 105 * 0.033
        assert building_component.demand['OperationDemand']['CellularCommunication'].initial_amount == 105 * 0.033
    
    def test_set_repair_demand(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_area(building_component, building_data)
        component_parameters_setter.set_repair_demand(building_component, 'Locality 1', 2)
        assert building_component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].current_amount == 11
        assert building_component.recovery_model.recovery_activities['Repair'].demand['RepairCrew'].initial_amount == 11
    
    def test_get_repair_crew_demand(self,
                               component_parameters_setter: SystemCreator.ComponentParametersSetter):
        assert component_parameters_setter.get_repair_crew_demand(0, 1000) == 0
        assert component_parameters_setter.get_repair_crew_demand(1, 5400) == 1
        assert component_parameters_setter.get_repair_crew_demand(2, 5400) == 1
        assert component_parameters_setter.get_repair_crew_demand(3, 5400) == 2
        assert component_parameters_setter.get_repair_crew_demand(4, 5400) == 2
        assert component_parameters_setter.get_repair_crew_demand(4, 5400000) == 50
    
    def test_set_repair_duration(self, building_component: Component.StandardiReCoDeSComponent, 
                               component_parameters_setter: SystemCreator.ComponentParametersSetter,
                               r2d_system_creator: SystemCreator.SystemCreator):
        r2d_system_creator.DEFAULT_REPAIR_DURATION_DICT['Lognormal']['Dispersion'] = 0
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_repair_duration(building_component, building_data)
        assert math.isclose(building_component.recovery_model.recovery_activities['Repair'].duration, 120)
    
    def test_get_median_repair_duration(self,
                                 component_parameters_setter: SystemCreator.ComponentParametersSetter,
                                 r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        assert component_parameters_setter.get_median_repair_duration(building_data) == 120
        r2d_system_creator.scenario_id = 44
        system_level_data = {key: r2d_system_creator.__dict__.get(key, None) for key in SystemCreator.R2DResidentialBuildingParametersSetter.SYSTEM_LEVEL_DATA_REQUIRED_FOR_BUILDINGS}
        component_parameters_setter.system_level_data = system_level_data
        assert component_parameters_setter.get_median_repair_duration(building_data) == 240
    
    def test_set_building_repair_cost(self, 
                                      component_parameters_setter: SystemCreator.ComponentParametersSetter,
                                      building_component: Component.StandardiReCoDeSComponent,
                                      r2d_system_creator: SystemCreator.SystemCreator):
        building_folder = r2d_system_creator.system_configuration['Content']['Locality 1']['ComponentsInLocality']['BuildingsInfoFolder']
        building_data = r2d_system_creator.load_building_data(8000, building_folder)
        component_parameters_setter.set_building_repair_cost(building_component, building_data)
        assert math.isclose(building_component.recovery_model.recovery_activities['Financing'].demand['Money'].current_amount, 3538328.367/building_component.recovery_model.recovery_activities['Financing'].duration)
        assert math.isclose(building_component.recovery_model.recovery_activities['Financing'].demand['Money'].initial_amount, 3538328.367/building_component.recovery_model.recovery_activities['Financing'].duration)

    
tests/test_SystemCreator.py
NikolaBlagojevic/pyrecodes
import os
from pyrecodes import System
from pyrecodes import main

# Define the path to the test JSON configuration files
TEST_DATA_PATH = "tests/test_inputs/"

def test_read_file():
    # Test reading a JSON file and check if the returned value is a dictionary
    file_name = os.path.join(TEST_DATA_PATH, "test_inputs_ThreeLocalitiesCommunity_Main.json")
    data = main.read_file(file_name)
    assert isinstance(data, dict)

def test_form_component_library():
    # Test forming a component library and check if the returned value is a dictionary
    input_dict = {
        "ComponentLibrary": {
            "ComponentLibraryCreatorClass": "JSONComponentLibraryCreator",
            "ComponentLibraryFile": "./tests/test_inputs/test_inputs_ThreeLocalitiesCommunity_ComponentLibrary.json"
        }
    }
    component_library = main.form_component_library(input_dict)
    assert isinstance(component_library, dict)

def test_create_system():
    # Test creating a system object and check if it is an instance of System.System
    file_name = os.path.join(TEST_DATA_PATH, "test_inputs_ThreeLocalitiesCommunity_Main.json")
    input_dict = main.read_file(file_name)
    system = main.create_system(input_dict)
    assert isinstance(system, System.System)

def test_run():
    # Test running the resilience assessment for the system and check if the returned value is a System.System object
    main_file = os.path.join(TEST_DATA_PATH, "test_inputs_ThreeLocalitiesCommunity_Main.json")
    system = main.run(main_file)
    assert isinstance(system, System.System)

def test_load_system():
    # Test loading a previously saved system and check if it is an instance of System.System
    loadname = os.path.join(TEST_DATA_PATH, "test_inputs_Example_3.pickle")
    system = main.load_system(loadname)
    assert isinstance(system, System.System)

tests/test_main.py
NikolaBlagojevic/pyrecodes