(*, content: bytes, limit: int, head: int, tail: int, char_in_line: int)
return tokens
`_tokenize_large_file_content` constructs the intermediate representations. They have no color infomation.
`_tokenize_large_file_content` constructs the intermediate representations. They have no color infomation.
_tokenize_large_file_content constructs the intermediate representations . They have no color infomation .
def _tokenize_large_file_content(*, content: bytes, limit: int, head: int, tail: int, char_in_line: int) -> List[_PrettyToken]:
    """`_tokenize_large_file_content` constructs the intermediate representations. They have no color infomation.
    """

    assert head + tail < limit

    def candidate_do_nothing(text: str) -> List[_PrettyToken]:
        tokens = []
        for line in text.splitlines(keepends=True):
            tokens += _tokenize_line(line)
        return tokens

    def candidate_line_based(text: str) -> List[_PrettyToken]:
        lines = text.splitlines(keepends=True)
        if len(lines) < limit:
            return candidate_do_nothing(text)

        tokens = []
        for line in lines[:head]:
            tokens += _tokenize_line(line)
        tokens.append(_PrettyToken(_PrettyTokenType.HINT, '... ({} lines) ...\n'.format(len(lines[head:-tail]))))
        for line in lines[-tail:]:
            tokens += _tokenize_line(line)
        return tokens

    def candidate_char_based(text: str) -> List[_PrettyToken]:
        if len(text) < char_in_line * limit:
            return candidate_do_nothing(text)

        l = len(text[:char_in_line * head].rstrip())
        r = len(text) - char_in_line * tail
        tokens = []
        for line in text[:l].splitlines(keepends=True):
            tokens += _tokenize_line(line)
        tokens.append(_PrettyToken(_PrettyTokenType.HINT, '... ({} chars) ...'.format(r - l)))
        for line in text[r:].splitlines(keepends=True):
            tokens += _tokenize_line(line)
        return tokens

    def count_size(tokens: Iterable[_PrettyToken]) -> int:
        size = 0
        for _, s in tokens:
            size += len(s)
        return size

    # Choose the shortest one from the three candidates.
    tokens, text = _decode_with_recovery(content)
    if text:
        candidates: List[List[_PrettyToken]] = [
            candidate_do_nothing(text),
            candidate_line_based(text),
            candidate_char_based(text),
        ]
        tokens.extend(min(candidates, key=count_size))
    tokens = _warn_if_empty(tokens)
    return tokens
def _tokenize_large_file_content ( * , content : bytes , limit : int , head : int , tail : int , char_in_line : int ) -> List [ _PrettyToken ] : assert head + tail < limit def candidate_do_nothing ( text : str ) -> List [ _PrettyToken ] : tokens = [ ] for line in text . splitlines ( keepends = True ) : tokens += _tokenize_line ( line ) return tokens def candidate_line_based ( text : str ) -> List [ _PrettyToken ] : lines = text . splitlines ( keepends = True ) if len ( lines ) < limit : return candidate_do_nothing ( text ) tokens = [ ] for line in lines [ : head ] : tokens += _tokenize_line ( line ) tokens . append ( _PrettyToken ( _PrettyTokenType . HINT , '... ({} lines) ...\n' . format ( len ( lines [ head : - tail ] ) ) ) ) for line in lines [ - tail : ] : tokens += _tokenize_line ( line ) return tokens def candidate_char_based ( text : str ) -> List [ _PrettyToken ] : if len ( text ) < char_in_line * limit : return candidate_do_nothing ( text ) l = len ( text [ : char_in_line * head ] . rstrip ( ) ) r = len ( text ) - char_in_line * tail tokens = [ ] for line in text [ : l ] . splitlines ( keepends = True ) : tokens += _tokenize_line ( line ) tokens . append ( _PrettyToken ( _PrettyTokenType . HINT , '... ({} chars) ...' . format ( r - l ) ) ) for line in text [ r : ] . splitlines ( keepends = True ) : tokens += _tokenize_line ( line ) return tokens def count_size ( tokens : Iterable [ _PrettyToken ] ) -> int : size = 0 for _ , s in tokens : size += len ( s ) return size # Choose the shortest one from the three candidates. tokens , text = _decode_with_recovery ( content ) if text : candidates : List [ List [ _PrettyToken ] ] = [ candidate_do_nothing ( text ) , candidate_line_based ( text ) , candidate_char_based ( text ) , ] tokens . extend ( min ( candidates , key = count_size ) ) tokens = _warn_if_empty ( tokens ) return tokens