(readline)
The generate_tokens() generator requires one argment, readline, which
    must be a callable object which provides the same interface as the
    readline() method of built-in file objects. Each call to the function
    should return one line of input as a string.  Alternately, readline
    can be a callable function terminating with StopIteration::

        readline = open(myfile).next    # Example of alternate readline

    The generator produces 5-tuples with these members: the token type; the
    token string; a 2-tuple (srow, scol) of ints specifying the row and
    column where the token begins in the source; a 2-tuple (erow, ecol) of
    ints specifying the row and column where the token ends in the source;
    and the line on which the token was found. The line passed is the
    logical line; continuation lines are included.
The generate_tokens() generator requires one argment, readline, which
    must be a callable object which provides the same interface as the
    readline() method of built-in file objects. Each call to the function
    should return one line of input as a string.  Alternately, readline
    can be a callable function terminating with StopIteration::
The generate_tokens () generator requires one argment readline which must be a callable object which provides the same interface as the readline () method of built - in file objects . Each call to the function should return one line of input as a string . Alternately readline can be a callable function terminating with StopIteration ::
def generate_tokens(readline):
    """
    The generate_tokens() generator requires one argment, readline, which
    must be a callable object which provides the same interface as the
    readline() method of built-in file objects. Each call to the function
    should return one line of input as a string.  Alternately, readline
    can be a callable function terminating with StopIteration::

        readline = open(myfile).next    # Example of alternate readline

    The generator produces 5-tuples with these members: the token type; the
    token string; a 2-tuple (srow, scol) of ints specifying the row and
    column where the token begins in the source; a 2-tuple (erow, ecol) of
    ints specifying the row and column where the token ends in the source;
    and the line on which the token was found. The line passed is the
    logical line; continuation lines are included.
    """
    lnum = parenlev = continued = 0
    namechars, numchars = string.ascii_letters + '_', '0123456789'
    contstr, needcont = '', 0
    contline = None
    indents = [0]

    while 1:                                   # loop over lines in stream
        try:
            line = readline()
        except StopIteration:
            line = ''
        lnum = lnum + 1
        pos, max = 0, len(line)

        if contstr:                            # continued string
            if not line:
                raise TokenError("EOF in multi-line string", strstart)
            endmatch = endprog.match(line)
            if endmatch:
                pos = end = endmatch.end(0)
                yield (STRING, contstr + line[:end],
                       strstart, (lnum, end), contline + line)
                contstr, needcont = '', 0
                contline = None
            elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
                yield (ERRORTOKEN, contstr + line,
                       strstart, (lnum, len(line)), contline)
                contstr = ''
                contline = None
                continue
            else:
                contstr = contstr + line
                contline = contline + line
                continue

        elif parenlev == 0 and not continued:  # new statement
            if not line:
                break
            column = 0
            while pos < max:                   # measure leading whitespace
                if line[pos] == ' ':
                    column = column + 1
                elif line[pos] == '\t':
                    column = (column/tabsize + 1)*tabsize
                elif line[pos] == '\f':
                    column = 0
                else:
                    break
                pos = pos + 1
            if pos == max:
                break

            if line[pos] in '#\r\n':           # skip comments or blank lines
                if line[pos] == '#':
                    comment_token = line[pos:].rstrip('\r\n')
                    nl_pos = pos + len(comment_token)
                    yield (COMMENT, comment_token,
                           (lnum, pos), (lnum, pos + len(comment_token)), line)
                    yield (NL, line[nl_pos:],
                           (lnum, nl_pos), (lnum, len(line)), line)
                else:
                    yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
                           (lnum, pos), (lnum, len(line)), line)
                continue

            if column > indents[-1]:           # count indents or dedents
                indents.append(column)
                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
            while column < indents[-1]:
                if column not in indents:
                    raise IndentationError(
                        "unindent does not match any outer indentation level",
                        ("<tokenize>", lnum, pos, line))
                indents = indents[:-1]
                yield (DEDENT, '', (lnum, pos), (lnum, pos), line)

        else:                                  # continued statement
            if not line:
                raise TokenError("EOF in multi-line statement", (lnum, 0))
            continued = 0

        while pos < max:
            pseudomatch = pseudoprog.match(line, pos)
            if pseudomatch:                                # scan for tokens
                start, end = pseudomatch.span(1)
                spos, epos, pos = (lnum, start), (lnum, end), end
                token, initial = line[start:end], line[start]

                if initial in numchars or \
                        (initial == '.' and token != '.'):      # ordinary number
                    yield (NUMBER, token, spos, epos, line)
                elif initial in '\r\n':
                    yield (NL if parenlev > 0 else NEWLINE, token, spos, epos, line)
                elif initial == '#':
                    if token.endswith("\n"):
                        raise ValueError("Token should not end with \n")
                    yield (COMMENT, token, spos, epos, line)
                elif token in triple_quoted:
                    endprog = endprogs[token]
                    endmatch = endprog.match(line, pos)
                    if endmatch:                           # all on one line
                        pos = endmatch.end(0)
                        token = line[start:pos]
                        yield (STRING, token, spos, (lnum, pos), line)
                    else:
                        strstart = (lnum, start)           # multiple lines
                        contstr = line[start:]
                        contline = line
                        break
                elif initial in single_quoted or \
                    token[:2] in single_quoted or \
                        token[:3] in single_quoted:
                    if token[-1] == '\n':                  # continued string
                        strstart = (lnum, start)
                        endprog = (endprogs[initial] or endprogs[token[1]] or
                                   endprogs[token[2]])
                        contstr, needcont = line[start:], 1
                        contline = line
                        break
                    else:                                  # ordinary string
                        yield (STRING, token, spos, epos, line)
                elif initial in namechars:                 # ordinary name
                    yield (NAME, token, spos, epos, line)
                elif initial == '\\':                      # continued stmt
                    continued = 1
                else:
                    if initial in '([{':
                        parenlev = parenlev + 1
                    elif initial in ')]}':
                        parenlev = parenlev - 1
                    yield (OP, token, spos, epos, line)
            else:
                yield (ERRORTOKEN, line[pos],
                       (lnum, pos), (lnum, pos + 1), line)
                pos = pos + 1

    for indent in indents[1:]:                 # pop remaining indent levels
        yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
    yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
def generate_tokens ( readline ) : lnum = parenlev = continued = 0 namechars , numchars = string . ascii_letters + '_' , '0123456789' contstr , needcont = '' , 0 contline = None indents = [ 0 ] while 1 : # loop over lines in stream try : line = readline ( ) except StopIteration : line = '' lnum = lnum + 1 pos , max = 0 , len ( line ) if contstr : # continued string if not line : raise TokenError ( "EOF in multi-line string" , strstart ) endmatch = endprog . match ( line ) if endmatch : pos = end = endmatch . end ( 0 ) yield ( STRING , contstr + line [ : end ] , strstart , ( lnum , end ) , contline + line ) contstr , needcont = '' , 0 contline = None elif needcont and line [ - 2 : ] != '\\\n' and line [ - 3 : ] != '\\\r\n' : yield ( ERRORTOKEN , contstr + line , strstart , ( lnum , len ( line ) ) , contline ) contstr = '' contline = None continue else : contstr = contstr + line contline = contline + line continue elif parenlev == 0 and not continued : # new statement if not line : break column = 0 while pos < max : # measure leading whitespace if line [ pos ] == ' ' : column = column + 1 elif line [ pos ] == '\t' : column = ( column / tabsize + 1 ) * tabsize elif line [ pos ] == '\f' : column = 0 else : break pos = pos + 1 if pos == max : break if line [ pos ] in '#\r\n' : # skip comments or blank lines if line [ pos ] == '#' : comment_token = line [ pos : ] . rstrip ( '\r\n' ) nl_pos = pos + len ( comment_token ) yield ( COMMENT , comment_token , ( lnum , pos ) , ( lnum , pos + len ( comment_token ) ) , line ) yield ( NL , line [ nl_pos : ] , ( lnum , nl_pos ) , ( lnum , len ( line ) ) , line ) else : yield ( ( NL , COMMENT ) [ line [ pos ] == '#' ] , line [ pos : ] , ( lnum , pos ) , ( lnum , len ( line ) ) , line ) continue if column > indents [ - 1 ] : # count indents or dedents indents . append ( column ) yield ( INDENT , line [ : pos ] , ( lnum , 0 ) , ( lnum , pos ) , line ) while column < indents [ - 1 ] : if column not in indents : raise IndentationError ( "unindent does not match any outer indentation level" , ( "<tokenize>" , lnum , pos , line ) ) indents = indents [ : - 1 ] yield ( DEDENT , '' , ( lnum , pos ) , ( lnum , pos ) , line ) else : # continued statement if not line : raise TokenError ( "EOF in multi-line statement" , ( lnum , 0 ) ) continued = 0 while pos < max : pseudomatch = pseudoprog . match ( line , pos ) if pseudomatch : # scan for tokens start , end = pseudomatch . span ( 1 ) spos , epos , pos = ( lnum , start ) , ( lnum , end ) , end token , initial = line [ start : end ] , line [ start ] if initial in numchars or ( initial == '.' and token != '.' ) : # ordinary number yield ( NUMBER , token , spos , epos , line ) elif initial in '\r\n' : yield ( NL if parenlev > 0 else NEWLINE , token , spos , epos , line ) elif initial == '#' : if token . endswith ( "\n" ) : raise ValueError ( "Token should not end with \n" ) yield ( COMMENT , token , spos , epos , line ) elif token in triple_quoted : endprog = endprogs [ token ] endmatch = endprog . match ( line , pos ) if endmatch : # all on one line pos = endmatch . end ( 0 ) token = line [ start : pos ] yield ( STRING , token , spos , ( lnum , pos ) , line ) else : strstart = ( lnum , start ) # multiple lines contstr = line [ start : ] contline = line break elif initial in single_quoted or token [ : 2 ] in single_quoted or token [ : 3 ] in single_quoted : if token [ - 1 ] == '\n' : # continued string strstart = ( lnum , start ) endprog = ( endprogs [ initial ] or endprogs [ token [ 1 ] ] or endprogs [ token [ 2 ] ] ) contstr , needcont = line [ start : ] , 1 contline = line break else : # ordinary string yield ( STRING , token , spos , epos , line ) elif initial in namechars : # ordinary name yield ( NAME , token , spos , epos , line ) elif initial == '\\' : # continued stmt continued = 1 else : if initial in '([{' : parenlev = parenlev + 1 elif initial in ')]}' : parenlev = parenlev - 1 yield ( OP , token , spos , epos , line ) else : yield ( ERRORTOKEN , line [ pos ] , ( lnum , pos ) , ( lnum , pos + 1 ) , line ) pos = pos + 1 for indent in indents [ 1 : ] : # pop remaining indent levels yield ( DEDENT , '' , ( lnum , 0 ) , ( lnum , 0 ) , '' ) yield ( ENDMARKER , '' , ( lnum , 0 ) , ( lnum , 0 ) , '' )