def scan_robots_txt(lines, logger):
    """Tokenizes the contents of a `robots.txt` file.

    lines
        Contents of a `robots.txt` file.
    logger
        Problems found while scanning are logged here.

    Yields:

    record: (lineno, token, value)*
        Records, where each record is a sequence of triples.
    """
    record = []
    for lineno, line in enumerate(lines, 1):
        stripped_line = line.lstrip()
        if stripped_line.startswith('#'):
            continue
        if not stripped_line:
            if record:
                yield record
                record = []
            continue
        if len(stripped_line) != len(line):
            logger.warning('Line %d has whitespace before field', lineno)
        nocomment_line = stripped_line.split('#', 1)[0]
        try:
            field, value = nocomment_line.split(':', 1)
        except ValueError:
            logger.error('Line %d contains no ":"; ignoring line', lineno)
        else:
            record.append((lineno, field.casefold(), value.strip()))
    if record:
        yield record
