(shakespeare_full)
return [play for play in plays if len(play[1]) > 1], discarded_lines
Splits the full data by play.
Splits the full data by play.
Splits the full data by play .
def _split_into_plays(shakespeare_full):
    """Splits the full data by play."""
    # List of tuples (play_name, dict from character to list of lines)
    plays = []
    discarded_lines = []  # Track discarded lines.
    slines = shakespeare_full.splitlines(True)[1:]

    # skip contents, the sonnets, and all's well that ends well
    author_count = 0
    start_i = 0
    for i, l in enumerate(slines):
        if 'by William Shakespeare' in l:
            author_count += 1
        if author_count == 2:
            start_i = i - 5
            break
    slines = slines[start_i:]

    current_character = None
    comedy_of_errors = False
    for i, line in enumerate(slines):
        # This marks the end of the plays in the file.
        if i > 124195 - start_i:
            break
        # This is a pretty good heuristic for detecting the start of a new play:
        if 'by William Shakespeare' in line:
            current_character = None
            characters = collections.defaultdict(list)
            # The title will be 2, 3, 4, 5, 6, or 7 lines above "by William Shakespeare".
            if slines[i - 2].strip():
                title = slines[i - 2]
            elif slines[i - 3].strip():
                title = slines[i - 3]
            elif slines[i - 4].strip():
                title = slines[i - 4]
            elif slines[i - 5].strip():
                title = slines[i - 5]
            elif slines[i - 6].strip():
                title = slines[i - 6]
            else:
                title = slines[i - 7]
            title = title.strip()

            assert title, (
                'Parsing error on line %d. Expecting title 2 or 3 lines above.' %
                i)
            comedy_of_errors = (title == 'THE COMEDY OF ERRORS')
            # Degenerate plays are removed at the end of the method.
            plays.append((title, characters))
            continue
        match = _match_character_regex(line, comedy_of_errors)
        if match:
            character, snippet = match.group(1), match.group(2)
            # Some character names are written with multiple casings, e.g., SIR_Toby
            # and SIR_TOBY. To normalize the character names, we uppercase each name.
            # Note that this was not done in the original preprocessing and is a
            # recent fix.
            character = character.upper()
            if not (comedy_of_errors and character.startswith('ACT ')):
                characters[character].append(snippet)
                current_character = character
                continue
            else:
                current_character = None
                continue
        elif current_character:
            match = _match_continuation_regex(line, comedy_of_errors)
            if match:
                if comedy_of_errors and match.group(1).startswith('<'):
                    current_character = None
                    continue
                else:
                    characters[current_character].append(match.group(1))
                    continue
        # Didn't consume the line.
        line = line.strip()
        if line and i > 2646:
            # Before 2646 are the sonnets, which we expect to discard.
            discarded_lines.append('%d:%s' % (i, line))
    # Remove degenerate "plays".
    return [play for play in plays if len(play[1]) > 1], discarded_lines
def _split_into_plays ( shakespeare_full ) : # List of tuples (play_name, dict from character to list of lines) plays = [ ] discarded_lines = [ ] # Track discarded lines. slines = shakespeare_full . splitlines ( True ) [ 1 : ] # skip contents, the sonnets, and all's well that ends well author_count = 0 start_i = 0 for i , l in enumerate ( slines ) : if 'by William Shakespeare' in l : author_count += 1 if author_count == 2 : start_i = i - 5 break slines = slines [ start_i : ] current_character = None comedy_of_errors = False for i , line in enumerate ( slines ) : # This marks the end of the plays in the file. if i > 124195 - start_i : break # This is a pretty good heuristic for detecting the start of a new play: if 'by William Shakespeare' in line : current_character = None characters = collections . defaultdict ( list ) # The title will be 2, 3, 4, 5, 6, or 7 lines above "by William Shakespeare". if slines [ i - 2 ] . strip ( ) : title = slines [ i - 2 ] elif slines [ i - 3 ] . strip ( ) : title = slines [ i - 3 ] elif slines [ i - 4 ] . strip ( ) : title = slines [ i - 4 ] elif slines [ i - 5 ] . strip ( ) : title = slines [ i - 5 ] elif slines [ i - 6 ] . strip ( ) : title = slines [ i - 6 ] else : title = slines [ i - 7 ] title = title . strip ( ) assert title , ( 'Parsing error on line %d. Expecting title 2 or 3 lines above.' % i ) comedy_of_errors = ( title == 'THE COMEDY OF ERRORS' ) # Degenerate plays are removed at the end of the method. plays . append ( ( title , characters ) ) continue match = _match_character_regex ( line , comedy_of_errors ) if match : character , snippet = match . group ( 1 ) , match . group ( 2 ) # Some character names are written with multiple casings, e.g., SIR_Toby # and SIR_TOBY. To normalize the character names, we uppercase each name. # Note that this was not done in the original preprocessing and is a # recent fix. character = character . upper ( ) if not ( comedy_of_errors and character . startswith ( 'ACT ' ) ) : characters [ character ] . append ( snippet ) current_character = character continue else : current_character = None continue elif current_character : match = _match_continuation_regex ( line , comedy_of_errors ) if match : if comedy_of_errors and match . group ( 1 ) . startswith ( '<' ) : current_character = None continue else : characters [ current_character ] . append ( match . group ( 1 ) ) continue # Didn't consume the line. line = line . strip ( ) if line and i > 2646 : # Before 2646 are the sonnets, which we expect to discard. discarded_lines . append ( '%d:%s' % ( i , line ) ) # Remove degenerate "plays". return [ play for play in plays if len ( play [ 1 ] ) > 1 ] , discarded_lines