(self, ins, outs)
return word
Convert a keyword to tokenised form.
Convert a keyword to tokenised form.
Convert a keyword to tokenised form .
def _tokenise_word(self, ins, outs):
        """Convert a keyword to tokenised form."""
        word = b''
        while True:
            c = ins.read(1)
            word += c.upper()
            # special cases 'GO     TO' -> 'GOTO', 'GO SUB' -> 'GOSUB'
            if word == b'GO':
                pos = ins.tell()
                # GO SUB allows 1 space
                if ins.peek(4).upper() == b' SUB':
                    word = tk.KW_GOSUB
                    ins.read(4)
                else:
                    # GOTO allows any number of spaces
                    nxt = ins.skip_blank()
                    if ins.read(2).upper() == b'TO':
                        word = tk.KW_GOTO
                    else:
                        ins.seek(pos)
                if word in (tk.KW_GOTO, tk.KW_GOSUB):
                    nxt = ins.peek()
                    if nxt and nxt in tk.NAME_CHARS:
                        ins.seek(pos)
                        word = b'GO'
            if word in self._keyword_to_token:
                # ignore if part of a longer name, except FN, SPC(, TAB(, USR
                if word not in (tk.KW_FN, tk.KW_SPC, tk.KW_TAB, tk.KW_USR):
                    nxt = ins.peek()
                    if nxt and nxt in tk.NAME_CHARS:
                        continue
                token = self._keyword_to_token[word]
                # handle special case ELSE -> :ELSE
                if word == tk.KW_ELSE:
                    outs.write(b':' + token)
                # handle special case WHILE -> WHILE+
                elif word == tk.KW_WHILE:
                    outs.write(token + tk.O_PLUS)
                else:
                    outs.write(token)
                break
            # allowed names: letter + (letters, numbers, .)
            elif not c:
                outs.write(word)
                break
            elif c not in tk.NAME_CHARS:
                word = word[:-1]
                ins.seek(-1, 1)
                outs.write(word)
                break
        return word
def _tokenise_word ( self , ins , outs ) : word = b'' while True : c = ins . read ( 1 ) word += c . upper ( ) # special cases 'GO     TO' -> 'GOTO', 'GO SUB' -> 'GOSUB' if word == b'GO' : pos = ins . tell ( ) # GO SUB allows 1 space if ins . peek ( 4 ) . upper ( ) == b' SUB' : word = tk . KW_GOSUB ins . read ( 4 ) else : # GOTO allows any number of spaces nxt = ins . skip_blank ( ) if ins . read ( 2 ) . upper ( ) == b'TO' : word = tk . KW_GOTO else : ins . seek ( pos ) if word in ( tk . KW_GOTO , tk . KW_GOSUB ) : nxt = ins . peek ( ) if nxt and nxt in tk . NAME_CHARS : ins . seek ( pos ) word = b'GO' if word in self . _keyword_to_token : # ignore if part of a longer name, except FN, SPC(, TAB(, USR if word not in ( tk . KW_FN , tk . KW_SPC , tk . KW_TAB , tk . KW_USR ) : nxt = ins . peek ( ) if nxt and nxt in tk . NAME_CHARS : continue token = self . _keyword_to_token [ word ] # handle special case ELSE -> :ELSE if word == tk . KW_ELSE : outs . write ( b':' + token ) # handle special case WHILE -> WHILE+ elif word == tk . KW_WHILE : outs . write ( token + tk . O_PLUS ) else : outs . write ( token ) break # allowed names: letter + (letters, numbers, .) elif not c : outs . write ( word ) break elif c not in tk . NAME_CHARS : word = word [ : - 1 ] ins . seek ( - 1 , 1 ) outs . write ( word ) break return word