(self, line)
return outs
Convert an ascii program line to tokenised form.
Convert an ascii program line to tokenised form.
Convert an ascii program line to tokenised form .
def tokenise_line(self, line):
        """Convert an ascii program line to tokenised form."""
        ins = PlainTextStream(line)
        outs = codestream.TokenisedStream()
        # skip whitespace at start of line
        d = ins.skip_blank()
        if d == b'':
            # empty line at EOF
            return outs
        # read the line number
        self._tokenise_line_number(ins, outs)
        # expect line number
        allow_jumpnum = False
        # expect number (6553 6 -> the 6 is encoded as \x17)
        allow_number = True
        # flag for SPC( or TAB( as numbers can follow the closing bracket
        spc_or_tab = False
        # parse through elements of line
        while True:
            # peek next character
            c = ins.peek()
            # anything after NUL is ignored till EOL
            if c == b'\0':
                ins.read(1)
                ins.read_to((b'', b'\r'))
                break
            # end of line
            elif c in (b'', b'\r'):
                break
            # handle whitespace
            elif c in ins.blanks:
                ins.read(1)
                outs.write(c)
            # handle string literals
            elif ins.peek() == b'"':
                outs.write(ins.read_string())
            # handle jump numbers
            elif allow_number and allow_jumpnum and c in DIGITS + b'.':
                self._tokenise_jump_number(ins, outs)
            # handle numbers
            # numbers following var names with no operator or token in between
            # should not be parsed, eg OPTION BASE 1
            # note we don't include leading signs, encoded as unary operators
            # number starting with & are always parsed
            elif c in (b'&', ) or (
                    allow_number and not allow_jumpnum and c in DIGITS + b'.'
                ):
                outs.write(self._tokenise_number(ins))
            # operator keywords ('+', '-', '=', '/', '\\', '^', '*', '<', '>'):
            elif c in self._ascii_operators:
                ins.read(1)
                # operators don't affect line number mode - can do line number
                # arithmetic and RENUM will do the strangest things
                # this allows for 'LIST 100-200' etc.
                outs.write(self._keyword_to_token[c])
                allow_number = True
            # special case ' -> :REM'
            elif c == b"'":
                ins.read(1)
                outs.write(b':' + tk.REM + tk.O_REM)
                self._tokenise_rem(ins, outs)
            # special case ? -> PRINT
            elif c == b'?':
                ins.read(1)
                outs.write(tk.PRINT)
                allow_number = True
            # keywords & variable names
            elif c in LETTERS:
                word = self._tokenise_word(ins, outs)
                # handle non-parsing modes
                if word in (tk.KW_REM, b"'"):
                    self._tokenise_rem(ins, outs)
                elif word == tk.KW_DATA:
                    self._tokenise_data(ins, outs)
                else:
                    allow_jumpnum = (word in self._linenum_words)
                    # numbers can follow tokenised keywords
                    # (which does not include the word 'AS')
                    allow_number = (word in self._keyword_to_token)
                    if word in (tk.KW_SPC, tk.KW_TAB):
                        spc_or_tab = True
            else:
                ins.read(1)
                if c in (b',', b'#', b';'):
                    # can separate numbers as well as jumpnums
                    allow_number = True
                elif c in (b'(', b'['):
                    allow_number = True
                elif c == b')':
                    if spc_or_tab:
                        spc_or_tab = False
                        allow_jumpnum = False
                    # closing bracket does not affect line number mode
                    allow_number = True
                else:
                    allow_jumpnum, allow_number = False, False
                # replace all other nonprinting chars by spaces;
                # HOUSE 0x7f is allowed.
                outs.write(c if ord(c) >= 32 and ord(c) <= 127 else b' ')
        outs.seek(0)
        return outs
def tokenise_line ( self , line ) : ins = PlainTextStream ( line ) outs = codestream . TokenisedStream ( ) # skip whitespace at start of line d = ins . skip_blank ( ) if d == b'' : # empty line at EOF return outs # read the line number self . _tokenise_line_number ( ins , outs ) # expect line number allow_jumpnum = False # expect number (6553 6 -> the 6 is encoded as \x17) allow_number = True # flag for SPC( or TAB( as numbers can follow the closing bracket spc_or_tab = False # parse through elements of line while True : # peek next character c = ins . peek ( ) # anything after NUL is ignored till EOL if c == b'\0' : ins . read ( 1 ) ins . read_to ( ( b'' , b'\r' ) ) break # end of line elif c in ( b'' , b'\r' ) : break # handle whitespace elif c in ins . blanks : ins . read ( 1 ) outs . write ( c ) # handle string literals elif ins . peek ( ) == b'"' : outs . write ( ins . read_string ( ) ) # handle jump numbers elif allow_number and allow_jumpnum and c in DIGITS + b'.' : self . _tokenise_jump_number ( ins , outs ) # handle numbers # numbers following var names with no operator or token in between # should not be parsed, eg OPTION BASE 1 # note we don't include leading signs, encoded as unary operators # number starting with & are always parsed elif c in ( b'&' , ) or ( allow_number and not allow_jumpnum and c in DIGITS + b'.' ) : outs . write ( self . _tokenise_number ( ins ) ) # operator keywords ('+', '-', '=', '/', '\\', '^', '*', '<', '>'): elif c in self . _ascii_operators : ins . read ( 1 ) # operators don't affect line number mode - can do line number # arithmetic and RENUM will do the strangest things # this allows for 'LIST 100-200' etc. outs . write ( self . _keyword_to_token [ c ] ) allow_number = True # special case ' -> :REM' elif c == b"'" : ins . read ( 1 ) outs . write ( b':' + tk . REM + tk . O_REM ) self . _tokenise_rem ( ins , outs ) # special case ? -> PRINT elif c == b'?' : ins . read ( 1 ) outs . write ( tk . PRINT ) allow_number = True # keywords & variable names elif c in LETTERS : word = self . _tokenise_word ( ins , outs ) # handle non-parsing modes if word in ( tk . KW_REM , b"'" ) : self . _tokenise_rem ( ins , outs ) elif word == tk . KW_DATA : self . _tokenise_data ( ins , outs ) else : allow_jumpnum = ( word in self . _linenum_words ) # numbers can follow tokenised keywords # (which does not include the word 'AS') allow_number = ( word in self . _keyword_to_token ) if word in ( tk . KW_SPC , tk . KW_TAB ) : spc_or_tab = True else : ins . read ( 1 ) if c in ( b',' , b'#' , b';' ) : # can separate numbers as well as jumpnums allow_number = True elif c in ( b'(' , b'[' ) : allow_number = True elif c == b')' : if spc_or_tab : spc_or_tab = False allow_jumpnum = False # closing bracket does not affect line number mode allow_number = True else : allow_jumpnum , allow_number = False , False # replace all other nonprinting chars by spaces; # HOUSE 0x7f is allowed. outs . write ( c if ord ( c ) >= 32 and ord ( c ) <= 127 else b' ' ) outs . seek ( 0 ) return outs