(self, s)
return i
Process a token's worth of chars from a string, if possible.

        If a token is found, it is added to self._tokens. A call might
        process characters but not emit a token.

        Args:
          s: The string to process.

        Returns:
          The number of characters processed from the beginning of the string.
Process a token's worth of chars from a string, if possible.
Process a token s worth of chars from a string if possible .
def _process_token(self, s):
        """Process a token's worth of chars from a string, if possible.

        If a token is found, it is added to self._tokens. A call might
        process characters but not emit a token.

        Args:
          s: The string to process.

        Returns:
          The number of characters processed from the beginning of the string.
        """
        i = 0

        # TODO: PICO-8 doesn't allow literal newlines in quoted strings,
        # so this probably shouldn't either.
        if self._in_string is not None:
            # Continue string literal.
            while i < len(s):
                c = s[i:i+1]

                if c == self._in_string_delim:
                    # End string literal.
                    self._tokens.append(
                        TokString(b''.join(self._in_string),
                                  self._in_string_lineno,
                                  self._in_string_charno,
                                  quote=self._in_string_delim))
                    self._in_string_delim = None
                    self._in_string_lineno = None
                    self._in_string_charno = None
                    self._in_string = None
                    i += 1
                    break

                if c == b'\\':
                    # Escape character.
                    num_m = re.match(br'\d{1,3}', s[i+1:])
                    if num_m:
                        c = bytes([int(num_m.group(0))])
                        i += len(num_m.group(0))
                    else:
                        next_c = s[i+1:i+2]
                        if next_c in _STRING_ESCAPES:
                            c = _STRING_ESCAPES[next_c]
                            i += 1

                self._in_string.append(c)
                i += 1

        elif self._in_multiline_comment is not None:
            try:
                i = s.index(b']]') + 2
                self._in_multiline_comment.append(s[:i])
                self._tokens.append(
                    TokComment(b''.join(self._in_multiline_comment),
                               self._in_multiline_comment_lineno,
                               self._in_multiline_comment_charno))
                self._in_multiline_comment = None
                self._in_multiline_comment_lineno = None
                self._in_multiline_comment_charno = None

            except ValueError:
                self._in_multiline_comment.append(s)
                i = len(s)

        elif self._in_multiline_string is not None:
            m = re.search(br'\]' + self._in_multiline_string_delim + br'\]', s)
            if m:
                i = m.end()
                self._in_multiline_string.append(s[:m.start()])
                self._tokens.append(
                    TokString(b''.join(self._in_multiline_string),
                              self._in_multiline_string_lineno,
                              self._in_multiline_string_charno,
                              multiline_quote=self._in_multiline_string_delim))
                self._in_multiline_string = None
                self._in_multiline_string_delim = None
                self._in_multiline_string_lineno = None
                self._in_multiline_string_charno = None

            else:
                self._in_multiline_string.append(s)
                i = len(s)

        elif s.startswith(b'--[['):
            # (Multiline comments do not support the [===[ thing that
            # multiline strings do, so we can match directly.)
            self._in_multiline_comment = [b'--[[']
            self._in_multiline_comment_lineno = self._cur_lineno
            self._in_multiline_comment_charno = self._cur_charno
            i = 4

        elif re.match(br'\[=*\[', s):
            m = re.match(br'\[(=*)\[', s)
            i = m.end()
            self._in_multiline_string = []
            self._in_multiline_string_delim = m.group(1)
            self._in_multiline_string_lineno = self._cur_lineno
            self._in_multiline_string_charno = self._cur_charno

        elif s.startswith(b"'") or s.startswith(b'"'):
            # Begin string literal.
            self._in_string_delim = s[0:1]
            self._in_string_lineno = self._cur_lineno
            self._in_string_charno = self._cur_charno
            self._in_string = []
            i = 1

        else:
            # Match one-line patterns.
            for (pat, tok_class) in _TOKEN_MATCHERS:
                m = pat.match(s)
                if m:
                    if tok_class is not None:
                        token = tok_class(m.group(0),
                                          self._cur_lineno,
                                          self._cur_charno)
                        self._tokens.append(token)
                    i = len(m.group(0))
                    break

        for c in s[:i]:
            # (b'\n'[0] == 10)
            if c == b'\n'[0]:
                self._cur_lineno += 1
                self._cur_charno = 0
            else:
                self._cur_charno += 1
        return i
def _process_token ( self , s ) : i = 0 # TODO: PICO-8 doesn't allow literal newlines in quoted strings, # so this probably shouldn't either. if self . _in_string is not None : # Continue string literal. while i < len ( s ) : c = s [ i : i + 1 ] if c == self . _in_string_delim : # End string literal. self . _tokens . append ( TokString ( b'' . join ( self . _in_string ) , self . _in_string_lineno , self . _in_string_charno , quote = self . _in_string_delim ) ) self . _in_string_delim = None self . _in_string_lineno = None self . _in_string_charno = None self . _in_string = None i += 1 break if c == b'\\' : # Escape character. num_m = re . match ( br'\d{1,3}' , s [ i + 1 : ] ) if num_m : c = bytes ( [ int ( num_m . group ( 0 ) ) ] ) i += len ( num_m . group ( 0 ) ) else : next_c = s [ i + 1 : i + 2 ] if next_c in _STRING_ESCAPES : c = _STRING_ESCAPES [ next_c ] i += 1 self . _in_string . append ( c ) i += 1 elif self . _in_multiline_comment is not None : try : i = s . index ( b']]' ) + 2 self . _in_multiline_comment . append ( s [ : i ] ) self . _tokens . append ( TokComment ( b'' . join ( self . _in_multiline_comment ) , self . _in_multiline_comment_lineno , self . _in_multiline_comment_charno ) ) self . _in_multiline_comment = None self . _in_multiline_comment_lineno = None self . _in_multiline_comment_charno = None except ValueError : self . _in_multiline_comment . append ( s ) i = len ( s ) elif self . _in_multiline_string is not None : m = re . search ( br'\]' + self . _in_multiline_string_delim + br'\]' , s ) if m : i = m . end ( ) self . _in_multiline_string . append ( s [ : m . start ( ) ] ) self . _tokens . append ( TokString ( b'' . join ( self . _in_multiline_string ) , self . _in_multiline_string_lineno , self . _in_multiline_string_charno , multiline_quote = self . _in_multiline_string_delim ) ) self . _in_multiline_string = None self . _in_multiline_string_delim = None self . _in_multiline_string_lineno = None self . _in_multiline_string_charno = None else : self . _in_multiline_string . append ( s ) i = len ( s ) elif s . startswith ( b'--[[' ) : # (Multiline comments do not support the [===[ thing that # multiline strings do, so we can match directly.) self . _in_multiline_comment = [ b'--[[' ] self . _in_multiline_comment_lineno = self . _cur_lineno self . _in_multiline_comment_charno = self . _cur_charno i = 4 elif re . match ( br'\[=*\[' , s ) : m = re . match ( br'\[(=*)\[' , s ) i = m . end ( ) self . _in_multiline_string = [ ] self . _in_multiline_string_delim = m . group ( 1 ) self . _in_multiline_string_lineno = self . _cur_lineno self . _in_multiline_string_charno = self . _cur_charno elif s . startswith ( b"'" ) or s . startswith ( b'"' ) : # Begin string literal. self . _in_string_delim = s [ 0 : 1 ] self . _in_string_lineno = self . _cur_lineno self . _in_string_charno = self . _cur_charno self . _in_string = [ ] i = 1 else : # Match one-line patterns. for ( pat , tok_class ) in _TOKEN_MATCHERS : m = pat . match ( s ) if m : if tok_class is not None : token = tok_class ( m . group ( 0 ) , self . _cur_lineno , self . _cur_charno ) self . _tokens . append ( token ) i = len ( m . group ( 0 ) ) break for c in s [ : i ] : # (b'\n'[0] == 10) if c == b'\n' [ 0 ] : self . _cur_lineno += 1 self . _cur_charno = 0 else : self . _cur_charno += 1 return i