(self, string, token_type, line, line_number, values=None)
Creates a new Token object.

    Args:
      string: The string of input the token contains.
      token_type: The type of token.
      line: The text of the line this token is in.
      line_number: The line number of the token.
      values: A dict of named values within the token.  For instance, a
        function declaration may have a value called 'name' which captures the
        name of the function.
Creates a new Token object.
Creates a new Token object .
def __init__(self, string, token_type, line, line_number, values=None):
    """Creates a new Token object.

    Args:
      string: The string of input the token contains.
      token_type: The type of token.
      line: The text of the line this token is in.
      line_number: The line number of the token.
      values: A dict of named values within the token.  For instance, a
        function declaration may have a value called 'name' which captures the
        name of the function.
    """
    self.type = token_type
    self.string = string
    self.length = len(string)
    self.line = line
    self.line_number = line_number
    self.values = values

    # These parts can only be computed when the file is fully tokenized
    self.previous = None
    self.next = None
    self.start_index = None

    # This part is set in statetracker.py
    # TODO(robbyw): Wrap this in to metadata
    self.attached_object = None

    # This part is set in *metadatapass.py
    self.metadata = None
def __init__ ( self , string , token_type , line , line_number , values = None ) : self . type = token_type self . string = string self . length = len ( string ) self . line = line self . line_number = line_number self . values = values # These parts can only be computed when the file is fully tokenized self . previous = None self . next = None self . start_index = None # This part is set in statetracker.py # TODO(robbyw): Wrap this in to metadata self . attached_object = None # This part is set in *metadatapass.py self . metadata = None