parameters: (self, type_, value, start_pos, prefix)
docstring: Add a token; return True if this is the end of the program.
docstring_summary: Add a token; return True if this is the end of the program.
docstring_tokens: Add a token ; return True if this is the end of the program .
function: def add_token(self, type_, value, start_pos, prefix):
          """Add a token; return True if this is the end of the program."""
          ilabel = token_to_ilabel(self.grammar, type_, value)
  
          # Loop until the token is shifted; may raise exceptions
          _gram = self.grammar
          _labels = _gram.labels
          _push = self._push
          _pop = self._pop
          _shift = self._shift
          while True:
              dfa, state, node = self.stack[-1]
              states, first = dfa
              arcs = states[state]
              # Look for a state with this label
              for i, newstate in arcs:
                  t, v = _labels[i]
                  if ilabel == i:
                      # Look it up in the list of labels
                      assert t < 256
                      # Shift a token; we're done with it
                      _shift(type_, value, newstate, prefix, start_pos)
                      # Pop while we are in an accept-only state
                      state = newstate
                      while states[state] == [(0, state)]:
                          _pop()
                          if not self.stack:
                              # Done parsing!
                              return True
                          dfa, state, node = self.stack[-1]
                          states, first = dfa
                      # Done with this token
                      return False
                  elif t >= 256:
                      # See if it's a symbol and if we're in its first set
                      itsdfa = _gram.dfas[t]
                      itsstates, itsfirst = itsdfa
                      if ilabel in itsfirst:
                          # Push a symbol
                          _push(t, itsdfa, newstate)
                          break  # To continue the outer while loop
              else:
                  if (0, state) in arcs:
                      # An accepting state, pop it and try something else
                      _pop()
                      if not self.stack:
                          # Done parsing, but another token is input
                          raise InternalParseError("too much input", type_, value, start_pos)
                  else:
                      self.error_recovery(self.grammar, self.stack, arcs, type_,
                                          value, start_pos, prefix, self.add_token)
                      break
function_tokens: def add_token ( self , type_ , value , start_pos , prefix ) : ilabel = token_to_ilabel ( self . grammar , type_ , value ) # Loop until the token is shifted; may raise exceptions _gram = self . grammar _labels = _gram . labels _push = self . _push _pop = self . _pop _shift = self . _shift while True : dfa , state , node = self . stack [ - 1 ] states , first = dfa arcs = states [ state ] # Look for a state with this label for i , newstate in arcs : t , v = _labels [ i ] if ilabel == i : # Look it up in the list of labels assert t < 256 # Shift a token; we're done with it _shift ( type_ , value , newstate , prefix , start_pos ) # Pop while we are in an accept-only state state = newstate while states [ state ] == [ ( 0 , state ) ] : _pop ( ) if not self . stack : # Done parsing! return True dfa , state , node = self . stack [ - 1 ] states , first = dfa # Done with this token return False elif t >= 256 : # See if it's a symbol and if we're in its first set itsdfa = _gram . dfas [ t ] itsstates , itsfirst = itsdfa if ilabel in itsfirst : # Push a symbol _push ( t , itsdfa , newstate ) break # To continue the outer while loop else : if ( 0 , state ) in arcs : # An accepting state, pop it and try something else _pop ( ) if not self . stack : # Done parsing, but another token is input raise InternalParseError ( "too much input" , type_ , value , start_pos ) else : self . error_recovery ( self . grammar , self . stack , arcs , type_ , value , start_pos , prefix , self . add_token ) break