code: def parse(self, lines):
          
          # states:
          #   0: start state
          #   1: saw user-agent line
          #   2: saw an allow or disallow line
          state = 0
          entry = Entry()
  
          for line in lines:
              if not line:
                  if state == 1:
                      entry = Entry()
                      state = 0
                  elif state == 2:
                      self._add_entry(entry)
                      entry = Entry()
                      state = 0
              # remove optional comment and strip line
              i = line.find('#')
              if i >= 0:
                  line = line[:i]
              line = line.strip()
              if not line:
                  continue
              line = line.split(':', 1)
              if len(line) == 2:
                  line[0] = line[0].strip().lower()
                  line[1] = urllib.parse.unquote(line[1].strip())
                  if line[0] == "user-agent":
                      if state == 2:
                          self._add_entry(entry)
                          entry = Entry()
                      entry.useragents.append(line[1])
                      state = 1
                  elif line[0] == "disallow":
                      if state != 0:
                          entry.rulelines.append(RuleLine(line[1], False))
                          state = 2
                  elif line[0] == "allow":
                      if state != 0:
                          entry.rulelines.append(RuleLine(line[1], True))
                          state = 2
          if state == 2:
              self._add_entry(entry)
docs: Parse the input lines from a robots.txt file.
  
          We allow that a user-agent: line is not preceded by
          one or more blank lines.