(reader)
Takes a file-like object and produces a stream of tokens following
        the LISP rules.

        This is the method doing the heavy-lifting of tokenization.
Takes a file-like object and produces a stream of tokens following
        the LISP rules.
Takes a file - like object and produces a stream of tokens following the LISP rules .
def create_generator(reader):
        """Takes a file-like object and produces a stream of tokens following
        the LISP rules.

        This is the method doing the heavy-lifting of tokenization.
        """
        spaces = set([" ", "\n", "\t"])
        separators = set(["(", ")", "|", "\""])
        specials = spaces | separators | set([";", ""])

        try:
            c = next(reader)
            eof = False
            while not eof:
                if c in specials:
                    # consume the spaces
                    if c in spaces:
                        c = next(reader)

                    elif c in separators:
                        if c == "|":
                            s = []
                            c = next(reader)
                            while c and c != "|":
                                if c == "\\": # This is a single '\'
                                    c = next(reader)
                                    if c != "|" and c != "\\":
                                        # Only \| and \\ are supported escapings
                                        raise PysmtSyntaxError(
                                            "Unknown escaping in quoted symbol: "
                                            "'\\%s'" % c, reader.pos_info)
                                s.append(c)
                                c = next(reader)
                            if not c:
                                raise PysmtSyntaxError("Expected '|'",
                                                       reader.pos_info)
                            yield "".join(s)
                            c = next(reader)

                        elif c == "\"":
                            # String literals
                            s = c
                            num_quotes = 0
                            while True:
                                c = next(reader)
                                if not c:
                                    raise PysmtSyntaxError("Expected '\"'",
                                                           reader.pos_info)

                                if c != "\"" and num_quotes % 2 != 0:
                                    break

                                s += c
                                if c == "\"":
                                    num_quotes += 1

                            yield s

                        else:
                            yield c
                            c = next(reader)

                    elif c == ";":
                        while c and c != "\n":
                            c = next(reader)
                        c = next(reader)

                    else:
                        # EOF
                        eof = True
                        assert len(c) == 0
                else:
                    tk = []
                    while c not in specials:
                        tk.append(c)
                        c = next(reader)
                    yield "".join(tk)
        except StopIteration:
            # No more data to read, close generator
            return
def create_generator ( reader ) : spaces = set ( [ " " , "\n" , "\t" ] ) separators = set ( [ "(" , ")" , "|" , "\"" ] ) specials = spaces | separators | set ( [ ";" , "" ] ) try : c = next ( reader ) eof = False while not eof : if c in specials : # consume the spaces if c in spaces : c = next ( reader ) elif c in separators : if c == "|" : s = [ ] c = next ( reader ) while c and c != "|" : if c == "\\" : # This is a single '\' c = next ( reader ) if c != "|" and c != "\\" : # Only \| and \\ are supported escapings raise PysmtSyntaxError ( "Unknown escaping in quoted symbol: " "'\\%s'" % c , reader . pos_info ) s . append ( c ) c = next ( reader ) if not c : raise PysmtSyntaxError ( "Expected '|'" , reader . pos_info ) yield "" . join ( s ) c = next ( reader ) elif c == "\"" : # String literals s = c num_quotes = 0 while True : c = next ( reader ) if not c : raise PysmtSyntaxError ( "Expected '\"'" , reader . pos_info ) if c != "\"" and num_quotes % 2 != 0 : break s += c if c == "\"" : num_quotes += 1 yield s else : yield c c = next ( reader ) elif c == ";" : while c and c != "\n" : c = next ( reader ) c = next ( reader ) else : # EOF eof = True assert len ( c ) == 0 else : tk = [ ] while c not in specials : tk . append ( c ) c = next ( reader ) yield "" . join ( tk ) except StopIteration : # No more data to read, close generator return