(cls, unprocessed, processed, state)
return tokens
Preprocess a single state definition.
Preprocess a single state definition.
Preprocess a single state definition .
def _process_state(cls, unprocessed, processed, state):
        """Preprocess a single state definition."""
        assert type(state) is str, "wrong state name %r" % state
        assert state[0] != '#', "invalid state name %r" % state
        if state in processed:
            return processed[state]
        tokens = processed[state] = []
        rflags = cls.flags
        for tdef in unprocessed[state]:
            if isinstance(tdef, include):
                # it's a state reference
                assert tdef != state, "circular state reference %r" % state
                tokens.extend(cls._process_state(unprocessed, processed,
                                                 str(tdef)))
                continue

            assert type(tdef) is tuple, "wrong rule def %r" % tdef

            try:
                rex = cls._process_regex(tdef[0], rflags)
            except Exception, err:
                raise ValueError("uncompilable regex %r in state %r of %r: %s" %
                                 (tdef[0], state, cls, err))

            token = cls._process_token(tdef[1])

            if len(tdef) == 2:
                new_state = None
            else:
                new_state = cls._process_new_state(tdef[2],
                                                   unprocessed, processed)

            tokens.append((rex, token, new_state))
        return tokens
def _process_state ( cls , unprocessed , processed , state ) : assert type ( state ) is str , "wrong state name %r" % state assert state [ 0 ] != '#' , "invalid state name %r" % state if state in processed : return processed [ state ] tokens = processed [ state ] = [ ] rflags = cls . flags for tdef in unprocessed [ state ] : if isinstance ( tdef , include ) : # it's a state reference assert tdef != state , "circular state reference %r" % state tokens . extend ( cls . _process_state ( unprocessed , processed , str ( tdef ) ) ) continue assert type ( tdef ) is tuple , "wrong rule def %r" % tdef try : rex = cls . _process_regex ( tdef [ 0 ] , rflags ) except Exception , err : raise ValueError ( "uncompilable regex %r in state %r of %r: %s" % ( tdef [ 0 ] , state , cls , err ) ) token = cls . _process_token ( tdef [ 1 ] ) if len ( tdef ) == 2 : new_state = None else : new_state = cls . _process_new_state ( tdef [ 2 ] , unprocessed , processed ) tokens . append ( ( rex , token , new_state ) ) return tokens