(self, data)
The token generator for the given string
The token generator for the given string
The token generator for the given string
def tokenize(self, data):
        """The token generator for the given string"""
        for match in re.finditer(self.scanner, data):
            for idx, capture in enumerate(match.groups()):
                if capture is not None:
                    rule = self.rules[idx]
                    symbol = rule.symbol
                    if rule.symbol is not None:
                        if rule.is_functional:
                            yield symbol(capture)
                        else:
                            yield symbol
                    break
        yield self.eoi
def tokenize ( self , data ) : for match in re . finditer ( self . scanner , data ) : for idx , capture in enumerate ( match . groups ( ) ) : if capture is not None : rule = self . rules [ idx ] symbol = rule . symbol if rule . symbol is not None : if rule . is_functional : yield symbol ( capture ) else : yield symbol break yield self . eoi