(self, value, positions=False, chars=False, keeporiginal=False,
                 removestops=True, start_pos=0, start_char=0, tokenize=True,
                  mode='', **kwargs)
:param value: The unicode string to tokenize.
        :param positions: Whether to record token positions in the token.
        :param chars: Whether to record character offsets in the token.
        :param start_pos: The position number of the first token. For example,
            if you set start_pos=2, the tokens will be numbered 2,3,4,...
            instead of 0,1,2,...
        :param start_char: The offset of the first character of the first
            token. For example, if you set start_char=2, the text "aaa bbb"
            will have chars (2,5),(6,9) instead (0,3),(4,7).
        :param tokenize: if True, the text should be tokenized.
:param value: The unicode string to tokenize.
        :param positions: Whether to record token positions in the token.
        :param chars: Whether to record character offsets in the token.
        :param start_pos: The position number of the first token. For example,
            if you set start_pos=2, the tokens will be numbered 2,3,4,...
            instead of 0,1,2,...
        :param start_char: The offset of the first character of the first
            token. For example, if you set start_char=2, the text "aaa bbb"
            will have chars (2,5),(6,9) instead (0,3),(4,7).
        :param tokenize: if True, the text should be tokenized.
: param value : The unicode string to tokenize . : param positions : Whether to record token positions in the token . : param chars : Whether to record character offsets in the token . : param start_pos : The position number of the first token . For example if you set start_pos = 2 the tokens will be numbered 2 3 4 ... instead of 0 1 2 ... : param start_char : The offset of the first character of the first token . For example if you set start_char = 2 the text aaa bbb will have chars ( 2 5 ) ( 6 9 ) instead ( 0 3 ) ( 4 7 ) . : param tokenize : if True the text should be tokenized .
def __call__(self, value, positions=False, chars=False, keeporiginal=False,
                 removestops=True, start_pos=0, start_char=0, tokenize=True,
                  mode='', **kwargs):
        """
        :param value: The unicode string to tokenize.
        :param positions: Whether to record token positions in the token.
        :param chars: Whether to record character offsets in the token.
        :param start_pos: The position number of the first token. For example,
            if you set start_pos=2, the tokens will be numbered 2,3,4,...
            instead of 0,1,2,...
        :param start_char: The offset of the first character of the first
            token. For example, if you set start_char=2, the text "aaa bbb"
            will have chars (2,5),(6,9) instead (0,3),(4,7).
        :param tokenize: if True, the text should be tokenized.
        """

        assert isinstance(value, text_type), "%r is not unicode" % value

        t = Token(positions, chars, removestops=removestops, mode=mode,
                  **kwargs)
        if not tokenize:
            t.original = t.text = value
            t.boost = 1.0
            if positions:
                t.pos = start_pos
            if chars:
                t.startchar = start_char
                t.endchar = start_char + len(value)
            yield t
        else:
            text = u("")
            charmap = self.charmap
            pos = start_pos
            startchar = currentchar = start_char
            for char in value:
                tchar = charmap[ord(char)]
                if tchar:
                    text += tchar
                else:
                    if currentchar > startchar:
                        t.text = text
                        t.boost = 1.0
                        if keeporiginal:
                            t.original = t.text
                        if positions:
                            t.pos = pos
                            pos += 1
                        if chars:
                            t.startchar = startchar
                            t.endchar = currentchar
                        yield t
                    startchar = currentchar + 1
                    text = u("")

                currentchar += 1

            if currentchar > startchar:
                t.text = value[startchar:currentchar]
                t.boost = 1.0
                if keeporiginal:
                    t.original = t.text
                if positions:
                    t.pos = pos
                if chars:
                    t.startchar = startchar
                    t.endchar = currentchar
                yield t
def __call__ ( self , value , positions = False , chars = False , keeporiginal = False , removestops = True , start_pos = 0 , start_char = 0 , tokenize = True , mode = '' , * * kwargs ) : assert isinstance ( value , text_type ) , "%r is not unicode" % value t = Token ( positions , chars , removestops = removestops , mode = mode , * * kwargs ) if not tokenize : t . original = t . text = value t . boost = 1.0 if positions : t . pos = start_pos if chars : t . startchar = start_char t . endchar = start_char + len ( value ) yield t else : text = u ( "" ) charmap = self . charmap pos = start_pos startchar = currentchar = start_char for char in value : tchar = charmap [ ord ( char ) ] if tchar : text += tchar else : if currentchar > startchar : t . text = text t . boost = 1.0 if keeporiginal : t . original = t . text if positions : t . pos = pos pos += 1 if chars : t . startchar = startchar t . endchar = currentchar yield t startchar = currentchar + 1 text = u ( "" ) currentchar += 1 if currentchar > startchar : t . text = value [ startchar : currentchar ] t . boost = 1.0 if keeporiginal : t . original = t . text if positions : t . pos = pos if chars : t . startchar = startchar t . endchar = currentchar yield t