(self, value, positions=False, chars=False, keeporiginal=False,
                 removestops=True, start_pos=0, start_char=0, tokenize=True,
                 mode='', **kwargs)
:param value: The unicode string to tokenize.
        :param positions: Whether to record token positions in the token.
        :param chars: Whether to record character offsets in the token.
        :param start_pos: The position number of the first token. For example,
            if you set start_pos=2, the tokens will be numbered 2,3,4,...
            instead of 0,1,2,...
        :param start_char: The offset of the first character of the first
            token. For example, if you set start_char=2, the text "aaa bbb"
            will have chars (2,5),(6,9) instead (0,3),(4,7).
        :param tokenize: if True, the text should be tokenized.
:param value: The unicode string to tokenize.
        :param positions: Whether to record token positions in the token.
        :param chars: Whether to record character offsets in the token.
        :param start_pos: The position number of the first token. For example,
            if you set start_pos=2, the tokens will be numbered 2,3,4,...
            instead of 0,1,2,...
        :param start_char: The offset of the first character of the first
            token. For example, if you set start_char=2, the text "aaa bbb"
            will have chars (2,5),(6,9) instead (0,3),(4,7).
        :param tokenize: if True, the text should be tokenized.
: param value : The unicode string to tokenize . : param positions : Whether to record token positions in the token . : param chars : Whether to record character offsets in the token . : param start_pos : The position number of the first token . For example if you set start_pos = 2 the tokens will be numbered 2 3 4 ... instead of 0 1 2 ... : param start_char : The offset of the first character of the first token . For example if you set start_char = 2 the text aaa bbb will have chars ( 2 5 ) ( 6 9 ) instead ( 0 3 ) ( 4 7 ) . : param tokenize : if True the text should be tokenized .
def __call__(self, value, positions=False, chars=False, keeporiginal=False,
                 removestops=True, start_pos=0, start_char=0, tokenize=True,
                 mode='', **kwargs):
        """
        :param value: The unicode string to tokenize.
        :param positions: Whether to record token positions in the token.
        :param chars: Whether to record character offsets in the token.
        :param start_pos: The position number of the first token. For example,
            if you set start_pos=2, the tokens will be numbered 2,3,4,...
            instead of 0,1,2,...
        :param start_char: The offset of the first character of the first
            token. For example, if you set start_char=2, the text "aaa bbb"
            will have chars (2,5),(6,9) instead (0,3),(4,7).
        :param tokenize: if True, the text should be tokenized.
        """

        assert isinstance(value, text_type), "%s is not unicode" % repr(value)

        t = Token(positions, chars, removestops=removestops, mode=mode,
                  **kwargs)
        if not tokenize:
            t.original = t.text = value
            t.boost = 1.0
            if positions:
                t.pos = start_pos
            if chars:
                t.startchar = start_char
                t.endchar = start_char + len(value)
            yield t
        elif not self.gaps:
            # The default: expression matches are used as tokens
            for pos, match in enumerate(self.expression.finditer(value)):
                t.text = match.group(0)
                t.boost = 1.0
                if keeporiginal:
                    t.original = t.text
                t.stopped = False
                if positions:
                    t.pos = start_pos + pos
                if chars:
                    t.startchar = start_char + match.start()
                    t.endchar = start_char + match.end()
                yield t
        else:
            # When gaps=True, iterate through the matches and
            # yield the text between them.
            prevend = 0
            pos = start_pos
            for match in self.expression.finditer(value):
                start = prevend
                end = match.start()
                text = value[start:end]
                if text:
                    t.text = text
                    t.boost = 1.0
                    if keeporiginal:
                        t.original = t.text
                    t.stopped = False
                    if positions:
                        t.pos = pos
                        pos += 1
                    if chars:
                        t.startchar = start_char + start
                        t.endchar = start_char + end

                    yield t

                prevend = match.end()

            # If the last "gap" was before the end of the text,
            # yield the last bit of text as a final token.
            if prevend < len(value):
                t.text = value[prevend:]
                t.boost = 1.0
                if keeporiginal:
                    t.original = t.text
                t.stopped = False
                if positions:
                    t.pos = pos
                if chars:
                    t.startchar = prevend
                    t.endchar = len(value)
                yield t
def __call__ ( self , value , positions = False , chars = False , keeporiginal = False , removestops = True , start_pos = 0 , start_char = 0 , tokenize = True , mode = '' , * * kwargs ) : assert isinstance ( value , text_type ) , "%s is not unicode" % repr ( value ) t = Token ( positions , chars , removestops = removestops , mode = mode , * * kwargs ) if not tokenize : t . original = t . text = value t . boost = 1.0 if positions : t . pos = start_pos if chars : t . startchar = start_char t . endchar = start_char + len ( value ) yield t elif not self . gaps : # The default: expression matches are used as tokens for pos , match in enumerate ( self . expression . finditer ( value ) ) : t . text = match . group ( 0 ) t . boost = 1.0 if keeporiginal : t . original = t . text t . stopped = False if positions : t . pos = start_pos + pos if chars : t . startchar = start_char + match . start ( ) t . endchar = start_char + match . end ( ) yield t else : # When gaps=True, iterate through the matches and # yield the text between them. prevend = 0 pos = start_pos for match in self . expression . finditer ( value ) : start = prevend end = match . start ( ) text = value [ start : end ] if text : t . text = text t . boost = 1.0 if keeporiginal : t . original = t . text t . stopped = False if positions : t . pos = pos pos += 1 if chars : t . startchar = start_char + start t . endchar = start_char + end yield t prevend = match . end ( ) # If the last "gap" was before the end of the text, # yield the last bit of text as a final token. if prevend < len ( value ) : t . text = value [ prevend : ] t . boost = 1.0 if keeporiginal : t . original = t . text t . stopped = False if positions : t . pos = pos if chars : t . startchar = prevend t . endchar = len ( value ) yield t