(self, tokenizer=lambda x: re.split(r'\s+',x), strict=True, lang='he', vtitle=None)
return index_list, ref_list
See TextChunk.text_index_map
        :param tokenizer:
        :param strict:
        :param lang:
        :return:
See TextChunk.text_index_map
        :param tokenizer:
        :param strict:
        :param lang:
        :return:
See TextChunk . text_index_map : param tokenizer : : param strict : : param lang : : return :
def text_index_map(self, tokenizer=lambda x: re.split(r'\s+',x), strict=True, lang='he', vtitle=None):
        """
        See TextChunk.text_index_map
        :param tokenizer:
        :param strict:
        :param lang:
        :return:
        """
        def traverse(node, callback, offset=0):
            index_list, ref_list, temp_offset = callback(node)
            if node.children:
                for child in node.children:
                    temp_index_list, temp_ref_list, temp_offset = traverse(child, callback, offset)
                    index_list += temp_index_list
                    ref_list += temp_ref_list
                    offset = temp_offset
            else:
                index_list = [i + offset for i in index_list]
                offset += temp_offset
            return index_list, ref_list, offset

        def callback(node):
            if not node.children:
                index_list, ref_list, total_len = node.ref().text(lang=lang, vtitle=vtitle).text_index_map(tokenizer,strict=strict)
                return index_list, ref_list, total_len
            else:
                return [],[], 0

        index_list, ref_list, _ = traverse(self, callback)
        return index_list, ref_list
def text_index_map ( self , tokenizer = lambda x : re . split ( r'\s+' , x ) , strict = True , lang = 'he' , vtitle = None ) : def traverse ( node , callback , offset = 0 ) : index_list , ref_list , temp_offset = callback ( node ) if node . children : for child in node . children : temp_index_list , temp_ref_list , temp_offset = traverse ( child , callback , offset ) index_list += temp_index_list ref_list += temp_ref_list offset = temp_offset else : index_list = [ i + offset for i in index_list ] offset += temp_offset return index_list , ref_list , offset def callback ( node ) : if not node . children : index_list , ref_list , total_len = node . ref ( ) . text ( lang = lang , vtitle = vtitle ) . text_index_map ( tokenizer , strict = strict ) return index_list , ref_list , total_len else : return [ ] , [ ] , 0 index_list , ref_list , _ = traverse ( self , callback ) return index_list , ref_list