( # 新增的方法
    # 目标按照RoBERTa的思路，使用DOC-SENTENCES，并会去掉NSP任务: 从一个文档中连续的获得文本，直到达到最大长度。如果是从下一个文档中获得，那么加上一个分隔符
    #  document即一整段话，包含多个句子。每个句子叫做segment.
    # 给定一个document即一整段话，生成一些instance.
        all_documents, document_index, max_seq_length, short_seq_prob,
        masked_lm_prob, max_predictions_per_seq, vocab_words, rng)
return instances
Creates `TrainingInstance`s for a single document.
Creates `TrainingInstance`s for a single document.
Creates TrainingInstance s for a single document .
def create_instances_from_document( # 新增的方法
    # 目标按照RoBERTa的思路，使用DOC-SENTENCES，并会去掉NSP任务: 从一个文档中连续的获得文本，直到达到最大长度。如果是从下一个文档中获得，那么加上一个分隔符
    #  document即一整段话，包含多个句子。每个句子叫做segment.
    # 给定一个document即一整段话，生成一些instance.
        all_documents, document_index, max_seq_length, short_seq_prob,
        masked_lm_prob, max_predictions_per_seq, vocab_words, rng):
    """Creates `TrainingInstance`s for a single document."""
    document = all_documents[document_index]

    # Account for [CLS], [SEP], [SEP]
    max_num_tokens = max_seq_length - 3

    # We *usually* want to fill up the entire sequence since we are padding
    # to `max_seq_length` anyways, so short sequences are generally wasted
    # computation. However, we *sometimes*
    # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter
    # sequences to minimize the mismatch between pre-training and fine-tuning.
    # The `target_seq_length` is just a rough target however, whereas
    # `max_seq_length` is a hard limit.

    #target_seq_length = max_num_tokens
    #if rng.random() < short_seq_prob:
    #    target_seq_length = rng.randint(2, max_num_tokens)

    instances = []
    raw_text_list_list=get_raw_instance(document, max_seq_length) # document即一整段话，包含多个句子。每个句子叫做segment.
    for j, raw_text_list in enumerate(raw_text_list_list):
        ####################################################################################################################
        raw_text_list = get_new_segment(raw_text_list) # 结合分词的中文的whole mask设置即在需要的地方加上“##”
        # 1、设置token, segment_ids
        is_random_next=True # this will not be used, so it's value doesn't matter
        tokens = []
        segment_ids = []
        tokens.append("[CLS]")
        segment_ids.append(0)
        for token in raw_text_list:
            tokens.append(token)
            segment_ids.append(0)
        tokens.append("[SEP]")
        segment_ids.append(0)
        ################################################################################################################
        # 2、调用原有的方法
        (tokens, masked_lm_positions,
         masked_lm_labels) = create_masked_lm_predictions(
            tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng)
        instance = TrainingInstance(
            tokens=tokens,
            segment_ids=segment_ids,
            is_random_next=is_random_next,
            masked_lm_positions=masked_lm_positions,
            masked_lm_labels=masked_lm_labels)
        instances.append(instance)

    return instances
def create_instances_from_document ( # 新增的方法 # 目标按照RoBERTa的思路，使用DOC-SENTENCES，并会去掉NSP任务: 从一个文档中连续的获得文本，直到达到最大长度。如果是从下一个文档中获得，那么加上一个分隔符 #  document即一整段话，包含多个句子。每个句子叫做segment. # 给定一个document即一整段话，生成一些instance. all_documents , document_index , max_seq_length , short_seq_prob , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) : document = all_documents [ document_index ] # Account for [CLS], [SEP], [SEP] max_num_tokens = max_seq_length - 3 # We *usually* want to fill up the entire sequence since we are padding # to `max_seq_length` anyways, so short sequences are generally wasted # computation. However, we *sometimes* # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter # sequences to minimize the mismatch between pre-training and fine-tuning. # The `target_seq_length` is just a rough target however, whereas # `max_seq_length` is a hard limit. #target_seq_length = max_num_tokens #if rng.random() < short_seq_prob: #    target_seq_length = rng.randint(2, max_num_tokens) instances = [ ] raw_text_list_list = get_raw_instance ( document , max_seq_length ) # document即一整段话，包含多个句子。每个句子叫做segment. for j , raw_text_list in enumerate ( raw_text_list_list ) : #################################################################################################################### raw_text_list = get_new_segment ( raw_text_list ) # 结合分词的中文的whole mask设置即在需要的地方加上“##” # 1、设置token, segment_ids is_random_next = True # this will not be used, so it's value doesn't matter tokens = [ ] segment_ids = [ ] tokens . append ( "[CLS]" ) segment_ids . append ( 0 ) for token in raw_text_list : tokens . append ( token ) segment_ids . append ( 0 ) tokens . append ( "[SEP]" ) segment_ids . append ( 0 ) ################################################################################################################ # 2、调用原有的方法 ( tokens , masked_lm_positions , masked_lm_labels ) = create_masked_lm_predictions ( tokens , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) instance = TrainingInstance ( tokens = tokens , segment_ids = segment_ids , is_random_next = is_random_next , masked_lm_positions = masked_lm_positions , masked_lm_labels = masked_lm_labels ) instances . append ( instance ) return instances