Update functions.py
Browse files- functions.py +2 -2
functions.py
CHANGED
@@ -337,8 +337,8 @@ def chunk_and_preprocess_text(text, model_name= 'philschmid/flan-t5-base-samsum'
|
|
337 |
count = -1
|
338 |
|
339 |
for sentence in sentences:
|
340 |
-
|
341 |
-
|
342 |
|
343 |
if combined_length <= tokenizer.max_len_single_sentence: # if it doesn't exceed
|
344 |
chunk += sentence + " " # add the sentence to the chunk
|
|
|
337 |
count = -1
|
338 |
|
339 |
for sentence in sentences:
|
340 |
+
count += 1
|
341 |
+
combined_length = len(tokenizer.tokenize(sentence)) + length # add the no. of sentence tokens to the length counter
|
342 |
|
343 |
if combined_length <= tokenizer.max_len_single_sentence: # if it doesn't exceed
|
344 |
chunk += sentence + " " # add the sentence to the chunk
|