Update functions.py
Browse files- functions.py +11 -11
functions.py
CHANGED
@@ -346,19 +346,19 @@ def chunk_and_preprocess_text(text, model_name= 'philschmid/flan-t5-base-samsum'
|
|
346 |
chunk += sentence + " " # add the sentence to the chunk
|
347 |
length = combined_length # update the length counter
|
348 |
|
349 |
-
|
350 |
-
|
351 |
-
|
352 |
|
353 |
-
|
354 |
-
|
355 |
-
|
356 |
-
|
357 |
-
|
358 |
|
359 |
-
|
360 |
-
|
361 |
-
|
362 |
|
363 |
return chunks
|
364 |
|
|
|
346 |
chunk += sentence + " " # add the sentence to the chunk
|
347 |
length = combined_length # update the length counter
|
348 |
|
349 |
+
# if it is the last sentence
|
350 |
+
if count == len(sentences) - 1:
|
351 |
+
chunks.append(chunk) # save the chunk
|
352 |
|
353 |
+
else:
|
354 |
+
chunks.append(chunk) # save the chunk
|
355 |
+
# reset
|
356 |
+
length = 0
|
357 |
+
chunk = ""
|
358 |
|
359 |
+
# take care of the overflow sentence
|
360 |
+
chunk += sentence + " "
|
361 |
+
length = len(tokenizer.tokenize(sentence))
|
362 |
|
363 |
return chunks
|
364 |
|