Co-authored-by: Minfeng Lu <minfenglu@Minfengs-MacBook-Pro.local>
Co-authored-by: Richard Beales <rich@richbeales.net>
This commit is contained in:
minfeng-ai
2023-05-08 06:10:58 -07:00
committed by GitHub
parent 0166eacb2b
commit 23e1e1ed53
4 changed files with 5 additions and 5 deletions

View File

@@ -30,10 +30,10 @@ def split_text(
Raises:
ValueError: If the text is longer than the maximum length
"""
flatened_paragraphs = " ".join(text.split("\n"))
flattened_paragraphs = " ".join(text.split("\n"))
nlp = spacy.load(CFG.browse_spacy_language_model)
nlp.add_pipe("sentencizer")
doc = nlp(flatened_paragraphs)
doc = nlp(flattened_paragraphs)
sentences = [sent.text.strip() for sent in doc.sents]
current_chunk = []