23 lines
548 B
Python
23 lines
548 B
Python
# -*- coding: utf-8 -*-
|
|
"""Utility condivise per chunking testo RAG."""
|
|
|
|
import re
|
|
|
|
CHUNK_SIZE = 900
|
|
CHUNK_OVERLAP = 150
|
|
|
|
|
|
def chunk_text(text: str) -> list[str]:
|
|
text = re.sub(r"\n{3,}", "\n\n", text.strip())
|
|
if len(text) <= CHUNK_SIZE:
|
|
return [text] if text else []
|
|
chunks = []
|
|
start = 0
|
|
while start < len(text):
|
|
end = min(len(text), start + CHUNK_SIZE)
|
|
chunks.append(text[start:end])
|
|
if end >= len(text):
|
|
break
|
|
start = max(end - CHUNK_OVERLAP, start + 1)
|
|
return chunks
|