LangChain 기반 문단 분리기

!pip install openai
!pip install gradio
!pip install faiss-cpu sentence-transformers
!pip install pdfplumber
!pip install tiktoken

from langchain.text_splitter import RecursiveCharacterTextSplitter
import pdfplumber

# ✅ PDF에서 텍스트 추출
def extract_text_from_pdf(file_path: str) -> str:
    full_text = ""
    with pdfplumber.open(file_path) as pdf:
        for page in pdf.pages:
            text = page.extract_text()
            if text:
                full_text += text + "\n"
    return full_text

# ✅ LangChain 분리기 설정
def split_text_with_langchain(text: str, chunk_size: int = 300, chunk_overlap: int = 50) -> List[str]:
    splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
        model_name="gpt-3.5-turbo",
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap
    )
    return splitter.split_text(text)

# ✅ 실행
pdf_path = "/content/sample.pdf"
text = extract_text_from_pdf(pdf_path)
chunks = split_text_with_langchain(text)

# ✅ 출력 확인
print(f"LangChain 방식: 총 {len(chunks)}개의 청크 생성됨")
print("\n🔹 첫 번째 청크 예시:\n", chunks[0])