Rethinking RAG: Document Ingestion ที่หลายคนมองข้าม

# hierarchical_chunking.py
from dataclasses import dataclass, field
 
@dataclass
class HierarchicalChunk:
    """
    Parent-Child structure:
    - ใช้ child (เล็ก) ตอน vector search → ความแม่นยำสูง
    - ดึง parent (ใหญ่) ให้ LLM อ่าน → บริบทครบ
    """
    parent_id: str
    child_id: str
    parent_text: str      # ส่งให้ LLM อ่าน (512-1024 tokens)
    child_text: str       # ใช้ทำ embedding (64-128 tokens)
    metadata: dict = field(default_factory=dict)
 
def create_hierarchical_chunks(
    text: str,
    parent_size: int = 1024,
    child_size: int = 128,
    overlap: int = 32
) -> list[HierarchicalChunk]:
    """
    สร้าง Parent-Child chunks
    1 parent → หลาย children
    """
    import hashlib
 
    def chunk_text(text, size, overlap):
        chunks, start = [], 0
        while start < len(text):
            end = min(start + size, len(text))
            chunks.append((start, text[start:end]))
            start += size - overlap
        return chunks
 
    parents = chunk_text(text, parent_size, overlap * 4)
    result = []
 
    for p_idx, (p_start, parent_text) in enumerate(parents):
        parent_id = hashlib.md5(f"parent_{p_idx}_{parent_text[:50]}".encode()).hexdigest()[:8]
        children = chunk_text(parent_text, child_size, overlap)
 
        for c_idx, (_, child_text) in enumerate(children):
            child_id = f"{parent_id}_c{c_idx}"
            result.append(HierarchicalChunk(
                parent_id=parent_id,
                child_id=child_id,
                parent_text=parent_text,
                child_text=child_text,
                metadata={
                    "parent_index": p_idx,
                    "child_index": c_idx,
                    "parent_char_start": p_start
                }
            ))
 
    return result
 
 
# Retrieval: ค้นด้วย child → ส่ง parent ให้ LLM
def retrieve_with_context(
    query: str,
    chunks: list[HierarchicalChunk],
    embedder,
    top_k: int = 3
) -> list[str]:
    """
    Search ด้วย child chunks (แม่นยำ)
    แต่ return parent chunks (มีบริบท)
    """
    child_embeddings = {
        c.child_id: embedder(c.child_text) for c in chunks
    }
    query_emb = embedder(query)
 
    from numpy import dot, linalg
    scores = {
        cid: dot(query_emb, emb) / (linalg.norm(query_emb) * linalg.norm(emb))
        for cid, emb in child_embeddings.items()
    }
    top_children = sorted(scores, key=scores.get, reverse=True)[:top_k]
 
    seen_parents = set()
    contexts = []
    for child_id in top_children:
        chunk = next(c for c in chunks if c.child_id == child_id)
        if chunk.parent_id not in seen_parents:
            seen_parents.add(chunk.parent_id)
            contexts.append(chunk.parent_text)
 
    return contexts

{
  "content": "...ข้อความ...",
  "metadata": {
    "source": "policy_v3.pdf",
    "doc_version": "3.0",
    "effective_date": "2025-01-01",
    "ingestion_ts": "2025-03-01T10:00:00Z",
    "chunk_index": 12,
    "parent_id": "a3f9b2c1"
  }
}

# adaptive_ingestion.py
import anthropic
from pathlib import Path
from typing import Literal
from dataclasses import dataclass
 
DocumentType = Literal["financial_report", "legal_doc", "research_paper", "chat_log", "unknown"]
 
@dataclass
class IngestionStrategy:
    doc_type: DocumentType
    chunking_method: str
    chunk_size: int
    overlap: int
    use_layout_parsing: bool
    metadata_fields: list[str]
 
class AdaptiveIngestionAgent:
    """
    ใช้ LLM วิเคราะห์เอกสารก่อน แล้วค่อยเลือก strategy ที่เหมาะสม
    แทนที่จะใช้วิธีเดียวกับทุกเอกสาร
    """
 

Input Files (PDF/DOCX/TXT)
         |
[Adaptive Ingestion Agent]  <-- Claude API (Document Classifier)
         | classify
  +----------------------------+
  | Layout Parser  |  Semantic |
  |  (Docling)     |  Chunker  |
  +----------------------------+
         | chunks + metadata
  [Hierarchical Index]
  Child chunks -> Qdrant (vector search)
  Parent chunks -> stored with parent_id
         | query time
  Search child -> retrieve parent -> LLM
         |
  Answer (with full context)

# requirements.txt
 
# Core LLM & Embeddings
anthropic>=0.40.0          # Claude API (LLM + Document Analysis)
openai>=1.50.0             # OpenAI Embeddings (text-embedding-3-large)
 
# Document Parsing (Layout-Aware)
docling>=2.0.0             # IBM Layout-Aware Parser (PDF tables, columns)
unstructured[pdf]>=0.15.0  # Fallback parser + file type detection
pymupdf>=1.24.0            # Fast PDF text extraction (fitz)
python-docx>=1.1.0         # Word document parsing
 
# Vector Store
qdrant-client>=1.10.0      # Qdrant (self-hosted / cloud)
 
# Orchestration / Pipeline
langchain>=0.3.0           # Pipeline orchestration
langchain-community>=0.3