feat: add versioned backend scoring workflow

backend/wisegrade/rag.py

@@ -0,0 +1,99 @@
+from functools import lru_cache
+from pathlib import Path
+
+from sqlalchemy import cast, delete, func, or_, select
+
+from .config import get_settings
+from .models import KnowledgeChunk, KnowledgeDocument
+
+
+def split_pages(pages: list[dict], *, size=650, overlap=100):
+    for page in pages:
+        blocks = [block.strip() for block in page.get("text", "").split("\n\n") if block.strip()]
+        for table in page.get("tables", []):
+            blocks.append("表格: " + str(table.get("rows", table)))
+        for block in blocks:
+            start = 0
+            while start < len(block):
+                yield {"page": page["page_number"], "section": page.get("section", ""), "text": block[start:start + size]}
+                if start + size >= len(block):
+                    break
+                start += size - overlap
+
+
+@lru_cache(maxsize=2)
+def local_encoder(model_path: str):
+    path = Path(model_path)
+    if not path.is_dir() or not any(path.iterdir()):
+        raise RuntimeError(f"离线嵌入模型不存在: {path}；请挂载本地模型包")
+    from sentence_transformers import SentenceTransformer
+    return SentenceTransformer(str(path), local_files_only=True, trust_remote_code=False)
+
+
+def embed(texts: list[str], *, model_path: str | None = None):
+    if not texts:
+        return []
+    model = local_encoder(model_path or str(get_settings().embedding_model_path))
+    vectors = model.encode(texts, normalize_embeddings=True)
+    output = []
+    for vector in vectors:
+        values = vector.tolist()
+        if len(values) > 1024:
+            raise RuntimeError("嵌入维度超过pgvector列容量1024")
+        output.append(values + [0.0] * (1024 - len(values)))
+    return output
+
+
+def ingest(db, *, course_version_id: int, project_id: int | None, submission_file_id: int | None,
+           title: str, category: str, source_hash: str, pages: list[dict], embedder=embed):
+    existing = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.course_version_id == course_version_id,
+                    KnowledgeDocument.project_id == project_id, KnowledgeDocument.submission_file_id == submission_file_id,
+                    KnowledgeDocument.title == title))
+    if existing and existing.source_hash == source_hash and existing.status == "ready":
+        return existing
+    if existing:
+        db.execute(delete(KnowledgeChunk).where(KnowledgeChunk.document_id == existing.id))
+        db.delete(existing)
+        db.flush()
+    doc = KnowledgeDocument(course_version_id=course_version_id, project_id=project_id, submission_file_id=submission_file_id,
+                            title=title, document_type=category, source_hash=source_hash, status="indexing")
+    db.add(doc)
+    db.flush()
+    chunks = list(split_pages(pages))
+    vectors = embedder([chunk["text"] for chunk in chunks])
+    for chunk, vector in zip(chunks, vectors):
+        db.add(KnowledgeChunk(document_id=doc.id, course_version_id=course_version_id, project_id=project_id,
+                              document_type=category, page=chunk["page"], section=chunk["section"], text=chunk["text"], embedding=vector))
+    doc.status = "ready"
+    db.commit()
+    return doc
+
+
+def retrieve(db, *, course_version_id: int, project_id: int, query: str, category: str | None = None,
+             limit=10, embedder=embed, include_public=True, public_only=False):
+    """Hybrid retrieval with project isolation and an optional public course namespace."""
+    scope = (KnowledgeChunk.project_id.is_(None) if public_only else
+             or_(KnowledgeChunk.project_id == project_id, KnowledgeChunk.project_id.is_(None)) if include_public else
+             KnowledgeChunk.project_id == project_id)
+    filters = [KnowledgeChunk.course_version_id == course_version_id, scope]
+    if category:
+        filters.append(KnowledgeChunk.document_type == category)
+    vector = embedder([query])[0]
+    if db.bind.dialect.name == "postgresql":
+        from pgvector.sqlalchemy import Vector
+        semantic = (1 - cast(KnowledgeChunk.embedding, Vector(1024)).cosine_distance(vector)).label("semantic")
+        lexical = func.ts_rank(func.to_tsvector("simple", KnowledgeChunk.text), func.plainto_tsquery("simple", query)).label("lexical")
+        rows = db.execute(select(KnowledgeChunk, semantic, lexical).where(*filters).order_by(semantic.desc(), lexical.desc()).limit(limit * 3)).all()
+        ranked = [(row[0], float(row.semantic or 0) * .7 + float(row.lexical or 0) * .3) for row in rows]
+    else:
+        chunks = db.scalars(select(KnowledgeChunk).where(*filters)).all()
+        terms = [query[i:i+2] for i in range(max(0, len(query)-1)) if query[i:i+2].strip()]
+        ranked = []
+        for chunk in chunks:
+            semantic = sum(a*b for a, b in zip(chunk.embedding or [], vector))
+            lexical = sum(term in chunk.text for term in terms) / max(len(terms), 1)
+            ranked.append((chunk, semantic * .7 + lexical * .3))
+    ranked.sort(key=lambda pair: pair[1], reverse=True)
+    return [{"chunk_id": chunk.id, "document_id": chunk.document_id, "project_id": chunk.project_id,
+             "document_type": chunk.document_type, "page": chunk.page, "section": chunk.section,
+             "text": chunk.text, "rank": float(rank)} for chunk, rank in ranked[:limit]]