123 lines
6.2 KiB
Python
123 lines
6.2 KiB
Python
"""Persistent RQ indexing jobs; scoring uses the same parser and file lock."""
|
|
from sqlalchemy import select
|
|
|
|
from .db import SessionLocal
|
|
from .models import IndexTask, KnowledgeDocument, SubmissionFile, now
|
|
from .rag import INDEX_VERSION
|
|
from .scoring import parse_submission
|
|
|
|
|
|
def run_index_task(task_id: int, *, session_factory=SessionLocal, embedder=None):
|
|
with session_factory() as db:
|
|
task = db.get(IndexTask, task_id)
|
|
if not task or task.status == "completed":
|
|
return
|
|
task.status, task.error = "running", ""
|
|
db.commit()
|
|
try:
|
|
if task.kind == "reindex":
|
|
# Rebuild documents whose index predates the current chunking, so a
|
|
# legacy index is never silently used as classified body text.
|
|
task.result = reindex_stale(db, task.course_version_id, embedder=embedder)
|
|
if task.result["failed"]:
|
|
raise ValueError("部分资料重建索引失败：" + "、".join(item["title"] for item in task.result["failed"]))
|
|
elif task.kind == "project":
|
|
file = db.get(SubmissionFile, task.submission_file_id)
|
|
if not file or file.recognition_status != "recognized":
|
|
raise ValueError("材料已取消选择，请重新预览")
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
parsed = parse_submission(db, file, **kwargs)
|
|
task.result = {"file_id": file.id, "pages": len(parsed.get("pages", []))}
|
|
elif task.kind == "course":
|
|
from .services import ingest_course
|
|
task.result = ingest_course(task.course_version_id, db)
|
|
if task.result["errors"]:
|
|
raise ValueError("部分课程资料入库失败：" + ", ".join(task.result["errors"]))
|
|
else:
|
|
raise ValueError("未知索引任务类型")
|
|
task.status = "completed"
|
|
except Exception as exc:
|
|
task.status, task.error = "failed", str(exc)[:1000]
|
|
task.updated_at = now()
|
|
db.commit()
|
|
|
|
|
|
def stale_documents(db, course_version_id: int | None = None, *, limit: int | None = None):
|
|
"""Documents whose stored index predates the current chunking or is not ready."""
|
|
query = select(KnowledgeDocument).where(
|
|
(KnowledgeDocument.index_version != INDEX_VERSION) | (KnowledgeDocument.status != "ready"))
|
|
if course_version_id is not None:
|
|
query = query.where(KnowledgeDocument.course_version_id == course_version_id)
|
|
query = query.order_by(KnowledgeDocument.id)
|
|
if limit:
|
|
query = query.limit(limit)
|
|
return db.scalars(query).all()
|
|
|
|
|
|
def reindex_stale(db, course_version_id: int | None = None, *, embedder=None, limit: int | None = None):
|
|
"""Rebuild every stale index from the saved parse output. Retryable and auditable.
|
|
|
|
Student material is rebuilt from ``submission_files.parsed``, which the parser saved
|
|
on disk; the original file is never re-read or modified. Course material follows the
|
|
normal ingest path, because its pages come from the mounted course documents.
|
|
"""
|
|
from .services import parse_submission
|
|
from .storage import sha256, secure_path
|
|
|
|
documents = stale_documents(db, course_version_id, limit=limit)
|
|
rebuilt, skipped, failed = [], [], []
|
|
for document in documents:
|
|
try:
|
|
if document.submission_file_id:
|
|
file = db.get(SubmissionFile, document.submission_file_id)
|
|
if not file:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "来源材料已不存在"})
|
|
continue
|
|
if sha256(secure_path(file.relative_path)) != file.sha256:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "源文件已变化，请重新扫描并确认导入"})
|
|
continue
|
|
file.parse_status = "pending"
|
|
db.commit()
|
|
parsed = parse_submission(db, file, embedder=embedder) if embedder else parse_submission(db, file)
|
|
rebuilt.append({"document_id": document.id, "title": document.title,
|
|
"pages": len(parsed.get("pages", []))})
|
|
elif document.project_id is None:
|
|
# Course material is re-ingested by the course task, not per document.
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "课程资料请通过课程入库重建"})
|
|
else:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "缺少可重建的解析结果"})
|
|
except Exception as exc:
|
|
failed.append({"document_id": document.id, "title": document.title, "error": str(exc)[:300]})
|
|
db.rollback()
|
|
return {"index_version": INDEX_VERSION, "rebuilt": rebuilt, "skipped": skipped,
|
|
"failed": failed, "remaining": len(stale_documents(db, course_version_id))}
|
|
|
|
|
|
def create_reindex_task(db, course_version_id: int):
|
|
"""Queue one reindex job for a course version, reusing an open task when there is one."""
|
|
from .services import enqueue_index_task
|
|
existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id,
|
|
IndexTask.kind == "reindex", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc()))
|
|
task = existing or IndexTask(course_version_id=course_version_id, kind="reindex", status="queued")
|
|
if not existing:
|
|
db.add(task)
|
|
db.commit()
|
|
enqueue_index_task(db, task.id)
|
|
return task
|
|
|
|
|
|
def create_course_index_task(db, course_version_id: int):
|
|
from .services import enqueue_index_task
|
|
existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id,
|
|
IndexTask.kind == "course", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc()))
|
|
task = existing or IndexTask(course_version_id=course_version_id, kind="course", status="queued")
|
|
if not existing:
|
|
db.add(task)
|
|
db.commit()
|
|
enqueue_index_task(db, task.id)
|
|
return task
|