"""Persistent RQ indexing jobs; scoring uses the same parser and file lock.""" from sqlalchemy import select from .db import SessionLocal from .models import IndexTask, KnowledgeDocument, SubmissionFile, now from .rag import INDEX_VERSION from .scoring import parse_submission def run_index_task(task_id: int, *, session_factory=SessionLocal, embedder=None): with session_factory() as db: task = db.get(IndexTask, task_id) if not task or task.status == "completed": return task.status, task.error = "running", "" db.commit() try: if task.kind == "reindex": # Rebuild documents whose index predates the current chunking, so a # legacy index is never silently used as classified body text. task.result = reindex_stale(db, task.course_version_id, embedder=embedder) if task.result["failed"]: raise ValueError("部分资料重建索引失败:" + "、".join(item["title"] for item in task.result["failed"])) elif task.kind == "project": file = db.get(SubmissionFile, task.submission_file_id) if not file or file.recognition_status != "recognized": raise ValueError("材料已取消选择,请重新预览") kwargs = {"embedder": embedder} if embedder else {} parsed = parse_submission(db, file, **kwargs) task.result = {"file_id": file.id, "pages": len(parsed.get("pages", []))} elif task.kind == "course": from .services import ingest_course task.result = ingest_course(task.course_version_id, db) if task.result["errors"]: raise ValueError("部分课程资料入库失败:" + ", ".join(task.result["errors"])) else: raise ValueError("未知索引任务类型") task.status = "completed" except Exception as exc: task.status, task.error = "failed", str(exc)[:1000] task.updated_at = now() db.commit() def stale_documents(db, course_version_id: int | None = None, *, limit: int | None = None): """Documents whose stored index predates the current chunking or is not ready.""" query = select(KnowledgeDocument).where( (KnowledgeDocument.index_version != INDEX_VERSION) | (KnowledgeDocument.status != "ready")) if course_version_id is not None: query = query.where(KnowledgeDocument.course_version_id == course_version_id) query = query.order_by(KnowledgeDocument.id) if limit: query = query.limit(limit) return db.scalars(query).all() def reindex_stale(db, course_version_id: int | None = None, *, embedder=None, limit: int | None = None): """Rebuild every stale index from the saved parse output. Retryable and auditable. Student material is rebuilt from ``submission_files.parsed``, which the parser saved on disk; the original file is never re-read or modified. Course material follows the normal ingest path, because its pages come from the mounted course documents. """ from .services import parse_submission from .storage import sha256, secure_path documents = stale_documents(db, course_version_id, limit=limit) rebuilt, skipped, failed = [], [], [] for document in documents: try: if document.submission_file_id: file = db.get(SubmissionFile, document.submission_file_id) if not file: skipped.append({"document_id": document.id, "title": document.title, "reason": "来源材料已不存在"}) continue if sha256(secure_path(file.relative_path)) != file.sha256: skipped.append({"document_id": document.id, "title": document.title, "reason": "源文件已变化,请重新扫描并确认导入"}) continue file.parse_status = "pending" db.commit() parsed = parse_submission(db, file, embedder=embedder) if embedder else parse_submission(db, file) rebuilt.append({"document_id": document.id, "title": document.title, "pages": len(parsed.get("pages", []))}) elif document.project_id is None: # Course material is re-ingested by the course task, not per document. skipped.append({"document_id": document.id, "title": document.title, "reason": "课程资料请通过课程入库重建"}) else: skipped.append({"document_id": document.id, "title": document.title, "reason": "缺少可重建的解析结果"}) except Exception as exc: failed.append({"document_id": document.id, "title": document.title, "error": str(exc)[:300]}) db.rollback() return {"index_version": INDEX_VERSION, "rebuilt": rebuilt, "skipped": skipped, "failed": failed, "remaining": len(stale_documents(db, course_version_id))} def create_reindex_task(db, course_version_id: int): """Queue one reindex job for a course version, reusing an open task when there is one.""" from .services import enqueue_index_task existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id, IndexTask.kind == "reindex", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc())) task = existing or IndexTask(course_version_id=course_version_id, kind="reindex", status="queued") if not existing: db.add(task) db.commit() enqueue_index_task(db, task.id) return task def create_course_index_task(db, course_version_id: int): from .services import enqueue_index_task existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id, IndexTask.kind == "course", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc())) task = existing or IndexTask(course_version_id=course_version_id, kind="course", status="queued") if not existing: db.add(task) db.commit() enqueue_index_task(db, task.id) return task