"""Persistent RQ indexing jobs; scoring uses the same parser and file lock.""" from sqlalchemy import select from .db import SessionLocal from .models import AuditLog, IndexTask, KnowledgeDocument, KnowledgePage, SubmissionFile, now from .rag import INDEX_VERSION from .submissions import file_index_lock, parse_submission def run_index_task(task_id: int, *, session_factory=SessionLocal, embedder=None): with session_factory() as db: task = db.get(IndexTask, task_id) if not task or task.status == "completed": return task.status, task.error = "running", "" db.commit() result = None try: if task.kind == "reindex": # Rebuild documents whose index predates the current chunking, so a # legacy index is never silently used as classified body text. result = reindex_stale(db, task.course_version_id, embedder=embedder) if result["failed"] or result["remaining"]: raise ValueError(f"索引重建尚未完成，剩余 {result['remaining']} 份；请核对失败和跳过记录") elif task.kind == "project": file = db.get(SubmissionFile, task.submission_file_id) if not file or file.recognition_status != "recognized": raise ValueError("材料已取消选择，请重新预览") kwargs = {"embedder": embedder} if embedder else {} parsed = parse_submission(db, file, **kwargs) result = {"file_id": file.id, "pages": len(parsed.get("pages", []))} elif task.kind == "course": from .services import ingest_course result = ingest_course(task.course_version_id, db) if result["errors"]: raise ValueError("部分课程资料入库失败：" + ", ".join(result["errors"])) else: raise ValueError("未知索引任务类型") task.status = "completed" except Exception as exc: db.rollback() task = db.get(IndexTask, task_id) if not task: return task.status, task.error = "failed", str(exc)[:1000] if result is not None: task.result = result task.updated_at = now() db.commit() def stale_documents(db, course_version_id: int | None = None, *, limit: int | None = None): """Documents whose stored index predates the current chunking or is not ready.""" query = select(KnowledgeDocument).where( KnowledgeDocument.status != "superseded", (KnowledgeDocument.index_version != INDEX_VERSION) | (KnowledgeDocument.status != "ready")) if course_version_id is not None: query = query.where(KnowledgeDocument.course_version_id == course_version_id) query = query.order_by(KnowledgeDocument.id) if limit: query = query.limit(limit) return db.scalars(query).all() def reindex_stale(db, course_version_id: int | None = None, *, embedder=None, limit: int | None = None): """Rebuild every stale index from the saved parse output. Retryable and auditable. Student material reuses ``submission_files.parsed`` after validating the source hash; public material reuses canonical pages. Old pages/chunks remain available under their original document IDs, while a new revision becomes searchable. """ from .storage import sha256, secure_path from .config import get_settings from .rag import ingest documents = stale_documents(db, course_version_id, limit=limit) rebuilt, skipped, failed = [], [], [] for document in documents: try: if document.submission_file_id: file = db.get(SubmissionFile, document.submission_file_id) if not file: skipped.append({"document_id": document.id, "title": document.title, "reason": "来源材料已不存在"}) continue if sha256(secure_path(file.relative_path)) != file.sha256: skipped.append({"document_id": document.id, "title": document.title, "reason": "源文件已变化，请重新扫描并确认导入"}) continue parsed = parse_submission(db, file, embedder=embedder) if embedder else parse_submission(db, file) replacement = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.submission_file_id == file.id, KnowledgeDocument.source_hash == file.sha256, KnowledgeDocument.status == "ready") .order_by(KnowledgeDocument.id.desc())) rebuilt.append({"document_id": document.id, "title": document.title, "replacement_document_id": replacement.id, "pages": len(parsed.get("pages", []))}) elif document.project_id is None: # Course import uses this same lock. Refresh after acquiring it so # overlapping jobs cannot rebuild a snapshot already replaced. with file_index_lock(-document.course_version_id): db.refresh(document) if document.status == "superseded" or (document.status == "ready" and document.index_version == INDEX_VERSION): skipped.append({"document_id": document.id, "title": document.title, "reason": "已由其他任务完成或替代"}) continue records = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id == document.id) .order_by(KnowledgePage.page)).all() if not records: skipped.append({"document_id": document.id, "title": document.title, "reason": "缺少课程页面，请通过课程入库重建"}) continue pages = [{"page_number": row.page, "text": row.text, "section": row.section, "rendered_image_path": str(get_settings().processed_path / row.image_path) if row.image_path else None} for row in records] kwargs = {"embedder": embedder} if embedder else {} replacement = ingest(db, course_version_id=document.course_version_id, project_id=None, submission_file_id=None, title=document.title, category=document.document_type, source_hash=document.source_hash, pages=pages, **kwargs) rebuilt.append({"document_id": document.id, "replacement_document_id": replacement.id, "title": document.title, "pages": len(pages)}) else: skipped.append({"document_id": document.id, "title": document.title, "reason": "缺少可重建的解析结果"}) except Exception as exc: failed.append({"document_id": document.id, "title": document.title, "error": str(exc)[:300]}) db.rollback() result = {"index_version": INDEX_VERSION, "rebuilt": rebuilt, "skipped": skipped, "failed": failed, "remaining": len(stale_documents(db, course_version_id))} db.add(AuditLog(action="reindex", entity_type="course_version", entity_id=course_version_id, details={"index_version": INDEX_VERSION, "rebuilt": len(rebuilt), "failed": len(failed), "skipped": len(skipped), "remaining": result["remaining"]})) db.commit() return result def create_reindex_task(db, course_version_id: int): """Queue one reindex job for a course version, reusing an open task when there is one.""" from .services import enqueue_index_task existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id, IndexTask.kind == "reindex", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc())) if existing: return existing task = existing or IndexTask(course_version_id=course_version_id, kind="reindex", status="queued") if not existing: db.add(task) db.commit() enqueue_index_task(db, task.id) return task def create_course_index_task(db, course_version_id: int): from .services import enqueue_index_task existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id, IndexTask.kind == "course", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc())) if existing: return existing task = existing or IndexTask(course_version_id=course_version_id, kind="course", status="queued") if not existing: db.add(task) db.commit() enqueue_index_task(db, task.id) return task