aerospaceboy
ebe33392ca
Preserve document revisions and evidence, harden indexing and export behavior, add explicit material selection and direct-folder import, validate model JSON schemas with safe error messages, and retain student selection while loading scoring data.
163 lines
8.8 KiB
Python
163 lines
8.8 KiB
Python
"""Persistent RQ indexing jobs; scoring uses the same parser and file lock."""
|
|
from sqlalchemy import select
|
|
|
|
from .db import SessionLocal
|
|
from .models import AuditLog, IndexTask, KnowledgeDocument, KnowledgePage, SubmissionFile, now
|
|
from .rag import INDEX_VERSION
|
|
from .submissions import file_index_lock, parse_submission
|
|
|
|
|
|
def run_index_task(task_id: int, *, session_factory=SessionLocal, embedder=None):
|
|
with session_factory() as db:
|
|
task = db.get(IndexTask, task_id)
|
|
if not task or task.status == "completed":
|
|
return
|
|
task.status, task.error = "running", ""
|
|
db.commit()
|
|
result = None
|
|
try:
|
|
if task.kind == "reindex":
|
|
# Rebuild documents whose index predates the current chunking, so a
|
|
# legacy index is never silently used as classified body text.
|
|
result = reindex_stale(db, task.course_version_id, embedder=embedder)
|
|
if result["failed"] or result["remaining"]:
|
|
raise ValueError(f"索引重建尚未完成，剩余 {result['remaining']} 份；请核对失败和跳过记录")
|
|
elif task.kind == "project":
|
|
file = db.get(SubmissionFile, task.submission_file_id)
|
|
if not file or file.recognition_status != "recognized":
|
|
raise ValueError("材料已取消选择，请重新预览")
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
parsed = parse_submission(db, file, **kwargs)
|
|
result = {"file_id": file.id, "pages": len(parsed.get("pages", []))}
|
|
elif task.kind == "course":
|
|
from .services import ingest_course
|
|
result = ingest_course(task.course_version_id, db)
|
|
if result["errors"]:
|
|
raise ValueError("部分课程资料入库失败：" + ", ".join(result["errors"]))
|
|
else:
|
|
raise ValueError("未知索引任务类型")
|
|
task.status = "completed"
|
|
except Exception as exc:
|
|
db.rollback()
|
|
task = db.get(IndexTask, task_id)
|
|
if not task:
|
|
return
|
|
task.status, task.error = "failed", str(exc)[:1000]
|
|
if result is not None:
|
|
task.result = result
|
|
task.updated_at = now()
|
|
db.commit()
|
|
|
|
|
|
def stale_documents(db, course_version_id: int | None = None, *, limit: int | None = None):
|
|
"""Documents whose stored index predates the current chunking or is not ready."""
|
|
query = select(KnowledgeDocument).where(
|
|
KnowledgeDocument.status != "superseded",
|
|
(KnowledgeDocument.index_version != INDEX_VERSION) | (KnowledgeDocument.status != "ready"))
|
|
if course_version_id is not None:
|
|
query = query.where(KnowledgeDocument.course_version_id == course_version_id)
|
|
query = query.order_by(KnowledgeDocument.id)
|
|
if limit:
|
|
query = query.limit(limit)
|
|
return db.scalars(query).all()
|
|
|
|
|
|
def reindex_stale(db, course_version_id: int | None = None, *, embedder=None, limit: int | None = None):
|
|
"""Rebuild every stale index from the saved parse output. Retryable and auditable.
|
|
|
|
Student material reuses ``submission_files.parsed`` after validating the source
|
|
hash; public material reuses canonical pages. Old pages/chunks remain available
|
|
under their original document IDs, while a new revision becomes searchable.
|
|
"""
|
|
from .storage import sha256, secure_path
|
|
from .config import get_settings
|
|
from .rag import ingest
|
|
|
|
documents = stale_documents(db, course_version_id, limit=limit)
|
|
rebuilt, skipped, failed = [], [], []
|
|
for document in documents:
|
|
try:
|
|
if document.submission_file_id:
|
|
file = db.get(SubmissionFile, document.submission_file_id)
|
|
if not file:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "来源材料已不存在"})
|
|
continue
|
|
if sha256(secure_path(file.relative_path)) != file.sha256:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "源文件已变化，请重新扫描并确认导入"})
|
|
continue
|
|
parsed = parse_submission(db, file, embedder=embedder) if embedder else parse_submission(db, file)
|
|
replacement = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.submission_file_id == file.id,
|
|
KnowledgeDocument.source_hash == file.sha256, KnowledgeDocument.status == "ready")
|
|
.order_by(KnowledgeDocument.id.desc()))
|
|
rebuilt.append({"document_id": document.id, "title": document.title,
|
|
"replacement_document_id": replacement.id,
|
|
"pages": len(parsed.get("pages", []))})
|
|
elif document.project_id is None:
|
|
# Course import uses this same lock. Refresh after acquiring it so
|
|
# overlapping jobs cannot rebuild a snapshot already replaced.
|
|
with file_index_lock(-document.course_version_id):
|
|
db.refresh(document)
|
|
if document.status == "superseded" or (document.status == "ready" and document.index_version == INDEX_VERSION):
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "已由其他任务完成或替代"})
|
|
continue
|
|
records = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id == document.id)
|
|
.order_by(KnowledgePage.page)).all()
|
|
if not records:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "缺少课程页面，请通过课程入库重建"})
|
|
continue
|
|
pages = [{"page_number": row.page, "text": row.text, "section": row.section,
|
|
"rendered_image_path": str(get_settings().processed_path / row.image_path)
|
|
if row.image_path else None} for row in records]
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
replacement = ingest(db, course_version_id=document.course_version_id, project_id=None,
|
|
submission_file_id=None, title=document.title, category=document.document_type,
|
|
source_hash=document.source_hash, pages=pages, **kwargs)
|
|
rebuilt.append({"document_id": document.id, "replacement_document_id": replacement.id,
|
|
"title": document.title, "pages": len(pages)})
|
|
else:
|
|
skipped.append({"document_id": document.id, "title": document.title,
|
|
"reason": "缺少可重建的解析结果"})
|
|
except Exception as exc:
|
|
failed.append({"document_id": document.id, "title": document.title, "error": str(exc)[:300]})
|
|
db.rollback()
|
|
result = {"index_version": INDEX_VERSION, "rebuilt": rebuilt, "skipped": skipped,
|
|
"failed": failed, "remaining": len(stale_documents(db, course_version_id))}
|
|
db.add(AuditLog(action="reindex", entity_type="course_version", entity_id=course_version_id,
|
|
details={"index_version": INDEX_VERSION, "rebuilt": len(rebuilt),
|
|
"failed": len(failed), "skipped": len(skipped), "remaining": result["remaining"]}))
|
|
db.commit()
|
|
return result
|
|
|
|
|
|
def create_reindex_task(db, course_version_id: int):
|
|
"""Queue one reindex job for a course version, reusing an open task when there is one."""
|
|
from .services import enqueue_index_task
|
|
existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id,
|
|
IndexTask.kind == "reindex", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc()))
|
|
if existing:
|
|
return existing
|
|
task = existing or IndexTask(course_version_id=course_version_id, kind="reindex", status="queued")
|
|
if not existing:
|
|
db.add(task)
|
|
db.commit()
|
|
enqueue_index_task(db, task.id)
|
|
return task
|
|
|
|
|
|
def create_course_index_task(db, course_version_id: int):
|
|
from .services import enqueue_index_task
|
|
existing = db.scalar(select(IndexTask).where(IndexTask.course_version_id == course_version_id,
|
|
IndexTask.kind == "course", IndexTask.status.in_(["queued", "running"])).order_by(IndexTask.id.desc()))
|
|
if existing:
|
|
return existing
|
|
task = existing or IndexTask(course_version_id=course_version_id, kind="course", status="queued")
|
|
if not existing:
|
|
db.add(task)
|
|
db.commit()
|
|
enqueue_index_task(db, task.id)
|
|
return task
|