"""Document preparation shared by scoring and indexing.""" from contextlib import contextmanager import errno import os import time from sqlalchemy import select from .config import get_settings from .models import KnowledgeDocument, Project, SubmissionFile from .parsing import parse_document, text_quality from .rag import INDEX_VERSION, ingest from .storage import secure_path, sha256 @contextmanager def file_index_lock(file_id: int): root = get_settings().processed_path root.mkdir(parents=True, exist_ok=True) with (root / f"file-{file_id}.lock").open("a+b") as handle: if os.name == "nt": import msvcrt if not handle.tell(): handle.write(b"0") handle.flush() while True: handle.seek(0) try: msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) break except OSError as exc: if exc.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK): raise time.sleep(.05) try: yield finally: handle.seek(0) msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) else: import fcntl fcntl.flock(handle, fcntl.LOCK_EX) try: yield finally: fcntl.flock(handle, fcntl.LOCK_UN) def parse_submission(db, file: SubmissionFile, *, embedder=None): with file_index_lock(file.id): db.refresh(file) if file.recognition_status != "recognized": raise ValueError("材料已取消选择,请重新预览") path = secure_path(file.relative_path) if sha256(path) != file.sha256: file.parse_status, file.parse_error = "changed", "源文件已变化,请重新扫描并确认导入" db.commit() raise ValueError(file.parse_error) try: # An indexing failure does not invalidate already saved parsing. parsed = file.parsed if file.parsed and file.parsed.get("pages") else parse_document( path, file.category, get_settings().processed_path / f"file-{file.id}" / file.sha256[:16]) parsed = {**parsed, "pages": [{**page, "text_quality": text_quality(page.get("text", ""))} for page in parsed["pages"]]} file.parsed = parsed doc = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.submission_file_id == file.id, KnowledgeDocument.source_hash == file.sha256, KnowledgeDocument.status != "superseded")) if not doc or doc.source_hash != file.sha256 or doc.index_version != INDEX_VERSION or doc.status != "ready": project = db.get(Project, file.project_id) kwargs = {"embedder": embedder} if embedder else {} ingest(db, course_version_id=project.course_version_id, project_id=project.id, submission_file_id=file.id, title=path.name, category=file.category, source_hash=file.sha256, pages=parsed["pages"], **kwargs) file.parse_status, file.parse_error = "ready", "" db.commit() return parsed except Exception as exc: db.rollback() file = db.get(SubmissionFile, file.id) # Preserve parse output on model/index errors for a later retry. if "parsed" in locals(): file.parsed = parsed file.parse_status, file.parse_error = "failed", str(exc)[:1000] db.commit() raise