aerospaceboy
ebe33392ca
Preserve document revisions and evidence, harden indexing and export behavior, add explicit material selection and direct-folder import, validate model JSON schemas with safe error messages, and retain student selection while loading scoring data.
86 lines
3.6 KiB
Python
86 lines
3.6 KiB
Python
"""Document preparation shared by scoring and indexing."""
|
|
from contextlib import contextmanager
|
|
import errno
|
|
import os
|
|
import time
|
|
|
|
from sqlalchemy import select
|
|
|
|
from .config import get_settings
|
|
from .models import KnowledgeDocument, Project, SubmissionFile
|
|
from .parsing import parse_document, text_quality
|
|
from .rag import INDEX_VERSION, ingest
|
|
from .storage import secure_path, sha256
|
|
|
|
|
|
@contextmanager
|
|
def file_index_lock(file_id: int):
|
|
root = get_settings().processed_path
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
with (root / f"file-{file_id}.lock").open("a+b") as handle:
|
|
if os.name == "nt":
|
|
import msvcrt
|
|
if not handle.tell():
|
|
handle.write(b"0")
|
|
handle.flush()
|
|
while True:
|
|
handle.seek(0)
|
|
try:
|
|
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
|
break
|
|
except OSError as exc:
|
|
if exc.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
|
|
raise
|
|
time.sleep(.05)
|
|
try:
|
|
yield
|
|
finally:
|
|
handle.seek(0)
|
|
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(handle, fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(handle, fcntl.LOCK_UN)
|
|
|
|
|
|
def parse_submission(db, file: SubmissionFile, *, embedder=None):
|
|
with file_index_lock(file.id):
|
|
db.refresh(file)
|
|
if file.recognition_status != "recognized":
|
|
raise ValueError("材料已取消选择，请重新预览")
|
|
path = secure_path(file.relative_path)
|
|
if sha256(path) != file.sha256:
|
|
file.parse_status, file.parse_error = "changed", "源文件已变化，请重新扫描并确认导入"
|
|
db.commit()
|
|
raise ValueError(file.parse_error)
|
|
try:
|
|
# An indexing failure does not invalidate already saved parsing.
|
|
parsed = file.parsed if file.parsed and file.parsed.get("pages") else parse_document(
|
|
path, file.category, get_settings().processed_path / f"file-{file.id}" / file.sha256[:16])
|
|
parsed = {**parsed, "pages": [{**page, "text_quality": text_quality(page.get("text", ""))}
|
|
for page in parsed["pages"]]}
|
|
file.parsed = parsed
|
|
doc = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.submission_file_id == file.id,
|
|
KnowledgeDocument.source_hash == file.sha256, KnowledgeDocument.status != "superseded"))
|
|
if not doc or doc.source_hash != file.sha256 or doc.index_version != INDEX_VERSION or doc.status != "ready":
|
|
project = db.get(Project, file.project_id)
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
ingest(db, course_version_id=project.course_version_id, project_id=project.id,
|
|
submission_file_id=file.id, title=path.name, category=file.category,
|
|
source_hash=file.sha256, pages=parsed["pages"], **kwargs)
|
|
file.parse_status, file.parse_error = "ready", ""
|
|
db.commit()
|
|
return parsed
|
|
except Exception as exc:
|
|
db.rollback()
|
|
file = db.get(SubmissionFile, file.id)
|
|
# Preserve parse output on model/index errors for a later retry.
|
|
if "parsed" in locals():
|
|
file.parsed = parsed
|
|
file.parse_status, file.parse_error = "failed", str(exc)[:1000]
|
|
db.commit()
|
|
raise
|