431 lines
23 KiB
Python
431 lines
23 KiB
Python
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from contextlib import contextmanager
|
|
import fcntl
|
|
|
|
from sqlalchemy import select, update
|
|
|
|
from .config import get_settings
|
|
from .db import SessionLocal
|
|
from .evidence import USE_CONTENT, content_targets, normalize_targets
|
|
from .knowledge_pages import page_image_path
|
|
from .llm import ConsistencyResult, FactResult, ModelError, ScoreResult, validate_score
|
|
from .models import (CriterionScore, KnowledgeChunk, KnowledgeDocument, KnowledgePage, Project,
|
|
ScoreEvidence, ScoringBatch, SubmissionFile, now)
|
|
from .pages import normalize_heading
|
|
from .parsing import parse_document
|
|
from .rag import (INDEX_VERSION, body_heading_index, ingest, retrieve, retrieve_targets)
|
|
from .storage import secure_path, sha256
|
|
from .settings_store import client_for_batch
|
|
|
|
#: Criterion statuses. ``needs_review`` is deliberately distinct from a missing
|
|
#: material, a parse failure and a model failure: the retrieval ran but only found
|
|
#: navigation pages where the rule requires body content.
|
|
STATUS_NEEDS_REVIEW = "needs_review"
|
|
STATUS_UNSCORABLE = "unscorable"
|
|
STATUS_FAILED = "failed"
|
|
STATUS_SCORED = "scored"
|
|
STATUS_REVIEWED = "reviewed"
|
|
#: Statuses that a batch counts as finished work.
|
|
DONE_STATUSES = (STATUS_SCORED, STATUS_REVIEWED, STATUS_NEEDS_REVIEW)
|
|
#: Statuses the dashboard shows as needing teacher attention.
|
|
ATTENTION_STATUSES = (STATUS_FAILED, STATUS_UNSCORABLE, STATUS_NEEDS_REVIEW)
|
|
|
|
|
|
@contextmanager
|
|
def file_index_lock(file_id: int):
|
|
root = get_settings().processed_path
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
with (root / f"file-{file_id}.lock").open("a+b") as handle:
|
|
fcntl.flock(handle, fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(handle, fcntl.LOCK_UN)
|
|
|
|
|
|
def parse_submission(db, file: SubmissionFile, *, embedder=None):
|
|
with file_index_lock(file.id):
|
|
db.refresh(file)
|
|
return _parse_submission_locked(db, file, embedder=embedder)
|
|
|
|
|
|
def _parse_submission_locked(db, file: SubmissionFile, *, embedder=None):
|
|
path = secure_path(file.relative_path)
|
|
if sha256(path) != file.sha256:
|
|
file.parse_status, file.parse_error = "changed", "源文件已变化，请重新扫描并确认导入"
|
|
db.commit()
|
|
raise ValueError(file.parse_error)
|
|
if file.parse_status == "ready" and file.parsed:
|
|
document = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.submission_file_id == file.id))
|
|
if not document or document.source_hash != file.sha256 or document.index_version != INDEX_VERSION or document.status != "ready":
|
|
project = db.get(Project, file.project_id)
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
try:
|
|
ingest(db, course_version_id=project.course_version_id, project_id=project.id,
|
|
submission_file_id=file.id, title=path.name, category=file.category,
|
|
source_hash=file.sha256, pages=file.parsed["pages"], **kwargs)
|
|
except Exception as exc:
|
|
file.parse_status, file.parse_error = "failed", str(exc)[:1000]
|
|
db.commit()
|
|
raise
|
|
return file.parsed
|
|
settings = get_settings()
|
|
try:
|
|
parsed = parse_document(path, file.category, settings.processed_path / f"file-{file.id}")
|
|
project = db.get(Project, file.project_id)
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
ingest(db, course_version_id=project.course_version_id, project_id=project.id,
|
|
submission_file_id=file.id, title=path.name, category=file.category,
|
|
source_hash=file.sha256, pages=parsed["pages"], **kwargs)
|
|
file.parsed, file.parse_status, file.parse_error = parsed, "ready", ""
|
|
db.commit()
|
|
return parsed
|
|
except Exception as exc:
|
|
file.parse_status, file.parse_error = "failed", str(exc)[:1000]
|
|
db.commit()
|
|
raise
|
|
|
|
|
|
def _project_facts(db, project, course_snapshot, client, *, retriever):
|
|
"""Extract project facts from body text only.
|
|
|
|
Implementation facts must not be inferred from a contents heading or an abstract
|
|
claim, so the search admits body and appendix pages exclusively. Project identity
|
|
(name, student number, topic) legitimately lives on the cover page and is read from
|
|
the teacher-confirmed import record instead of from retrieved text.
|
|
"""
|
|
prompts = course_snapshot["prompts"]
|
|
categories = ("stage1", "stage2", "stage3", "final_report", "presentation")
|
|
snippets = []
|
|
for category in categories:
|
|
snippets.extend(retriever(db, course_version_id=project.course_version_id, project_id=project.id,
|
|
category=category, query="项目名称 目标 指标 硬件 软件 实验 数据 成本 结论",
|
|
limit=3, include_public=False, allowed_page_types=["body", "appendix"],
|
|
substantive_only=True))
|
|
if not snippets:
|
|
return {}
|
|
if client is None:
|
|
raise ModelError("未配置或无法连接评分模型")
|
|
facts = client.json_call(prompts["facts"], {"evidence": snippets}, FactResult).model_dump()
|
|
from .models import Student
|
|
student = db.get(Student, project.student_id)
|
|
warnings = []
|
|
for key, expected in (("student_no", student.student_no), ("name", student.name), ("topic", project.topic)):
|
|
if facts.get(key) and facts[key] != expected:
|
|
warnings.append(f"模型抽取的{key}与教师确认信息不一致")
|
|
consistency = client.json_call(prompts["consistency"], {"facts": facts, "evidence": snippets}, ConsistencyResult)
|
|
known = {snippet["chunk_id"] for snippet in snippets}
|
|
for issue in consistency.issues:
|
|
if not set(issue.chunk_ids) <= known:
|
|
raise ModelError("一致性检查引用了未提供的证据")
|
|
project.facts = {**facts, "identity_warnings": warnings,
|
|
"consistency_issues": [issue.model_dump() for issue in consistency.issues]}
|
|
db.commit()
|
|
return project.facts
|
|
|
|
|
|
def _body_navigator(db, *, course_version_id, project_id, category):
|
|
"""Find real body pages for a target whose contents entries name missing sections.
|
|
|
|
The contents page is used only as a map: its headings are matched against headings
|
|
that actually occur in body text, and the body sections and their neighbouring pages
|
|
are then searched. The final quotation always lands on a real body page.
|
|
"""
|
|
headings = body_heading_index(db, course_version_id=course_version_id, project_id=project_id, category=category)
|
|
if not headings:
|
|
return None
|
|
|
|
def navigate(target):
|
|
label = normalize_heading(target.get("label") or "")
|
|
if not label:
|
|
return []
|
|
candidates = []
|
|
for heading, places in headings.items():
|
|
if not heading:
|
|
continue
|
|
if heading.startswith(label) or label.startswith(heading) or label in heading or heading in label:
|
|
candidates.extend(places)
|
|
if not candidates:
|
|
return []
|
|
pages = sorted({(place["document_id"], place["page"]) for place in candidates})
|
|
neighbours = set()
|
|
for document_id, page in pages:
|
|
neighbours.update({(document_id, page), (document_id, page + 1)})
|
|
wanted = sorted(neighbours)
|
|
rows = db.scalars(select(KnowledgeChunk).where(
|
|
KnowledgeChunk.course_version_id == course_version_id, KnowledgeChunk.project_id == project_id,
|
|
KnowledgeChunk.page_type == "body", KnowledgeChunk.block_type != "toc",
|
|
KnowledgeChunk.document_id.in_([item[0] for item in wanted]),
|
|
KnowledgeChunk.page.in_([item[1] for item in wanted])).order_by(KnowledgeChunk.id)).all()
|
|
hits = []
|
|
for chunk in rows:
|
|
hits.append({"chunk_id": chunk.id, "document_id": chunk.document_id, "project_id": chunk.project_id,
|
|
"document_type": chunk.document_type, "page": chunk.page, "section": chunk.section,
|
|
"chapter": chunk.chapter, "page_type": chunk.page_type, "block_type": chunk.block_type,
|
|
"text": chunk.text, "rank": 0.5})
|
|
return hits
|
|
|
|
return navigate
|
|
|
|
|
|
def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_retriever=retrieve_targets):
|
|
score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
|
|
CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
|
|
if score and score.status in (STATUS_SCORED, STATUS_REVIEWED):
|
|
return score
|
|
if not score:
|
|
score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
|
|
db.add(score)
|
|
db.flush()
|
|
category = "presentation" if criterion["criterion_id"] == "final.presentation" else (
|
|
"final_report" if criterion["category"] == "final" else criterion["category"])
|
|
files = db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id,
|
|
SubmissionFile.category == category, SubmissionFile.recognition_status == "recognized")).all()
|
|
ready = [file for file in files if file.parse_status == "ready"]
|
|
if len(files) != 1 or len(ready) != 1:
|
|
score.status, score.error = STATUS_UNSCORABLE, "材料缺失、重复、待确认或解析失败"
|
|
db.commit()
|
|
return score
|
|
|
|
plan = batch.rubric_snapshot.get("evidence_plan") or {}
|
|
targets = (plan.get("criteria") or {}).get(criterion["criterion_id"]) or normalize_targets(
|
|
criterion.get("required_evidence"))
|
|
navigator = _body_navigator(db, course_version_id=batch.course_version_id,
|
|
project_id=project.id, category=category)
|
|
# The single ``retriever`` stays the seam: each per-target search goes through it,
|
|
# so a caller that injects a retriever (tests, alternate index) still applies.
|
|
selection = multi_retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
|
|
targets=targets, category=category, navigator=navigator,
|
|
retrieve_fn=retriever)
|
|
chunks = _attach_page_types(db, selection["evidence"])
|
|
gaps = selection["uncovered_targets"]
|
|
if not chunks:
|
|
# Nothing admissible was found for any target. Content targets go to review;
|
|
# a purely structural criterion only depends on navigation pages, which exist.
|
|
score.status = STATUS_NEEDS_REVIEW if content_targets(criterion) else STATUS_UNSCORABLE
|
|
score.error = "未检索到允许作为证据的正文页"
|
|
score.model_score = None
|
|
score.missing_evidence = [target["label"] for target in gaps]
|
|
_record_evidence_gaps(score, gaps)
|
|
db.commit()
|
|
return score
|
|
if client is None:
|
|
raise ModelError("未配置或无法连接评分模型")
|
|
|
|
content_gaps = [target for target in gaps if target["use"] == USE_CONTENT]
|
|
if content_gaps:
|
|
# The model is not asked to grade from navigation pages. Record the gap and keep
|
|
# the model score null so a teacher decides, rather than asserting a low score.
|
|
score.model_score, score.confidence, score.status = None, None, STATUS_NEEDS_REVIEW
|
|
score.model_comment = "检索仅命中目录、摘要或章节标题，缺少可用于内容判断的正文证据，转教师复核。"
|
|
score.missing_evidence = [target["label"] for target in content_gaps]
|
|
score.suggestion = "请补充该章节的正文页，或由教师直接核定该项分数。"
|
|
score.error = "缺少正文证据：" + "、".join(target["label"] for target in content_gaps)
|
|
_record_evidence_gaps(score, content_gaps, searched=_searched_page_types(plan, content_gaps))
|
|
db.commit()
|
|
return score
|
|
|
|
# Page images go to the model as well, so prefer substantive body pages over the
|
|
# navigation pages that may legitimately appear in a structural target's evidence.
|
|
image_sources = list(dict.fromkeys(
|
|
(chunk["document_id"], chunk["page"]) for chunk in chunks
|
|
if chunk.get("page_type") == "body"))
|
|
if not image_sources:
|
|
image_sources = list(dict.fromkeys((chunk["document_id"], chunk["page"]) for chunk in chunks))
|
|
images = []
|
|
for document_id, page in image_sources[:3]:
|
|
doc = db.get(KnowledgeDocument, document_id)
|
|
path = page_image_path(db, doc, page) if doc else None
|
|
if path:
|
|
images.append(str(path))
|
|
query = criterion["description"] + " " + " ".join(target["label"] for target in targets)
|
|
public_context = retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
|
|
category=None, query=query, limit=6, public_only=True)
|
|
payload = {"criterion": criterion,
|
|
"required_evidence_targets": [{"target_id": target["target_id"], "label": target["label"],
|
|
"use": target["use"],
|
|
"allowed_page_types": target["allowed_page_types"]}
|
|
for target in targets],
|
|
"evidence_policy": {
|
|
"structure": "目录、封面和参考文献页可以证明章节结构和报告完整性，不能证明章节内容或完成质量。",
|
|
"content": "内容判断只能使用正文或附录中的实质段落、数据、表格；目录、摘要和单独章节标题不能作为内容证据。"},
|
|
"project_facts": project.facts, "course_context": public_context, "evidence": chunks,
|
|
"required_output": {"criterion_id": "string", "score": "number or null", "max_score": "number",
|
|
"reason": "string",
|
|
"evidence": [{"chunk_id": "integer", "page": "integer",
|
|
"quote": "exact substring", "target_id": "string"}],
|
|
"missing_evidence": ["string"], "suggestion": "string", "confidence": "0..1",
|
|
"note": "证据不足以判断内容时 score 填 null，并在 missing_evidence 中列出缺少的正文目标。"}}
|
|
prompt = (batch.course_snapshot["prompts"]["score"] +
|
|
" 证据quote必须从所引用chunk_id的text逐字复制连续片段，保留原文标点和换行；page必须等于该片段的page；"
|
|
"target_id必须取自required_evidence_targets。不要改写、缩写或拼接引文。"
|
|
"不得把目录、摘要或章节标题当作内容证据；无法据此判断内容时把score设为null。")
|
|
error = ""
|
|
result = None
|
|
for attempt in range(2):
|
|
try:
|
|
result = validate_score(client.json_call(prompt, payload, ScoreResult, images=images), criterion, chunks,
|
|
targets=targets)
|
|
break
|
|
except ModelError as exc:
|
|
error = str(exc)
|
|
if attempt:
|
|
raise
|
|
prompt += f" 上次回答未通过校验（{error}）；请改用允许页面类型中的实质正文片段，并逐字复制quote。"
|
|
score.model_score, score.model_comment = result.score, result.reason
|
|
score.confidence = result.confidence
|
|
score.missing_evidence, score.suggestion = result.missing_evidence, result.suggestion
|
|
if result.score is None:
|
|
# The retrieval found body pages but the model judged them insufficient.
|
|
score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项，转教师复核。"
|
|
else:
|
|
score.status, score.error = STATUS_SCORED, ""
|
|
for evidence in db.scalars(select(ScoreEvidence).where(ScoreEvidence.score_id == score.id)).all():
|
|
db.delete(evidence)
|
|
by_id = {chunk["chunk_id"]: chunk for chunk in chunks}
|
|
for evidence in result.evidence:
|
|
chunk = by_id[evidence.chunk_id]
|
|
doc = db.get(KnowledgeDocument, chunk["document_id"])
|
|
path = page_image_path(db, doc, evidence.page)
|
|
target = next((item for item in targets if item["target_id"] == evidence.target_id), None)
|
|
db.add(ScoreEvidence(score_id=score.id, document_id=doc.id, submission_file_id=doc.submission_file_id,
|
|
page=evidence.page, quote=evidence.quote, image_path=str(path) if path else None,
|
|
target_id=evidence.target_id, target_label=target["label"] if target else "",
|
|
page_type=chunk.get("page_type") or "unknown",
|
|
source_role=target["use"] if target else USE_CONTENT))
|
|
db.commit()
|
|
return score
|
|
|
|
|
|
def _attach_page_types(db, chunks: list[dict]) -> list[dict]:
|
|
"""Attach the stored page classification to each retrieved chunk.
|
|
|
|
The page type recorded in ``knowledge_pages`` is the authority for the evidence
|
|
policy. Reading it here, rather than trusting whatever a retriever happened to put
|
|
in the dict, means a statement about what a page is cannot be forged by a caller
|
|
or lost by an alternate index.
|
|
"""
|
|
keys = {(chunk.get("document_id"), chunk.get("page")) for chunk in chunks}
|
|
keys.discard((None, None))
|
|
if not keys:
|
|
return chunks
|
|
documents = {document_id for document_id, _ in keys}
|
|
pages = {page for _, page in keys}
|
|
records = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id.in_(documents),
|
|
KnowledgePage.page.in_(pages))).all()
|
|
by_key = {(item.document_id, item.page): item for item in records}
|
|
output = []
|
|
for chunk in chunks:
|
|
record = by_key.get((chunk.get("document_id"), chunk.get("page")))
|
|
if record is None:
|
|
output.append({**chunk, "page_type": chunk.get("page_type") or "unknown"})
|
|
continue
|
|
output.append({**chunk, "page_type": record.page_type or "unknown",
|
|
"section": chunk.get("section") or record.section or "",
|
|
"chapter": chunk.get("chapter") or record.chapter or ""})
|
|
return output
|
|
|
|
|
|
def _searched_page_types(plan, targets):
|
|
allowed = set()
|
|
for target in targets:
|
|
allowed.update(target.get("allowed_page_types") or [])
|
|
return sorted(allowed or (plan.get("content_allowed_page_types") or []))
|
|
|
|
|
|
def _record_evidence_gaps(score, gaps, searched=None):
|
|
"""Store which content targets lacked body evidence and where retrieval looked.
|
|
|
|
The record is kept through teacher review so the original retrieval problem stays
|
|
auditable next to the score the teacher finally entered.
|
|
"""
|
|
score.evidence_gaps = [
|
|
{"target_id": target.get("target_id"), "target_label": target.get("label"),
|
|
"searched_page_types": searched or target.get("allowed_page_types") or [],
|
|
"reason": "补检后仍未找到正文证据"}
|
|
for target in gaps]
|
|
|
|
|
|
def _run_project(batch_id, project_id, session_factory, client_factory, embedder, retriever,
|
|
multi_retriever=retrieve_targets):
|
|
"""Score one project. Returns ``(failures, reviews)``.
|
|
|
|
Failures are missing material, parse problems and model errors. Reviews count the
|
|
items that only found navigation pages where content evidence is required; those
|
|
need a teacher decision rather than a retry.
|
|
"""
|
|
failures = 0
|
|
reviews = 0
|
|
with session_factory() as db:
|
|
batch = db.get(ScoringBatch, batch_id)
|
|
project = db.get(Project, project_id)
|
|
try:
|
|
client = client_factory(db, batch)
|
|
except Exception:
|
|
client = None
|
|
files = db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id)).all()
|
|
for file in files:
|
|
if file.recognition_status == "recognized":
|
|
try:
|
|
parse_submission(db, file, embedder=embedder)
|
|
except Exception:
|
|
failures += 1
|
|
try:
|
|
_project_facts(db, project, batch.course_snapshot, client, retriever=retriever)
|
|
except Exception as exc:
|
|
project.facts = {"error": str(exc)[:500]}
|
|
db.commit()
|
|
failures += 1
|
|
for criterion in batch.rubric_snapshot["criteria"]:
|
|
try:
|
|
result = _score_criterion(db, batch, project, criterion, client, retriever=retriever,
|
|
multi_retriever=multi_retriever)
|
|
if result.status == STATUS_NEEDS_REVIEW:
|
|
reviews += 1
|
|
elif result.status not in (STATUS_SCORED, STATUS_REVIEWED):
|
|
failures += 1
|
|
except Exception as exc:
|
|
score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
|
|
CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
|
|
if not score:
|
|
score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
|
|
db.add(score)
|
|
score.status, score.error = STATUS_FAILED, str(exc)[:500]
|
|
db.commit()
|
|
failures += 1
|
|
db.execute(update(ScoringBatch).where(ScoringBatch.id == batch_id).values(progress_done=ScoringBatch.progress_done + 1))
|
|
db.commit()
|
|
return failures, reviews
|
|
|
|
|
|
def run_batch(batch_id: int, *, session_factory=SessionLocal, client_factory=client_for_batch, embedder=None,
|
|
retriever=retrieve, multi_retriever=retrieve_targets):
|
|
with session_factory() as db:
|
|
batch = db.get(ScoringBatch, batch_id)
|
|
if not batch:
|
|
raise ValueError("评分批次不存在")
|
|
project_ids = batch.project_ids if batch.project_ids is not None else db.scalars(
|
|
select(Project.id).where(Project.course_version_id == batch.course_version_id).order_by(Project.id)).all()
|
|
criteria = batch.rubric_snapshot["criteria"]
|
|
batch.status, batch.progress_total, batch.progress_done, batch.finished_at = "running", len(project_ids) * len(criteria), 0, None
|
|
db.commit()
|
|
failures = 0
|
|
reviews = 0
|
|
with ThreadPoolExecutor(max_workers=max(1, int(batch.model_config_snapshot.get("concurrency", 1)))) as pool:
|
|
futures = [pool.submit(_run_project, batch_id, project_id, session_factory, client_factory,
|
|
embedder, retriever, multi_retriever) for project_id in project_ids]
|
|
for future in as_completed(futures):
|
|
try:
|
|
project_failures, project_reviews = future.result()
|
|
failures += project_failures
|
|
reviews += project_reviews
|
|
except Exception as exc:
|
|
failures += 1
|
|
batch.error = (batch.error + f" 项目任务失败: {type(exc).__name__}")[:1000]
|
|
# A batch with unresolved review items is finished running, but it is not a
|
|
# completed grading result: the teacher still has to decide those criteria.
|
|
batch.status = "partial" if failures else ("review_required" if reviews else "completed")
|
|
batch.finished_at = now()
|
|
db.commit()
|
|
return {"failures": failures, "reviews": reviews, "status": batch.status}
|