feat: separate contents, abstract and body evidence

backend/wisegrade/scoring.py

@@ -6,15 +6,30 @@ from sqlalchemy import select, update
 
 from .config import get_settings
 from .db import SessionLocal
+from .evidence import USE_CONTENT, content_targets, normalize_targets
 from .knowledge_pages import page_image_path
 from .llm import ConsistencyResult, FactResult, ModelError, ScoreResult, validate_score
-from .models import (CriterionScore, KnowledgeDocument, Project, ScoreEvidence, ScoringBatch,
-                     SubmissionFile, now)
+from .models import (CriterionScore, KnowledgeChunk, KnowledgeDocument, KnowledgePage, Project,
+                     ScoreEvidence, ScoringBatch, SubmissionFile, now)
+from .pages import normalize_heading
 from .parsing import parse_document
-from .rag import INDEX_VERSION, ingest, retrieve
+from .rag import (INDEX_VERSION, body_heading_index, ingest, retrieve, retrieve_targets)
 from .storage import secure_path, sha256
 from .settings_store import client_for_batch
 
+#: Criterion statuses. ``needs_review`` is deliberately distinct from a missing
+#: material, a parse failure and a model failure: the retrieval ran but only found
+#: navigation pages where the rule requires body content.
+STATUS_NEEDS_REVIEW = "needs_review"
+STATUS_UNSCORABLE = "unscorable"
+STATUS_FAILED = "failed"
+STATUS_SCORED = "scored"
+STATUS_REVIEWED = "reviewed"
+#: Statuses that a batch counts as finished work.
+DONE_STATUSES = (STATUS_SCORED, STATUS_REVIEWED, STATUS_NEEDS_REVIEW)
+#: Statuses the dashboard shows as needing teacher attention.
+ATTENTION_STATUSES = (STATUS_FAILED, STATUS_UNSCORABLE, STATUS_NEEDS_REVIEW)
+
 
 @contextmanager
 def file_index_lock(file_id: int):
@@ -72,12 +87,21 @@ def _parse_submission_locked(db, file: SubmissionFile, *, embedder=None):
 
 
 def _project_facts(db, project, course_snapshot, client, *, retriever):
+    """Extract project facts from body text only.
+
+    Implementation facts must not be inferred from a contents heading or an abstract
+    claim, so the search admits body and appendix pages exclusively. Project identity
+    (name, student number, topic) legitimately lives on the cover page and is read from
+    the teacher-confirmed import record instead of from retrieved text.
+    """
     prompts = course_snapshot["prompts"]
     categories = ("stage1", "stage2", "stage3", "final_report", "presentation")
     snippets = []
     for category in categories:
         snippets.extend(retriever(db, course_version_id=project.course_version_id, project_id=project.id,
-                                  category=category, query="项目名称 目标 指标 硬件 软件 实验 数据 成本 结论", limit=3, include_public=False))
+                                  category=category, query="项目名称 目标 指标 硬件 软件 实验 数据 成本 结论",
+                                  limit=3, include_public=False, allowed_page_types=["body", "appendix"],
+                                  substantive_only=True))
     if not snippets:
         return {}
     if client is None:
@@ -100,10 +124,54 @@ def _project_facts(db, project, course_snapshot, client, *, retriever):
     return project.facts
 
 
-def _score_criterion(db, batch, project, criterion, client, *, retriever):
+def _body_navigator(db, *, course_version_id, project_id, category):
+    """Find real body pages for a target whose contents entries name missing sections.
+
+    The contents page is used only as a map: its headings are matched against headings
+    that actually occur in body text, and the body sections and their neighbouring pages
+    are then searched. The final quotation always lands on a real body page.
+    """
+    headings = body_heading_index(db, course_version_id=course_version_id, project_id=project_id, category=category)
+    if not headings:
+        return None
+
+    def navigate(target):
+        label = normalize_heading(target.get("label") or "")
+        if not label:
+            return []
+        candidates = []
+        for heading, places in headings.items():
+            if not heading:
+                continue
+            if heading.startswith(label) or label.startswith(heading) or label in heading or heading in label:
+                candidates.extend(places)
+        if not candidates:
+            return []
+        pages = sorted({(place["document_id"], place["page"]) for place in candidates})
+        neighbours = set()
+        for document_id, page in pages:
+            neighbours.update({(document_id, page), (document_id, page + 1)})
+        wanted = sorted(neighbours)
+        rows = db.scalars(select(KnowledgeChunk).where(
+            KnowledgeChunk.course_version_id == course_version_id, KnowledgeChunk.project_id == project_id,
+            KnowledgeChunk.page_type == "body", KnowledgeChunk.block_type != "toc",
+            KnowledgeChunk.document_id.in_([item[0] for item in wanted]),
+            KnowledgeChunk.page.in_([item[1] for item in wanted])).order_by(KnowledgeChunk.id)).all()
+        hits = []
+        for chunk in rows:
+            hits.append({"chunk_id": chunk.id, "document_id": chunk.document_id, "project_id": chunk.project_id,
+                         "document_type": chunk.document_type, "page": chunk.page, "section": chunk.section,
+                         "chapter": chunk.chapter, "page_type": chunk.page_type, "block_type": chunk.block_type,
+                         "text": chunk.text, "rank": 0.5})
+        return hits
+
+    return navigate
+
+
+def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_retriever=retrieve_targets):
     score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
                       CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
-    if score and score.status in ("scored", "reviewed"):
+    if score and score.status in (STATUS_SCORED, STATUS_REVIEWED):
         return score
     if not score:
         score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
@@ -115,44 +183,103 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever):
                        SubmissionFile.category == category, SubmissionFile.recognition_status == "recognized")).all()
     ready = [file for file in files if file.parse_status == "ready"]
     if len(files) != 1 or len(ready) != 1:
-        score.status, score.error = "unscorable", "材料缺失、重复、待确认或解析失败"
+        score.status, score.error = STATUS_UNSCORABLE, "材料缺失、重复、待确认或解析失败"
         db.commit()
         return score
-    query = criterion["description"] + " " + " ".join(criterion["required_evidence"])
-    chunks = retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
-                       category=category, query=query, limit=10, include_public=False)
+
+    plan = batch.rubric_snapshot.get("evidence_plan") or {}
+    targets = (plan.get("criteria") or {}).get(criterion["criterion_id"]) or normalize_targets(
+        criterion.get("required_evidence"))
+    navigator = _body_navigator(db, course_version_id=batch.course_version_id,
+                                project_id=project.id, category=category)
+    # The single ``retriever`` stays the seam: each per-target search goes through it,
+    # so a caller that injects a retriever (tests, alternate index) still applies.
+    selection = multi_retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
+                                targets=targets, category=category, navigator=navigator,
+                                retrieve_fn=retriever)
+    chunks = _attach_page_types(db, selection["evidence"])
+    gaps = selection["uncovered_targets"]
     if not chunks:
-        score.status, score.error = "unscorable", "无可检索的材料证据"
+        # Nothing admissible was found for any target. Content targets go to review;
+        # a purely structural criterion only depends on navigation pages, which exist.
+        score.status = STATUS_NEEDS_REVIEW if content_targets(criterion) else STATUS_UNSCORABLE
+        score.error = "未检索到允许作为证据的正文页"
+        score.model_score = None
+        score.missing_evidence = [target["label"] for target in gaps]
+        _record_evidence_gaps(score, gaps)
         db.commit()
         return score
     if client is None:
         raise ModelError("未配置或无法连接评分模型")
-    image_sources = dict.fromkeys((chunk["document_id"], chunk["page"]) for chunk in chunks[:2])
+
+    content_gaps = [target for target in gaps if target["use"] == USE_CONTENT]
+    if content_gaps:
+        # The model is not asked to grade from navigation pages. Record the gap and keep
+        # the model score null so a teacher decides, rather than asserting a low score.
+        score.model_score, score.confidence, score.status = None, None, STATUS_NEEDS_REVIEW
+        score.model_comment = "检索仅命中目录、摘要或章节标题，缺少可用于内容判断的正文证据，转教师复核。"
+        score.missing_evidence = [target["label"] for target in content_gaps]
+        score.suggestion = "请补充该章节的正文页，或由教师直接核定该项分数。"
+        score.error = "缺少正文证据：" + "、".join(target["label"] for target in content_gaps)
+        _record_evidence_gaps(score, content_gaps, searched=_searched_page_types(plan, content_gaps))
+        db.commit()
+        return score
+
+    # Page images go to the model as well, so prefer substantive body pages over the
+    # navigation pages that may legitimately appear in a structural target's evidence.
+    image_sources = list(dict.fromkeys(
+        (chunk["document_id"], chunk["page"]) for chunk in chunks
+        if chunk.get("page_type") == "body"))
+    if not image_sources:
+        image_sources = list(dict.fromkeys((chunk["document_id"], chunk["page"]) for chunk in chunks))
     images = []
-    for document_id, page in image_sources:
+    for document_id, page in image_sources[:3]:
         doc = db.get(KnowledgeDocument, document_id)
         path = page_image_path(db, doc, page) if doc else None
         if path:
             images.append(str(path))
+    query = criterion["description"] + " " + " ".join(target["label"] for target in targets)
     public_context = retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
                                category=None, query=query, limit=6, public_only=True)
-    payload = {"criterion": criterion, "project_facts": project.facts, "course_context": public_context,
-               "evidence": chunks,
-               "required_output": {"criterion_id": "string", "score": "number", "max_score": "number", "reason": "string",
-                                   "evidence": [{"chunk_id": "integer", "page": "integer", "quote": "exact substring"}],
-                                   "missing_evidence": ["string"], "suggestion": "string", "confidence": "0..1"}}
+    payload = {"criterion": criterion,
+               "required_evidence_targets": [{"target_id": target["target_id"], "label": target["label"],
+                                              "use": target["use"],
+                                              "allowed_page_types": target["allowed_page_types"]}
+                                             for target in targets],
+               "evidence_policy": {
+                   "structure": "目录、封面和参考文献页可以证明章节结构和报告完整性，不能证明章节内容或完成质量。",
+                   "content": "内容判断只能使用正文或附录中的实质段落、数据、表格；目录、摘要和单独章节标题不能作为内容证据。"},
+               "project_facts": project.facts, "course_context": public_context, "evidence": chunks,
+               "required_output": {"criterion_id": "string", "score": "number or null", "max_score": "number",
+                                   "reason": "string",
+                                   "evidence": [{"chunk_id": "integer", "page": "integer",
+                                                 "quote": "exact substring", "target_id": "string"}],
+                                   "missing_evidence": ["string"], "suggestion": "string", "confidence": "0..1",
+                                   "note": "证据不足以判断内容时 score 填 null，并在 missing_evidence 中列出缺少的正文目标。"}}
     prompt = (batch.course_snapshot["prompts"]["score"] +
-              " 证据quote必须从所引用chunk_id的text逐字复制连续片段，保留原文标点和换行；page必须等于该片段的page。不要改写、缩写或拼接引文。")
+              " 证据quote必须从所引用chunk_id的text逐字复制连续片段，保留原文标点和换行；page必须等于该片段的page；"
+              "target_id必须取自required_evidence_targets。不要改写、缩写或拼接引文。"
+              "不得把目录、摘要或章节标题当作内容证据；无法据此判断内容时把score设为null。")
+    error = ""
+    result = None
     for attempt in range(2):
         try:
-            result = validate_score(client.json_call(prompt, payload, ScoreResult, images=images), criterion, chunks)
+            result = validate_score(client.json_call(prompt, payload, ScoreResult, images=images), criterion, chunks,
+                                    targets=targets)
             break
-        except ModelError:
+        except ModelError as exc:
+            error = str(exc)
             if attempt:
                 raise
-            prompt += " 上次回答的证据未通过原文和页码校验；请重新选择片段并逐字复制quote。"
-    score.model_score, score.model_comment, score.confidence = result.score, result.reason, result.confidence
-    score.missing_evidence, score.suggestion, score.status, score.error = result.missing_evidence, result.suggestion, "scored", ""
+            prompt += f" 上次回答未通过校验（{error}）；请改用允许页面类型中的实质正文片段，并逐字复制quote。"
+    score.model_score, score.model_comment = result.score, result.reason
+    score.confidence = result.confidence
+    score.missing_evidence, score.suggestion = result.missing_evidence, result.suggestion
+    if result.score is None:
+        # The retrieval found body pages but the model judged them insufficient.
+        score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项，转教师复核。"
+    else:
+        score.status, score.error = STATUS_SCORED, ""
     for evidence in db.scalars(select(ScoreEvidence).where(ScoreEvidence.score_id == score.id)).all():
         db.delete(evidence)
     by_id = {chunk["chunk_id"]: chunk for chunk in chunks}
@@ -160,14 +287,75 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever):
         chunk = by_id[evidence.chunk_id]
         doc = db.get(KnowledgeDocument, chunk["document_id"])
         path = page_image_path(db, doc, evidence.page)
+        target = next((item for item in targets if item["target_id"] == evidence.target_id), None)
         db.add(ScoreEvidence(score_id=score.id, document_id=doc.id, submission_file_id=doc.submission_file_id,
-                             page=evidence.page, quote=evidence.quote, image_path=str(path) if path else None))
+                             page=evidence.page, quote=evidence.quote, image_path=str(path) if path else None,
+                             target_id=evidence.target_id, target_label=target["label"] if target else "",
+                             page_type=chunk.get("page_type") or "unknown",
+                             source_role=target["use"] if target else USE_CONTENT))
     db.commit()
     return score
 
 
-def _run_project(batch_id, project_id, session_factory, client_factory, embedder, retriever):
+def _attach_page_types(db, chunks: list[dict]) -> list[dict]:
+    """Attach the stored page classification to each retrieved chunk.
+
+    The page type recorded in ``knowledge_pages`` is the authority for the evidence
+    policy. Reading it here, rather than trusting whatever a retriever happened to put
+    in the dict, means a statement about what a page is cannot be forged by a caller
+    or lost by an alternate index.
+    """
+    keys = {(chunk.get("document_id"), chunk.get("page")) for chunk in chunks}
+    keys.discard((None, None))
+    if not keys:
+        return chunks
+    documents = {document_id for document_id, _ in keys}
+    pages = {page for _, page in keys}
+    records = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id.in_(documents),
+                                                     KnowledgePage.page.in_(pages))).all()
+    by_key = {(item.document_id, item.page): item for item in records}
+    output = []
+    for chunk in chunks:
+        record = by_key.get((chunk.get("document_id"), chunk.get("page")))
+        if record is None:
+            output.append({**chunk, "page_type": chunk.get("page_type") or "unknown"})
+            continue
+        output.append({**chunk, "page_type": record.page_type or "unknown",
+                       "section": chunk.get("section") or record.section or "",
+                       "chapter": chunk.get("chapter") or record.chapter or ""})
+    return output
+
+
+def _searched_page_types(plan, targets):
+    allowed = set()
+    for target in targets:
+        allowed.update(target.get("allowed_page_types") or [])
+    return sorted(allowed or (plan.get("content_allowed_page_types") or []))
+
+
+def _record_evidence_gaps(score, gaps, searched=None):
+    """Store which content targets lacked body evidence and where retrieval looked.
+
+    The record is kept through teacher review so the original retrieval problem stays
+    auditable next to the score the teacher finally entered.
+    """
+    score.evidence_gaps = [
+        {"target_id": target.get("target_id"), "target_label": target.get("label"),
+         "searched_page_types": searched or target.get("allowed_page_types") or [],
+         "reason": "补检后仍未找到正文证据"}
+        for target in gaps]
+
+
+def _run_project(batch_id, project_id, session_factory, client_factory, embedder, retriever,
+                 multi_retriever=retrieve_targets):
+    """Score one project. Returns ``(failures, reviews)``.
+
+    Failures are missing material, parse problems and model errors. Reviews count the
+    items that only found navigation pages where content evidence is required; those
+    need a teacher decision rather than a retry.
+    """
     failures = 0
+    reviews = 0
     with session_factory() as db:
         batch = db.get(ScoringBatch, batch_id)
         project = db.get(Project, project_id)
@@ -190,8 +378,11 @@ def _run_project(batch_id, project_id, session_factory, client_factory, embedder
             failures += 1
         for criterion in batch.rubric_snapshot["criteria"]:
             try:
-                result = _score_criterion(db, batch, project, criterion, client, retriever=retriever)
-                if result.status not in ("scored", "reviewed"):
+                result = _score_criterion(db, batch, project, criterion, client, retriever=retriever,
+                                          multi_retriever=multi_retriever)
+                if result.status == STATUS_NEEDS_REVIEW:
+                    reviews += 1
+                elif result.status not in (STATUS_SCORED, STATUS_REVIEWED):
                     failures += 1
             except Exception as exc:
                 score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
@@ -199,15 +390,16 @@ def _run_project(batch_id, project_id, session_factory, client_factory, embedder
                 if not score:
                     score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
                     db.add(score)
-                score.status, score.error = "failed", str(exc)[:500]
+                score.status, score.error = STATUS_FAILED, str(exc)[:500]
                 db.commit()
                 failures += 1
             db.execute(update(ScoringBatch).where(ScoringBatch.id == batch_id).values(progress_done=ScoringBatch.progress_done + 1))
             db.commit()
-    return failures
+    return failures, reviews
 
 
-def run_batch(batch_id: int, *, session_factory=SessionLocal, client_factory=client_for_batch, embedder=None, retriever=retrieve):
+def run_batch(batch_id: int, *, session_factory=SessionLocal, client_factory=client_for_batch, embedder=None,
+              retriever=retrieve, multi_retriever=retrieve_targets):
     with session_factory() as db:
         batch = db.get(ScoringBatch, batch_id)
         if not batch:
@@ -218,13 +410,21 @@ def run_batch(batch_id: int, *, session_factory=SessionLocal, client_factory=cli
         batch.status, batch.progress_total, batch.progress_done, batch.finished_at = "running", len(project_ids) * len(criteria), 0, None
         db.commit()
         failures = 0
+        reviews = 0
         with ThreadPoolExecutor(max_workers=max(1, int(batch.model_config_snapshot.get("concurrency", 1)))) as pool:
-            futures = [pool.submit(_run_project, batch_id, project_id, session_factory, client_factory, embedder, retriever) for project_id in project_ids]
+            futures = [pool.submit(_run_project, batch_id, project_id, session_factory, client_factory,
+                                   embedder, retriever, multi_retriever) for project_id in project_ids]
             for future in as_completed(futures):
                 try:
-                    failures += future.result()
+                    project_failures, project_reviews = future.result()
+                    failures += project_failures
+                    reviews += project_reviews
                 except Exception as exc:
                     failures += 1
                     batch.error = (batch.error + f" 项目任务失败: {type(exc).__name__}")[:1000]
-        batch.status, batch.finished_at = ("partial" if failures else "completed"), now()
+        # A batch with unresolved review items is finished running, but it is not a
+        # completed grading result: the teacher still has to decide those criteria.
+        batch.status = "partial" if failures else ("review_required" if reviews else "completed")
+        batch.finished_at = now()
         db.commit()
+        return {"failures": failures, "reviews": reviews, "status": batch.status}