From ef9edecff29bfa4243c702c8fad00d5d8bb4fd71 Mon Sep 17 00:00:00 2001 From: aerospaceboy <790286734@qq.com> Date: Sat, 17 Oct 2026 20:17:19 +0800 Subject: [PATCH] fix: resolve legacy evidence targets without guessing and guide retries --- backend/tests/test_optimization.py | 65 ++++++++++++++++++++++++++++++ backend/wisegrade/llm.py | 12 +++++- backend/wisegrade/scoring.py | 8 +++- 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/backend/tests/test_optimization.py b/backend/tests/test_optimization.py index b20d8b9..c3f4d05 100644 --- a/backend/tests/test_optimization.py +++ b/backend/tests/test_optimization.py @@ -563,6 +563,71 @@ def test_valid_target_cannot_be_attached_to_another_targets_chunk(): validate_score(result, criterion, [chunk]) +def test_legacy_missing_target_uses_unique_chunk_association(): + criterion = {"criterion_id": "x", "max_score": 10, "required_evidence": ["实验数据", "实现方法"]} + chunk = {"chunk_id": 1, "page": 1, "page_type": "body", "text": "采用视觉传感器实现位置反馈。", "target_ids": ["t1"]} + result = ScoreResult(criterion_id="x", score=8, max_score=10, reason="有实现方法", confidence=.8, + evidence=[{"chunk_id": 1, "page": 1, "quote": chunk["text"]}]) + assert validate_score(result, criterion, [chunk]).evidence[0].target_id == "t1" + + +def test_target_mismatch_feedback_identifies_the_legal_chunk_target(): + criterion = {"criterion_id": "x", "max_score": 10, "required_evidence": ["实验数据", "实现方法"]} + chunk = {"chunk_id": 42, "page": 1, "page_type": "body", "text": "采用视觉传感器实现位置反馈。", "target_ids": ["t1"]} + result = ScoreResult(criterion_id="x", score=8, max_score=10, reason="错误关联", confidence=.8, + evidence=[{"chunk_id": 42, "page": 1, "quote": chunk["text"], "target_id": "t0"}]) + with pytest.raises(ModelError) as failure: + validate_score(result, criterion, [chunk]) + assert "chunk_id=42" in str(failure.value) + assert "允许 target_id=t1" in str(failure.value) + + +@pytest.mark.parametrize("rich", [False, True]) +def test_missing_target_is_not_guessed_when_ambiguous_or_required(rich): + required = [{"label": "实验数据"}, {"label": "实现方法"}] if rich else ["实验数据", "实现方法"] + criterion = {"criterion_id": "x", "max_score": 10, "required_evidence": required} + chunk = {"chunk_id": 1, "page": 1, "page_type": "body", "text": "采用视觉传感器实现位置反馈。", "target_ids": ["t0", "t1"]} + result = ScoreResult(criterion_id="x", score=8, max_score=10, reason="需要明确目标", confidence=.8, + evidence=[{"chunk_id": 1, "page": 1, "quote": chunk["text"]}]) + with pytest.raises(ModelError, match="缺少 target_id"): + validate_score(result, criterion, [chunk]) + + +def test_scoring_retry_carries_target_contract_and_specific_repair_feedback(sqlite_db): + db = sqlite_db + item = project(db) + file = SubmissionFile(project_id=item.id, category="stage3", relative_path="test.pdf", mime_type="application/pdf", + size=1, mtime=1, sha256="test", recognition_status="recognized", parse_status="ready") + db.add(file) + db.commit() + doc = ingest(db, course_version_id=1, project_id=item.id, submission_file_id=file.id, title="test.pdf", + category="stage3", source_hash="test", pages=[{"page_number": 1, "text": "采用视觉传感器实现位置反馈。"}], embedder=FLAT) + chunk = db.scalar(select(KnowledgeChunk).where(KnowledgeChunk.document_id == doc.id)) + criterion = {"criterion_id": "stage3.design", "category": "stage3", "max_score": 20, + "description": "实现方法", "required_evidence": ["实验数据", "实现方法"]} + created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db) + batch = db.get(ScoringBatch, created["id"]) + class Client: + calls = 0 + def json_call(self, prompt, payload, *args, **kwargs): + self.calls += 1 + assert "每条引文的 target_id 必须属于该片段的 target_ids" in prompt + if self.calls == 2: + assert payload["validation_feedback"]["error"].endswith("允许 target_id=t1") + return ScoreResult(criterion_id=criterion["criterion_id"], score=15, max_score=20, + reason="有实现方法", confidence=.8, evidence=[{ + "chunk_id": chunk.id, "page": 1, "quote": chunk.text, + "target_id": "t0" if self.calls == 1 else "t1"}]) + client = Client() + def selected(*args, **kwargs): + return {"evidence": [{"chunk_id": chunk.id, "document_id": doc.id, "page": 1, + "text": chunk.text, "page_type": "body", "target_ids": ["t1"]}], + "uncovered_targets": []} + score = _score_criterion(db, batch, item, criterion, client, + retriever=lambda *args, **kwargs: [], multi_retriever=selected) + assert score.status == "scored" and client.calls == 2 + + @pytest.mark.parametrize("version", [None, 1]) def test_worker_rejects_old_policy_without_touching_scores(sqlite_db, version): from wisegrade.scoring import run_batch diff --git a/backend/wisegrade/llm.py b/backend/wisegrade/llm.py index 7c32162..7691dbd 100644 --- a/backend/wisegrade/llm.py +++ b/backend/wisegrade/llm.py @@ -238,11 +238,19 @@ def validate_score(result: ScoreResult, criterion: dict, chunks: list[dict], *, # Empty identifiers are accepted only for pre-object legacy rubrics. if any(isinstance(value, dict) for value in criterion.get("required_evidence", [])): raise ModelError("模型证据缺少 target_id") - target = default_content or (targets[0] if targets else None) + associated = [by_target[key] for key in chunk.get("target_ids", []) if key in by_target] + if associated: + if len(associated) != 1: + raise ModelError(f"模型证据缺少 target_id：chunk_id={chunk['chunk_id']} 对应多个检索目标，请明确选择") + target = associated[0] + else: + target = default_content or (targets[0] if targets else None) if target is None: raise ModelError("模型证据未关联任何必备证据目标") if chunk.get("target_ids") and target["target_id"] not in chunk["target_ids"]: - raise ModelError("模型证据未关联该片段的检索目标") + allowed = ",".join(str(key) for key in chunk["target_ids"] if key in by_target) + raise ModelError(f"模型证据未关联该片段的检索目标：chunk_id={chunk['chunk_id']}，" + f"target_id={target['target_id']}；允许 target_id={allowed}") if not page_type_allowed(page_type, target["allowed_page_types"]): raise ModelError(f"模型证据来自不允许的页面类型（{page_type}），不能支持「{target['label']}」") if target["use"] == USE_CONTENT and not content_evidence_allowed(page_type): diff --git a/backend/wisegrade/scoring.py b/backend/wisegrade/scoring.py index a7af4a6..5f34560 100644 --- a/backend/wisegrade/scoring.py +++ b/backend/wisegrade/scoring.py @@ -227,6 +227,8 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_ prompt = (batch.course_snapshot["prompts"]["score"] + " 证据quote必须从所引用chunk_id的text逐字复制连续片段，保留原文标点和换行；page必须等于该片段的page；" "target_id必须取自required_evidence_targets。不要改写、缩写或拼接引文。" + "每条引文的 target_id 必须属于该片段的 target_ids；同一片段可对应多个目标，" + "请按实际内容选择，不要按证据列表顺序分配编号。" "不得把目录、摘要或章节标题当作内容证据；无法据此判断内容时把score设为null。") error = "" result = None @@ -239,7 +241,11 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_ error = str(exc) if attempt: raise - prompt += f" 上次回答未通过校验（{error}）；请改用允许页面类型中的实质正文片段，并逐字复制quote。" + payload["validation_feedback"] = { + "error": error, + "instruction": "根据具体错误修正输出；引用目标只能从对应片段的 target_ids 选择，" + "quote须为该片段原文。没有足够依据时保留空分，不要编造引用。"} + prompt += " 上次回答未通过校验；请按 validation_feedback 修正，不要重复错误引用。" score = db.scalar(select(CriterionScore).where(CriterionScore.id == score.id) .with_for_update().execution_options(populate_existing=True)) if score.status == STATUS_REVIEWED or score.teacher_score is not None: