fix: resolve legacy evidence targets without guessing and guide retries

This commit is contained in:
2026-10-17 20:17:19 +08:00
parent 93ff9c8c8d
commit ef9edecff2
3 changed files with 82 additions and 3 deletions
+65
View File
@@ -563,6 +563,71 @@ def test_valid_target_cannot_be_attached_to_another_targets_chunk():
validate_score(result, criterion, [chunk])
def test_legacy_missing_target_uses_unique_chunk_association():
criterion = {"criterion_id": "x", "max_score": 10, "required_evidence": ["实验数据", "实现方法"]}
chunk = {"chunk_id": 1, "page": 1, "page_type": "body", "text": "采用视觉传感器实现位置反馈。", "target_ids": ["t1"]}
result = ScoreResult(criterion_id="x", score=8, max_score=10, reason="有实现方法", confidence=.8,
evidence=[{"chunk_id": 1, "page": 1, "quote": chunk["text"]}])
assert validate_score(result, criterion, [chunk]).evidence[0].target_id == "t1"
def test_target_mismatch_feedback_identifies_the_legal_chunk_target():
criterion = {"criterion_id": "x", "max_score": 10, "required_evidence": ["实验数据", "实现方法"]}
chunk = {"chunk_id": 42, "page": 1, "page_type": "body", "text": "采用视觉传感器实现位置反馈。", "target_ids": ["t1"]}
result = ScoreResult(criterion_id="x", score=8, max_score=10, reason="错误关联", confidence=.8,
evidence=[{"chunk_id": 42, "page": 1, "quote": chunk["text"], "target_id": "t0"}])
with pytest.raises(ModelError) as failure:
validate_score(result, criterion, [chunk])
assert "chunk_id=42" in str(failure.value)
assert "允许 target_id=t1" in str(failure.value)
@pytest.mark.parametrize("rich", [False, True])
def test_missing_target_is_not_guessed_when_ambiguous_or_required(rich):
required = [{"label": "实验数据"}, {"label": "实现方法"}] if rich else ["实验数据", "实现方法"]
criterion = {"criterion_id": "x", "max_score": 10, "required_evidence": required}
chunk = {"chunk_id": 1, "page": 1, "page_type": "body", "text": "采用视觉传感器实现位置反馈。", "target_ids": ["t0", "t1"]}
result = ScoreResult(criterion_id="x", score=8, max_score=10, reason="需要明确目标", confidence=.8,
evidence=[{"chunk_id": 1, "page": 1, "quote": chunk["text"]}])
with pytest.raises(ModelError, match="缺少 target_id"):
validate_score(result, criterion, [chunk])
def test_scoring_retry_carries_target_contract_and_specific_repair_feedback(sqlite_db):
db = sqlite_db
item = project(db)
file = SubmissionFile(project_id=item.id, category="stage3", relative_path="test.pdf", mime_type="application/pdf",
size=1, mtime=1, sha256="test", recognition_status="recognized", parse_status="ready")
db.add(file)
db.commit()
doc = ingest(db, course_version_id=1, project_id=item.id, submission_file_id=file.id, title="test.pdf",
category="stage3", source_hash="test", pages=[{"page_number": 1, "text": "采用视觉传感器实现位置反馈。"}], embedder=FLAT)
chunk = db.scalar(select(KnowledgeChunk).where(KnowledgeChunk.document_id == doc.id))
criterion = {"criterion_id": "stage3.design", "category": "stage3", "max_score": 20,
"description": "实现方法", "required_evidence": ["实验数据", "实现方法"]}
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
batch = db.get(ScoringBatch, created["id"])
class Client:
calls = 0
def json_call(self, prompt, payload, *args, **kwargs):
self.calls += 1
assert "每条引文的 target_id 必须属于该片段的 target_ids" in prompt
if self.calls == 2:
assert payload["validation_feedback"]["error"].endswith("允许 target_id=t1")
return ScoreResult(criterion_id=criterion["criterion_id"], score=15, max_score=20,
reason="有实现方法", confidence=.8, evidence=[{
"chunk_id": chunk.id, "page": 1, "quote": chunk.text,
"target_id": "t0" if self.calls == 1 else "t1"}])
client = Client()
def selected(*args, **kwargs):
return {"evidence": [{"chunk_id": chunk.id, "document_id": doc.id, "page": 1,
"text": chunk.text, "page_type": "body", "target_ids": ["t1"]}],
"uncovered_targets": []}
score = _score_criterion(db, batch, item, criterion, client,
retriever=lambda *args, **kwargs: [], multi_retriever=selected)
assert score.status == "scored" and client.calls == 2
@pytest.mark.parametrize("version", [None, 1])
def test_worker_rejects_old_policy_without_touching_scores(sqlite_db, version):
from wisegrade.scoring import run_batch