fix: attribute descriptive review gaps to the named evidence target

This commit is contained in:
2026-10-17 20:44:50 +08:00
parent 14cef5c929
commit 5a6e5d47a0
2 changed files with 35 additions and 1 deletions
+28
View File
@@ -373,6 +373,34 @@ def test_model_declining_score_records_original_gap(sqlite_db):
assert score.evidence_gaps[0]["reason"] == "只有装置介绍" assert score.evidence_gaps[0]["reason"] == "只有装置介绍"
@pytest.mark.parametrize("missing", [["误差分析(t1):缺少误差来源和统计数据"],
["尚缺 t1 对应的正文"], ["当前资料无法判断"]])
def test_declined_score_attributes_descriptive_missing_evidence_to_its_target(sqlite_db, missing):
db = sqlite_db
item = project(db)
file = SubmissionFile(project_id=item.id, category="stage3", relative_path="test.pdf", mime_type="application/pdf",
size=1, mtime=1, sha256="test", recognition_status="recognized", parse_status="ready")
db.add(file)
db.commit()
ingest(db, course_version_id=1, project_id=item.id, submission_file_id=file.id, title="test.pdf",
category="stage3", source_hash="test", pages=[{"page_number": 1, "text": "实验测得响应时间80毫秒,但未解释误差来源。"}], embedder=FLAT)
criterion = {"criterion_id": "stage3.data", "category": "stage3", "max_score": 20,
"description": "实验数据与误差分析", "required_evidence": ["实验数据", "误差分析"]}
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
class Client:
def json_call(self, *args, **kwargs):
return ScoreResult(criterion_id="stage3.data", score=None, max_score=20,
reason="误差分析尚不足以判断", missing_evidence=missing, confidence=.2)
def retriever(session, **kwargs):
if kwargs.get("public_only"):
return []
return retrieve(session, **{**kwargs, "embedder": FLAT})
score = _score_criterion(db, db.get(ScoringBatch, created["id"]), item, criterion, Client(), retriever=retriever)
expected = ["t0", "t1"] if missing == ["当前资料无法判断"] else ["t1"]
assert [gap["target_id"] for gap in score.evidence_gaps] == expected
assert score.status == "needs_review" and score.model_score is None
def test_failed_model_call_cannot_overwrite_a_concurrent_teacher_review(sqlite_db, monkeypatch): def test_failed_model_call_cannot_overwrite_a_concurrent_teacher_review(sqlite_db, monkeypatch):
from wisegrade import scoring from wisegrade import scoring
db = sqlite_db db = sqlite_db
+7 -1
View File
@@ -1,4 +1,5 @@
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
import re
from sqlalchemy import and_, or_, select, update from sqlalchemy import and_, or_, select, update
@@ -258,8 +259,13 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
if result.score is None: if result.score is None:
# The retrieval found body pages but the model judged them insufficient. # The retrieval found body pages but the model judged them insufficient.
score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项,转教师复核。" score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项,转教师复核。"
# Models normally write "误差分析(t1):缺少…", not just the bare
# label. Attribute those explanations to their target instead of marking
# every target as missing. Unknown explanations remain conservative.
declined = [target for target in targets if target["use"] == USE_CONTENT and declined = [target for target in targets if target["use"] == USE_CONTENT and
(not result.missing_evidence or target["label"] in result.missing_evidence)] any(target["label"] in note or re.search(
r"(?<![A-Za-z0-9_])" + re.escape(target["target_id"]) + r"(?![A-Za-z0-9_])", note)
for note in result.missing_evidence)]
_record_evidence_gaps(score, declined or [target for target in targets if target["use"] == USE_CONTENT]) _record_evidence_gaps(score, declined or [target for target in targets if target["use"] == USE_CONTENT])
for gap in score.evidence_gaps: for gap in score.evidence_gaps:
gap["reason"] = result.reason gap["reason"] = result.reason