fix: attribute descriptive review gaps to the named evidence target
This commit is contained in:
@@ -373,6 +373,34 @@ def test_model_declining_score_records_original_gap(sqlite_db):
|
||||
assert score.evidence_gaps[0]["reason"] == "只有装置介绍"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing", [["误差分析(t1):缺少误差来源和统计数据"],
|
||||
["尚缺 t1 对应的正文"], ["当前资料无法判断"]])
|
||||
def test_declined_score_attributes_descriptive_missing_evidence_to_its_target(sqlite_db, missing):
|
||||
db = sqlite_db
|
||||
item = project(db)
|
||||
file = SubmissionFile(project_id=item.id, category="stage3", relative_path="test.pdf", mime_type="application/pdf",
|
||||
size=1, mtime=1, sha256="test", recognition_status="recognized", parse_status="ready")
|
||||
db.add(file)
|
||||
db.commit()
|
||||
ingest(db, course_version_id=1, project_id=item.id, submission_file_id=file.id, title="test.pdf",
|
||||
category="stage3", source_hash="test", pages=[{"page_number": 1, "text": "实验测得响应时间80毫秒,但未解释误差来源。"}], embedder=FLAT)
|
||||
criterion = {"criterion_id": "stage3.data", "category": "stage3", "max_score": 20,
|
||||
"description": "实验数据与误差分析", "required_evidence": ["实验数据", "误差分析"]}
|
||||
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
||||
class Client:
|
||||
def json_call(self, *args, **kwargs):
|
||||
return ScoreResult(criterion_id="stage3.data", score=None, max_score=20,
|
||||
reason="误差分析尚不足以判断", missing_evidence=missing, confidence=.2)
|
||||
def retriever(session, **kwargs):
|
||||
if kwargs.get("public_only"):
|
||||
return []
|
||||
return retrieve(session, **{**kwargs, "embedder": FLAT})
|
||||
score = _score_criterion(db, db.get(ScoringBatch, created["id"]), item, criterion, Client(), retriever=retriever)
|
||||
expected = ["t0", "t1"] if missing == ["当前资料无法判断"] else ["t1"]
|
||||
assert [gap["target_id"] for gap in score.evidence_gaps] == expected
|
||||
assert score.status == "needs_review" and score.model_score is None
|
||||
|
||||
|
||||
def test_failed_model_call_cannot_overwrite_a_concurrent_teacher_review(sqlite_db, monkeypatch):
|
||||
from wisegrade import scoring
|
||||
db = sqlite_db
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import re
|
||||
|
||||
from sqlalchemy import and_, or_, select, update
|
||||
|
||||
@@ -258,8 +259,13 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
|
||||
if result.score is None:
|
||||
# The retrieval found body pages but the model judged them insufficient.
|
||||
score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项,转教师复核。"
|
||||
# Models normally write "误差分析(t1):缺少…", not just the bare
|
||||
# label. Attribute those explanations to their target instead of marking
|
||||
# every target as missing. Unknown explanations remain conservative.
|
||||
declined = [target for target in targets if target["use"] == USE_CONTENT and
|
||||
(not result.missing_evidence or target["label"] in result.missing_evidence)]
|
||||
any(target["label"] in note or re.search(
|
||||
r"(?<![A-Za-z0-9_])" + re.escape(target["target_id"]) + r"(?![A-Za-z0-9_])", note)
|
||||
for note in result.missing_evidence)]
|
||||
_record_evidence_gaps(score, declined or [target for target in targets if target["use"] == USE_CONTENT])
|
||||
for gap in score.evidence_gaps:
|
||||
gap["reason"] = result.reason
|
||||
|
||||
Reference in New Issue
Block a user