fix: attribute descriptive review gaps to the named evidence target
This commit is contained in:
@@ -373,6 +373,34 @@ def test_model_declining_score_records_original_gap(sqlite_db):
|
|||||||
assert score.evidence_gaps[0]["reason"] == "只有装置介绍"
|
assert score.evidence_gaps[0]["reason"] == "只有装置介绍"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("missing", [["误差分析(t1):缺少误差来源和统计数据"],
|
||||||
|
["尚缺 t1 对应的正文"], ["当前资料无法判断"]])
|
||||||
|
def test_declined_score_attributes_descriptive_missing_evidence_to_its_target(sqlite_db, missing):
|
||||||
|
db = sqlite_db
|
||||||
|
item = project(db)
|
||||||
|
file = SubmissionFile(project_id=item.id, category="stage3", relative_path="test.pdf", mime_type="application/pdf",
|
||||||
|
size=1, mtime=1, sha256="test", recognition_status="recognized", parse_status="ready")
|
||||||
|
db.add(file)
|
||||||
|
db.commit()
|
||||||
|
ingest(db, course_version_id=1, project_id=item.id, submission_file_id=file.id, title="test.pdf",
|
||||||
|
category="stage3", source_hash="test", pages=[{"page_number": 1, "text": "实验测得响应时间80毫秒,但未解释误差来源。"}], embedder=FLAT)
|
||||||
|
criterion = {"criterion_id": "stage3.data", "category": "stage3", "max_score": 20,
|
||||||
|
"description": "实验数据与误差分析", "required_evidence": ["实验数据", "误差分析"]}
|
||||||
|
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
||||||
|
class Client:
|
||||||
|
def json_call(self, *args, **kwargs):
|
||||||
|
return ScoreResult(criterion_id="stage3.data", score=None, max_score=20,
|
||||||
|
reason="误差分析尚不足以判断", missing_evidence=missing, confidence=.2)
|
||||||
|
def retriever(session, **kwargs):
|
||||||
|
if kwargs.get("public_only"):
|
||||||
|
return []
|
||||||
|
return retrieve(session, **{**kwargs, "embedder": FLAT})
|
||||||
|
score = _score_criterion(db, db.get(ScoringBatch, created["id"]), item, criterion, Client(), retriever=retriever)
|
||||||
|
expected = ["t0", "t1"] if missing == ["当前资料无法判断"] else ["t1"]
|
||||||
|
assert [gap["target_id"] for gap in score.evidence_gaps] == expected
|
||||||
|
assert score.status == "needs_review" and score.model_score is None
|
||||||
|
|
||||||
|
|
||||||
def test_failed_model_call_cannot_overwrite_a_concurrent_teacher_review(sqlite_db, monkeypatch):
|
def test_failed_model_call_cannot_overwrite_a_concurrent_teacher_review(sqlite_db, monkeypatch):
|
||||||
from wisegrade import scoring
|
from wisegrade import scoring
|
||||||
db = sqlite_db
|
db = sqlite_db
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
import re
|
||||||
|
|
||||||
from sqlalchemy import and_, or_, select, update
|
from sqlalchemy import and_, or_, select, update
|
||||||
|
|
||||||
@@ -258,8 +259,13 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
|
|||||||
if result.score is None:
|
if result.score is None:
|
||||||
# The retrieval found body pages but the model judged them insufficient.
|
# The retrieval found body pages but the model judged them insufficient.
|
||||||
score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项,转教师复核。"
|
score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项,转教师复核。"
|
||||||
|
# Models normally write "误差分析(t1):缺少…", not just the bare
|
||||||
|
# label. Attribute those explanations to their target instead of marking
|
||||||
|
# every target as missing. Unknown explanations remain conservative.
|
||||||
declined = [target for target in targets if target["use"] == USE_CONTENT and
|
declined = [target for target in targets if target["use"] == USE_CONTENT and
|
||||||
(not result.missing_evidence or target["label"] in result.missing_evidence)]
|
any(target["label"] in note or re.search(
|
||||||
|
r"(?<![A-Za-z0-9_])" + re.escape(target["target_id"]) + r"(?![A-Za-z0-9_])", note)
|
||||||
|
for note in result.missing_evidence)]
|
||||||
_record_evidence_gaps(score, declined or [target for target in targets if target["use"] == USE_CONTENT])
|
_record_evidence_gaps(score, declined or [target for target in targets if target["use"] == USE_CONTENT])
|
||||||
for gap in score.evidence_gaps:
|
for gap in score.evidence_gaps:
|
||||||
gap["reason"] = result.reason
|
gap["reason"] = result.reason
|
||||||
|
|||||||
Reference in New Issue
Block a user