perf: skip fact extraction for fully graded projects on batch retries
This commit is contained in:
@@ -402,6 +402,43 @@ def test_failed_model_call_cannot_overwrite_a_concurrent_teacher_review(sqlite_d
|
|||||||
assert db.get(ScoringBatch, batch.id).progress_done == 1
|
assert db.get(ScoringBatch, batch.id).progress_done == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("complete", [True, False])
|
||||||
|
def test_retry_skips_facts_only_for_fully_graded_projects(sqlite_db, monkeypatch, complete):
|
||||||
|
from wisegrade import scoring
|
||||||
|
db = sqlite_db
|
||||||
|
item = project(db)
|
||||||
|
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
||||||
|
batch = db.get(ScoringBatch, created["id"])
|
||||||
|
criteria = batch.rubric_snapshot["criteria"][:2]
|
||||||
|
batch.rubric_snapshot = {**batch.rubric_snapshot, "criteria": criteria}
|
||||||
|
batch.status, batch.progress_total, batch.progress_done = "running", 2, 0
|
||||||
|
item.facts = {"conclusions": ["已抽取并确认的事实"]}
|
||||||
|
db.add_all([
|
||||||
|
CriterionScore(batch_id=batch.id, project_id=item.id, criterion_id=criteria[0]["criterion_id"], status="scored", model_score=0),
|
||||||
|
CriterionScore(batch_id=batch.id, project_id=item.id, criterion_id=criteria[1]["criterion_id"],
|
||||||
|
status="reviewed" if complete else "needs_review", teacher_score=0 if complete else None),
|
||||||
|
])
|
||||||
|
db.commit()
|
||||||
|
calls = []
|
||||||
|
def client_factory(*args):
|
||||||
|
calls.append("client")
|
||||||
|
def facts(*args, **kwargs):
|
||||||
|
calls.append("facts")
|
||||||
|
return {}
|
||||||
|
def scored(session, batch, item, criterion, *args, **kwargs):
|
||||||
|
return session.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
|
||||||
|
CriterionScore.project_id == item.id, CriterionScore.criterion_id == criterion["criterion_id"]))
|
||||||
|
monkeypatch.setattr(scoring, "_project_facts", facts)
|
||||||
|
monkeypatch.setattr(scoring, "_score_criterion", scored)
|
||||||
|
factory = sessionmaker(bind=db.get_bind(), expire_on_commit=False)
|
||||||
|
failures, reviews = scoring._run_project(batch.id, item.id, factory, client_factory, FLAT, lambda *a, **k: [])
|
||||||
|
assert calls == ([] if complete else ["client", "facts"])
|
||||||
|
assert failures == 0 and reviews == (0 if complete else 1)
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(ScoringBatch, batch.id).progress_done == 2
|
||||||
|
assert db.get(Project, item.id).facts == {"conclusions": ["已抽取并确认的事实"]}
|
||||||
|
|
||||||
|
|
||||||
def test_unrebuilt_course_document_does_not_complete_task(sqlite_db):
|
def test_unrebuilt_course_document_does_not_complete_task(sqlite_db):
|
||||||
db = sqlite_db
|
db = sqlite_db
|
||||||
doc = KnowledgeDocument(course_version_id=1, title="无页面资料", document_type="course",
|
doc = KnowledgeDocument(course_version_id=1, title="无页面资料", document_type="course",
|
||||||
|
|||||||
@@ -358,6 +358,18 @@ def _run_project(batch_id, project_id, session_factory, client_factory, embedder
|
|||||||
with session_factory() as db:
|
with session_factory() as db:
|
||||||
batch = db.get(ScoringBatch, batch_id)
|
batch = db.get(ScoringBatch, batch_id)
|
||||||
project = db.get(Project, project_id)
|
project = db.get(Project, project_id)
|
||||||
|
existing = db.scalars(select(CriterionScore).where(CriterionScore.batch_id == batch_id,
|
||||||
|
CriterionScore.project_id == project_id)).all()
|
||||||
|
graded = {score.criterion_id for score in existing
|
||||||
|
if score.status in (STATUS_SCORED, STATUS_REVIEWED)}
|
||||||
|
criteria = batch.rubric_snapshot["criteria"]
|
||||||
|
if all(criterion["criterion_id"] in graded for criterion in criteria):
|
||||||
|
# A retry in a multi-student batch must not spend two more model calls
|
||||||
|
# extracting facts for a student whose scores will all be skipped.
|
||||||
|
db.execute(update(ScoringBatch).where(ScoringBatch.id == batch_id).values(
|
||||||
|
progress_done=ScoringBatch.progress_done + len(criteria)))
|
||||||
|
db.commit()
|
||||||
|
return 0, 0
|
||||||
try:
|
try:
|
||||||
client = client_factory(db, batch)
|
client = client_factory(db, batch)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
Reference in New Issue
Block a user