fix: resolve legacy evidence targets without guessing and guide retries

This commit is contained in:
2026-10-17 20:17:19 +08:00
parent 93ff9c8c8d
commit ef9edecff2
3 changed files with 82 additions and 3 deletions
+10 -2
View File
@@ -238,11 +238,19 @@ def validate_score(result: ScoreResult, criterion: dict, chunks: list[dict], *,
# Empty identifiers are accepted only for pre-object legacy rubrics.
if any(isinstance(value, dict) for value in criterion.get("required_evidence", [])):
raise ModelError("模型证据缺少 target_id")
target = default_content or (targets[0] if targets else None)
associated = [by_target[key] for key in chunk.get("target_ids", []) if key in by_target]
if associated:
if len(associated) != 1:
raise ModelError(f"模型证据缺少 target_id:chunk_id={chunk['chunk_id']} 对应多个检索目标,请明确选择")
target = associated[0]
else:
target = default_content or (targets[0] if targets else None)
if target is None:
raise ModelError("模型证据未关联任何必备证据目标")
if chunk.get("target_ids") and target["target_id"] not in chunk["target_ids"]:
raise ModelError("模型证据未关联该片段的检索目标")
allowed = ",".join(str(key) for key in chunk["target_ids"] if key in by_target)
raise ModelError(f"模型证据未关联该片段的检索目标:chunk_id={chunk['chunk_id']},"
f"target_id={target['target_id']};允许 target_id={allowed}")
if not page_type_allowed(page_type, target["allowed_page_types"]):
raise ModelError(f"模型证据来自不允许的页面类型({page_type}),不能支持「{target['label']}」")
if target["use"] == USE_CONTENT and not content_evidence_allowed(page_type):
+7 -1
View File
@@ -227,6 +227,8 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
prompt = (batch.course_snapshot["prompts"]["score"] +
" 证据quote必须从所引用chunk_id的text逐字复制连续片段,保留原文标点和换行;page必须等于该片段的page;"
"target_id必须取自required_evidence_targets。不要改写、缩写或拼接引文。"
"每条引文的 target_id 必须属于该片段的 target_ids;同一片段可对应多个目标,"
"请按实际内容选择,不要按证据列表顺序分配编号。"
"不得把目录、摘要或章节标题当作内容证据;无法据此判断内容时把score设为null。")
error = ""
result = None
@@ -239,7 +241,11 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
error = str(exc)
if attempt:
raise
prompt += f" 上次回答未通过校验({error});请改用允许页面类型中的实质正文片段,并逐字复制quote。"
payload["validation_feedback"] = {
"error": error,
"instruction": "根据具体错误修正输出;引用目标只能从对应片段的 target_ids 选择,"
"quote须为该片段原文。没有足够依据时保留空分,不要编造引用。"}
prompt += " 上次回答未通过校验;请按 validation_feedback 修正,不要重复错误引用。"
score = db.scalar(select(CriterionScore).where(CriterionScore.id == score.id)
.with_for_update().execution_options(populate_existing=True))
if score.status == STATUS_REVIEWED or score.teacher_score is not None: