fix: resolve legacy evidence targets without guessing and guide retries
This commit is contained in:
@@ -238,11 +238,19 @@ def validate_score(result: ScoreResult, criterion: dict, chunks: list[dict], *,
|
||||
# Empty identifiers are accepted only for pre-object legacy rubrics.
|
||||
if any(isinstance(value, dict) for value in criterion.get("required_evidence", [])):
|
||||
raise ModelError("模型证据缺少 target_id")
|
||||
target = default_content or (targets[0] if targets else None)
|
||||
associated = [by_target[key] for key in chunk.get("target_ids", []) if key in by_target]
|
||||
if associated:
|
||||
if len(associated) != 1:
|
||||
raise ModelError(f"模型证据缺少 target_id:chunk_id={chunk['chunk_id']} 对应多个检索目标,请明确选择")
|
||||
target = associated[0]
|
||||
else:
|
||||
target = default_content or (targets[0] if targets else None)
|
||||
if target is None:
|
||||
raise ModelError("模型证据未关联任何必备证据目标")
|
||||
if chunk.get("target_ids") and target["target_id"] not in chunk["target_ids"]:
|
||||
raise ModelError("模型证据未关联该片段的检索目标")
|
||||
allowed = ",".join(str(key) for key in chunk["target_ids"] if key in by_target)
|
||||
raise ModelError(f"模型证据未关联该片段的检索目标:chunk_id={chunk['chunk_id']},"
|
||||
f"target_id={target['target_id']};允许 target_id={allowed}")
|
||||
if not page_type_allowed(page_type, target["allowed_page_types"]):
|
||||
raise ModelError(f"模型证据来自不允许的页面类型({page_type}),不能支持「{target['label']}」")
|
||||
if target["use"] == USE_CONTENT and not content_evidence_allowed(page_type):
|
||||
|
||||
@@ -227,6 +227,8 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
|
||||
prompt = (batch.course_snapshot["prompts"]["score"] +
|
||||
" 证据quote必须从所引用chunk_id的text逐字复制连续片段,保留原文标点和换行;page必须等于该片段的page;"
|
||||
"target_id必须取自required_evidence_targets。不要改写、缩写或拼接引文。"
|
||||
"每条引文的 target_id 必须属于该片段的 target_ids;同一片段可对应多个目标,"
|
||||
"请按实际内容选择,不要按证据列表顺序分配编号。"
|
||||
"不得把目录、摘要或章节标题当作内容证据;无法据此判断内容时把score设为null。")
|
||||
error = ""
|
||||
result = None
|
||||
@@ -239,7 +241,11 @@ def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_
|
||||
error = str(exc)
|
||||
if attempt:
|
||||
raise
|
||||
prompt += f" 上次回答未通过校验({error});请改用允许页面类型中的实质正文片段,并逐字复制quote。"
|
||||
payload["validation_feedback"] = {
|
||||
"error": error,
|
||||
"instruction": "根据具体错误修正输出;引用目标只能从对应片段的 target_ids 选择,"
|
||||
"quote须为该片段原文。没有足够依据时保留空分,不要编造引用。"}
|
||||
prompt += " 上次回答未通过校验;请按 validation_feedback 修正,不要重复错误引用。"
|
||||
score = db.scalar(select(CriterionScore).where(CriterionScore.id == score.id)
|
||||
.with_for_update().execution_options(populate_existing=True))
|
||||
if score.status == STATUS_REVIEWED or score.teacher_score is not None:
|
||||
|
||||
Reference in New Issue
Block a user