feat: separate contents, abstract and body evidence

2026-10-01 00:07:40 +08:00
parent 540d0e2918
commit b7634c8374
25 changed files with 2592 additions and 172 deletions
@@ -15,25 +15,39 @@ class EvidenceResult(BaseModel):
    chunk_id: int
    page: int = Field(ge=1)
    quote: str = Field(min_length=1)
+    #: Which required-evidence target this quotation is offered to support.
+    target_id: str = ""


 class ScoreResult(BaseModel):
    criterion_id: str
-    score: float = Field(ge=0)
+    #: ``None`` means the model declined to score because admissible evidence is absent.
+    score: float | None = Field(default=None, ge=0)
    max_score: float = Field(gt=0)
    reason: str = Field(min_length=1)
-    evidence: list[EvidenceResult]
+    evidence: list[EvidenceResult] = Field(default_factory=list)
    missing_evidence: list[str] = Field(default_factory=list)
    suggestion: str = ""
    confidence: float = Field(ge=0, le=1)

    @model_validator(mode="after")
    def valid_range(self):
-        if self.score > self.max_score or (self.score > 0 and not self.evidence):
-            raise ValueError("score exceeds maximum or lacks evidence")
+        if self.score is not None and self.score > self.max_score:
+            raise ValueError("score exceeds maximum")
+        if self.score and not self.evidence:
+            raise ValueError("a positive score requires evidence")
        return self


+class EvidenceGap(BaseModel):
+    """A content target that still has no body evidence after targeted re-checking."""
+
+    target_id: str = ""
+    target_label: str = ""
+    searched_page_types: list[str] = Field(default_factory=list)
+    reason: str = ""
+
+
 class FactResult(BaseModel):
    student_no: str = ""
    name: str = ""
@@ -132,33 +146,58 @@ class DeepSeekClient:
            raise ModelError("当前端点或模型未返回原生 tool_calls；请配置支持工具调用的端点和模型")


-def validate_score(result: ScoreResult, criterion: dict, chunks: list[dict]):
+def locate_quote(text: str, quote: str):
+    """Locate a model-written quotation inside the retrieved text.
+
+    PDF line breaks and punctuation differ from what the model reproduces, so matching
+    is done on NFKC-normalized text with whitespace and punctuation removed, while the
+    returned string is still the exact source substring.
+    """
+    if not text or not quote:
+        return None
+    if quote in text:
+        return quote
+
+    def comparable(value: str):
+        return "".join(char for original in value for char in unicodedata.normalize("NFKC", original)
+                       if not char.isspace() and not unicodedata.category(char).startswith("P"))
+
+    needle = comparable(quote)
+    if len(needle) < 4:
+        return None
+    positions = []
+    normalized = []
+    for index, original in enumerate(text):
+        for char in unicodedata.normalize("NFKC", original):
+            if not char.isspace() and not unicodedata.category(char).startswith("P"):
+                normalized.append(char)
+                positions.append(index)
+    start = "".join(normalized).find(needle)
+    if start < 0:
+        return None
+    return text[positions[start]:positions[start + len(needle) - 1] + 1]
+
+
+def validate_score(result: ScoreResult, criterion: dict, chunks: list[dict], *, targets=None,
+                   require_content_evidence: bool | None = None):
+    """Check identity, quotation fidelity and the evidence-source policy.
+
+    The model may understand evidence and propose re-checks; the page permissions are
+    enforced here, in code, and cannot be argued away by the model's own reasoning.
+    """
+    from .evidence import (USE_CONTENT, content_evidence_allowed, heading_only_text,
+                           normalize_targets, page_type_allowed)
+
    if result.criterion_id != criterion["criterion_id"] or result.max_score != criterion["max_score"]:
        raise ModelError("模型评分项或满分与规则版本不一致")
+    if require_content_evidence is None:
+        require_content_evidence = any(target["use"] == USE_CONTENT
+                                       for target in normalize_targets(criterion.get("required_evidence")))
+    targets = targets if targets is not None else normalize_targets(criterion.get("required_evidence"))
+    by_target = {target["target_id"]: target for target in targets}
+    default_content = next((target for target in targets if target["use"] == USE_CONTENT), None)
    by_id = {chunk["chunk_id"]: chunk for chunk in chunks}

-    def locate_quote(text: str, quote: str):
-        def comparable(value: str):
-            return "".join(char for original in value for char in unicodedata.normalize("NFKC", original)
-                           if not char.isspace() and not unicodedata.category(char).startswith("P"))
-
-        if quote in text:
-            return quote
-        needle = comparable(quote)
-        if len(needle) < 4:
-            return None
-        positions = []
-        normalized = []
-        for index, original in enumerate(text):
-            for char in unicodedata.normalize("NFKC", original):
-                if not char.isspace() and not unicodedata.category(char).startswith("P"):
-                    normalized.append(char)
-                    positions.append(index)
-        start = "".join(normalized).find(needle)
-        if start < 0:
-            return None
-        return text[positions[start]:positions[start + len(needle) - 1] + 1]
-
    for evidence in result.evidence:
        chunk = by_id.get(evidence.chunk_id)
        quote = locate_quote(chunk["text"], evidence.quote) if chunk else None
@@ -169,5 +208,17 @@ def validate_score(result: ScoreResult, criterion: dict, chunks: list[dict]):
                chunk, quote = matches[0]
        if quote is None:
            raise ModelError("模型证据未能匹配检索原文")
+        page_type = chunk.get("page_type") or "unknown"
+        target = by_target.get(evidence.target_id) or default_content or (targets[0] if targets else None)
+        if target is None:
+            raise ModelError("模型证据未关联任何必备证据目标")
+        if not page_type_allowed(page_type, target["allowed_page_types"]):
+            raise ModelError(f"模型证据来自不允许的页面类型（{page_type}），不能支持「{target['label']}」")
+        if target["use"] == USE_CONTENT and not content_evidence_allowed(page_type):
+            raise ModelError(f"目录、摘要或参考文献页不能作为「{target['label']}」的内容证据")
+        if target["use"] == USE_CONTENT and heading_only_text(quote):
+            raise ModelError(f"「{target['label']}」的证据只有章节标题，没有实质内容")
        evidence.chunk_id, evidence.page, evidence.quote = chunk["chunk_id"], chunk["page"], quote
+        evidence.target_id = target["target_id"]
    return result
+