WiseGrade/backend/wisegrade/scoring.py

from concurrent.futures import ThreadPoolExecutor, as_completed

from sqlalchemy import and_, or_, select, update

from .db import SessionLocal
from .evidence import POLICY_VERSION, USE_CONTENT, content_targets, normalize_targets
from .knowledge_pages import page_image_path
from .llm import ConsistencyResult, FactResult, ModelError, ScoreResult, validate_score
from .models import (CriterionScore, KnowledgeChunk, KnowledgeDocument, KnowledgePage, Project,
                     ScoreEvidence, ScoringBatch, SubmissionFile, now)
from .pages import normalize_heading
from .submissions import file_index_lock, parse_submission
from .rag import (INDEX_VERSION, body_heading_index, current_document_ids, retrieve, retrieve_targets)
from .settings_store import client_for_batch

#: Criterion statuses. ``needs_review`` is deliberately distinct from a missing
#: material, a parse failure and a model failure: the retrieval ran but only found
#: navigation pages where the rule requires body content.
STATUS_NEEDS_REVIEW = "needs_review"
STATUS_UNSCORABLE = "unscorable"
STATUS_FAILED = "failed"
STATUS_SCORED = "scored"
STATUS_REVIEWED = "reviewed"
#: Statuses that a batch counts as finished work.
DONE_STATUSES = (STATUS_SCORED, STATUS_REVIEWED, STATUS_NEEDS_REVIEW)
#: Statuses the dashboard shows as needing teacher attention.
ATTENTION_STATUSES = (STATUS_FAILED, STATUS_UNSCORABLE, STATUS_NEEDS_REVIEW)


def refresh_batch_status(db, batch):
    """Recompute a finished batch after review without disturbing running workers."""
    if batch.status in ("queued", "running"):
        return batch.status
    project_ids = batch.project_ids if batch.project_ids is not None else db.scalars(
        select(Project.id).where(Project.course_version_id == batch.course_version_id)).all()
    wanted = {(project_id, criterion["criterion_id"]) for project_id in project_ids
              for criterion in batch.rubric_snapshot["criteria"]}
    scores = db.scalars(select(CriterionScore).where(CriterionScore.batch_id == batch.id)).all()
    states = {(score.project_id, score.criterion_id): score.status for score in scores}
    statuses = [states.get(key, "pending") for key in wanted]
    if any(status not in DONE_STATUSES for status in statuses):
        batch.status = "partial"
    elif STATUS_NEEDS_REVIEW in statuses:
        batch.status = "review_required"
    else:
        batch.status = "completed"
    return batch.status


def _project_facts(db, project, course_snapshot, client, *, retriever):
    """Extract project facts from body text only.

    Implementation facts must not be inferred from a contents heading or an abstract
    claim, so the search admits body and appendix pages exclusively. Project identity
    (name, student number, topic) legitimately lives on the cover page and is read from
    the teacher-confirmed import record instead of from retrieved text.
    """
    prompts = course_snapshot["prompts"]
    categories = ("stage1", "stage2", "stage3", "final_report", "presentation")
    snippets = []
    for category in categories:
        snippets.extend(retriever(db, course_version_id=project.course_version_id, project_id=project.id,
                                  category=category, query="项目名称 目标 指标 硬件 软件 实验 数据 成本 结论",
                                  limit=3, include_public=False, allowed_page_types=["body", "appendix"],
                                  substantive_only=True))
    if not snippets:
        return {}
    if client is None:
        raise ModelError("未配置或无法连接评分模型")
    facts = client.json_call(prompts["facts"], {"evidence": snippets}, FactResult).model_dump()
    from .models import Student
    student = db.get(Student, project.student_id)
    warnings = []
    for key, expected in (("student_no", student.student_no), ("name", student.name), ("topic", project.topic)):
        if facts.get(key) and facts[key] != expected:
            warnings.append(f"模型抽取的{key}与教师确认信息不一致")
    consistency = client.json_call(prompts["consistency"], {"facts": facts, "evidence": snippets}, ConsistencyResult)
    known = {snippet["chunk_id"] for snippet in snippets}
    for issue in consistency.issues:
        if not set(issue.chunk_ids) <= known:
            raise ModelError("一致性检查引用了未提供的证据")
    project.facts = {**facts, "identity_warnings": warnings,
                     "consistency_issues": [issue.model_dump() for issue in consistency.issues]}
    db.commit()
    return project.facts


def _body_navigator(db, *, course_version_id, project_id, category):
    """Find real body pages for a target whose contents entries name missing sections.

    The contents page is used only as a map: its headings are matched against headings
    that actually occur in body text, and the body sections and their neighbouring pages
    are then searched. The final quotation always lands on a real body page.
    """
    headings = body_heading_index(db, course_version_id=course_version_id, project_id=project_id, category=category)
    if not headings:
        return None

    def navigate(target):
        label = normalize_heading(target.get("label") or "")
        if not label:
            return []
        candidates = []
        for heading, places in headings.items():
            if not heading:
                continue
            if heading.startswith(label) or label.startswith(heading) or label in heading or heading in label:
                candidates.extend(places)
        if not candidates:
            return []
        pages = sorted({(place["document_id"], place["page"]) for place in candidates})
        neighbours = set()
        for document_id, page in pages:
            neighbours.update({(document_id, page), (document_id, page + 1)})
        wanted = sorted(neighbours)
        rows = db.scalars(select(KnowledgeChunk).where(
            KnowledgeChunk.course_version_id == course_version_id, KnowledgeChunk.project_id == project_id,
            KnowledgeChunk.page_type == "body", KnowledgeChunk.block_type != "toc",
            KnowledgeChunk.document_id.in_(current_document_ids()),
            or_(*(and_(KnowledgeChunk.document_id == document_id, KnowledgeChunk.page == page)
                  for document_id, page in wanted))).order_by(KnowledgeChunk.id)).all()
        hits = []
        for chunk in rows:
            hits.append({"chunk_id": chunk.id, "document_id": chunk.document_id, "project_id": chunk.project_id,
                         "document_type": chunk.document_type, "page": chunk.page, "section": chunk.section,
                         "chapter": chunk.chapter, "page_type": chunk.page_type, "block_type": chunk.block_type,
                         "text": chunk.text, "rank": 0.5})
        return hits

    return navigate


def _score_criterion(db, batch, project, criterion, client, *, retriever, multi_retriever=retrieve_targets):
    score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
                      CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
    if score and score.status in (STATUS_SCORED, STATUS_REVIEWED):
        return score
    if not score:
        score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
        db.add(score)
        db.flush()
    category = "presentation" if criterion["criterion_id"] == "final.presentation" else (
        "final_report" if criterion["category"] == "final" else criterion["category"])
    files = db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id,
                       SubmissionFile.category == category, SubmissionFile.recognition_status == "recognized")).all()
    ready = [file for file in files if file.parse_status == "ready"]
    if len(files) != 1 or len(ready) != 1:
        score.status, score.error = STATUS_UNSCORABLE, "材料缺失、重复、待确认或解析失败"
        db.commit()
        return score

    plan = batch.rubric_snapshot.get("evidence_plan") or {}
    targets = (plan.get("criteria") or {}).get(criterion["criterion_id"]) or normalize_targets(
        criterion.get("required_evidence"))
    navigator = _body_navigator(db, course_version_id=batch.course_version_id,
                                project_id=project.id, category=category)
    # The single ``retriever`` stays the seam: each per-target search goes through it,
    # so a caller that injects a retriever (tests, alternate index) still applies.
    selection = multi_retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
                                targets=targets, category=category, navigator=navigator,
                                retrieve_fn=retriever)
    chunks = _attach_page_types(db, selection["evidence"])
    gaps = selection["uncovered_targets"]
    if not chunks:
        # Nothing admissible was found for any target. Content targets go to review;
        # a purely structural criterion only depends on navigation pages, which exist.
        score.status = STATUS_NEEDS_REVIEW if content_targets(criterion) else STATUS_UNSCORABLE
        score.error = "未检索到允许作为证据的正文页"
        score.model_score = None
        score.missing_evidence = [target["label"] for target in gaps]
        _record_evidence_gaps(score, gaps)
        db.commit()
        return score
    if client is None:
        raise ModelError("未配置或无法连接评分模型")

    content_gaps = [target for target in gaps if target["use"] == USE_CONTENT]
    if content_gaps:
        # The model is not asked to grade from navigation pages. Record the gap and keep
        # the model score null so a teacher decides, rather than asserting a low score.
        score.model_score, score.confidence, score.status = None, None, STATUS_NEEDS_REVIEW
        score.model_comment = "检索仅命中目录、摘要或章节标题，缺少可用于内容判断的正文证据，转教师复核。"
        score.missing_evidence = [target["label"] for target in content_gaps]
        score.suggestion = "请补充该章节的正文页，或由教师直接核定该项分数。"
        score.error = "缺少正文证据：" + "、".join(target["label"] for target in content_gaps)
        _record_evidence_gaps(score, content_gaps, searched=_searched_page_types(plan, content_gaps))
        db.commit()
        return score

    # Page images go to the model as well, so prefer substantive body pages over the
    # navigation pages that may legitimately appear in a structural target's evidence.
    image_sources = list(dict.fromkeys(
        (chunk["document_id"], chunk["page"]) for chunk in chunks
        if chunk.get("page_type") == "body"))
    if not image_sources:
        image_sources = list(dict.fromkeys((chunk["document_id"], chunk["page"]) for chunk in chunks))
    images = []
    for document_id, page in image_sources[:3]:
        doc = db.get(KnowledgeDocument, document_id)
        path = page_image_path(db, doc, page) if doc else None
        if path:
            images.append(str(path))
    query = criterion["description"] + " " + " ".join(target["label"] for target in targets)
    public_context = retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
                               category=None, query=query, limit=6, public_only=True)
    payload = {"criterion": criterion,
               "required_evidence_targets": [{"target_id": target["target_id"], "label": target["label"],
                                              "use": target["use"],
                                              "allowed_page_types": target["allowed_page_types"]}
                                             for target in targets],
               "evidence_policy": {
                   "structure": "目录、封面和参考文献页可以证明章节结构和报告完整性，不能证明章节内容或完成质量。",
                   "content": "内容判断只能使用正文或附录中的实质段落、数据、表格；目录、摘要和单独章节标题不能作为内容证据。"},
               "project_facts": project.facts, "course_context": public_context, "evidence": chunks,
               "required_output": {"criterion_id": "string", "score": "number or null", "max_score": "number",
                                   "reason": "string",
                                   "evidence": [{"chunk_id": "integer", "page": "integer",
                                                 "quote": "exact substring", "target_id": "string"}],
                                   "missing_evidence": ["string"], "suggestion": "string", "confidence": "0..1",
                                   "note": "证据不足以判断内容时 score 填 null，并在 missing_evidence 中列出缺少的正文目标。"}}
    prompt = (batch.course_snapshot["prompts"]["score"] +
              " 证据quote必须从所引用chunk_id的text逐字复制连续片段，保留原文标点和换行；page必须等于该片段的page；"
              "target_id必须取自required_evidence_targets。不要改写、缩写或拼接引文。"
              "不得把目录、摘要或章节标题当作内容证据；无法据此判断内容时把score设为null。")
    error = ""
    result = None
    for attempt in range(2):
        try:
            result = validate_score(client.json_call(prompt, payload, ScoreResult, images=images), criterion, chunks,
                                    targets=targets)
            break
        except ModelError as exc:
            error = str(exc)
            if attempt:
                raise
            prompt += f" 上次回答未通过校验（{error}）；请改用允许页面类型中的实质正文片段，并逐字复制quote。"
    score = db.scalar(select(CriterionScore).where(CriterionScore.id == score.id)
                      .with_for_update().execution_options(populate_existing=True))
    if score.status == STATUS_REVIEWED or score.teacher_score is not None:
        return score  # A teacher decision made during the model call wins.
    score.model_score, score.model_comment = result.score, result.reason
    score.confidence = result.confidence
    score.missing_evidence, score.suggestion = result.missing_evidence, result.suggestion
    if result.score is None:
        # The retrieval found body pages but the model judged them insufficient.
        score.status, score.error = STATUS_NEEDS_REVIEW, "模型认为现有正文证据不足以判断该项，转教师复核。"
        declined = [target for target in targets if target["use"] == USE_CONTENT and
                    (not result.missing_evidence or target["label"] in result.missing_evidence)]
        _record_evidence_gaps(score, declined or [target for target in targets if target["use"] == USE_CONTENT])
        for gap in score.evidence_gaps:
            gap["reason"] = result.reason
    else:
        score.status, score.error = STATUS_SCORED, ""
    for evidence in db.scalars(select(ScoreEvidence).where(ScoreEvidence.score_id == score.id)).all():
        db.delete(evidence)
    by_id = {chunk["chunk_id"]: chunk for chunk in chunks}
    for evidence in result.evidence:
        chunk = by_id[evidence.chunk_id]
        doc = db.get(KnowledgeDocument, chunk["document_id"])
        path = page_image_path(db, doc, evidence.page)
        target = next((item for item in targets if item["target_id"] == evidence.target_id), None)
        db.add(ScoreEvidence(score_id=score.id, document_id=doc.id, submission_file_id=doc.submission_file_id,
                             page=evidence.page, quote=evidence.quote, image_path=str(path) if path else None,
                             target_id=evidence.target_id, target_label=target["label"] if target else "",
                             page_type=chunk.get("page_type") or "unknown",
                             source_role=target["use"] if target else USE_CONTENT))
    db.commit()
    return score


def _attach_page_types(db, chunks: list[dict]) -> list[dict]:
    """Attach the stored page classification to each retrieved chunk.

    The page type recorded in ``knowledge_pages`` is the authority for the evidence
    policy. Reading it here, rather than trusting whatever a retriever happened to put
    in the dict, means a statement about what a page is cannot be forged by a caller
    or lost by an alternate index.
    """
    keys = {(chunk.get("document_id"), chunk.get("page")) for chunk in chunks}
    keys.discard((None, None))
    if not keys:
        return chunks
    documents = {document_id for document_id, _ in keys}
    pages = {page for _, page in keys}
    records = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id.in_(documents),
                                                     KnowledgePage.page.in_(pages))).all()
    by_key = {(item.document_id, item.page): item for item in records}
    stored = {item.id: item for item in db.scalars(select(KnowledgeChunk).where(
        KnowledgeChunk.id.in_([chunk["chunk_id"] for chunk in chunks if chunk.get("chunk_id") is not None])))}
    output = []
    for chunk in chunks:
        record = by_key.get((chunk.get("document_id"), chunk.get("page")))
        if record is None:
            output.append({**chunk, "page_type": chunk.get("page_type") or "unknown"})
            continue
        source_type = record.page_type or "unknown"
        saved = stored.get(chunk.get("chunk_id"))
        if (saved is not None and saved.document_id == record.document_id and saved.page == record.page
                and saved.page_type == "unknown"):
            source_type = "unknown"
        if (record.mixed and saved is not None and saved.document_id == record.document_id
                and saved.page == record.page and saved.block_type == "body"
                and saved.page_type in ("body", "appendix")):
            source_type = saved.page_type
        if any(char in chunk["text"] for char in ("\ufffd", "\ufffe", "\uffff")):
            source_type = "unknown"
        output.append({**chunk, "page_type": source_type,
                       "section": chunk.get("section") or record.section or "",
                       "chapter": chunk.get("chapter") or record.chapter or ""})
    return output


def _searched_page_types(plan, targets):
    allowed = set()
    for target in targets:
        allowed.update(target.get("allowed_page_types") or [])
    return sorted(allowed or (plan.get("content_allowed_page_types") or []))


def _record_evidence_gaps(score, gaps, searched=None):
    """Store which content targets lacked body evidence and where retrieval looked.

    The record is kept through teacher review so the original retrieval problem stays
    auditable next to the score the teacher finally entered.
    """
    score.evidence_gaps = [
        {"target_id": target.get("target_id"), "target_label": target.get("label"),
         "searched_page_types": searched or target.get("allowed_page_types") or [],
         "reason": "补检后仍未找到正文证据"}
        for target in gaps]


def _run_project(batch_id, project_id, session_factory, client_factory, embedder, retriever,
                 multi_retriever=retrieve_targets):
    """Score one project. Returns ``(failures, reviews)``.

    Failures are missing material, parse problems and model errors. Reviews count the
    items that only found navigation pages where content evidence is required; those
    need a teacher decision rather than a retry.
    """
    failures = 0
    reviews = 0
    with session_factory() as db:
        batch = db.get(ScoringBatch, batch_id)
        project = db.get(Project, project_id)
        try:
            client = client_factory(db, batch)
        except Exception:
            client = None
        files = db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id)).all()
        for file in files:
            if file.recognition_status == "recognized":
                try:
                    parse_submission(db, file, embedder=embedder)
                except Exception:
                    failures += 1
        try:
            _project_facts(db, project, batch.course_snapshot, client, retriever=retriever)
        except Exception as exc:
            db.rollback()
            project.facts = {"error": str(exc)[:500]}
            db.commit()
            failures += 1
        for criterion in batch.rubric_snapshot["criteria"]:
            try:
                result = _score_criterion(db, batch, project, criterion, client, retriever=retriever,
                                          multi_retriever=multi_retriever)
                if result.status == STATUS_NEEDS_REVIEW:
                    reviews += 1
                elif result.status not in (STATUS_SCORED, STATUS_REVIEWED):
                    failures += 1
            except Exception as exc:
                db.rollback()
                score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
                                  CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
                if not score:
                    score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
                    db.add(score)
                score.status, score.error = STATUS_FAILED, str(exc)[:500]
                db.commit()
                failures += 1
            db.execute(update(ScoringBatch).where(ScoringBatch.id == batch_id).values(progress_done=ScoringBatch.progress_done + 1))
            db.commit()
    return failures, reviews


def run_batch(batch_id: int, *, session_factory=SessionLocal, client_factory=client_for_batch, embedder=None,
              retriever=retrieve, multi_retriever=retrieve_targets):
    with session_factory() as db:
        batch = db.get(ScoringBatch, batch_id)
        if not batch:
            raise ValueError("评分批次不存在")
        if batch.rubric_snapshot.get("evidence_plan", {}).get("version") != POLICY_VERSION:
            batch.status, batch.error = "failed", "该批次使用旧证据政策，请保留历史结果并创建新批次"
            db.commit()
            return {"failures": 1, "reviews": 0, "status": "failed"}
        project_ids = batch.project_ids if batch.project_ids is not None else db.scalars(
            select(Project.id).where(Project.course_version_id == batch.course_version_id).order_by(Project.id)).all()
        criteria = batch.rubric_snapshot["criteria"]
        batch.status, batch.progress_total, batch.progress_done, batch.finished_at = "running", len(project_ids) * len(criteria), 0, None
        db.commit()
        failures = 0
        reviews = 0
        with ThreadPoolExecutor(max_workers=max(1, int(batch.model_config_snapshot.get("concurrency", 1)))) as pool:
            futures = [pool.submit(_run_project, batch_id, project_id, session_factory, client_factory,
                                   embedder, retriever, multi_retriever) for project_id in project_ids]
            for future in as_completed(futures):
                try:
                    project_failures, project_reviews = future.result()
                    failures += project_failures
                    reviews += project_reviews
                except Exception as exc:
                    failures += 1
                    batch.error = (batch.error + f" 项目任务失败: {type(exc).__name__}")[:1000]
        # A batch with unresolved review items is finished running, but it is not a
        # completed grading result: the teacher still has to decide those criteria.
        batch.status = "partial" if failures else ("review_required" if reviews else "completed")
        if not failures:
            refresh_batch_status(db, batch)
        batch.finished_at = now()
        db.commit()
        return {"failures": failures, "reviews": reviews, "status": batch.status}