"""Regression tests for page-typed evidence retrieval and scoring. The real course reports in ``docs/`` are used where the acceptance criteria name them, plus synthetic pages for the cases those files do not contain (a genuinely mixed page, a contents continuation, a repeated header, an unsettled page). A constant embedding vector is used on purpose: it leaves semantic similarity flat, so the tests prove the Chinese keyword recall and the page-type filtering, which is where the reported defect came from. """ from pathlib import Path import pymupdf as fitz import pytest from sqlalchemy import select from wisegrade.evidence import normalize_targets from wisegrade.llm import ModelError, ScoreResult, validate_score from wisegrade.models import CriterionScore, KnowledgeChunk, Project, ScoreEvidence, Student from wisegrade.pages import classify_document from wisegrade.rag import retrieve, retrieve_targets from wisegrade.scoring import _score_criterion #: Constant vectors: semantic ranking contributes nothing, so recall is lexical. FLAT = lambda texts: [[0.1] * 1024 for _ in texts] COURSE_DOCS = ("阶段报告1.pdf", "阶段报告3.pdf", "自动化专业综合实践报告.pdf") def docs_root() -> Path: for candidate in (Path("/app/course-docs"), Path(__file__).resolve().parents[2] / "docs", Path(__file__).resolve().parents[1] / "docs"): if all((candidate / name).is_file() for name in COURSE_DOCS): return candidate pytest.skip("课程示例报告不可用") def report_pages(name: str) -> list[dict]: with fitz.open(docs_root() / name) as pdf: return [{"page_number": index + 1, "text": page.get_text(sort=True)} for index, page in enumerate(pdf)] def add_project(db, number: str): student = Student(student_no=number, name=f"测试学生{number[-2:]}") db.add(student) db.flush() project = Project(course_version_id=1, student_id=student.id, topic="证据治理测试", source_dir=f"tests/evidence-{number}") db.add(project) db.flush() return project def ingest_pages(db, project, pages, *, category="stage1", title="报告.pdf", source_hash="hash"): from wisegrade.rag import ingest return ingest(db, course_version_id=1, project_id=project.id, submission_file_id=None, title=title, category=category, source_hash=source_hash, pages=pages, embedder=FLAT) def page_types(db, project): from wisegrade.models import KnowledgeDocument, KnowledgePage document = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.project_id == project.id)) rows = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id == document.id) .order_by(KnowledgePage.page)).all() return {row.page: row.page_type for row in rows} # --------------------------------------------------------------------------- # # 1. Stage report 1: contents pages versus the literature review body # # --------------------------------------------------------------------------- # def test_stage1_contents_is_separated_from_literature_body(db): project = add_project(db, "123460001") pages = report_pages("阶段报告1.pdf") ingest_pages(db, project, pages) types = page_types(db, project) # The report's contents spans pages 2 and 3; the literature review is pages 6-7. assert types[1] == "cover" assert types[2] == types[3] == "toc" assert types[6] == types[7] == "body" assert types[22] == "references" criterion = {"criterion_id": "stage1.literature", "category": "stage1", "max_score": 20, "description": "文献调研和技术现状总结", "required_evidence": ["参考文献", "技术现状"]} targets = normalize_targets(criterion["required_evidence"]) selection = retrieve_targets(db, course_version_id=1, project_id=project.id, targets=targets, category="stage1", embedder=FLAT) pages_hit = {chunk["page"] for chunk in selection["evidence"]} # The literature analysis really exists on pages 6-7 and must be recalled there. assert pages_hit & {6, 7}, pages_hit content_types = {chunk["page_type"] for chunk in selection["evidence"] if chunk["target_use"] == "content"} assert content_types <= {"body", "appendix"}, content_types # The structured target may cite the reference list, and it is labelled as such. structure = [chunk for chunk in selection["evidence"] if chunk["target_use"] == "structure"] assert all(chunk["page_type"] in ("toc", "references", "body", "cover", "abstract") for chunk in structure) def test_stage1_literature_analysis_is_not_taken_from_the_contents_page(): pages = report_pages("阶段报告1.pdf") classified = {item["page"]: item for item in classify_document(pages)} toc_text = "\n".join(block["text"] for block in classified[2]["blocks"] if block["kind"] == "toc") assert "文献调研与技术现状" in toc_text # the heading really is in the contents body_text = "\n".join(block["text"] for block in classified[6]["blocks"] if block["kind"] == "body") assert "PID 控制因其结构简单" in body_text # the analysis really is in the body # The contents page carries no body block, so it cannot supply content evidence. assert not any(block["kind"] == "body" and block.get("substantive") for block in classified[2]["blocks"]) # --------------------------------------------------------------------------- # # 2. Stage report 3: contents versus the testing body # # --------------------------------------------------------------------------- # def test_stage3_contents_is_separated_from_testing_body(db): project = add_project(db, "123460002") ingest_pages(db, project, report_pages("阶段报告3.pdf"), category="stage3") types = page_types(db, project) assert types[2] == types[3] == "toc" assert types[11] == types[12] == "body" criterion = {"criterion_id": "stage3.data", "category": "stage3", "max_score": 20, "description": "实验数据和初步分析", "required_evidence": ["实验数据", "分析"]} selection = retrieve_targets(db, course_version_id=1, project_id=project.id, targets=normalize_targets(criterion["required_evidence"]), category="stage3", embedder=FLAT) hits = [(chunk["page"], chunk["page_type"], chunk["text"][:40]) for chunk in selection["evidence"]] pages_hit = {chunk["page"] for chunk in selection["evidence"]} # The testing description on pages 11-12 must be reachable, not the contents. assert pages_hit & {11, 12}, hits assert all(chunk["page_type"] in ("body", "appendix") for chunk in selection["evidence"]), hits # Finding the page is not a score: the evidence only has to be reachable. assert any("测试" in chunk["text"] or "试验" in chunk["text"] for chunk in selection["evidence"]) # --------------------------------------------------------------------------- # # 3. Final report: abstract, cross-page contents, body # # --------------------------------------------------------------------------- # def test_final_report_abstract_cross_page_contents_and_body(db): project = add_project(db, "123460003") ingest_pages(db, project, report_pages("自动化专业综合实践报告.pdf"), category="final_report") types = page_types(db, project) assert types[1] == "cover" assert types[2] == types[3] == "abstract" # The contents continues onto a second page (roman numeral IV) and is still contents. assert types[4] == types[5] == "toc" assert types[6] == "body" assert types[49] == "references" # Appendix pages stay usable for content, and are labelled as appendices. assert types[50] == "appendix" def test_contents_page_still_supports_chapter_completeness(): """A structural criterion keeps using the contents page; that is its purpose.""" pages = report_pages("自动化专业综合实践报告.pdf") classified = {item["page"]: item for item in classify_document(pages)} from wisegrade.pages import toc_entries entries = toc_entries(classified[4]["blocks"]) assert len(entries) >= 20 assert any("第1 章" in entry or "第1章" in entry for entry in entries) criterion = {"criterion_id": "stage1.format", "max_score": 5, "description": "报告完整性和规范性", "required_evidence": ["标题", "章节", "参考文献"]} targets = normalize_targets(criterion["required_evidence"]) assert all(target["use"] == "structure" for target in targets) # A structure target admits the contents page; a content target never does. assert all("toc" in target["allowed_page_types"] for target in targets) # --------------------------------------------------------------------------- # # 4. Mixed pages, continuations, repeated headers, unsettled pages # # --------------------------------------------------------------------------- # def test_mixed_page_contents_continuation_header_and_uncertain_page(): pages = [ {"page_number": 1, "text": "自动化专业综合实践\n阶段性报告\n姓名：张三\n学号：1234567890"}, # A contents page that also carries the start of the first chapter. {"page_number": 2, "text": "目录\n第1 章 项目背景.......... 1\n第2 章 需求分析.......... 3\n" "第1 章 项目背景\n本项目面向智能检测场景，通过多传感器融合实现目标识别。"}, # Contents continued: dotted entries and nothing else. {"page_number": 3, "text": "3.2 性能测试.......... 9\n3.3 结果分析.......... 11\n参考文献.......... 15"}, # A repeated header plus real prose. {"page_number": 4, "text": "自动化专业综合实践\n2.1 需求分析\n" "系统需要完成目标检测、定位与结果输出，并在标准场地内稳定运行。\n" "自动化专业综合实践"}, # A section title with no prose at all. {"page_number": 5, "text": "自动化专业综合实践\n3.1 功能测试"}, ] classified = {item["page"]: item for item in classify_document(pages)} # The repeated header is stripped from indexed text. from wisegrade.pages import clean_page_lines, strip_repeated_lines repeated = strip_repeated_lines(pages) assert "自动化专业综合实践" in repeated assert "自动化专业综合实践" not in clean_page_lines(pages[3], repeated) assert classified[1]["page_type"] == "cover" assert classified[2]["page_type"] == "toc" and classified[2]["mixed"] is True # A mixed page keeps a usable body block instead of dropping the whole page. body_blocks = [b for b in classified[2]["blocks"] if b["kind"] == "body" and b.get("substantive")] assert body_blocks and "多传感器融合" in body_blocks[0]["text"] # The contents continuation is recognised as contents, not body. assert classified[3]["page_type"] == "toc" assert classified[4]["page_type"] == "body" # A page with only a section title is not silently ordinary body text. assert classified[5]["page_type"] == "unknown" def test_mixed_page_body_block_is_indexed_as_body_not_navigation(db): project = add_project(db, "123460004") pages = [{"page_number": 2, "text": "目录\n第1 章 项目背景.......... 1\n" "第1 章 项目背景\n本项目面向智能检测场景，通过多传感器融合实现目标识别。"}] ingest_pages(db, project, pages) chunk = db.scalar(select(KnowledgeChunk).where(KnowledgeChunk.document_id.isnot(None), KnowledgeChunk.page == 2, KnowledgeChunk.block_type == "body")) # The body half of the mixed page survives as its own searchable chunk. assert chunk is not None and "多传感器融合" in chunk.text navigation = db.scalars(select(KnowledgeChunk).where(KnowledgeChunk.page == 2, KnowledgeChunk.block_type == "toc")).all() assert navigation and all("多传感器融合" not in row.text for row in navigation) def test_uncertain_page_is_not_offered_as_body_content(db): project = add_project(db, "123460005") pages = [{"page_number": 1, "text": "封面"}, {"page_number": 2, "text": "第1 章 项目背景与需求分析"}, {"page_number": 3, "text": "第1 章 项目背景与需求分析\n系统面向智能检测场景，完成目标识别与定位。"}] ingest_pages(db, project, pages) types = page_types(db, project) assert types[2] == "unknown" body_chunks = db.scalars(select(KnowledgeChunk).where(KnowledgeChunk.page_type == "body")).all() assert all(row.page != 2 for row in body_chunks) # --------------------------------------------------------------------------- # # 5. Content judgements cannot rest on navigation pages # # --------------------------------------------------------------------------- # def test_content_judgement_rejects_navigation_sources_but_structure_accepts_them(): criterion = {"criterion_id": "stage3.data", "max_score": 20, "description": "实验数据和初步分析", "required_evidence": ["实验数据", "分析"]} toc = {"chunk_id": 1, "page": 2, "page_type": "toc", "text": "第3 章 系统测试与验证 ......... 8"} result = ScoreResult(criterion_id="stage3.data", score=18, max_score=20, reason="看目录", evidence=[{"chunk_id": 1, "page": 2, "quote": "系统测试与验证", "target_id": "t0"}], confidence=.9) with pytest.raises(ModelError): validate_score(result, criterion, [toc]) structure = {"criterion_id": "stage1.format", "max_score": 5, "description": "报告完整性和规范性", "required_evidence": ["章节"]} ok = ScoreResult(criterion_id="stage1.format", score=4, max_score=5, reason="章节完整", evidence=[{"chunk_id": 1, "page": 2, "quote": "第3 章 系统测试与验证", "target_id": "t0"}], confidence=.9) assert validate_score(ok, structure, [toc]).score == 4 # --------------------------------------------------------------------------- # # 6. Insufficient body evidence becomes a teacher review, never a low score # # --------------------------------------------------------------------------- # def test_insufficient_body_evidence_goes_to_review_with_a_null_model_score(db): from types import SimpleNamespace from wisegrade import services from wisegrade.models import ScoringBatch, SubmissionFile project = add_project(db, "123460006") created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db) batch = db.get(ScoringBatch, created["id"]) criterion = next(item for item in batch.rubric_snapshot["criteria"] if item["criterion_id"] == "stage3.data") file = SubmissionFile(project_id=project.id, category="stage3", relative_path="test/review/report.pdf", mime_type="application/pdf", size=1, mtime=1, sha256="c" * 64, recognition_status="recognized", parse_status="ready") db.add(file) db.flush() # The material only contains a cover page and a contents page: no body at all. pages = [{"page_number": 1, "text": "阶段性报告\n姓名：张三\n学号：1234567890"}, {"page_number": 2, "text": "目录\n第3 章 系统测试与验证.......... 8\n第4 章 结论.......... 12"}] file.parsed = {"pages": pages} ingest_pages(db, project, pages, category="stage3", title="report.pdf", source_hash="c" * 64) class NeverCalled: def json_call(self, *_, **__): raise AssertionError("缺少正文证据时不应调用评分模型") scored = _score_criterion(db, batch, project, criterion, NeverCalled(), retriever=lambda *_, **kw: [] if kw.get("public_only") else retrieve( *_, **{**kw, "embedder": FLAT})) assert scored.status == "needs_review" assert scored.model_score is None assert scored.evidence_gaps and scored.evidence_gaps[0]["target_label"] assert scored.evidence_gaps[0]["searched_page_types"] # The criterion shows as awaiting review, and the batch is not reported as complete. counts = {row.status for row in db.scalars(select(CriterionScore).where( CriterionScore.batch_id == batch.id)).all()} assert "needs_review" in counts def test_teacher_can_review_a_needs_review_item_and_the_gap_record_survives(db): from types import SimpleNamespace from wisegrade import services project = add_project(db, "123460007") created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db) score = CriterionScore(batch_id=created["id"], project_id=project.id, criterion_id="stage3.data", model_score=None, status="needs_review", evidence_gaps=[{"target_id": "t0", "target_label": "实验数据", "searched_page_types": ["body", "appendix"], "reason": "补检后仍未找到正文证据"}]) db.add(score) db.commit() result = services.review_score(score.id, SimpleNamespace(score=17, comment="教师核对原始材料", reason="材料确实存在测试章节"), db) assert result["final_score"] == 17 and result["model_score"] is None reviewed = db.get(CriterionScore, score.id) assert reviewed.status == "reviewed" # The original retrieval problem stays on the record after the teacher decides. assert reviewed.evidence_gaps[0]["target_label"] == "实验数据" # --------------------------------------------------------------------------- # # 7. Agent citations only carry sources that were actually used # # --------------------------------------------------------------------------- # def test_search_tools_label_page_types_and_mark_historical_citations(db): from wisegrade import agent_tools from wisegrade.models import Conversation project = add_project(db, "123460008") ingest_pages(db, project, [ {"page_number": 2, "text": "目录\n第3 章 系统测试与验证.......... 8"}, {"page_number": 11, "text": "第3 章 系统测试与验证\n在自主巡线测试中，底盘记录了直线段与弯道的时间数据。"}, ], category="stage3") conversation = Conversation(title="检索", course_version_id=1, project_id=project.id) db.add(conversation) db.commit() original = agent_tools.retrieve agent_tools.retrieve = lambda *args, **kwargs: original(*args, **kwargs, embedder=FLAT) try: content = agent_tools.execute(db, conversation, "search_project_material", {"project_id": project.id, "query": "系统测试与验证"}, "") assert content["found"] # Content search never returns the contents page as body evidence. assert all(item["page_type"] in ("body", "appendix") for item in content["citations"]) assert all(item["source_role"] == "content" for item in content["citations"]) assert all(item["citation_id"].startswith("chunk:") for item in content["citations"]) structure = agent_tools.execute(db, conversation, "search_project_structure", {"project_id": project.id, "query": "系统测试"}, "") assert structure["found"] assert all(item["source_role"] == "structure" for item in structure["citations"]) # A structural hit is explicitly declared as unable to prove chapter content. assert any(item["page_type"] == "toc" for item in structure["citations"]) assert "不能证明章节内容" in structure["note"] finally: agent_tools.retrieve = original def test_scores_tool_marks_historical_score_evidence(db): from types import SimpleNamespace from wisegrade import agent_tools, services from wisegrade.models import Conversation project = add_project(db, "123460009") created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db) score = CriterionScore(batch_id=created["id"], project_id=project.id, criterion_id="stage1.format", model_score=4, status="scored") db.add(score) db.flush() db.add(ScoreEvidence(score_id=score.id, document_id=None, submission_file_id=None, page=2, quote="第1 章 项目背景与需求分析", image_path=None, page_type="toc", target_label="章节", historical=True)) conversation = Conversation(title="评分", course_version_id=1, batch_id=created["id"]) db.add(conversation) db.commit() result = agent_tools.execute(db, conversation, "get_scores", {"batch_id": created["id"], "project_id": project.id}, "") citations = result["citations"] assert citations and citations[0]["historical"] is True assert citations[0]["source"] == "historical_score_evidence" assert citations[0]["usage"] == "历史评分引用" # --------------------------------------------------------------------------- # # 8. Isolation and existing behaviour # # --------------------------------------------------------------------------- # def test_page_filtering_happens_before_the_candidate_limit(db): """A contents page must not crowd a body page out of the candidate window.""" project = add_project(db, "123460010") pages = [{"page_number": 2, "text": "目录\n第3 章 系统测试与验证.......... 8"}] for number in range(3, 25): pages.append({"page_number": number, "text": f"第3 章 系统测试与验证\n第{number} 页记录了系统测试的实测数据与分析过程。"}) ingest_pages(db, project, pages, category="stage3") unrestricted = retrieve(db, course_version_id=1, project_id=project.id, query="系统测试与验证", category="stage3", limit=3, embedder=FLAT, allowed_page_types=None) filtered = retrieve(db, course_version_id=1, project_id=project.id, query="系统测试与验证", category="stage3", limit=3, embedder=FLAT, allowed_page_types=["body", "appendix"], substantive_only=True) assert filtered and all(chunk["page_type"] in ("body", "appendix") for chunk in filtered) assert {chunk["page"] for chunk in filtered} != {2} # Without the filter the contents page is a candidate; the filter removes it before # the limit is applied, so three real body pages still come back. assert len(filtered) == 3 assert any(chunk["page_type"] == "toc" for chunk in unrestricted) def test_project_and_version_isolation_is_unchanged(db): first = add_project(db, "123460011") second = add_project(db, "123460012") ingest_pages(db, first, [{"page_number": 3, "text": "第3 章 系统测试与验证\n一号项目记录了巡线测试数据。"}], category="stage3", title="一号.pdf", source_hash="one") ingest_pages(db, second, [{"page_number": 3, "text": "第3 章 系统测试与验证\n二号项目的私有测试结果。"}], category="stage3", title="二号.pdf", source_hash="two") hits = retrieve(db, course_version_id=1, project_id=first.id, category="stage3", query="系统测试与验证", embedder=FLAT, allowed_page_types=["body", "appendix"]) assert hits and all(hit["project_id"] == first.id for hit in hits) assert "私有" not in str(hits) # Course material stays in its own namespace and is not mixed into project search. public = retrieve(db, course_version_id=1, project_id=first.id, query="系统测试与验证", embedder=FLAT, public_only=True) assert all(hit["project_id"] is None for hit in public) def test_batch_snapshot_records_the_evidence_policy(db): from types import SimpleNamespace from wisegrade import services from wisegrade.models import ScoringBatch created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db) assert created["id"] batch = db.get(ScoringBatch, created["id"]) plan = batch.rubric_snapshot["evidence_plan"] assert plan["version"] == 1 assert plan["content_allowed_page_types"] == ["body", "appendix"] targets = plan["criteria"]["stage3.data"] assert [target["use"] for target in targets] == ["content", "content"] assert all("toc" not in target["allowed_page_types"] for target in targets) # The legacy string form is read into typed targets without being rewritten. assert [item["required_evidence"] for item in batch.rubric_snapshot["criteria"] if item["criterion_id"] == "stage3.data"] == [["实验数据", "分析"]]