396 lines
22 KiB
Python
396 lines
22 KiB
Python
import json
|
|
from datetime import timedelta
|
|
from types import SimpleNamespace
|
|
|
|
import httpx
|
|
import pymupdf
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import select
|
|
|
|
from wisegrade import agent_tools, chat_api, services
|
|
from wisegrade.actions import confirm, prepare
|
|
from wisegrade.agent_tools import execute
|
|
from wisegrade.db import get_db
|
|
from wisegrade.indexing import run_index_task
|
|
from wisegrade.llm import DeepSeekClient, ModelError
|
|
from wisegrade.main import app
|
|
from wisegrade.models import (AgentRun, ChatMessage, Conversation, CourseVersion, CriterionScore, IndexTask, PendingAction,
|
|
Project, RubricVersion, ScoringBatch, Student, TeacherRevision, now)
|
|
from wisegrade.rag import ingest
|
|
from wisegrade.scoring import run_batch
|
|
from wisegrade.storage import scan
|
|
|
|
|
|
def add_project(db, number, *, version_id=1):
|
|
student = Student(student_no=number, name=f"学生{number[-1]}")
|
|
db.add(student)
|
|
db.flush()
|
|
project = Project(course_version_id=version_id, student_id=student.id, topic="控制系统", source_dir=f"test/{number}")
|
|
db.add(project)
|
|
db.commit()
|
|
return project
|
|
|
|
|
|
def test_tool_scope_snapshots_and_project_citations(db):
|
|
first = add_project(db, "123450001")
|
|
second = add_project(db, "123450002")
|
|
vectors = lambda texts: [[0.1] * 1024 for _ in texts]
|
|
for project, text in ((first, "一号项目电机控制方案"), (second, "二号项目视觉方案")):
|
|
ingest(db, course_version_id=1, project_id=project.id, submission_file_id=None,
|
|
title=f"{project.id}.pdf", category="final_report", source_hash=str(project.id),
|
|
pages=[{"page_number": 2, "section": "方案", "text": text, "tables": []}], embedder=vectors)
|
|
conversation = Conversation(title="范围", course_version_id=1, project_id=first.id)
|
|
db.add(conversation)
|
|
db.commit()
|
|
from wisegrade import agent_tools
|
|
original = agent_tools.retrieve
|
|
agent_tools.retrieve = lambda *args, **kwargs: original(*args, **kwargs, embedder=vectors)
|
|
try:
|
|
found = execute(db, conversation, "search_project_material", {"project_id": first.id, "query": "电机控制"}, "")
|
|
assert found["found"] and len(found["citations"]) == 1
|
|
assert found["citations"][0]["quote"] == "一号项目电机控制方案"
|
|
assert found["citations"][0]["page"] == 2
|
|
assert "/pages/2" in found["citations"][0]["url"]
|
|
with pytest.raises(HTTPException):
|
|
execute(db, conversation, "search_project_material", {"project_id": second.id, "query": "视觉"}, "")
|
|
with pytest.raises(Exception):
|
|
execute(db, conversation, "get_project", {"project_id": first.id, "arbitrary_sql": "DROP TABLE"}, "")
|
|
finally:
|
|
agent_tools.retrieve = original
|
|
batch = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
|
snapshot = db.get(ScoringBatch, batch["id"])
|
|
before = snapshot.rubric_snapshot["criteria"][0]["name"]
|
|
rubric = db.get(RubricVersion, 1)
|
|
rubric.criteria = [{**rubric.criteria[0], "name": "后来修改"}, *rubric.criteria[1:]]
|
|
db.commit()
|
|
rules = execute(db, conversation, "get_course_rules", {"batch_id": snapshot.id}, "")
|
|
assert rules["rubric"]["criteria"][0]["name"] == before
|
|
assert rules["source"] == "batch_snapshot"
|
|
|
|
|
|
def test_get_scores_pages_projects_and_criteria(db, monkeypatch):
|
|
first = add_project(db, "123450010")
|
|
second = add_project(db, "123450011")
|
|
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
|
conversation = Conversation(title="分页评分", batch_id=created["id"])
|
|
db.add(conversation)
|
|
db.commit()
|
|
|
|
def result(_db, _batch, item):
|
|
return {"student_no": f"{item.id}", "name": "学生", "totals": {"overall": 80},
|
|
"rows": [{"criterion": {"criterion_id": f"C{index}", "name": f"评分项{index}", "max_score": 10},
|
|
"score_id": index, "model_score": 8, "teacher_score": None, "final_score": 8,
|
|
"status": "scored", "error": "", "evidence": [{"page": index + 1,
|
|
"quote": "证据" * 200, "document": "报告.pdf",
|
|
"image_url": f"/api/assets/{item.id}/1" if index == 0 else None}]}
|
|
for index in range(14)]}
|
|
|
|
monkeypatch.setattr(agent_tools, "project_result", result)
|
|
first_page = execute(db, conversation, "get_scores", {"batch_id": created["id"]}, "")
|
|
assert first_page["count"] == 2 and len(first_page["results"]) == 1
|
|
assert first_page["results"][0]["project_id"] == first.id
|
|
assert len(first_page["results"][0]["rows"]) == 6
|
|
assert first_page["next_criterion_offset"] == 6 and first_page["next_project_offset"] == 1
|
|
assert first_page["citations"][0]["url"] == f"/?page=scores&batch_id={created['id']}&project_id={first.id}"
|
|
assert first_page["citations"][0]["image_url"] == f"/api/assets/{first.id}/1"
|
|
assert first_page["citations"][1]["url"] == first_page["citations"][0]["url"]
|
|
assert first_page["citations"][1]["image_url"] is None
|
|
assert len(json.dumps(first_page, ensure_ascii=False)) < 12000
|
|
later = execute(db, conversation, "get_scores", {"batch_id": created["id"],
|
|
"project_offset": 1, "criterion_offset": 12}, "")
|
|
assert later["results"][0]["project_id"] == second.id
|
|
assert [row["criterion_id"] for row in later["results"][0]["rows"]] == ["C12", "C13"]
|
|
assert later["next_project_offset"] is None and later["next_criterion_offset"] is None
|
|
|
|
|
|
def test_import_confirmation_hash_change_index_and_idempotency(environment, db, monkeypatch, session_factory):
|
|
folder = environment / "inbox" / "batch" / "123450003_张三_机器人"
|
|
folder.mkdir(parents=True)
|
|
file = folder / "阶段报告1.pdf"
|
|
def write_pdf(text):
|
|
pdf = pymupdf.open()
|
|
page = pdf.new_page()
|
|
page.insert_text((72, 72), (text + " with enough searchable report detail for indexing. ") * 4)
|
|
pdf.save(file)
|
|
write_pdf("First report content")
|
|
scanned = scan("batch")["projects"][0]
|
|
payload = {"course_version_id": 1, "projects": [{"relative_dir": scanned["relative_dir"],
|
|
"student_no": scanned["student_no"], "name": scanned["name"], "topic": scanned["topic"],
|
|
"files": [{"relative_path": item["relative_path"], "category": item["category"],
|
|
"sha256": item["sha256"], "status": "recognized"} for item in scanned["files"]]}]}
|
|
conversation = Conversation(title="导入", course_version_id=1)
|
|
db.add(conversation)
|
|
db.commit()
|
|
action = prepare(db, conversation, "import", payload)
|
|
assert db.scalar(select(Project.id)) is None
|
|
write_pdf("Changed report content")
|
|
with pytest.raises(HTTPException) as exc:
|
|
confirm(db, conversation, action.id)
|
|
assert exc.value.status_code == 409 and db.scalar(select(Project.id)) is None
|
|
scanned = scan("batch")["projects"][0]
|
|
payload["projects"][0]["files"][0]["sha256"] = scanned["files"][0]["sha256"]
|
|
monkeypatch.setattr(services, "enqueue_index_task", lambda db, task_id: db.get(IndexTask, task_id))
|
|
action = prepare(db, conversation, "import", payload)
|
|
result = confirm(db, conversation, action.id)
|
|
assert confirm(db, conversation, action.id) == result
|
|
assert len(result["project_ids"]) == len(result["index_task_ids"]) == 1
|
|
task = db.get(IndexTask, result["index_task_ids"][0])
|
|
assert task.status == "queued"
|
|
vectors = lambda texts: [[0.1] * 1024 for _ in texts]
|
|
run_index_task(task.id, session_factory=session_factory, embedder=vectors)
|
|
db.refresh(task)
|
|
assert task.status == "completed" and task.result["pages"] == 1, task.error
|
|
failed = IndexTask(course_version_id=1, submission_file_id=task.submission_file_id,
|
|
kind="project", status="failed", error="temporary")
|
|
db.add(failed)
|
|
db.commit()
|
|
first_retry = services.retry_index_task(db, failed.id)
|
|
assert services.retry_index_task(db, failed.id).id == first_retry.id
|
|
|
|
|
|
def test_duplicate_scan_requires_teacher_file_choice(environment, db):
|
|
folder = environment / "inbox" / "dup" / "123450009_王五_控制"
|
|
folder.mkdir(parents=True)
|
|
for name in ("阶段报告1-A.pdf", "阶段报告1-B.pdf"):
|
|
pdf = pymupdf.open()
|
|
page = pdf.new_page()
|
|
page.insert_text((72, 72), "Long report text for reliable detection. " * 4)
|
|
pdf.save(folder / name)
|
|
scanned = scan("dup")["projects"][0]
|
|
assert all(file["status"] == "duplicate" for file in scanned["files"])
|
|
chosen = scanned["files"][0]
|
|
payload = {"course_version_id": 1, "projects": [{"relative_dir": scanned["relative_dir"],
|
|
"student_no": scanned["student_no"], "name": scanned["name"], "topic": scanned["topic"],
|
|
"files": [{"relative_path": chosen["relative_path"], "category": "stage1",
|
|
"sha256": chosen["sha256"], "status": "recognized"}]}]}
|
|
conversation = Conversation(title="重复文件", course_version_id=1)
|
|
db.add(conversation)
|
|
db.commit()
|
|
generated = execute(db, conversation, "prepare_import", {"course_version_id": 1, "relative_dir": "dup"}, "准备导入")
|
|
generated_action = db.get(PendingAction, generated["pending_action"]["id"])
|
|
assert generated_action.preview["requires_edit"] is True
|
|
assert all(file["status"] == "ignored" for file in generated_action.payload["projects"][0]["files"])
|
|
proposed = prepare(db, conversation, "import", payload)
|
|
assert proposed.preview["requires_edit"] is True
|
|
with pytest.raises(HTTPException):
|
|
confirm(db, conversation, proposed.id)
|
|
assert db.scalar(select(Project.id)) is None
|
|
corrected = prepare(db, conversation, "import", payload, teacher_edited=True)
|
|
assert corrected.id != proposed.id and corrected.preview["requires_edit"] is False
|
|
|
|
|
|
def test_fixed_batch_membership_duplicate_queue_and_review(db, monkeypatch, session_factory):
|
|
first = add_project(db, "123450004")
|
|
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
|
batch = db.get(ScoringBatch, created["id"])
|
|
assert batch.status == "created" and batch.project_ids == [first.id]
|
|
add_project(db, "123450005")
|
|
jobs = []
|
|
monkeypatch.setattr(services, "Queue", lambda *args, **kwargs: SimpleNamespace(enqueue=lambda *a, **k: jobs.append(k)))
|
|
services.start_batch(batch.id, db)
|
|
with pytest.raises(HTTPException) as exc:
|
|
services.start_batch(batch.id, db)
|
|
assert exc.value.status_code == 409 and len(jobs) == 1
|
|
run_batch(batch.id, session_factory=session_factory, client_factory=lambda *_: SimpleNamespace())
|
|
# The fixed list excludes the project imported after creation.
|
|
db.refresh(batch)
|
|
assert batch.progress_total == len(batch.rubric_snapshot["criteria"])
|
|
score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id))
|
|
score.model_score, score.status, score.error = None, "failed", "证据不足"
|
|
db.commit()
|
|
conversation = Conversation(title="复核", batch_id=batch.id, project_id=first.id)
|
|
db.add(conversation)
|
|
db.commit()
|
|
with pytest.raises(HTTPException):
|
|
execute(db, conversation, "prepare_review", {"score_id": score.id, "score": 3,
|
|
"comment": "人工核定", "reason": "核查纸质材料"}, "请复核")
|
|
prepared = execute(db, conversation, "prepare_review", {"score_id": score.id, "score": 3,
|
|
"comment": "人工核定", "reason": "核查纸质材料"}, "给 3 分")
|
|
action_id = prepared["pending_action"]["id"]
|
|
assert score.teacher_score is None
|
|
student = db.get(Student, first.student_id)
|
|
student.name = "身份变化"
|
|
db.commit()
|
|
with pytest.raises(HTTPException):
|
|
confirm(db, conversation, action_id)
|
|
assert score.teacher_score is None
|
|
student.name = "学生4"
|
|
db.commit()
|
|
prepared = execute(db, conversation, "prepare_review", {"score_id": score.id, "score": 3,
|
|
"comment": "人工核定", "reason": "核查纸质材料"}, "给 3 分")
|
|
action_id = prepared["pending_action"]["id"]
|
|
confirm(db, conversation, action_id)
|
|
db.refresh(score)
|
|
assert score.teacher_score == 3 and score.model_score is None and score.status == "reviewed"
|
|
assert db.scalar(select(TeacherRevision).where(TeacherRevision.score_id == score.id)).new_score == 3
|
|
summary = execute(db, conversation, "summarize_batch", {"batch_id": batch.id}, "")
|
|
assert summary["project_count"] == 1 and summary["complete_count"] == 0
|
|
assert summary["incomplete_count"] == 1 and not summary["overall_distribution"]
|
|
|
|
|
|
def test_batch_and_export_are_previews_until_confirmed(db):
|
|
project = add_project(db, "123450006")
|
|
conversation = Conversation(title="确认", course_version_id=1)
|
|
db.add(conversation)
|
|
db.commit()
|
|
action = prepare(db, conversation, "create_batch", {"course_version_id": 1, "rubric_version_id": 1})
|
|
assert action.preview["project_ids"] == [project.id]
|
|
assert db.scalar(select(ScoringBatch.id)) is None
|
|
first = confirm(db, conversation, action.id)
|
|
assert confirm(db, conversation, action.id) == first
|
|
assert len(db.scalars(select(ScoringBatch.id)).all()) == 1
|
|
export = prepare(db, conversation, "export", {"batch_id": first["id"], "format": "excel"})
|
|
assert export.preview["incomplete_projects"] == [project.id]
|
|
from wisegrade.models import ExportTask
|
|
assert db.scalar(select(ExportTask.id)) is None
|
|
|
|
|
|
def test_sse_persists_messages_and_tool_failure(session_factory, monkeypatch):
|
|
calls = []
|
|
class FakeClient:
|
|
def tool_call(self, messages, tools):
|
|
calls.append(messages)
|
|
if len(calls) == 1:
|
|
return {"role": "assistant", "content": None, "tool_calls": [
|
|
{"id": f"call{index}", "type": "function",
|
|
"function": {"name": "list_courses", "arguments": "{}"}} for index in range(9)]}
|
|
return {"role": "assistant", "content": "已列出课程版本。", "tool_calls": []}
|
|
monkeypatch.setattr(chat_api, "client_for_chat", lambda db: FakeClient())
|
|
def override_db():
|
|
with session_factory() as session:
|
|
yield session
|
|
app.dependency_overrides[get_db] = override_db
|
|
try:
|
|
client = TestClient(app)
|
|
created = client.post("/api/conversations", json={"title": "测试对话"}).json()
|
|
with client.stream("POST", f"/api/conversations/{created['id']}/messages", json={"content": "列出课程"}) as response:
|
|
body = "".join(response.iter_text())
|
|
assert body.count("event: tool") == 18 and "event: message" in body and "event: done" in body
|
|
restored = client.get(f"/api/conversations/{created['id']}").json()
|
|
assert [m["role"] for m in restored["messages"]] == ["user", "assistant"]
|
|
assert len(restored["messages"][1]["tool_calls"]) == 9
|
|
assert len({call["id"] for call in restored["messages"][1]["tool_calls"]}) == 9
|
|
assert [call["status"] for call in restored["messages"][1]["tool_calls"]] == ["completed", *["skipped"] * 8]
|
|
assert all(call["result"] for call in restored["messages"][1]["tool_calls"])
|
|
assert restored["runs"][0]["status"] == "completed"
|
|
monkeypatch.setattr(chat_api, "client_for_chat", lambda db: (_ for _ in ()).throw(ModelError("不支持工具调用")))
|
|
with client.stream("POST", f"/api/conversations/{created['id']}/messages", json={"content": "再试"}) as response:
|
|
body = "".join(response.iter_text())
|
|
assert "event: error" in body
|
|
restored = client.get(f"/api/conversations/{created['id']}").json()
|
|
assert len(restored["messages"]) == 3 and restored["runs"][0]["status"] == "failed"
|
|
with session_factory() as db:
|
|
db.add(AgentRun(conversation_id=created["id"], status="running",
|
|
created_at=now() - timedelta(hours=1)))
|
|
db.commit()
|
|
recovered = client.get(f"/api/conversations/{created['id']}").json()
|
|
assert recovered["runs"][0]["status"] == "interrupted"
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_tool_arguments_cover_registered_fields_without_review_prose():
|
|
numbers = {"course_version_id", "batch_id", "project_id", "score_id", "task_id", "rubric_version_id",
|
|
"score", "offset", "limit", "project_offset", "criterion_offset"}
|
|
strings = {"query", "criterion_id", "format", "status", "student_no", "relative_dir"}
|
|
redacted = {"comment", "reason"}
|
|
fields = set().union(*(tool.schema.model_fields for tool in agent_tools.TOOLS.values()))
|
|
assert fields == numbers | strings | redacted
|
|
args = {key: 8 for key in numbers} | {key: "电机控制" for key in strings} | {key: "学生原文" for key in redacted}
|
|
saved = chat_api._safe_arguments(args)
|
|
assert set(saved) == fields
|
|
assert saved["query"] == "电机控制" and saved["criterion_offset"] == 8
|
|
assert saved["comment"] == saved["reason"] == "[文本已省略]"
|
|
assert "学生原文" not in json.dumps(saved, ensure_ascii=False)
|
|
assert chat_api._safe_arguments({"api_key": "secret"}) == {"omitted_fields": 1}
|
|
|
|
|
|
def test_identical_tool_calls_are_skipped_but_distinct_pages_and_queries_run(session_factory, monkeypatch):
|
|
requested = [
|
|
("get_scores", {"batch_id": 3, "project_id": 1}),
|
|
("get_scores", {"batch_id": 3, "project_id": 1, "project_offset": 0, "criterion_offset": 0}),
|
|
("get_scores", {"batch_id": 3, "project_id": 1, "criterion_offset": 6}),
|
|
("search_project_material", {"project_id": 1, "query": "电机控制"}),
|
|
("search_project_material", {"query": "电机控制", "project_id": 1}),
|
|
("search_project_material", {"project_id": 1, "query": "视觉方案"}),
|
|
]
|
|
class FakeClient:
|
|
def tool_call(self, messages, _tools):
|
|
if any(message["role"] == "tool" for message in messages):
|
|
return {"role": "assistant", "content": "查询完成", "tool_calls": []}
|
|
return {"role": "assistant", "content": None, "tool_calls": [
|
|
{"id": f"call-{index}", "type": "function",
|
|
"function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}
|
|
for index, (name, args) in enumerate(requested)]}
|
|
executed = []
|
|
def fake_execute(_db, _conversation, name, arguments, _text):
|
|
executed.append((name, arguments))
|
|
return {"found": True, "citations": []} if name.startswith("search_") else {
|
|
"count": 1, "results": [], "project_offset": arguments.get("project_offset", 0),
|
|
"criterion_offset": arguments.get("criterion_offset", 0)}
|
|
monkeypatch.setattr(chat_api, "client_for_chat", lambda _db: FakeClient())
|
|
monkeypatch.setattr(chat_api, "execute", fake_execute)
|
|
def override_db():
|
|
with session_factory() as session:
|
|
yield session
|
|
app.dependency_overrides[get_db] = override_db
|
|
try:
|
|
client = TestClient(app)
|
|
created = client.post("/api/conversations", json={"title": "查重"}).json()
|
|
with client.stream("POST", f"/api/conversations/{created['id']}/messages", json={"content": "查询"}) as response:
|
|
body = "".join(response.iter_text())
|
|
assert "event: done" in body
|
|
assert [(name, args.get("criterion_offset", 0), args.get("query")) for name, args in executed] == [
|
|
("get_scores", 0, None), ("get_scores", 6, None),
|
|
("search_project_material", 0, "电机控制"), ("search_project_material", 0, "视觉方案")]
|
|
restored = client.get(f"/api/conversations/{created['id']}").json()
|
|
calls = restored["messages"][1]["tool_calls"]
|
|
assert [call["status"] for call in calls] == ["completed", "skipped", "completed", "completed", "skipped", "completed"]
|
|
assert calls[0]["arguments"]["criterion_offset"] == 0
|
|
assert calls[2]["arguments"]["criterion_offset"] == 6
|
|
assert calls[3]["arguments"]["query"] == "电机控制"
|
|
assert calls[4]["result"]["previous_call_id"] == calls[3]["id"]
|
|
assert restored["runs"][0]["tool_calls"] == calls
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_failed_tool_calls_count_toward_limit(monkeypatch):
|
|
class RepeatingClient:
|
|
def tool_call(self, messages, tools):
|
|
return {"role": "assistant", "content": None, "tool_calls": [
|
|
{"id": f"call{len(messages)}", "type": "function",
|
|
"function": {"name": "unknown_tool", "arguments": "{}"}}]}
|
|
|
|
conversation = SimpleNamespace(id=1, course_version_id=None, batch_id=None, project_id=None, updated_at=None)
|
|
run = SimpleNamespace(id=1, status="running", error=None, finished_at=None)
|
|
class FakeSession:
|
|
def __enter__(self):
|
|
return self
|
|
def __exit__(self, *_args):
|
|
pass
|
|
def get(self, model, _id):
|
|
return conversation if model is Conversation else run
|
|
def scalars(self, _query):
|
|
return SimpleNamespace(all=lambda: [])
|
|
def commit(self):
|
|
pass
|
|
def rollback(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(chat_api, "client_for_chat", lambda db: RepeatingClient())
|
|
monkeypatch.setattr(chat_api, "get_settings", lambda: SimpleNamespace(chat_max_tool_calls=9))
|
|
body = "".join(chat_api._run_agent(FakeSession, 1, 1, "循环调用工具"))
|
|
assert body.count("event: tool") == 18
|
|
assert "本次消息工具调用超过上限 9 次" in body
|
|
assert run.status == "failed"
|
|
|
|
|
|
def test_native_tool_calls_unsupported_endpoint():
|
|
transport = httpx.MockTransport(lambda request: httpx.Response(400, json={"error": "unsupported"}))
|
|
client = DeepSeekClient(api_key="test-key", transport=transport)
|
|
with pytest.raises(ModelError, match="tool_calls"):
|
|
client.tool_call([{"role": "user", "content": "hello"}], [])
|