feat: add persistent chat agent and indexed batch workflows
This commit is contained in:
@@ -0,0 +1,247 @@
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pymupdf
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from wisegrade import chat_api, services
|
||||
from wisegrade.actions import confirm, prepare
|
||||
from wisegrade.agent_tools import execute
|
||||
from wisegrade.db import get_db
|
||||
from wisegrade.indexing import run_index_task
|
||||
from wisegrade.llm import DeepSeekClient, ModelError
|
||||
from wisegrade.main import app
|
||||
from wisegrade.models import (AgentRun, ChatMessage, Conversation, CourseVersion, CriterionScore, IndexTask, PendingAction,
|
||||
Project, RubricVersion, ScoringBatch, Student, TeacherRevision, now)
|
||||
from wisegrade.rag import ingest
|
||||
from wisegrade.scoring import run_batch
|
||||
from wisegrade.storage import scan
|
||||
|
||||
|
||||
def add_project(db, number, *, version_id=1):
|
||||
student = Student(student_no=number, name=f"学生{number[-1]}")
|
||||
db.add(student)
|
||||
db.flush()
|
||||
project = Project(course_version_id=version_id, student_id=student.id, topic="控制系统", source_dir=f"test/{number}")
|
||||
db.add(project)
|
||||
db.commit()
|
||||
return project
|
||||
|
||||
|
||||
def test_tool_scope_snapshots_and_project_citations(db):
|
||||
first = add_project(db, "123450001")
|
||||
second = add_project(db, "123450002")
|
||||
vectors = lambda texts: [[0.1] * 1024 for _ in texts]
|
||||
for project, text in ((first, "一号项目电机控制方案"), (second, "二号项目视觉方案")):
|
||||
ingest(db, course_version_id=1, project_id=project.id, submission_file_id=None,
|
||||
title=f"{project.id}.pdf", category="final_report", source_hash=str(project.id),
|
||||
pages=[{"page_number": 2, "section": "方案", "text": text, "tables": []}], embedder=vectors)
|
||||
conversation = Conversation(title="范围", course_version_id=1, project_id=first.id)
|
||||
db.add(conversation)
|
||||
db.commit()
|
||||
from wisegrade import agent_tools
|
||||
original = agent_tools.retrieve
|
||||
agent_tools.retrieve = lambda *args, **kwargs: original(*args, **kwargs, embedder=vectors)
|
||||
try:
|
||||
found = execute(db, conversation, "search_project_material", {"project_id": first.id, "query": "电机控制"}, "")
|
||||
assert found["found"] and len(found["citations"]) == 1
|
||||
assert found["citations"][0]["quote"] == "一号项目电机控制方案"
|
||||
assert found["citations"][0]["page"] == 2
|
||||
assert "/pages/2" in found["citations"][0]["url"]
|
||||
with pytest.raises(HTTPException):
|
||||
execute(db, conversation, "search_project_material", {"project_id": second.id, "query": "视觉"}, "")
|
||||
with pytest.raises(Exception):
|
||||
execute(db, conversation, "get_project", {"project_id": first.id, "arbitrary_sql": "DROP TABLE"}, "")
|
||||
finally:
|
||||
agent_tools.retrieve = original
|
||||
batch = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
||||
snapshot = db.get(ScoringBatch, batch["id"])
|
||||
before = snapshot.rubric_snapshot["criteria"][0]["name"]
|
||||
rubric = db.get(RubricVersion, 1)
|
||||
rubric.criteria = [{**rubric.criteria[0], "name": "后来修改"}, *rubric.criteria[1:]]
|
||||
db.commit()
|
||||
rules = execute(db, conversation, "get_course_rules", {"batch_id": snapshot.id}, "")
|
||||
assert rules["rubric"]["criteria"][0]["name"] == before
|
||||
assert rules["source"] == "batch_snapshot"
|
||||
|
||||
|
||||
def test_import_confirmation_hash_change_index_and_idempotency(environment, db, monkeypatch, session_factory):
|
||||
folder = environment / "inbox" / "batch" / "123450003_张三_机器人"
|
||||
folder.mkdir(parents=True)
|
||||
file = folder / "阶段报告1.pdf"
|
||||
def write_pdf(text):
|
||||
pdf = pymupdf.open()
|
||||
page = pdf.new_page()
|
||||
page.insert_text((72, 72), (text + " with enough searchable report detail for indexing. ") * 4)
|
||||
pdf.save(file)
|
||||
write_pdf("First report content")
|
||||
scanned = scan("batch")["projects"][0]
|
||||
payload = {"course_version_id": 1, "projects": [{"relative_dir": scanned["relative_dir"],
|
||||
"student_no": scanned["student_no"], "name": scanned["name"], "topic": scanned["topic"],
|
||||
"files": [{"relative_path": item["relative_path"], "category": item["category"],
|
||||
"sha256": item["sha256"], "status": "recognized"} for item in scanned["files"]]}]}
|
||||
conversation = Conversation(title="导入", course_version_id=1)
|
||||
db.add(conversation)
|
||||
db.commit()
|
||||
action = prepare(db, conversation, "import", payload)
|
||||
assert db.scalar(select(Project.id)) is None
|
||||
write_pdf("Changed report content")
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
confirm(db, conversation, action.id)
|
||||
assert exc.value.status_code == 409 and db.scalar(select(Project.id)) is None
|
||||
scanned = scan("batch")["projects"][0]
|
||||
payload["projects"][0]["files"][0]["sha256"] = scanned["files"][0]["sha256"]
|
||||
monkeypatch.setattr(services, "enqueue_index_task", lambda db, task_id: db.get(IndexTask, task_id))
|
||||
action = prepare(db, conversation, "import", payload)
|
||||
result = confirm(db, conversation, action.id)
|
||||
assert confirm(db, conversation, action.id) == result
|
||||
assert len(result["project_ids"]) == len(result["index_task_ids"]) == 1
|
||||
task = db.get(IndexTask, result["index_task_ids"][0])
|
||||
assert task.status == "queued"
|
||||
vectors = lambda texts: [[0.1] * 1024 for _ in texts]
|
||||
run_index_task(task.id, session_factory=session_factory, embedder=vectors)
|
||||
db.refresh(task)
|
||||
assert task.status == "completed" and task.result["pages"] == 1, task.error
|
||||
failed = IndexTask(course_version_id=1, submission_file_id=task.submission_file_id,
|
||||
kind="project", status="failed", error="temporary")
|
||||
db.add(failed)
|
||||
db.commit()
|
||||
first_retry = services.retry_index_task(db, failed.id)
|
||||
assert services.retry_index_task(db, failed.id).id == first_retry.id
|
||||
|
||||
|
||||
def test_duplicate_scan_requires_teacher_file_choice(environment, db):
|
||||
folder = environment / "inbox" / "dup" / "123450009_王五_控制"
|
||||
folder.mkdir(parents=True)
|
||||
for name in ("阶段报告1-A.pdf", "阶段报告1-B.pdf"):
|
||||
pdf = pymupdf.open()
|
||||
page = pdf.new_page()
|
||||
page.insert_text((72, 72), "Long report text for reliable detection. " * 4)
|
||||
pdf.save(folder / name)
|
||||
scanned = scan("dup")["projects"][0]
|
||||
assert all(file["status"] == "duplicate" for file in scanned["files"])
|
||||
chosen = scanned["files"][0]
|
||||
payload = {"course_version_id": 1, "projects": [{"relative_dir": scanned["relative_dir"],
|
||||
"student_no": scanned["student_no"], "name": scanned["name"], "topic": scanned["topic"],
|
||||
"files": [{"relative_path": chosen["relative_path"], "category": "stage1",
|
||||
"sha256": chosen["sha256"], "status": "recognized"}]}]}
|
||||
conversation = Conversation(title="重复文件", course_version_id=1)
|
||||
db.add(conversation)
|
||||
db.commit()
|
||||
generated = execute(db, conversation, "prepare_import", {"course_version_id": 1, "relative_dir": "dup"}, "准备导入")
|
||||
generated_action = db.get(PendingAction, generated["pending_action"]["id"])
|
||||
assert generated_action.preview["requires_edit"] is True
|
||||
assert all(file["status"] == "ignored" for file in generated_action.payload["projects"][0]["files"])
|
||||
proposed = prepare(db, conversation, "import", payload)
|
||||
assert proposed.preview["requires_edit"] is True
|
||||
with pytest.raises(HTTPException):
|
||||
confirm(db, conversation, proposed.id)
|
||||
assert db.scalar(select(Project.id)) is None
|
||||
corrected = prepare(db, conversation, "import", payload, teacher_edited=True)
|
||||
assert corrected.id != proposed.id and corrected.preview["requires_edit"] is False
|
||||
|
||||
|
||||
def test_fixed_batch_membership_duplicate_queue_and_review(db, monkeypatch, session_factory):
|
||||
first = add_project(db, "123450004")
|
||||
created = services.create_batch(SimpleNamespace(course_version_id=1, rubric_version_id=1), db)
|
||||
batch = db.get(ScoringBatch, created["id"])
|
||||
assert batch.status == "created" and batch.project_ids == [first.id]
|
||||
add_project(db, "123450005")
|
||||
jobs = []
|
||||
monkeypatch.setattr(services, "Queue", lambda *args, **kwargs: SimpleNamespace(enqueue=lambda *a, **k: jobs.append(k)))
|
||||
services.start_batch(batch.id, db)
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
services.start_batch(batch.id, db)
|
||||
assert exc.value.status_code == 409 and len(jobs) == 1
|
||||
run_batch(batch.id, session_factory=session_factory, client_factory=lambda *_: SimpleNamespace())
|
||||
# The fixed list excludes the project imported after creation.
|
||||
db.refresh(batch)
|
||||
assert batch.progress_total == len(batch.rubric_snapshot["criteria"])
|
||||
score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id))
|
||||
score.model_score, score.status, score.error = None, "failed", "证据不足"
|
||||
db.commit()
|
||||
conversation = Conversation(title="复核", batch_id=batch.id, project_id=first.id)
|
||||
db.add(conversation)
|
||||
db.commit()
|
||||
with pytest.raises(HTTPException):
|
||||
execute(db, conversation, "prepare_review", {"score_id": score.id, "score": 3,
|
||||
"comment": "人工核定", "reason": "核查纸质材料"}, "请复核")
|
||||
prepared = execute(db, conversation, "prepare_review", {"score_id": score.id, "score": 3,
|
||||
"comment": "人工核定", "reason": "核查纸质材料"}, "给 3 分")
|
||||
action_id = prepared["pending_action"]["id"]
|
||||
assert score.teacher_score is None
|
||||
confirm(db, conversation, action_id)
|
||||
db.refresh(score)
|
||||
assert score.teacher_score == 3 and score.model_score is None and score.status == "reviewed"
|
||||
assert db.scalar(select(TeacherRevision).where(TeacherRevision.score_id == score.id)).new_score == 3
|
||||
summary = execute(db, conversation, "summarize_batch", {"batch_id": batch.id}, "")
|
||||
assert summary["project_count"] == 1 and summary["complete_count"] == 0
|
||||
assert summary["incomplete_count"] == 1 and not summary["overall_distribution"]
|
||||
|
||||
|
||||
def test_batch_and_export_are_previews_until_confirmed(db):
|
||||
project = add_project(db, "123450006")
|
||||
conversation = Conversation(title="确认", course_version_id=1)
|
||||
db.add(conversation)
|
||||
db.commit()
|
||||
action = prepare(db, conversation, "create_batch", {"course_version_id": 1, "rubric_version_id": 1})
|
||||
assert action.preview["project_ids"] == [project.id]
|
||||
assert db.scalar(select(ScoringBatch.id)) is None
|
||||
first = confirm(db, conversation, action.id)
|
||||
assert confirm(db, conversation, action.id) == first
|
||||
assert len(db.scalars(select(ScoringBatch.id)).all()) == 1
|
||||
export = prepare(db, conversation, "export", {"batch_id": first["id"], "format": "excel"})
|
||||
assert export.preview["incomplete_projects"] == [project.id]
|
||||
from wisegrade.models import ExportTask
|
||||
assert db.scalar(select(ExportTask.id)) is None
|
||||
|
||||
|
||||
def test_sse_persists_messages_and_tool_failure(session_factory, monkeypatch):
|
||||
calls = []
|
||||
class FakeClient:
|
||||
def tool_call(self, messages, tools):
|
||||
calls.append(messages)
|
||||
if len(calls) == 1:
|
||||
return {"role": "assistant", "content": None, "tool_calls": [{"id": "call1", "type": "function",
|
||||
"function": {"name": "list_courses", "arguments": "{}"}}]}
|
||||
return {"role": "assistant", "content": "已列出课程版本。", "tool_calls": []}
|
||||
monkeypatch.setattr(chat_api, "client_for_chat", lambda db: FakeClient())
|
||||
def override_db():
|
||||
with session_factory() as session:
|
||||
yield session
|
||||
app.dependency_overrides[get_db] = override_db
|
||||
try:
|
||||
client = TestClient(app)
|
||||
created = client.post("/api/conversations", json={"title": "测试对话"}).json()
|
||||
with client.stream("POST", f"/api/conversations/{created['id']}/messages", json={"content": "列出课程"}) as response:
|
||||
body = "".join(response.iter_text())
|
||||
assert "event: tool" in body and "event: message" in body
|
||||
restored = client.get(f"/api/conversations/{created['id']}").json()
|
||||
assert [m["role"] for m in restored["messages"]] == ["user", "assistant"]
|
||||
assert restored["runs"][0]["status"] == "completed"
|
||||
monkeypatch.setattr(chat_api, "client_for_chat", lambda db: (_ for _ in ()).throw(ModelError("不支持工具调用")))
|
||||
with client.stream("POST", f"/api/conversations/{created['id']}/messages", json={"content": "再试"}) as response:
|
||||
body = "".join(response.iter_text())
|
||||
assert "event: error" in body
|
||||
restored = client.get(f"/api/conversations/{created['id']}").json()
|
||||
assert len(restored["messages"]) == 3 and restored["runs"][0]["status"] == "failed"
|
||||
with session_factory() as db:
|
||||
db.add(AgentRun(conversation_id=created["id"], status="running",
|
||||
created_at=now() - timedelta(hours=1)))
|
||||
db.commit()
|
||||
recovered = client.get(f"/api/conversations/{created['id']}").json()
|
||||
assert recovered["runs"][0]["status"] == "interrupted"
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_native_tool_calls_unsupported_endpoint():
|
||||
transport = httpx.MockTransport(lambda request: httpx.Response(400, json={"error": "unsupported"}))
|
||||
client = DeepSeekClient(api_key="test-key", transport=transport)
|
||||
with pytest.raises(ModelError, match="tool_calls"):
|
||||
client.tool_call([{"role": "user", "content": "hello"}], [])
|
||||
@@ -10,7 +10,7 @@ from wisegrade.storage import classify, extract_identity, parse_student_folder,
|
||||
|
||||
def test_postgresql_migrations_and_vector_extension(db):
|
||||
assert db.bind.dialect.name == "postgresql"
|
||||
assert db.scalar(text("SELECT version_num FROM alembic_version")) == "0002"
|
||||
assert db.scalar(text("SELECT version_num FROM alembic_version")) == "0003"
|
||||
assert db.scalar(text("SELECT extname FROM pg_extension WHERE extname = 'vector'")) == "vector"
|
||||
|
||||
|
||||
|
||||
@@ -48,7 +48,15 @@ def test_docs_examples_end_to_end(environment, session_factory):
|
||||
batch_id = response.json()["id"]
|
||||
run_batch(batch_id, session_factory=session_factory)
|
||||
progress = client.get(f"/api/scoring-batches/{batch_id}/progress").json()
|
||||
assert progress["status"] == "completed", progress
|
||||
if progress["status"] == "partial":
|
||||
# A model quote may fail exact evidence matching on one attempt;
|
||||
# retry only the unfinished items while retaining successful work.
|
||||
run_batch(batch_id, session_factory=session_factory)
|
||||
progress = client.get(f"/api/scoring-batches/{batch_id}/progress").json()
|
||||
if progress["status"] != "completed":
|
||||
rows = client.get(f"/api/projects/{project_id}/scores?batch_id={batch_id}").json()["rows"]
|
||||
issues = [(row["criterion"]["criterion_id"], row["status"], row["error"]) for row in rows if row["status"] != "scored"]
|
||||
pytest.fail(f"评分未完成: {progress}; 异常评分项: {issues}")
|
||||
assert progress["progress_done"] == progress["progress_total"] == 22
|
||||
result = client.get(f"/api/projects/{project_id}/scores?batch_id={batch_id}").json()
|
||||
assert result["totals"]["overall"] is not None
|
||||
|
||||
Reference in New Issue
Block a user