feat: add persistent chat agent and indexed batch workflows
This commit is contained in:
+58
-167
@@ -1,29 +1,23 @@
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from redis import Redis
|
||||
from rq import Queue, Retry
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from . import services
|
||||
from .config import get_settings
|
||||
from .db import get_db
|
||||
from .exports import export_excel, export_project_pdf, project_result
|
||||
from .models import (AppSetting, AuditLog, Course, CourseVersion, CriterionScore, ExportTask,
|
||||
KnowledgeChunk, KnowledgeDocument, Project, RubricVersion, ScoringBatch,
|
||||
Student, Submission, SubmissionFile, TeacherRevision)
|
||||
from .exports import batch_projects, export_excel, export_project_pdf, project_result
|
||||
from .models import (AuditLog, Course, CourseVersion, CriterionScore, ExportTask,
|
||||
IndexTask, KnowledgeChunk, KnowledgeDocument, Project, RubricVersion, ScoringBatch,
|
||||
Student, SubmissionFile, TeacherRevision)
|
||||
from .materialized import sync_course_entities, sync_rubric_entities
|
||||
from .parsing import parse_document
|
||||
from .rag import ingest
|
||||
from .rules import RubricData
|
||||
from .scoring import run_batch
|
||||
from .settings_store import key_configured, model_config, save_model_config
|
||||
from .storage import CATEGORIES, SUPPORTED, directories, scan, secure_path, sha256
|
||||
from .storage import CATEGORIES, directories, scan
|
||||
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
@@ -39,13 +33,6 @@ def audit(db, action, entity_type, entity_id, details=None):
|
||||
db.add(AuditLog(action=action, entity_type=entity_type, entity_id=entity_id, details=details or {}))
|
||||
|
||||
|
||||
def remove_file_index(db, file):
|
||||
for document in db.scalars(select(KnowledgeDocument).where(KnowledgeDocument.submission_file_id == file.id)).all():
|
||||
db.execute(delete(KnowledgeChunk).where(KnowledgeChunk.document_id == document.id))
|
||||
db.delete(document)
|
||||
file.parsed, file.parse_status = {}, "pending"
|
||||
|
||||
|
||||
def version_dict(item):
|
||||
return {key: getattr(item, key) for key in ("id", "course_id", "version", "title", "objectives", "indicators", "directions", "prompts", "templates", "published", "created_at")}
|
||||
|
||||
@@ -152,6 +139,11 @@ def list_versions(course_id: int, db: Session = Depends(get_db)):
|
||||
return [version_dict(v) for v in db.scalars(select(CourseVersion).where(CourseVersion.course_id == course_id).order_by(CourseVersion.version.desc()))]
|
||||
|
||||
|
||||
@router.get("/course-versions/{version_id}")
|
||||
def get_version(version_id: int, db: Session = Depends(get_db)):
|
||||
return version_dict(found(db, CourseVersion, version_id))
|
||||
|
||||
|
||||
@router.post("/courses/{course_id}/versions")
|
||||
def copy_version(course_id: int, data: VersionInput, db: Session = Depends(get_db)):
|
||||
found(db, Course, course_id)
|
||||
@@ -298,73 +290,7 @@ def scan_import(data: ImportScan):
|
||||
|
||||
@router.post("/import/confirm")
|
||||
def confirm_import(data: ImportConfirm, db: Session = Depends(get_db)):
|
||||
version = found(db, CourseVersion, data.course_version_id)
|
||||
if not version.published:
|
||||
raise HTTPException(400, "请先发布课程版本")
|
||||
imported = []
|
||||
for item in data.projects:
|
||||
folder = secure_path(item.relative_dir)
|
||||
if not folder.is_dir():
|
||||
raise ValueError("学生目录不存在")
|
||||
student = db.scalar(select(Student).where(Student.student_no == item.student_no))
|
||||
if not student:
|
||||
student = Student(student_no=item.student_no, name=item.name)
|
||||
db.add(student)
|
||||
db.flush()
|
||||
else:
|
||||
student.name = item.name
|
||||
project = db.scalar(select(Project).where(Project.course_version_id == version.id, Project.source_dir == item.relative_dir))
|
||||
if not project:
|
||||
project = Project(course_version_id=version.id, student_id=student.id, source_dir=item.relative_dir,
|
||||
topic=item.topic, direction=item.direction)
|
||||
db.add(project)
|
||||
db.flush()
|
||||
else:
|
||||
project.student_id, project.topic, project.direction = student.id, item.topic, item.direction
|
||||
selected = set()
|
||||
selected_paths = set()
|
||||
for file in item.files:
|
||||
path = secure_path(file.relative_path)
|
||||
if path.parent != folder or not path.is_file() or path.suffix.lower() not in SUPPORTED:
|
||||
raise ValueError("材料必须是学生目录内的PDF、DOCX或PPTX文件")
|
||||
if sha256(path) != file.sha256:
|
||||
raise ValueError("文件已变化，请重新扫描")
|
||||
if file.status == "ignored":
|
||||
continue
|
||||
if file.category not in CATEGORIES or file.category in selected or file.relative_path in selected_paths:
|
||||
raise ValueError("请为每类材料只确认一个文件")
|
||||
selected.add(file.category)
|
||||
selected_paths.add(file.relative_path)
|
||||
stat = path.stat()
|
||||
record = db.scalar(select(SubmissionFile).where(SubmissionFile.project_id == project.id, SubmissionFile.relative_path == file.relative_path))
|
||||
if not record:
|
||||
record = SubmissionFile(project_id=project.id, relative_path=file.relative_path, category=file.category,
|
||||
mime_type=mimetypes.guess_type(path.name)[0] or "application/octet-stream",
|
||||
size=stat.st_size, mtime=stat.st_mtime, sha256=file.sha256, recognition_status="recognized")
|
||||
db.add(record)
|
||||
else:
|
||||
if record.sha256 != file.sha256 or record.category != file.category:
|
||||
remove_file_index(db, record)
|
||||
record.category, record.sha256, record.size, record.mtime, record.recognition_status = file.category, file.sha256, stat.st_size, stat.st_mtime, "recognized"
|
||||
for old in db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id)).all():
|
||||
if old.relative_path not in selected_paths:
|
||||
old.recognition_status = "ignored"
|
||||
if old.parse_status == "ready":
|
||||
remove_file_index(db, old)
|
||||
for category in CATEGORIES:
|
||||
submission = db.scalar(select(Submission).where(Submission.project_id == project.id, Submission.category == category))
|
||||
if not submission:
|
||||
submission = Submission(project_id=project.id, category=category, status="missing")
|
||||
db.add(submission)
|
||||
db.flush()
|
||||
submission.status = "submitted" if category in selected else "missing"
|
||||
for file in db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id,
|
||||
SubmissionFile.category == category, SubmissionFile.recognition_status == "recognized")).all():
|
||||
file.submission_id = submission.id
|
||||
audit(db, "import", "project", project.id, {"files": len(selected)})
|
||||
imported.append(project.id)
|
||||
db.commit()
|
||||
return {"project_ids": imported}
|
||||
return services.confirm_import(data, db)
|
||||
|
||||
|
||||
@router.get("/projects")
|
||||
@@ -401,99 +327,73 @@ def knowledge(course_version_id: int, project_id: int | None = None, db: Session
|
||||
"chunks": db.scalar(select(func.count(KnowledgeChunk.id)).where(KnowledgeChunk.document_id == doc.id))} for doc in db.scalars(query)]
|
||||
|
||||
|
||||
@router.get("/knowledge/documents/{document_id}/pages/{page}")
|
||||
def knowledge_page(document_id: int, page: int, db: Session = Depends(get_db)):
|
||||
doc = found(db, KnowledgeDocument, document_id)
|
||||
chunks = db.scalars(select(KnowledgeChunk).where(KnowledgeChunk.document_id == document_id,
|
||||
KnowledgeChunk.page == page).order_by(KnowledgeChunk.id)).all()
|
||||
if not chunks:
|
||||
raise HTTPException(404, "资料页面不存在")
|
||||
return {"document_id": doc.id, "title": doc.title, "course_version_id": doc.course_version_id,
|
||||
"project_id": doc.project_id, "page": page, "text": "\n".join(chunk.text for chunk in chunks),
|
||||
"image_url": f"/api/assets/{doc.submission_file_id}/{page}" if doc.submission_file_id else None}
|
||||
|
||||
|
||||
@router.post("/knowledge/ingest-course/{course_version_id}")
|
||||
def ingest_course(course_version_id: int, db: Session = Depends(get_db)):
|
||||
version = found(db, CourseVersion, course_version_id)
|
||||
root = Path("/app/course-docs")
|
||||
processed = get_settings().processed_path / f"course-{course_version_id}"
|
||||
completed, errors = [], {}
|
||||
for filename in version.templates:
|
||||
if Path(filename).name != filename:
|
||||
errors[filename] = "非法课程资料路径"
|
||||
continue
|
||||
path = root / filename
|
||||
if not path.is_file():
|
||||
errors[filename] = "课程资料文件不存在"
|
||||
continue
|
||||
try:
|
||||
category = "stage1" if "阶段报告1" in filename else "stage2" if "阶段报告2" in filename else "stage3" if "阶段报告3" in filename else "course"
|
||||
parsed = parse_document(path, category, processed / hashlib.sha256(filename.encode()).hexdigest()[:16])
|
||||
ingest(db, course_version_id=version.id, project_id=None, submission_file_id=None,
|
||||
title=filename, category=category, source_hash=sha256(path), pages=parsed["pages"])
|
||||
completed.append(filename)
|
||||
except Exception as exc:
|
||||
errors[filename] = str(exc)[:500]
|
||||
rubric = db.scalar(select(RubricVersion).where(RubricVersion.course_version_id == version.id,
|
||||
RubricVersion.published.is_(True)).order_by(RubricVersion.version.desc()))
|
||||
virtual = [
|
||||
("课程目标、毕业要求与选题方向", {"objectives": version.objectives, "indicators": version.indicators,
|
||||
"directions": version.directions}),
|
||||
("已发布评分规则", {"criteria": rubric.criteria, "weights": rubric.weights} if rubric else {}),
|
||||
]
|
||||
for title, payload in virtual:
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
content = json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
ingest(db, course_version_id=version.id, project_id=None, submission_file_id=None, title=title,
|
||||
category="course", source_hash=hashlib.sha256(content.encode()).hexdigest(),
|
||||
pages=[{"page_number": 1, "section": title, "text": content, "tables": []}])
|
||||
completed.append(title)
|
||||
except Exception as exc:
|
||||
errors[title] = str(exc)[:500]
|
||||
return {"completed": completed, "errors": errors}
|
||||
return services.ingest_course(course_version_id, db)
|
||||
|
||||
|
||||
@router.get("/index-tasks")
|
||||
def list_index_tasks(project_id: int | None = None, db: Session = Depends(get_db)):
|
||||
query = select(IndexTask).order_by(IndexTask.id.desc()).limit(100)
|
||||
if project_id is not None:
|
||||
file_ids = select(SubmissionFile.id).where(SubmissionFile.project_id == project_id)
|
||||
query = query.where(IndexTask.submission_file_id.in_(file_ids))
|
||||
return [{key: getattr(task, key) for key in ("id", "course_version_id", "submission_file_id", "kind", "status", "result", "error", "created_at", "updated_at")}
|
||||
for task in db.scalars(query)]
|
||||
|
||||
|
||||
@router.get("/index-tasks/{task_id}")
|
||||
def get_index_task(task_id: int, db: Session = Depends(get_db)):
|
||||
task = found(db, IndexTask, task_id)
|
||||
return {key: getattr(task, key) for key in ("id", "course_version_id", "submission_file_id", "kind", "status", "result", "error", "created_at", "updated_at")}
|
||||
|
||||
|
||||
@router.post("/index-tasks/{task_id}/retry")
|
||||
def retry_index_task(task_id: int, db: Session = Depends(get_db)):
|
||||
retry = services.retry_index_task(db, task_id)
|
||||
return get_index_task(retry.id, db)
|
||||
|
||||
|
||||
@router.post("/scoring-batches")
|
||||
def create_batch(data: BatchInput, db: Session = Depends(get_db)):
|
||||
version = found(db, CourseVersion, data.course_version_id)
|
||||
rubric = found(db, RubricVersion, data.rubric_version_id)
|
||||
if not version.published or not rubric.published or rubric.course_version_id != version.id:
|
||||
raise HTTPException(400, "请选择同一课程版本下已发布的规则")
|
||||
settings = get_settings()
|
||||
batch = ScoringBatch(course_version_id=version.id, rubric_version_id=rubric.id,
|
||||
course_snapshot={key: getattr(version, key) for key in ("version", "title", "objectives", "indicators", "directions", "prompts", "templates")},
|
||||
rubric_snapshot={"version": rubric.version, "criteria": rubric.criteria, "weights": rubric.weights},
|
||||
model_config_snapshot=model_config(db), embedding_model_name=settings.embedding_model_name,
|
||||
embedding_model_version=settings.embedding_model_version)
|
||||
db.add(batch)
|
||||
db.flush()
|
||||
audit(db, "create", "scoring_batch", batch.id)
|
||||
db.commit()
|
||||
return {"id": batch.id, "status": batch.status}
|
||||
return services.create_batch(data, db)
|
||||
|
||||
|
||||
@router.get("/scoring-batches")
|
||||
def list_batches(db: Session = Depends(get_db)):
|
||||
return [{key: getattr(batch, key) for key in ("id", "course_version_id", "rubric_version_id", "status", "progress_done", "progress_total", "error", "created_at", "finished_at")}
|
||||
return [{**{key: getattr(batch, key) for key in ("id", "course_version_id", "rubric_version_id", "status", "progress_done", "progress_total", "error", "created_at", "finished_at")},
|
||||
"project_ids": batch.project_ids, "legacy_membership": batch.project_ids is None}
|
||||
for batch in db.scalars(select(ScoringBatch).order_by(ScoringBatch.id.desc()))]
|
||||
|
||||
|
||||
@router.get("/scoring-batches/{batch_id}/progress")
|
||||
def batch_progress(batch_id: int, db: Session = Depends(get_db)):
|
||||
batch = found(db, ScoringBatch, batch_id)
|
||||
return {key: getattr(batch, key) for key in ("id", "status", "progress_done", "progress_total", "error", "created_at", "finished_at")}
|
||||
return {**{key: getattr(batch, key) for key in ("id", "status", "progress_done", "progress_total", "error", "created_at", "finished_at")},
|
||||
"project_ids": batch.project_ids, "legacy_membership": batch.project_ids is None}
|
||||
|
||||
|
||||
@router.post("/scoring-batches/{batch_id}/start")
|
||||
def start_batch(batch_id: int, db: Session = Depends(get_db)):
|
||||
batch = found(db, ScoringBatch, batch_id)
|
||||
if batch.status == "running":
|
||||
raise HTTPException(409, "批次正在评分")
|
||||
try:
|
||||
Queue("wisegrade", connection=Redis.from_url(get_settings().redis_url)).enqueue(run_batch, batch_id,
|
||||
job_timeout="12h", retry=Retry(max=1, interval=[60]))
|
||||
except Exception as exc:
|
||||
raise HTTPException(503, f"无法提交后台任务: {type(exc).__name__}") from exc
|
||||
batch.status, batch.error, batch.finished_at = "queued", "", None
|
||||
db.commit()
|
||||
return {"id": batch.id, "status": batch.status}
|
||||
return services.start_batch(batch_id, db)
|
||||
|
||||
|
||||
@router.get("/scoring-batches/{batch_id}/scores")
|
||||
def batch_scores(batch_id: int, db: Session = Depends(get_db)):
|
||||
batch = found(db, ScoringBatch, batch_id)
|
||||
projects = db.scalars(select(Project).where(Project.course_version_id == batch.course_version_id).order_by(Project.id)).all()
|
||||
projects = batch_projects(db, batch)
|
||||
return [project_result(db, batch, project) for project in projects]
|
||||
|
||||
|
||||
@@ -502,23 +402,14 @@ def project_scores(project_id: int, batch_id: int, db: Session = Depends(get_db)
|
||||
batch, project = found(db, ScoringBatch, batch_id), found(db, Project, project_id)
|
||||
if project.course_version_id != batch.course_version_id:
|
||||
raise HTTPException(400, "项目不属于评分批次课程版本")
|
||||
if batch.project_ids is not None and project.id not in batch.project_ids:
|
||||
raise HTTPException(400, "项目不属于该批次固定项目清单")
|
||||
return project_result(db, batch, project)
|
||||
|
||||
|
||||
@router.put("/criterion-scores/{score_id}/review")
|
||||
def review_score(score_id: int, data: ReviewInput, db: Session = Depends(get_db)):
|
||||
score = found(db, CriterionScore, score_id)
|
||||
batch = found(db, ScoringBatch, score.batch_id)
|
||||
criterion = next((item for item in batch.rubric_snapshot["criteria"] if item["criterion_id"] == score.criterion_id), None)
|
||||
if not criterion or data.score > criterion["max_score"]:
|
||||
raise HTTPException(400, "教师分数超出规则满分")
|
||||
db.add(TeacherRevision(score_id=score.id, old_score=score.teacher_score, new_score=data.score,
|
||||
old_comment=score.teacher_comment, new_comment=data.comment, reason=data.reason))
|
||||
score.teacher_score, score.teacher_comment, score.status = data.score, data.comment, "reviewed"
|
||||
audit(db, "review", "criterion_score", score.id, {"reason": data.reason})
|
||||
db.commit()
|
||||
return {"id": score.id, "model_score": score.model_score, "teacher_score": score.teacher_score,
|
||||
"final_score": score.final_score, "teacher_comment": score.teacher_comment}
|
||||
return services.review_score(score_id, data, db)
|
||||
|
||||
|
||||
@router.get("/criterion-scores/{score_id}/revisions")
|
||||
|
||||
Reference in New Issue
Block a user