181 lines
9.9 KiB
Python
181 lines
9.9 KiB
Python
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
from sqlalchemy import select, update
|
|
|
|
from .config import get_settings
|
|
from .db import SessionLocal
|
|
from .llm import ConsistencyResult, FactResult, ModelError, ScoreResult, validate_score
|
|
from .models import (CriterionScore, KnowledgeDocument, Project, ScoreEvidence, ScoringBatch,
|
|
SubmissionFile, now)
|
|
from .parsing import parse_document
|
|
from .rag import ingest, retrieve
|
|
from .storage import secure_path, sha256
|
|
from .settings_store import client_for_batch
|
|
|
|
|
|
def parse_submission(db, file: SubmissionFile, *, embedder=None):
|
|
path = secure_path(file.relative_path)
|
|
if sha256(path) != file.sha256:
|
|
file.parse_status, file.parse_error = "changed", "源文件已变化，请重新扫描并确认导入"
|
|
db.commit()
|
|
raise ValueError(file.parse_error)
|
|
if file.parse_status == "ready" and file.parsed:
|
|
return file.parsed
|
|
settings = get_settings()
|
|
try:
|
|
parsed = parse_document(path, file.category, settings.processed_path / f"file-{file.id}")
|
|
project = db.get(Project, file.project_id)
|
|
kwargs = {"embedder": embedder} if embedder else {}
|
|
ingest(db, course_version_id=project.course_version_id, project_id=project.id,
|
|
submission_file_id=file.id, title=path.name, category=file.category,
|
|
source_hash=file.sha256, pages=parsed["pages"], **kwargs)
|
|
file.parsed, file.parse_status, file.parse_error = parsed, "ready", ""
|
|
db.commit()
|
|
return parsed
|
|
except Exception as exc:
|
|
file.parse_status, file.parse_error = "failed", str(exc)[:1000]
|
|
db.commit()
|
|
raise
|
|
|
|
|
|
def _project_facts(db, project, course_snapshot, client, *, retriever):
|
|
prompts = course_snapshot["prompts"]
|
|
categories = ("stage1", "stage2", "stage3", "final_report", "presentation")
|
|
snippets = []
|
|
for category in categories:
|
|
snippets.extend(retriever(db, course_version_id=project.course_version_id, project_id=project.id,
|
|
category=category, query="项目名称 目标 指标 硬件 软件 实验 数据 成本 结论", limit=3, include_public=False))
|
|
if not snippets:
|
|
return {}
|
|
facts = client.json_call(prompts["facts"], {"evidence": snippets}, FactResult).model_dump()
|
|
from .models import Student
|
|
student = db.get(Student, project.student_id)
|
|
warnings = []
|
|
for key, expected in (("student_no", student.student_no), ("name", student.name), ("topic", project.topic)):
|
|
if facts.get(key) and facts[key] != expected:
|
|
warnings.append(f"模型抽取的{key}与教师确认信息不一致")
|
|
consistency = client.json_call(prompts["consistency"], {"facts": facts, "evidence": snippets}, ConsistencyResult)
|
|
known = {snippet["chunk_id"] for snippet in snippets}
|
|
for issue in consistency.issues:
|
|
if not set(issue.chunk_ids) <= known:
|
|
raise ModelError("一致性检查引用了未提供的证据")
|
|
project.facts = {**facts, "identity_warnings": warnings,
|
|
"consistency_issues": [issue.model_dump() for issue in consistency.issues]}
|
|
db.commit()
|
|
return project.facts
|
|
|
|
|
|
def _score_criterion(db, batch, project, criterion, client, *, retriever):
|
|
score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
|
|
CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
|
|
if score and score.status in ("scored", "reviewed"):
|
|
return score
|
|
if not score:
|
|
score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
|
|
db.add(score)
|
|
db.flush()
|
|
category = "presentation" if criterion["criterion_id"] == "final.presentation" else (
|
|
"final_report" if criterion["category"] == "final" else criterion["category"])
|
|
files = db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id,
|
|
SubmissionFile.category == category, SubmissionFile.recognition_status == "recognized")).all()
|
|
ready = [file for file in files if file.parse_status == "ready"]
|
|
if len(files) != 1 or len(ready) != 1:
|
|
score.status, score.error = "unscorable", "材料缺失、重复、待确认或解析失败"
|
|
db.commit()
|
|
return score
|
|
query = criterion["description"] + " " + " ".join(criterion["required_evidence"])
|
|
chunks = retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
|
|
category=category, query=query, limit=10, include_public=False)
|
|
if not chunks:
|
|
score.status, score.error = "unscorable", "无可检索的材料证据"
|
|
db.commit()
|
|
return score
|
|
page_paths = {page["page_number"]: page.get("rendered_image_path") for page in ready[0].parsed["pages"]}
|
|
images = [page_paths[page] for page in dict.fromkeys(chunk["page"] for chunk in chunks[:2]) if page_paths.get(page)]
|
|
public_context = retriever(db, course_version_id=batch.course_version_id, project_id=project.id,
|
|
category=None, query=query, limit=6, public_only=True)
|
|
payload = {"criterion": criterion, "project_facts": project.facts, "course_context": public_context,
|
|
"evidence": chunks,
|
|
"required_output": {"criterion_id": "string", "score": "number", "max_score": "number", "reason": "string",
|
|
"evidence": [{"chunk_id": "integer", "page": "integer", "quote": "exact substring"}],
|
|
"missing_evidence": ["string"], "suggestion": "string", "confidence": "0..1"}}
|
|
result = validate_score(client.json_call(batch.course_snapshot["prompts"]["score"], payload, ScoreResult, images=images), criterion, chunks)
|
|
score.model_score, score.model_comment, score.confidence = result.score, result.reason, result.confidence
|
|
score.missing_evidence, score.suggestion, score.status, score.error = result.missing_evidence, result.suggestion, "scored", ""
|
|
for evidence in db.scalars(select(ScoreEvidence).where(ScoreEvidence.score_id == score.id)).all():
|
|
db.delete(evidence)
|
|
by_id = {chunk["chunk_id"]: chunk for chunk in chunks}
|
|
for evidence in result.evidence:
|
|
chunk = by_id[evidence.chunk_id]
|
|
doc = db.get(KnowledgeDocument, chunk["document_id"])
|
|
db.add(ScoreEvidence(score_id=score.id, document_id=doc.id, submission_file_id=doc.submission_file_id,
|
|
page=evidence.page, quote=evidence.quote, image_path=page_paths.get(evidence.page)))
|
|
db.commit()
|
|
return score
|
|
|
|
|
|
def _run_project(batch_id, project_id, session_factory, client_factory, embedder, retriever):
|
|
failures = 0
|
|
with session_factory() as db:
|
|
batch = db.get(ScoringBatch, batch_id)
|
|
project = db.get(Project, project_id)
|
|
client = client_factory(db, batch)
|
|
files = db.scalars(select(SubmissionFile).where(SubmissionFile.project_id == project.id)).all()
|
|
for file in files:
|
|
if file.recognition_status == "recognized" and file.parse_status != "ready":
|
|
try:
|
|
parse_submission(db, file, embedder=embedder)
|
|
except Exception:
|
|
failures += 1
|
|
try:
|
|
_project_facts(db, project, batch.course_snapshot, client, retriever=retriever)
|
|
except Exception as exc:
|
|
project.facts = {"error": str(exc)[:500]}
|
|
db.commit()
|
|
failures += 1
|
|
for criterion in batch.rubric_snapshot["criteria"]:
|
|
try:
|
|
result = _score_criterion(db, batch, project, criterion, client, retriever=retriever)
|
|
if result.status not in ("scored", "reviewed"):
|
|
failures += 1
|
|
except Exception as exc:
|
|
score = db.scalar(select(CriterionScore).where(CriterionScore.batch_id == batch.id,
|
|
CriterionScore.project_id == project.id, CriterionScore.criterion_id == criterion["criterion_id"]))
|
|
if not score:
|
|
score = CriterionScore(batch_id=batch.id, project_id=project.id, criterion_id=criterion["criterion_id"])
|
|
db.add(score)
|
|
score.status, score.error = "failed", str(exc)[:500]
|
|
db.commit()
|
|
failures += 1
|
|
db.execute(update(ScoringBatch).where(ScoringBatch.id == batch_id).values(progress_done=ScoringBatch.progress_done + 1))
|
|
db.commit()
|
|
return failures
|
|
|
|
|
|
def run_batch(batch_id: int, *, session_factory=SessionLocal, client_factory=client_for_batch, embedder=None, retriever=retrieve):
|
|
with session_factory() as db:
|
|
batch = db.get(ScoringBatch, batch_id)
|
|
if not batch:
|
|
raise ValueError("评分批次不存在")
|
|
project_ids = db.scalars(select(Project.id).where(Project.course_version_id == batch.course_version_id).order_by(Project.id)).all()
|
|
criteria = batch.rubric_snapshot["criteria"]
|
|
batch.status, batch.progress_total, batch.progress_done = "running", len(project_ids) * len(criteria), 0
|
|
db.commit()
|
|
try:
|
|
client_factory(db, batch)
|
|
except Exception as exc:
|
|
batch.status, batch.error, batch.finished_at = "failed", str(exc), now()
|
|
db.commit()
|
|
return
|
|
failures = 0
|
|
with ThreadPoolExecutor(max_workers=max(1, int(batch.model_config_snapshot.get("concurrency", 1)))) as pool:
|
|
futures = [pool.submit(_run_project, batch_id, project_id, session_factory, client_factory, embedder, retriever) for project_id in project_ids]
|
|
for future in as_completed(futures):
|
|
try:
|
|
failures += future.result()
|
|
except Exception as exc:
|
|
failures += 1
|
|
batch.error = (batch.error + f" 项目任务失败: {type(exc).__name__}")[:1000]
|
|
batch.status, batch.finished_at = ("partial" if failures else "completed"), now()
|
|
db.commit()
|