feat: separate contents, abstract and body evidence

backend/wisegrade/api.py

@@ -10,13 +10,16 @@ from sqlalchemy.orm import Session
 from . import services
 from .config import get_settings
 from .db import get_db
-from .exports import batch_projects, export_excel, export_project_pdf, project_result
+from .evidence import CONTENT_ALLOWED
+from .exports import (BATCH_STATUS_LABELS, batch_projects, export_excel,
+                      export_project_pdf, page_type_label, project_result)
 from .knowledge_pages import page_image_path, page_links
 from .models import (AuditLog, Course, CourseVersion, CriterionScore, ExportTask,
                      IndexTask, KnowledgeChunk, KnowledgeDocument, KnowledgePage, Project, RubricVersion, ScoringBatch,
                      Student, SubmissionFile, TeacherRevision)
 from .materialized import sync_course_entities, sync_rubric_entities
 from .rules import RubricData
+from .scoring import ATTENTION_STATUSES
 from .settings_store import key_configured, model_config, save_model_config
 from .storage import CATEGORIES, directories, scan
 
@@ -108,9 +111,20 @@ class ModelSettingsInput(BaseModel):
 
 @router.get("/dashboard")
 def dashboard(db: Session = Depends(get_db)):
+    """Overview counts. Items awaiting teacher review are counted as attention needed.
+
+    A ``needs_review`` item is not a failure: retrieval ran and found only navigation
+    pages where the rule requires body content, so a teacher decides it.
+    """
+    counts = dict(db.execute(select(CriterionScore.status, func.count(CriterionScore.id))
+                             .group_by(CriterionScore.status)).all())
     return {"courses": db.scalar(select(func.count(Course.id))), "projects": db.scalar(select(func.count(Project.id))),
             "batches": db.scalar(select(func.count(ScoringBatch.id))),
-            "needs_attention": db.scalar(select(func.count(CriterionScore.id)).where(CriterionScore.status.in_(["failed", "unscorable"]))) }
+            "needs_attention": sum(counts.get(status, 0) for status in ATTENTION_STATUSES),
+            "needs_review": counts.get("needs_review", 0),
+            "unscorable": counts.get("unscorable", 0),
+            "failed": counts.get("failed", 0),
+            "status_counts": counts}
 
 
 @router.get("/courses")
@@ -337,9 +351,32 @@ def knowledge_page(document_id: int, page: int, db: Session = Depends(get_db)):
     return {"document_id": doc.id, "title": doc.title, "course_version_id": doc.course_version_id,
             "project_id": doc.project_id, "page": page,
             "text": record.text,
+            "page_type": record.page_type or "unknown",
+            "page_type_label": page_type_label(record.page_type),
+            "mixed": bool(record.mixed),
+            "section": record.section or "", "chapter": record.chapter or "",
+            "confidence": record.confidence,
+            "classification": record.classification or {},
+            "content_allowed": (record.page_type or "unknown") in CONTENT_ALLOWED,
             "image_url": page_links(db, document_id, page)["image_url"]}
 
 
+@router.get("/knowledge/documents/{document_id}/pages")
+def knowledge_document_pages(document_id: int, db: Session = Depends(get_db)):
+    """Page inventory with types, so the teacher can see how a document was classified."""
+    doc = found(db, KnowledgeDocument, document_id)
+    records = db.scalars(select(KnowledgePage).where(KnowledgePage.document_id == document_id)
+                         .order_by(KnowledgePage.page)).all()
+    return {"document_id": doc.id, "title": doc.title, "index_version": doc.index_version,
+            "pages": [{"page": item.page, "page_type": item.page_type or "unknown",
+                       "page_type_label": page_type_label(item.page_type),
+                       "mixed": bool(item.mixed), "section": item.section or "",
+                       "chapter": item.chapter or "", "confidence": item.confidence,
+                       "classification": item.classification or {},
+                       "content_allowed": (item.page_type or "unknown") in CONTENT_ALLOWED}
+                      for item in records]}
+
+
 @router.get("/knowledge/documents/{document_id}/pages/{page}/image")
 def knowledge_page_image(document_id: int, page: int, db: Session = Depends(get_db)):
     doc = found(db, KnowledgeDocument, document_id)
@@ -354,6 +391,28 @@ def ingest_course(course_version_id: int, db: Session = Depends(get_db)):
     return services.ingest_course(course_version_id, db)
 
 
+@router.get("/knowledge/stale/{course_version_id}")
+def stale_knowledge(course_version_id: int, db: Session = Depends(get_db)):
+    """Documents still on an older index; they are excluded from retrieval until rebuilt."""
+    from .indexing import stale_documents
+    from .rag import INDEX_VERSION
+    found(db, CourseVersion, course_version_id)
+    documents = stale_documents(db, course_version_id)
+    return {"index_version": INDEX_VERSION,
+            "documents": [{"id": item.id, "title": item.title, "type": item.document_type,
+                           "project_id": item.project_id, "index_version": item.index_version,
+                           "status": item.status} for item in documents]}
+
+
+@router.post("/knowledge/reindex/{course_version_id}")
+def reindex_knowledge(course_version_id: int, db: Session = Depends(get_db)):
+    """Queue a rebuild of every stale index for this course version."""
+    from .indexing import create_reindex_task
+    found(db, CourseVersion, course_version_id)
+    task = create_reindex_task(db, course_version_id)
+    return get_index_task(task.id, db)
+
+
 @router.get("/index-tasks")
 def list_index_tasks(project_id: int | None = None, db: Session = Depends(get_db)):
     query = select(IndexTask).order_by(IndexTask.id.desc()).limit(100)
@@ -381,10 +440,20 @@ def create_batch(data: BatchInput, db: Session = Depends(get_db)):
     return services.create_batch(data, db)
 
 
+def batch_status_counts(db, batch_id: int) -> dict:
+    rows = db.execute(select(CriterionScore.status, func.count(CriterionScore.id))
+                      .where(CriterionScore.batch_id == batch_id).group_by(CriterionScore.status)).all()
+    counts = {status: count for status, count in rows}
+    return {"status_counts": counts, "needs_review": counts.get("needs_review", 0),
+            "unscorable": counts.get("unscorable", 0), "failed": counts.get("failed", 0)}
+
+
 @router.get("/scoring-batches")
 def list_batches(db: Session = Depends(get_db)):
     return [{**{key: getattr(batch, key) for key in ("id", "course_version_id", "rubric_version_id", "status", "progress_done", "progress_total", "error", "created_at", "finished_at")},
-             "project_ids": batch.project_ids, "legacy_membership": batch.project_ids is None}
+             "status_label": BATCH_STATUS_LABELS.get(batch.status, batch.status),
+             "project_ids": batch.project_ids, "legacy_membership": batch.project_ids is None,
+             **batch_status_counts(db, batch.id)}
             for batch in db.scalars(select(ScoringBatch).order_by(ScoringBatch.id.desc()))]
 
 
@@ -392,7 +461,9 @@ def list_batches(db: Session = Depends(get_db)):
 def batch_progress(batch_id: int, db: Session = Depends(get_db)):
     batch = found(db, ScoringBatch, batch_id)
     return {**{key: getattr(batch, key) for key in ("id", "status", "progress_done", "progress_total", "error", "created_at", "finished_at")},
-            "project_ids": batch.project_ids, "legacy_membership": batch.project_ids is None}
+            "status_label": BATCH_STATUS_LABELS.get(batch.status, batch.status),
+            "project_ids": batch.project_ids, "legacy_membership": batch.project_ids is None,
+            **batch_status_counts(db, batch_id)}
 
 
 @router.post("/scoring-batches/{batch_id}/start")