fix: improve import, evidence indexing and material-scoped grading
Preserve document revisions and evidence, harden indexing and export behavior, add explicit material selection and direct-folder import, validate model JSON schemas with safe error messages, and retain student selection while loading scoring data.
This commit is contained in:
+92
-83
@@ -14,14 +14,16 @@ from functools import lru_cache
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from sqlalchemy import case, cast, delete, func, literal, or_, select
|
||||
from sqlalchemy import case, cast, func, literal, or_, select
|
||||
|
||||
from .config import get_settings
|
||||
from .models import KnowledgeChunk, KnowledgeDocument, KnowledgePage
|
||||
from .pages import (BLOCK_BODY, BLOCK_TOC, classify_document, is_toc_entry,
|
||||
from .models import KnowledgeChunk, KnowledgeDocument, KnowledgePage, SubmissionFile
|
||||
from .knowledge_pages import page_text, relative_image_path
|
||||
from .pages import (BLOCK_BODY, BLOCK_TOC, is_toc_entry,
|
||||
normalize_heading, searchable_pages)
|
||||
from .evidence import heading_only_text
|
||||
|
||||
INDEX_VERSION = 3
|
||||
INDEX_VERSION = 5
|
||||
|
||||
#: Fusion constant for reciprocal rank fusion; the usual value from the IR literature.
|
||||
RRF_K = 60
|
||||
@@ -60,34 +62,10 @@ def query_terms(query: str) -> tuple[list[str], list[str]]:
|
||||
return sorted(set(ascii_tokens)), bigrams
|
||||
|
||||
|
||||
def _keyword_query(db, filters, query: str, limit: int):
|
||||
"""Keyword recall ranked by bigram/token overlap over the already-filtered rows.
|
||||
|
||||
PostgreSQL's ``to_tsvector('simple', …)`` cannot segment Chinese, so it is not used
|
||||
for recall. The overlap is computed with ``strpos``/``substr``/``generate_series``
|
||||
so the same expression works on every dialect SQLAlchemy targets here.
|
||||
"""
|
||||
ascii_tokens, bigrams = query_terms(query)
|
||||
if not ascii_tokens and not bigrams:
|
||||
return []
|
||||
total = len(ascii_tokens) + len(bigrams)
|
||||
score = None
|
||||
for token in ascii_tokens:
|
||||
hit = case((func.strpos(func.lower(KnowledgeChunk.text), literal(token)) > 0, 1.0), else_=0.0)
|
||||
score = hit if score is None else score + hit
|
||||
if bigrams:
|
||||
series = select(func.generate_series(1, func.greatest(func.length(KnowledgeChunk.text) - 1, 0)).label("i")).subquery("series")
|
||||
matches = (select(func.count()).select_from(series)
|
||||
.where(func.substr(KnowledgeChunk.text, series.c.i, 2).in_(bigrams)).scalar_subquery())
|
||||
score = matches if score is None else score + matches
|
||||
score = (score / float(total)).label("keyword")
|
||||
rows = db.execute(select(KnowledgeChunk.id, score).where(*filters)
|
||||
.order_by(score.desc(), KnowledgeChunk.id.asc()).limit(limit)).all()
|
||||
return [row[0] for row in rows if float(row[1] or 0) > 0]
|
||||
|
||||
|
||||
def split_pages(pages: list[dict], *, size=650, overlap=100):
|
||||
"""Split pages into searchable chunks carrying page type and section path."""
|
||||
if size <= 0 or overlap < 0 or overlap >= size:
|
||||
raise ValueError("切块大小必须为正且重叠小于块大小")
|
||||
for page in pages:
|
||||
page_type = page.get("page_type") or "body"
|
||||
chapter = page.get("chapter", "")
|
||||
@@ -98,6 +76,9 @@ def split_pages(pages: list[dict], *, size=650, overlap=100):
|
||||
blocks.extend({"kind": BLOCK_BODY, "text": "表格: " + str(table.get("rows", table))}
|
||||
for table in page.get("tables", []))
|
||||
for block in blocks:
|
||||
source_type = block.get("page_type", page_type)
|
||||
block_chapter = block.get("chapter", chapter)
|
||||
block_section = block.get("section", section)
|
||||
if block["kind"] == BLOCK_TOC:
|
||||
# A contents page is navigation, so its text is not body content.
|
||||
yield {"page": page["page_number"], "page_type": "toc", "block_type": BLOCK_TOC,
|
||||
@@ -112,8 +93,11 @@ def split_pages(pages: list[dict], *, size=650, overlap=100):
|
||||
# Table rows of a page are appended once, on the page's first body chunk.
|
||||
start = 0
|
||||
while start < len(text_block):
|
||||
yield {"page": page["page_number"], "page_type": page_type, "block_type": BLOCK_BODY,
|
||||
"chapter": chapter, "section": section, "text": text_block[start:start + size]}
|
||||
part = text_block[start:start + size]
|
||||
unsafe = any(c in part for c in ("\ufffd", "\ufffe", "\uffff")) or heading_only_text(part)
|
||||
safe_type = "unknown" if unsafe and source_type in ("body", "appendix") else source_type
|
||||
yield {"page": page["page_number"], "page_type": safe_type, "block_type": BLOCK_BODY,
|
||||
"chapter": block_chapter, "section": block_section, "text": part}
|
||||
if start + size >= len(text_block):
|
||||
break
|
||||
start += size - overlap
|
||||
@@ -150,25 +134,26 @@ def embed(texts: list[str], *, model_path: str | None = None):
|
||||
|
||||
def ingest(db, *, course_version_id: int, project_id: int | None, submission_file_id: int | None,
|
||||
title: str, category: str, source_hash: str, pages: list[dict], embedder=embed):
|
||||
existing = db.scalar(select(KnowledgeDocument).where(KnowledgeDocument.course_version_id == course_version_id,
|
||||
history = db.scalars(select(KnowledgeDocument).where(KnowledgeDocument.course_version_id == course_version_id,
|
||||
KnowledgeDocument.project_id == project_id, KnowledgeDocument.submission_file_id == submission_file_id,
|
||||
KnowledgeDocument.title == title))
|
||||
if existing and existing.source_hash == source_hash and existing.index_version == INDEX_VERSION and existing.status == "ready":
|
||||
KnowledgeDocument.title == title).order_by(KnowledgeDocument.id.desc())).all()
|
||||
existing = next((doc for doc in history if doc.source_hash == source_hash and doc.status == "ready"
|
||||
and doc.index_version == INDEX_VERSION and doc.document_type == category), None)
|
||||
if existing:
|
||||
return existing
|
||||
prepared = searchable_pages(pages)
|
||||
chunks = list(split_pages(prepared))
|
||||
vectors = embedder([chunk["text"] for chunk in chunks])
|
||||
if len(vectors) != len(chunks):
|
||||
raise RuntimeError("嵌入向量数量与知识片段数量不一致")
|
||||
if existing:
|
||||
db.execute(delete(KnowledgePage).where(KnowledgePage.document_id == existing.id))
|
||||
db.execute(delete(KnowledgeChunk).where(KnowledgeChunk.document_id == existing.id))
|
||||
doc = existing
|
||||
doc.document_type, doc.source_hash = category, source_hash
|
||||
else:
|
||||
doc = KnowledgeDocument(course_version_id=course_version_id, project_id=project_id, submission_file_id=submission_file_id,
|
||||
title=title, document_type=category, source_hash=source_hash)
|
||||
db.add(doc)
|
||||
# Pages and chunk IDs are immutable citation sources. Even an unchanged file
|
||||
# needs a new document revision when its index or material category changes.
|
||||
revision = (db.scalar(select(func.max(KnowledgeDocument.revision)).where(
|
||||
KnowledgeDocument.course_version_id == course_version_id, KnowledgeDocument.project_id == project_id,
|
||||
KnowledgeDocument.submission_file_id == submission_file_id, KnowledgeDocument.source_hash == source_hash)) or 0) + 1
|
||||
doc = KnowledgeDocument(course_version_id=course_version_id, project_id=project_id, submission_file_id=submission_file_id,
|
||||
title=title, document_type=category, source_hash=source_hash, revision=revision)
|
||||
db.add(doc)
|
||||
doc.index_version, doc.status = INDEX_VERSION, "indexing"
|
||||
db.flush()
|
||||
for source, prepared_page in zip(pages, prepared):
|
||||
@@ -188,6 +173,8 @@ def ingest(db, *, course_version_id: int, project_id: int | None, submission_fil
|
||||
document_type=category, page=chunk["page"], section=chunk["section"],
|
||||
chapter=chunk["chapter"], page_type=chunk["page_type"],
|
||||
block_type=chunk["block_type"], text=chunk["text"], embedding=vector))
|
||||
for old in history:
|
||||
old.status = "superseded"
|
||||
doc.status = "ready"
|
||||
db.commit()
|
||||
return doc
|
||||
@@ -201,10 +188,24 @@ def _scope_filter(project_id: int | None, include_public: bool, public_only: boo
|
||||
return KnowledgeChunk.project_id == project_id
|
||||
|
||||
|
||||
def current_document_ids():
|
||||
selected_file = select(SubmissionFile.id).where(
|
||||
SubmissionFile.id == KnowledgeDocument.submission_file_id,
|
||||
SubmissionFile.recognition_status == "recognized", SubmissionFile.parse_status == "ready",
|
||||
SubmissionFile.sha256 == KnowledgeDocument.source_hash).exists()
|
||||
return select(KnowledgeDocument.id).where(KnowledgeDocument.status == "ready",
|
||||
KnowledgeDocument.index_version == INDEX_VERSION,
|
||||
or_(KnowledgeDocument.submission_file_id.is_(None), selected_file))
|
||||
|
||||
|
||||
def _base_filters(*, course_version_id: int, project_id: int | None, category, categories,
|
||||
include_public: bool, public_only: bool, allowed_page_types, exclude_document_ids):
|
||||
include_public: bool, public_only: bool, allowed_page_types, exclude_document_ids, document_id=None):
|
||||
filters = [KnowledgeChunk.course_version_id == course_version_id,
|
||||
_scope_filter(project_id, include_public, public_only)]
|
||||
_scope_filter(project_id, include_public, public_only),
|
||||
KnowledgeChunk.block_type != "legacy",
|
||||
KnowledgeChunk.document_id.in_(current_document_ids())]
|
||||
if document_id is not None:
|
||||
filters.append(KnowledgeChunk.document_id == document_id)
|
||||
if category:
|
||||
filters.append(KnowledgeChunk.document_type == category)
|
||||
if categories:
|
||||
@@ -226,19 +227,23 @@ def _keyword_query(db, filters, query: str, limit: int):
|
||||
ascii_tokens, bigrams = query_terms(query)
|
||||
if not ascii_tokens and not bigrams:
|
||||
return []
|
||||
if db.bind.dialect.name != "postgresql":
|
||||
rows = db.execute(select(KnowledgeChunk.id, KnowledgeChunk.text).where(*filters)).all()
|
||||
ranked = [(row[0], keyword_overlap(row[1], ascii_tokens, bigrams)) for row in rows]
|
||||
ranked.sort(key=lambda row: (-row[1], row[0]))
|
||||
return [chunk_id for chunk_id, score in ranked[:limit] if score > 0]
|
||||
total = len(ascii_tokens) + len(bigrams)
|
||||
score = None
|
||||
for token in ascii_tokens:
|
||||
term = literal(token)
|
||||
hit = case((func.strpos(func.lower(KnowledgeChunk.text), term) > 0, 1.0), else_=0.0)
|
||||
pattern = r"(^|[^a-z0-9_+.-])" + re.escape(token) + r"($|[^a-z0-9_+.-])"
|
||||
hit = case((func.lower(KnowledgeChunk.text).op("~")(pattern), 1.0), else_=0.0)
|
||||
score = hit if score is None else score + hit
|
||||
if bigrams:
|
||||
start = func.generate_series(1, func.greatest(func.length(KnowledgeChunk.text) - 1, 0))
|
||||
series = select(start.label("i")).subquery("series")
|
||||
matches = (select(func.count()).select_from(series)
|
||||
.where(func.substr(KnowledgeChunk.text, series.c.i, 2).in_(bigrams))
|
||||
.scalar_subquery())
|
||||
score = matches if score is None else score + matches
|
||||
# Count each query term once, matching keyword_overlap and avoiding a bonus
|
||||
# for repeated words on an otherwise irrelevant page.
|
||||
for bigram in bigrams:
|
||||
hit = case((func.strpos(KnowledgeChunk.text, literal(bigram)) > 0, 1.0), else_=0.0)
|
||||
score = hit if score is None else score + hit
|
||||
score = (score / float(total)).label("keyword")
|
||||
rows = db.execute(select(KnowledgeChunk.id, score).where(*filters)
|
||||
.order_by(score.desc(), KnowledgeChunk.id.asc()).limit(limit)).all()
|
||||
@@ -249,7 +254,8 @@ def keyword_overlap(text_value: str, ascii_tokens: list[str], bigrams: list[str]
|
||||
"""Bigram/token overlap used by the non-PostgreSQL path and by tests."""
|
||||
value = str(text_value or "")
|
||||
lower = value.lower()
|
||||
hits = sum(1 for token in ascii_tokens if token in lower)
|
||||
tokens = set(_ASCII_TOKEN.findall(lower))
|
||||
hits = sum(1 for token in ascii_tokens if token in tokens)
|
||||
total = len(ascii_tokens)
|
||||
hits += sum(1 for bigram in bigrams if bigram in value)
|
||||
total += len(bigrams)
|
||||
@@ -305,7 +311,7 @@ def normalize_hit(hit: dict) -> dict:
|
||||
def retrieve(db, *, course_version_id: int, project_id: int, query: str, category: str | None = None,
|
||||
categories=None, limit=10, embedder=embed, include_public=True, public_only=False,
|
||||
allowed_page_types=None, exclude_document_ids=(), substantive_only=False,
|
||||
semantic_pool: int | None = None, keyword_pool: int | None = None):
|
||||
semantic_pool: int | None = None, keyword_pool: int | None = None, document_id=None):
|
||||
"""Filtered hybrid retrieval, then rank fusion over the two independent recalls.
|
||||
|
||||
Page filtering happens inside the query, so it always precedes the candidate limit:
|
||||
@@ -313,7 +319,8 @@ def retrieve(db, *, course_version_id: int, project_id: int, query: str, categor
|
||||
"""
|
||||
filters = _base_filters(course_version_id=course_version_id, project_id=project_id, category=category,
|
||||
categories=categories, include_public=include_public, public_only=public_only,
|
||||
allowed_page_types=allowed_page_types, exclude_document_ids=exclude_document_ids)
|
||||
allowed_page_types=allowed_page_types, exclude_document_ids=exclude_document_ids,
|
||||
document_id=document_id)
|
||||
if substantive_only:
|
||||
filters.append(KnowledgeChunk.block_type != BLOCK_TOC)
|
||||
vector = embedder([query])[0]
|
||||
@@ -339,11 +346,11 @@ def _page_diverse(chunks: list[dict], limit: int, *, page_limit: int, section_ca
|
||||
"""Spread the first pass over pages, then allow a bounded number per section."""
|
||||
picked: list[dict] = []
|
||||
per_page: dict[tuple, int] = {}
|
||||
per_section: dict[str, int] = {}
|
||||
per_section: dict[tuple, int] = {}
|
||||
remaining = []
|
||||
for chunk in chunks:
|
||||
page_key = (chunk.get("document_id"), chunk.get("page"))
|
||||
section = str(chunk.get("section") or "")
|
||||
section = (chunk.get("document_id"), str(chunk.get("section") or "")) if chunk.get("section") else None
|
||||
if len(picked) < limit and per_page.get(page_key, 0) < page_limit:
|
||||
if section_cap and section and per_section.get(section, 0) >= section_cap:
|
||||
remaining.append(chunk)
|
||||
@@ -373,13 +380,13 @@ def retrieve_targets(db, *, course_version_id: int, project_id: int, targets: li
|
||||
nothing admissible, which is what decides a teacher review.
|
||||
"""
|
||||
merged: list[dict] = []
|
||||
seen: set[int] = set()
|
||||
seen: dict[int, dict] = {}
|
||||
per_target: dict[str, list[dict]] = {}
|
||||
for target in targets:
|
||||
allowed = target.get("allowed_page_types")
|
||||
raw_hits = retrieve_fn(db, course_version_id=course_version_id, project_id=project_id, category=category,
|
||||
categories=categories, query=target.get("query") or target.get("label") or "",
|
||||
limit=per_target_limit or max(limit, 8), embedder=embed,
|
||||
limit=per_target_limit or max(limit, 8), embedder=embedder,
|
||||
include_public=include_public, allowed_page_types=allowed,
|
||||
substantive_only=target.get("use") != "structure",
|
||||
semantic_pool=SEMANTIC_POOL, keyword_pool=KEYWORD_POOL)
|
||||
@@ -392,19 +399,37 @@ def retrieve_targets(db, *, course_version_id: int, project_id: int, targets: li
|
||||
per_target[target["target_id"]] = hits
|
||||
for hit in hits:
|
||||
if hit["chunk_id"] not in seen:
|
||||
seen.add(hit["chunk_id"])
|
||||
merged.append({**hit, "target_id": target["target_id"], "target_label": target["label"],
|
||||
"target_use": target["use"]})
|
||||
item = {**hit, "target_id": target["target_id"], "target_ids": [target["target_id"]],
|
||||
"target_label": target["label"], "target_use": target["use"]}
|
||||
seen[hit["chunk_id"]] = item
|
||||
merged.append(item)
|
||||
else:
|
||||
seen[hit["chunk_id"]]["target_ids"].append(target["target_id"])
|
||||
merged.sort(key=lambda chunk: -chunk["rank"])
|
||||
selected = _page_diverse(merged, limit, page_limit=page_limit, section_cap=section_cap)
|
||||
uncovered = [target for target in targets if not per_target.get(target["target_id"])]
|
||||
# Reserve coverage before filling the remaining budget by rank. One shared
|
||||
# passage may cover several targets, so it consumes only one slot.
|
||||
selected = []
|
||||
covered = set()
|
||||
for target in targets:
|
||||
if target["target_id"] in covered or len(selected) >= limit:
|
||||
continue
|
||||
candidates = [chunk for chunk in merged if target["target_id"] in chunk["target_ids"]]
|
||||
if candidates:
|
||||
selected.append(candidates[0])
|
||||
covered.update(candidates[0]["target_ids"])
|
||||
chosen = {chunk["chunk_id"] for chunk in selected}
|
||||
remaining = [chunk for chunk in merged if chunk["chunk_id"] not in chosen]
|
||||
selected.extend(_page_diverse(remaining, limit - len(selected), page_limit=page_limit, section_cap=section_cap))
|
||||
covered = {target_id for chunk in selected for target_id in chunk["target_ids"]}
|
||||
uncovered = [target for target in targets if target["target_id"] not in covered]
|
||||
return {"evidence": selected, "per_target": per_target, "uncovered_targets": uncovered}
|
||||
|
||||
|
||||
def body_heading_index(db, *, course_version_id: int, project_id: int, category: str | None = None):
|
||||
"""Sections present in real body text, used to navigate from a contents page."""
|
||||
filters = [KnowledgeChunk.course_version_id == course_version_id, KnowledgeChunk.project_id == project_id,
|
||||
KnowledgeChunk.page_type == "body"]
|
||||
KnowledgeChunk.page_type == "body", KnowledgeChunk.block_type != "legacy",
|
||||
KnowledgeChunk.document_id.in_(current_document_ids())]
|
||||
if category:
|
||||
filters.append(KnowledgeChunk.document_type == category)
|
||||
rows = db.execute(select(KnowledgeChunk.document_id, KnowledgeChunk.page, KnowledgeChunk.section,
|
||||
@@ -417,19 +442,3 @@ def body_heading_index(db, *, course_version_id: int, project_id: int, category:
|
||||
index.setdefault(key, []).append({"document_id": document_id, "page": page,
|
||||
"section": section, "chapter": chapter})
|
||||
return index
|
||||
|
||||
|
||||
def page_text(source: dict) -> str:
|
||||
blocks = [str(source.get("text") or "").strip()]
|
||||
blocks.extend("表格: " + str(table.get("rows", table)) for table in source.get("tables", []))
|
||||
return "\n".join(block for block in blocks if block)
|
||||
|
||||
|
||||
def relative_image_path(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
root = get_settings().processed_path.resolve()
|
||||
path = Path(value).resolve()
|
||||
if not path.is_relative_to(root):
|
||||
raise ValueError("页面图像路径不在处理目录内")
|
||||
return str(path.relative_to(root))
|
||||
|
||||
Reference in New Issue
Block a user