SkyLiCoder/WiseGrade
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: improve import, evidence indexing and material-scoped grading

Browse Source 
Preserve document revisions and evidence, harden indexing and export behavior, add explicit material selection and direct-folder import, validate model JSON schemas with safe error messages, and retain student selection while loading scoring data.
This commit is contained in:
aerospaceboy
2026-10-14 13:52:56 +08:00
parent 29c95ce5ad
commit ebe33392ca
42 changed files with 2332 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/wisegrade/pages.py
+76 -25
View File
@@ -15,7 +15,7 @@ from __future__ import annotations
import re
import unicodedata
CLASSIFIER_VERSION = 3
CLASSIFIER_VERSION = 5
PAGE_TYPES = ("cover", "abstract", "toc", "body", "appendix", "references", "unknown")
@@ -76,12 +76,19 @@ def normalize_heading(text: str) -> str:
return "".join(char for char in value if not char.isspace() and char not in "．.·•…")
def is_toc_entry(line: str) -> bool:
"""A dotted leader plus a trailing page or slide number."""
def is_toc_entry(line: str, *, undotted=False) -> bool:
"""Dotted navigation, plus aligned undotted titles in a contents context."""
stripped = (line or "").strip()
if not stripped or not _DOT_LEADER.search(stripped):
if not stripped:
return False
return bool(_TRAILING_ANCHOR.search(stripped))
if _DOT_LEADER.search(stripped):
return bool(_TRAILING_ANCHOR.search(stripped))
if not undotted:
return False
match = re.match(r"^\d+(?:\.\d+)*[、.．]?\s+(.+?)(?:[ \t]{2,}|\t)(\d{1,4})$", stripped)
title = re.sub(r"\s+", "", match.group(1)) if match else ""
return bool(title and _CJK.search(title) and len(title) <= MAX_HEADING_TITLE_CHARS
and not any(char in title for char in _PROSE_PUNCTUATION))
def heading_syntax(line: str) -> tuple[int, str] | None:
@@ -156,7 +163,8 @@ def _signals(page: dict, *, index: int, total: int, toc_titles: set[str] | None
# Prose detection ignores repeated page headers and footers: a running header is not
# something the student wrote, so it must not make an empty page look like content.
prose_source = [line for line in lines if line not in (repeated or ())]
entries = [line for line in lines if is_toc_entry(line)]
contents_context = any(_TOC_TITLE.match(line) for line in lines[:3]) or sum(is_toc_entry(line) for line in lines) >= 2
entries = [line for line in lines if is_toc_entry(line, undotted=contents_context)]
headings = [item for item in (heading_level(line, toc_titles) for line in lines) if item]
body_lines = [line for line in lines if line not in set(entries)]
plain_chars = sum(len(line) for line in body_lines)
@@ -189,7 +197,7 @@ def _signals(page: dict, *, index: int, total: int, toc_titles: set[str] | None
# Prose punctuation separates running text from a label-style cover page.
"prose_punctuation": any(char in text for char in _PROSE_PUNCTUATION),
# Lines that read as running text rather than headings, labels or dotted entries.
"prose_lines": [line for line in prose_source if _is_prose_line(line)],
"prose_lines": [line for line in prose_source if line not in entries and _is_prose_line(line)],
}
@@ -238,15 +246,24 @@ def _is_prose_line(line: str) -> bool:
return len(stripped) >= MIN_PROSE_CHARS
def split_blocks(lines: list[str]) -> list[dict]:
def split_blocks(lines: list[str], *, undotted=False) -> list[dict]:
"""Split page lines into TOC and body blocks so a mixed page keeps both parts."""
blocks: list[dict] = []
current: dict | None = None
contents_entries = set()
in_contents = undotted
for line in lines:
stripped = line.strip()
# A contents title and its dotted entries are one navigation block. A lone
# dotted line with no title is not a table of contents, so it stays as body.
kind = BLOCK_TOC if is_toc_entry(line) or _TOC_TITLE.match(stripped) else BLOCK_BODY
entry = is_toc_entry(line, undotted=in_contents)
if entry:
contents_entries.add(line)
kind = BLOCK_TOC if entry or _TOC_TITLE.match(stripped) else BLOCK_BODY
if kind == BLOCK_BODY and contents_entries:
# Once genuine body text/headings begin, aligned numeric data is no
# longer an undotted contents row, even on a mixed page.
in_contents = False
if current is None or current["kind"] != kind:
current = {"kind": kind, "lines": []}
blocks.append(current)
@@ -256,7 +273,7 @@ def split_blocks(lines: list[str]) -> list[dict]:
text = "\n".join(block["lines"]).strip()
if not text:
continue
has_entry = any(is_toc_entry(line) for line in block["lines"])
has_entry = any(line in contents_entries for line in block["lines"])
if block["kind"] == BLOCK_TOC and not has_entry:
# Only a title line, with no entries under it: this is a heading, not contents.
output.append({"kind": BLOCK_BODY, "text": text, "substantive": False})
@@ -323,8 +340,12 @@ def classify_document(pages: list[dict]) -> list[dict]:
# real headings; this keeps enumerated list items and equations out of the path.
toc_titles = set()
for page in pages:
for line in _lines(str(page.get("text") or "")):
if is_toc_entry(line):
page_lines = _lines(str(page.get("text") or ""))
is_contents = any(_TOC_TITLE.match(line) for line in page_lines[:3]) or sum(is_toc_entry(line) for line in page_lines) >= 2
for line in page_lines:
# Top-level entries often have no leaders, unlike the subheadings.
plain_entry = re.match(r"^\d+(?:\.\d+)*\s+\S.*\s+\d{1,4}$", line) if is_contents else None
if is_toc_entry(line) or plain_entry:
title = normalize_heading(line)
if title:
toc_titles.add(title)
@@ -338,7 +359,9 @@ def classify_document(pages: list[dict]) -> list[dict]:
for position, page in enumerate(pages, start=1):
signal = signals[position - 1]
lines = clean_page_lines(page, repeated)
blocks = split_blocks(lines)
contents_context = bool(_TOC_TITLE.match(signal["first_line"])) or _is_toc_page(
signal, previous_is_toc=(previous == "toc"))
blocks = split_blocks(lines, undotted=contents_context)
toc_blocks = [block for block in blocks if block["kind"] == BLOCK_TOC]
body_blocks = [block for block in blocks if block["kind"] == BLOCK_BODY and block["substantive"]]
first = signal["first_line"]
@@ -346,7 +369,7 @@ def classify_document(pages: list[dict]) -> list[dict]:
appendix = signal["appendix_title"]
if _TOC_TITLE.match(first) and signal["toc_entries"] >= 1:
page_type, confidence = "toc", 0.95
elif _ABSTRACT_TITLE.match(first) or (previous == "abstract" and signal["footer"] in ("roman", "none")
elif any(_ABSTRACT_TITLE.match(line) for line in lines[:3]) or (previous == "abstract" and signal["footer"] in ("roman", "none")
and signal["headings"] == 0 and signal["chars"] >= 100):
page_type, confidence = "abstract", 0.9
elif _REFERENCE_TITLE.match(first) or _ACK_TITLE.match(first):
@@ -376,18 +399,31 @@ def classify_document(pages: list[dict]) -> list[dict]:
mixed = page_type == "toc" and bool(body_blocks)
if mixed:
confidence = 0.7
if page_type in ("body", "appendix", "unknown") or mixed:
# Structure-extracted headings come from body text only, never from a TOC.
for block in body_blocks:
for line in block["text"].splitlines():
found = heading_level(line, toc_titles)
if not found:
continue
# Store section paths on blocks, rather than assigning the last heading on a
# page to all its text. Paths continue across pages, but not across navigation.
typed_blocks = []
for block in blocks:
kind = block["kind"]
source_type = "body" if mixed and kind == BLOCK_BODY and block.get("substantive") else page_type
can_navigate = kind == BLOCK_BODY and (page_type in ("body", "appendix", "unknown") or mixed)
buffer = []
def flush():
if buffer:
substantive = any(_is_prose_line(line) for line in buffer)
typed_blocks.append({**block, "text": "\n".join(buffer), "page_type": source_type,
"chapter": chapter, "section": section, "substantive": substantive})
buffer.clear()
for line in block["text"].splitlines():
found = heading_level(line, toc_titles) if can_navigate else None
if found:
flush()
level, title = found
if level == 1:
chapter, section = title, title
else:
section = title
chapter = title
section = title
buffer.append(line)
flush()
blocks = typed_blocks
# A table of contents may only name the chapter path; the body supplies the
# section wording. Navigation pages therefore carry the chapter, not a section.
section_for_page = section if page_type in ("body", "unknown") or mixed else chapter
@@ -399,7 +435,8 @@ def classify_document(pages: list[dict]) -> list[dict]:
"section": section_for_page,
"blocks": blocks,
"confidence": round(confidence, 2),
"signals": {**signal, "repeated_line_count": len(repeated), "toc_blocks": len(toc_blocks)},
"signals": {**signal, "repeated_line_count": len(repeated), "toc_blocks": len(toc_blocks),
"text_quality": page.get("text_quality", {})},
"classifier_version": CLASSIFIER_VERSION,
})
previous = page_type
@@ -428,6 +465,20 @@ def searchable_pages(pages: list[dict]) -> list[dict]:
output = []
for page, item in zip(pages, classification):
body = block_text(item["blocks"])
# Parser tables may also carry values absent from the page's text layer.
# Do not duplicate tables whose cells are already represented in that text.
compact = re.sub(r"\s+", "", body)
for table in page.get("tables", []):
if isinstance(table, dict) and table.get("text"):
# PDF table text is already part of the page's text layer. Keep the
# inferred grid as metadata, not as a fabricated verbatim quotation.
continue
rows = table.get("rows", []) if isinstance(table, dict) else table
values = [str(cell) for row in rows for cell in row if cell is not None and str(cell).strip()]
if values and any(re.sub(r"\s+", "", cell) not in compact for cell in values):
item["blocks"].append({"kind": BLOCK_BODY, "text": "表格: " + str(rows),
"page_type": item["page_type"], "chapter": item["chapter"],
"section": item["section"], "substantive": True})
output.append({
"page_number": page.get("page_number"),
"section": item["section"] or page.get("section", ""),