SkyLiCoder/WiseGrade
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: separate contents, abstract and body evidence

Browse Source
This commit is contained in:
李傲琦
2026-10-01 00:07:40 +08:00
parent 540d0e2918
commit b7634c8374
25 changed files with 2592 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/wisegrade/pages.py
+443
View File
@@ -0,0 +1,443 @@
"""Page-level classification, block splitting and real section paths.
A page carries one ``page_type``: cover, abstract, toc, body, references, unknown.
A page may additionally be ``mixed`` when a table of contents and body text truly
share one page; only that page is split into blocks so the body part is not lost.
Classification never depends on a fixed page number or on the single word "目录".
It combines the page's own signals (heading title, dot-leader entry density,
trailing page numbers, abstract/reference wording, cover fields, footer style)
with document-level continuity for continuation pages (TOC continued, abstract
continued, references spanning pages).
"""
from __future__ import annotations
import re
import unicodedata
CLASSIFIER_VERSION = 3
PAGE_TYPES = ("cover", "abstract", "toc", "body", "appendix", "references", "unknown")
#: Page types whose text may support a content judgement about what a student did.
CONTENT_PAGE_TYPES = ("body", "appendix")
#: Page types whose text may support a structural judgement (chapter completeness etc.).
STRUCTURE_PAGE_TYPES = ("cover", "abstract", "toc", "references", "body", "appendix")
#: Page types that may never be used as content evidence on their own.
NAVIGATION_PAGE_TYPES = ("cover", "abstract", "toc", "references")
#: Page types a page may be stored as, used to validate rule configuration.
STORE_PAGE_TYPES = PAGE_TYPES
BLOCK_BODY = "body"
BLOCK_TOC = "toc"
BLOCK_TITLE = "title"
_DOT_LEADER = re.compile(r"(?:[.·•…․…]\s*){3,}")
_TRAILING_ANCHOR = re.compile(r"(\d{1,4})\s*$")
_CHAPTER = re.compile(r"^第\s*([0-9一二三四五六七八九十百]+)\s*章\s*(.*)$")
_APPENDIX = re.compile(r"^附\s*录\s*([A-Za-z0-9一二三四五六七八九十]+)?\s*(.*)$")
_NUMBERED = re.compile(r"^(\d+(?:\.\d+)*)[\s、.．]+(\S.*)$")
_COVER_FIELD = re.compile(r"(学\s*号|姓\s*名|题\s*目|指导教师|专\s*业|学\s*院|日\s*期)\s*[:：]")
_PAGE_NUMBER_LINE = re.compile(r"^\s*(?:\d{1,3}|[IVXLC]{1,5}|第\s*\d+\s*页)\s*$")
_ABSTRACT_TITLE = re.compile(r"^\s*(摘\s*要|abstract)\s*$", re.IGNORECASE)
_REFERENCE_TITLE = re.compile(r"^\s*(参\s*考\s*文\s*献|references?|参考文献与资料)\s*$", re.IGNORECASE)
_ACK_TITLE = re.compile(r"^\s*(致\s*谢|acknowledge?ments?)\s*$", re.IGNORECASE)
_TOC_TITLE = re.compile(r"^\s*(目\s*录|contents?|table\s+of\s+contents)\s*$", re.IGNORECASE)
_REFERENCE_ENTRY = re.compile(r"^\s*\[\d{1,3}\]")
_CJK = re.compile(r"[㐀-䶿一-鿿]")
_HEADING_NUMBER = re.compile(r"^\s*(第\s*[0-9一二三四五六七八九十百]+\s*章|\d+(\.\d+)*|附\s*录\s*[A-Za-z0-9]?)\s*\S")
#: A body block must carry at least this many non-space characters to count as substance.
MIN_BODY_CHARS = 60
#: Heading-only text shorter than this never counts as content evidence on its own.
MIN_CONTENT_QUOTE_CHARS = 25
#: A line at least this long, outside a heading, reads as running prose. The bar is low
#: on purpose: the rule separates text from headings and labels, not long from short
#: text, so a brief but real sentence is still content.
MIN_PROSE_CHARS = 4
#: A numbered heading title must stay this short; longer text is prose or a list item.
MAX_HEADING_TITLE_CHARS = 40
#: Wording that marks the line as running prose rather than a heading.
_PROSE_PUNCTUATION = "。，；！？,;"
#: The documents are Chinese engineering reports; the keyword recall uses these units.
BIGRAM_MIN = 2
def _lines(text: str) -> list[str]:
return [line.strip() for line in (text or "").splitlines() if line.strip()]
def normalize_heading(text: str) -> str:
"""Compare headings across the TOC and the body: unified width, no spaces/dots."""
value = unicodedata.normalize("NFKC", text or "")
value = _DOT_LEADER.sub("", value)
value = _TRAILING_ANCHOR.sub("", value.strip())
return "".join(char for char in value if not char.isspace() and char not in "．.·•…")
def is_toc_entry(line: str) -> bool:
"""A dotted leader plus a trailing page or slide number."""
stripped = (line or "").strip()
if not stripped or not _DOT_LEADER.search(stripped):
return False
return bool(_TRAILING_ANCHOR.search(stripped))
def heading_syntax(line: str) -> tuple[int, str] | None:
"""Return ``(level, title)`` when the line has the shape of a heading.
This is a purely syntactic reading. Numbered prose such as
``4. 实时图像传输：将摄像头画面…`` and enumerated list items are not headings: a
numbered heading has a short Chinese title, no sentence punctuation and no trailing
colon. Equation and matrix rows such as ``3.3.3`` or ``1 0 0 1 Zc`` are never
headings, and a line cut mid-formula (trailing combining mark) is not either.
"""
stripped = (line or "").strip()
if not stripped or is_toc_entry(stripped) or len(stripped) > 120:
return None
match = _CHAPTER.match(stripped)
if match:
return 1, stripped
match = _APPENDIX.match(stripped)
if match and not _COVER_FIELD.search(stripped):
return 2, stripped
if _ABSTRACT_TITLE.match(stripped) or _REFERENCE_TITLE.match(stripped) or _ACK_TITLE.match(stripped):
return 1, stripped
match = _NUMBERED.match(stripped)
if not match:
return None
title = match.group(2).strip()
if not title or title.endswith(":") or title.endswith("："):
return None
if len(title) > MAX_HEADING_TITLE_CHARS or any(char in title for char in _PROSE_PUNCTUATION):
return None
if not _CJK.search(title):
return None
if unicodedata.category(title[-1]).startswith("M"):
return None
return 1 + match.group(1).count("."), stripped
def heading_level(line: str, toc_titles: set[str] | None = None) -> tuple[int, str] | None:
"""A heading for the section path: syntactic shape, confirmed by the contents list.
When the document carries its own table of contents, that list is the authority on
which numbered lines are real headings. That is what keeps an enumerated list item or
a numbered paragraph from becoming a chapter in the extracted section path. Callers
that only need to know whether a line reads as a heading use ``heading_syntax``.
"""
found = heading_syntax(line)
if not found:
return None
if toc_titles and normalize_heading(line) not in toc_titles:
return None
return found
def _footer_style(lines: list[str]) -> str:
"""Classify the trailing line so repeated headers/footers can be removed."""
if not lines:
return "none"
last = lines[-1]
if re.fullmatch(r"[IVXLC]{1,5}", last):
return "roman"
if re.fullmatch(r"\d{1,3}", last):
return "arabic"
if re.fullmatch(r"第\s*\d+\s*页", last):
return "page_cn"
return "text"
def _signals(page: dict, *, index: int, total: int, toc_titles: set[str] | None = None,
repeated: set[str] | None = None) -> dict:
text = str(page.get("text") or "")
lines = _lines(text)
# Prose detection ignores repeated page headers and footers: a running header is not
# something the student wrote, so it must not make an empty page look like content.
prose_source = [line for line in lines if line not in (repeated or ())]
entries = [line for line in lines if is_toc_entry(line)]
headings = [item for item in (heading_level(line, toc_titles) for line in lines) if item]
body_lines = [line for line in lines if line not in set(entries)]
plain_chars = sum(len(line) for line in body_lines)
first = lines[0] if lines else ""
reference_entries = [line for line in lines if _REFERENCE_ENTRY.match(line)]
titles = [position for position, line in enumerate(lines)
if _REFERENCE_TITLE.match(line) or _ACK_TITLE.match(line)]
# A bibliography page is dense reference entries, or a reference title near the top.
reference_ratio = len(reference_entries) / len(lines) if lines else 0.0
reference_page = (len(reference_entries) >= 5 and reference_ratio >= 0.5
or any(position <= 2 for position in titles))
return {
"chars": len(text.strip()),
"lines": len(lines),
"toc_entries": len(entries),
"toc_ratio": round(len(entries) / len(lines), 3) if lines else 0.0,
"headings": len(headings),
"chapter_headings": sum(1 for level, _ in headings if level == 1),
"body_chars": plain_chars,
"first_line": first[:120],
"position": index,
"total": total,
"footer": _footer_style(lines),
"cover_fields": len(_COVER_FIELD.findall(text)),
"has_table": bool(page.get("tables")),
"reference_entries": len(reference_entries),
"reference_ratio": round(reference_ratio, 3),
"reference_page": reference_page,
"appendix_title": any(_APPENDIX.match(line.strip()) for line in lines),
# Prose punctuation separates running text from a label-style cover page.
"prose_punctuation": any(char in text for char in _PROSE_PUNCTUATION),
# Lines that read as running text rather than headings, labels or dotted entries.
"prose_lines": [line for line in prose_source if _is_prose_line(line)],
}
def strip_repeated_lines(pages: list[dict]) -> set[str]:
"""Header/footer lines repeated across many pages, removed from searchable text."""
counts: dict[str, int] = {}
for page in pages:
lines = _lines(str(page.get("text") or ""))
if not lines:
continue
candidates = {lines[0][:120]}
if len(lines) > 1:
candidates.add(lines[-1][:60])
for candidate in candidates:
if candidate and not is_toc_entry(candidate):
counts[candidate] = counts.get(candidate, 0) + 1
threshold = max(3, int(len(pages) * 0.4 + 0.5))
return {line for line, count in counts.items() if count >= threshold}
def clean_page_lines(page: dict, repeated: set[str]) -> list[str]:
"""Lines kept for indexing: repeated headers/footers and bare page numbers removed."""
lines = _lines(str(page.get("text") or ""))
kept = []
for position, line in enumerate(lines):
if line in repeated and (position == 0 or position == len(lines) - 1):
continue
if _PAGE_NUMBER_LINE.match(line) and position == len(lines) - 1:
continue
kept.append(line)
return kept
def _is_prose_line(line: str) -> bool:
"""A line that reads as running text rather than a heading, label or dotted entry.
The syntactic heading check is used, not the contents-confirmed one: a heading that
the contents list happens not to mention is still a heading, so it must not be
counted as prose when deciding whether a page carries content.
"""
stripped = line.strip()
if is_toc_entry(stripped) or _TOC_TITLE.match(stripped) or heading_syntax(stripped):
return False
if _PAGE_NUMBER_LINE.match(stripped):
return False
return len(stripped) >= MIN_PROSE_CHARS
def split_blocks(lines: list[str]) -> list[dict]:
"""Split page lines into TOC and body blocks so a mixed page keeps both parts."""
blocks: list[dict] = []
current: dict | None = None
for line in lines:
stripped = line.strip()
# A contents title and its dotted entries are one navigation block. A lone
# dotted line with no title is not a table of contents, so it stays as body.
kind = BLOCK_TOC if is_toc_entry(line) or _TOC_TITLE.match(stripped) else BLOCK_BODY
if current is None or current["kind"] != kind:
current = {"kind": kind, "lines": []}
blocks.append(current)
current["lines"].append(line)
output = []
for block in blocks:
text = "\n".join(block["lines"]).strip()
if not text:
continue
has_entry = any(is_toc_entry(line) for line in block["lines"])
if block["kind"] == BLOCK_TOC and not has_entry:
# Only a title line, with no entries under it: this is a heading, not contents.
output.append({"kind": BLOCK_BODY, "text": text, "substantive": False})
continue
output.append({"kind": block["kind"], "text": text})
# A block made only of headings carries navigation, not content.
for block in output:
if block["kind"] == BLOCK_BODY:
block["substantive"] = (any(_is_prose_line(line) for line in block["text"].splitlines())
or len(block["text"]) >= MIN_BODY_CHARS)
else:
block["substantive"] = False
return output
def classifier_signals(pages: list[dict]) -> list[dict]:
"""Per-page signals, shared by classification and by audit output."""
total = len(pages)
return [_signals(page, index=position, total=total) for position, page in enumerate(pages, start=1)]
def _is_cover(signal: dict) -> bool:
"""A cover page carries identity fields, not prose.
The weak rule requires a first page of a document that actually has a body, with
label-like lines and no sentence punctuation. Without those guards a short
single-page document would be mistaken for a cover page and lose all its content.
"""
if signal["cover_fields"] >= 2 and signal["body_chars"] < 600:
return True
return (signal["position"] == 1 and signal["total"] >= 3 and signal["chars"] <= 400
and signal["lines"] <= 8 and signal["toc_entries"] == 0 and signal["headings"] == 0
and not signal["has_table"] and not signal["prose_punctuation"])
def _is_toc_page(signal: dict, *, previous_is_toc: bool) -> bool:
if signal["toc_entries"] >= 4 and signal["toc_ratio"] >= 0.4:
return True
if _TOC_TITLE.match(signal["first_line"]) and signal["toc_entries"] >= 2:
return True
# Continuation page: the previous page is a table of contents and this page is
# still almost nothing but dotted entries.
return bool(previous_is_toc and signal["toc_entries"] >= 2 and signal["toc_ratio"] >= 0.5)
def _looks_heading_only(signal: dict, blocks: list[dict]) -> bool:
"""A page that carries no running prose at all, so it proves no content.
The test is structural, not a length test: a short page with one real sentence is
body text, a page with a table carries data, while a page of nothing but headings
or labels cannot support a judgement about what was done.
"""
if signal["has_table"]:
return False
if any(block["kind"] == BLOCK_BODY and block.get("substantive") for block in blocks):
return False
return not signal["prose_lines"]
def classify_document(pages: list[dict]) -> list[dict]:
"""Classify every page and extract real section paths from body headings only."""
repeated = strip_repeated_lines(pages)
# The document's own contents list is the authority on which numbered lines are
# real headings; this keeps enumerated list items and equations out of the path.
toc_titles = set()
for page in pages:
for line in _lines(str(page.get("text") or "")):
if is_toc_entry(line):
title = normalize_heading(line)
if title:
toc_titles.add(title)
toc_titles = toc_titles or None
signals = [_signals(page, index=position, total=len(pages), toc_titles=toc_titles, repeated=repeated)
for position, page in enumerate(pages, start=1)]
results: list[dict] = []
previous = None
chapter = ""
section = ""
for position, page in enumerate(pages, start=1):
signal = signals[position - 1]
lines = clean_page_lines(page, repeated)
blocks = split_blocks(lines)
toc_blocks = [block for block in blocks if block["kind"] == BLOCK_TOC]
body_blocks = [block for block in blocks if block["kind"] == BLOCK_BODY and block["substantive"]]
first = signal["first_line"]
appendix = signal["appendix_title"]
if _TOC_TITLE.match(first) and signal["toc_entries"] >= 1:
page_type, confidence = "toc", 0.95
elif _ABSTRACT_TITLE.match(first) or (previous == "abstract" and signal["footer"] in ("roman", "none")
and signal["headings"] == 0 and signal["chars"] >= 100):
page_type, confidence = "abstract", 0.9
elif _REFERENCE_TITLE.match(first) or _ACK_TITLE.match(first):
page_type, confidence = "references", 0.9
elif _is_toc_page(signal, previous_is_toc=(previous == "toc")):
page_type, confidence = "toc", 0.85 if signal["toc_ratio"] >= 0.4 else 0.75
elif previous == "references" and signal["headings"] == 0 and signal["toc_entries"] == 0:
page_type, confidence = "references", 0.7
elif signal["reference_page"] and signal["toc_entries"] == 0 and signal["chapter_headings"] <= 1:
page_type, confidence = "references", 0.8
elif appendix:
# An appendix carries substantive material, so it counts as content, but it
# is labelled separately so the interface can show where a quotation came from.
page_type, confidence = "appendix", 0.8
elif _is_cover(signal):
# Checked before the heading-only rule: a cover page carries identity fields
# and no prose, so it would otherwise be mistaken for an unsettled page.
page_type, confidence = "cover", 0.8
elif _looks_heading_only(signal, blocks) and signal["toc_entries"] == 0:
# A body page that carries only a section title cannot support a content
# judgement on its own, so it is not silently treated as ordinary body text.
page_type, confidence = "unknown", 0.5
else:
page_type, confidence = "body", 0.6
confidence = 0.85 if body_blocks and signal["toc_entries"] == 0 else confidence
mixed = page_type == "toc" and bool(body_blocks)
if mixed:
confidence = 0.7
if page_type in ("body", "appendix", "unknown") or mixed:
# Structure-extracted headings come from body text only, never from a TOC.
for block in body_blocks:
for line in block["text"].splitlines():
found = heading_level(line, toc_titles)
if not found:
continue
level, title = found
if level == 1:
chapter, section = title, title
else:
section = title
# A table of contents may only name the chapter path; the body supplies the
# section wording. Navigation pages therefore carry the chapter, not a section.
section_for_page = section if page_type in ("body", "unknown") or mixed else chapter
results.append({
"page": position,
"page_type": page_type,
"mixed": mixed,
"chapter": chapter,
"section": section_for_page,
"blocks": blocks,
"confidence": round(confidence, 2),
"signals": {**signal, "repeated_line_count": len(repeated), "toc_blocks": len(toc_blocks)},
"classifier_version": CLASSIFIER_VERSION,
})
previous = page_type
return results
def block_text(blocks: list[dict], *, kinds: tuple[str, ...] = (BLOCK_BODY,)) -> str:
return "\n".join(block["text"] for block in blocks if block["kind"] in kinds).strip()
def toc_entries(blocks: list[dict]) -> list[str]:
"""Heading-shaped entries read out of a table of contents page."""
entries = []
for block in blocks:
if block["kind"] != BLOCK_TOC:
continue
for line in block["text"].splitlines():
if is_toc_entry(line):
entries.append(line)
return entries
def searchable_pages(pages: list[dict]) -> list[dict]:
"""Pages prepared for indexing: cleaned lines, typed blocks and section paths."""
classification = classify_document(pages)
output = []
for page, item in zip(pages, classification):
body = block_text(item["blocks"])
output.append({
"page_number": page.get("page_number"),
"section": item["section"] or page.get("section", ""),
"chapter": item["chapter"],
"text": item.get("text", ""),
"tables": page.get("tables", []),
"rendered_image_path": page.get("rendered_image_path"),
"page_type": item["page_type"],
"blocks": item["blocks"],
"classification": item,
"index_text": body,
})
return output