444 lines
20 KiB
Python
444 lines
20 KiB
Python
"""Page-level classification, block splitting and real section paths.
|
||
|
||
A page carries one ``page_type``: cover, abstract, toc, body, references, unknown.
|
||
A page may additionally be ``mixed`` when a table of contents and body text truly
|
||
share one page; only that page is split into blocks so the body part is not lost.
|
||
|
||
Classification never depends on a fixed page number or on the single word "目录".
|
||
It combines the page's own signals (heading title, dot-leader entry density,
|
||
trailing page numbers, abstract/reference wording, cover fields, footer style)
|
||
with document-level continuity for continuation pages (TOC continued, abstract
|
||
continued, references spanning pages).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import unicodedata
|
||
|
||
CLASSIFIER_VERSION = 3
|
||
|
||
PAGE_TYPES = ("cover", "abstract", "toc", "body", "appendix", "references", "unknown")
|
||
|
||
#: Page types whose text may support a content judgement about what a student did.
|
||
CONTENT_PAGE_TYPES = ("body", "appendix")
|
||
#: Page types whose text may support a structural judgement (chapter completeness etc.).
|
||
STRUCTURE_PAGE_TYPES = ("cover", "abstract", "toc", "references", "body", "appendix")
|
||
#: Page types that may never be used as content evidence on their own.
|
||
NAVIGATION_PAGE_TYPES = ("cover", "abstract", "toc", "references")
|
||
#: Page types a page may be stored as, used to validate rule configuration.
|
||
STORE_PAGE_TYPES = PAGE_TYPES
|
||
|
||
BLOCK_BODY = "body"
|
||
BLOCK_TOC = "toc"
|
||
BLOCK_TITLE = "title"
|
||
|
||
_DOT_LEADER = re.compile(r"(?:[.·•…․…]\s*){3,}")
|
||
_TRAILING_ANCHOR = re.compile(r"(\d{1,4})\s*$")
|
||
_CHAPTER = re.compile(r"^第\s*([0-9一二三四五六七八九十百]+)\s*章\s*(.*)$")
|
||
_APPENDIX = re.compile(r"^附\s*录\s*([A-Za-z0-9一二三四五六七八九十]+)?\s*(.*)$")
|
||
_NUMBERED = re.compile(r"^(\d+(?:\.\d+)*)[\s、.．]+(\S.*)$")
|
||
_COVER_FIELD = re.compile(r"(学\s*号|姓\s*名|题\s*目|指导教师|专\s*业|学\s*院|日\s*期)\s*[:：]")
|
||
_PAGE_NUMBER_LINE = re.compile(r"^\s*(?:\d{1,3}|[IVXLC]{1,5}|第\s*\d+\s*页)\s*$")
|
||
_ABSTRACT_TITLE = re.compile(r"^\s*(摘\s*要|abstract)\s*$", re.IGNORECASE)
|
||
_REFERENCE_TITLE = re.compile(r"^\s*(参\s*考\s*文\s*献|references?|参考文献与资料)\s*$", re.IGNORECASE)
|
||
_ACK_TITLE = re.compile(r"^\s*(致\s*谢|acknowledge?ments?)\s*$", re.IGNORECASE)
|
||
_TOC_TITLE = re.compile(r"^\s*(目\s*录|contents?|table\s+of\s+contents)\s*$", re.IGNORECASE)
|
||
_REFERENCE_ENTRY = re.compile(r"^\s*\[\d{1,3}\]")
|
||
_CJK = re.compile(r"[㐀-䶿一-鿿]")
|
||
_HEADING_NUMBER = re.compile(r"^\s*(第\s*[0-9一二三四五六七八九十百]+\s*章|\d+(\.\d+)*|附\s*录\s*[A-Za-z0-9]?)\s*\S")
|
||
|
||
#: A body block must carry at least this many non-space characters to count as substance.
|
||
MIN_BODY_CHARS = 60
|
||
#: Heading-only text shorter than this never counts as content evidence on its own.
|
||
MIN_CONTENT_QUOTE_CHARS = 25
|
||
#: A line at least this long, outside a heading, reads as running prose. The bar is low
|
||
#: on purpose: the rule separates text from headings and labels, not long from short
|
||
#: text, so a brief but real sentence is still content.
|
||
MIN_PROSE_CHARS = 4
|
||
#: A numbered heading title must stay this short; longer text is prose or a list item.
|
||
MAX_HEADING_TITLE_CHARS = 40
|
||
#: Wording that marks the line as running prose rather than a heading.
|
||
_PROSE_PUNCTUATION = "。，；！？,;"
|
||
|
||
#: The documents are Chinese engineering reports; the keyword recall uses these units.
|
||
BIGRAM_MIN = 2
|
||
|
||
|
||
def _lines(text: str) -> list[str]:
|
||
return [line.strip() for line in (text or "").splitlines() if line.strip()]
|
||
|
||
|
||
def normalize_heading(text: str) -> str:
|
||
"""Compare headings across the TOC and the body: unified width, no spaces/dots."""
|
||
value = unicodedata.normalize("NFKC", text or "")
|
||
value = _DOT_LEADER.sub("", value)
|
||
value = _TRAILING_ANCHOR.sub("", value.strip())
|
||
return "".join(char for char in value if not char.isspace() and char not in "．.·•…")
|
||
|
||
|
||
def is_toc_entry(line: str) -> bool:
|
||
"""A dotted leader plus a trailing page or slide number."""
|
||
stripped = (line or "").strip()
|
||
if not stripped or not _DOT_LEADER.search(stripped):
|
||
return False
|
||
return bool(_TRAILING_ANCHOR.search(stripped))
|
||
|
||
|
||
def heading_syntax(line: str) -> tuple[int, str] | None:
|
||
"""Return ``(level, title)`` when the line has the shape of a heading.
|
||
|
||
This is a purely syntactic reading. Numbered prose such as
|
||
``4. 实时图像传输：将摄像头画面…`` and enumerated list items are not headings: a
|
||
numbered heading has a short Chinese title, no sentence punctuation and no trailing
|
||
colon. Equation and matrix rows such as ``3.3.3`` or ``1 0 0 1 Zc`` are never
|
||
headings, and a line cut mid-formula (trailing combining mark) is not either.
|
||
"""
|
||
stripped = (line or "").strip()
|
||
if not stripped or is_toc_entry(stripped) or len(stripped) > 120:
|
||
return None
|
||
match = _CHAPTER.match(stripped)
|
||
if match:
|
||
return 1, stripped
|
||
match = _APPENDIX.match(stripped)
|
||
if match and not _COVER_FIELD.search(stripped):
|
||
return 2, stripped
|
||
if _ABSTRACT_TITLE.match(stripped) or _REFERENCE_TITLE.match(stripped) or _ACK_TITLE.match(stripped):
|
||
return 1, stripped
|
||
match = _NUMBERED.match(stripped)
|
||
if not match:
|
||
return None
|
||
title = match.group(2).strip()
|
||
if not title or title.endswith(":") or title.endswith("："):
|
||
return None
|
||
if len(title) > MAX_HEADING_TITLE_CHARS or any(char in title for char in _PROSE_PUNCTUATION):
|
||
return None
|
||
if not _CJK.search(title):
|
||
return None
|
||
if unicodedata.category(title[-1]).startswith("M"):
|
||
return None
|
||
return 1 + match.group(1).count("."), stripped
|
||
|
||
|
||
def heading_level(line: str, toc_titles: set[str] | None = None) -> tuple[int, str] | None:
|
||
"""A heading for the section path: syntactic shape, confirmed by the contents list.
|
||
|
||
When the document carries its own table of contents, that list is the authority on
|
||
which numbered lines are real headings. That is what keeps an enumerated list item or
|
||
a numbered paragraph from becoming a chapter in the extracted section path. Callers
|
||
that only need to know whether a line reads as a heading use ``heading_syntax``.
|
||
"""
|
||
found = heading_syntax(line)
|
||
if not found:
|
||
return None
|
||
if toc_titles and normalize_heading(line) not in toc_titles:
|
||
return None
|
||
return found
|
||
|
||
|
||
def _footer_style(lines: list[str]) -> str:
|
||
"""Classify the trailing line so repeated headers/footers can be removed."""
|
||
if not lines:
|
||
return "none"
|
||
last = lines[-1]
|
||
if re.fullmatch(r"[IVXLC]{1,5}", last):
|
||
return "roman"
|
||
if re.fullmatch(r"\d{1,3}", last):
|
||
return "arabic"
|
||
if re.fullmatch(r"第\s*\d+\s*页", last):
|
||
return "page_cn"
|
||
return "text"
|
||
|
||
|
||
def _signals(page: dict, *, index: int, total: int, toc_titles: set[str] | None = None,
|
||
repeated: set[str] | None = None) -> dict:
|
||
text = str(page.get("text") or "")
|
||
lines = _lines(text)
|
||
# Prose detection ignores repeated page headers and footers: a running header is not
|
||
# something the student wrote, so it must not make an empty page look like content.
|
||
prose_source = [line for line in lines if line not in (repeated or ())]
|
||
entries = [line for line in lines if is_toc_entry(line)]
|
||
headings = [item for item in (heading_level(line, toc_titles) for line in lines) if item]
|
||
body_lines = [line for line in lines if line not in set(entries)]
|
||
plain_chars = sum(len(line) for line in body_lines)
|
||
first = lines[0] if lines else ""
|
||
reference_entries = [line for line in lines if _REFERENCE_ENTRY.match(line)]
|
||
titles = [position for position, line in enumerate(lines)
|
||
if _REFERENCE_TITLE.match(line) or _ACK_TITLE.match(line)]
|
||
# A bibliography page is dense reference entries, or a reference title near the top.
|
||
reference_ratio = len(reference_entries) / len(lines) if lines else 0.0
|
||
reference_page = (len(reference_entries) >= 5 and reference_ratio >= 0.5
|
||
or any(position <= 2 for position in titles))
|
||
return {
|
||
"chars": len(text.strip()),
|
||
"lines": len(lines),
|
||
"toc_entries": len(entries),
|
||
"toc_ratio": round(len(entries) / len(lines), 3) if lines else 0.0,
|
||
"headings": len(headings),
|
||
"chapter_headings": sum(1 for level, _ in headings if level == 1),
|
||
"body_chars": plain_chars,
|
||
"first_line": first[:120],
|
||
"position": index,
|
||
"total": total,
|
||
"footer": _footer_style(lines),
|
||
"cover_fields": len(_COVER_FIELD.findall(text)),
|
||
"has_table": bool(page.get("tables")),
|
||
"reference_entries": len(reference_entries),
|
||
"reference_ratio": round(reference_ratio, 3),
|
||
"reference_page": reference_page,
|
||
"appendix_title": any(_APPENDIX.match(line.strip()) for line in lines),
|
||
# Prose punctuation separates running text from a label-style cover page.
|
||
"prose_punctuation": any(char in text for char in _PROSE_PUNCTUATION),
|
||
# Lines that read as running text rather than headings, labels or dotted entries.
|
||
"prose_lines": [line for line in prose_source if _is_prose_line(line)],
|
||
}
|
||
|
||
|
||
def strip_repeated_lines(pages: list[dict]) -> set[str]:
|
||
"""Header/footer lines repeated across many pages, removed from searchable text."""
|
||
counts: dict[str, int] = {}
|
||
for page in pages:
|
||
lines = _lines(str(page.get("text") or ""))
|
||
if not lines:
|
||
continue
|
||
candidates = {lines[0][:120]}
|
||
if len(lines) > 1:
|
||
candidates.add(lines[-1][:60])
|
||
for candidate in candidates:
|
||
if candidate and not is_toc_entry(candidate):
|
||
counts[candidate] = counts.get(candidate, 0) + 1
|
||
threshold = max(3, int(len(pages) * 0.4 + 0.5))
|
||
return {line for line, count in counts.items() if count >= threshold}
|
||
|
||
|
||
def clean_page_lines(page: dict, repeated: set[str]) -> list[str]:
|
||
"""Lines kept for indexing: repeated headers/footers and bare page numbers removed."""
|
||
lines = _lines(str(page.get("text") or ""))
|
||
kept = []
|
||
for position, line in enumerate(lines):
|
||
if line in repeated and (position == 0 or position == len(lines) - 1):
|
||
continue
|
||
if _PAGE_NUMBER_LINE.match(line) and position == len(lines) - 1:
|
||
continue
|
||
kept.append(line)
|
||
return kept
|
||
|
||
|
||
def _is_prose_line(line: str) -> bool:
|
||
"""A line that reads as running text rather than a heading, label or dotted entry.
|
||
|
||
The syntactic heading check is used, not the contents-confirmed one: a heading that
|
||
the contents list happens not to mention is still a heading, so it must not be
|
||
counted as prose when deciding whether a page carries content.
|
||
"""
|
||
stripped = line.strip()
|
||
if is_toc_entry(stripped) or _TOC_TITLE.match(stripped) or heading_syntax(stripped):
|
||
return False
|
||
if _PAGE_NUMBER_LINE.match(stripped):
|
||
return False
|
||
return len(stripped) >= MIN_PROSE_CHARS
|
||
|
||
|
||
def split_blocks(lines: list[str]) -> list[dict]:
|
||
"""Split page lines into TOC and body blocks so a mixed page keeps both parts."""
|
||
blocks: list[dict] = []
|
||
current: dict | None = None
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
# A contents title and its dotted entries are one navigation block. A lone
|
||
# dotted line with no title is not a table of contents, so it stays as body.
|
||
kind = BLOCK_TOC if is_toc_entry(line) or _TOC_TITLE.match(stripped) else BLOCK_BODY
|
||
if current is None or current["kind"] != kind:
|
||
current = {"kind": kind, "lines": []}
|
||
blocks.append(current)
|
||
current["lines"].append(line)
|
||
output = []
|
||
for block in blocks:
|
||
text = "\n".join(block["lines"]).strip()
|
||
if not text:
|
||
continue
|
||
has_entry = any(is_toc_entry(line) for line in block["lines"])
|
||
if block["kind"] == BLOCK_TOC and not has_entry:
|
||
# Only a title line, with no entries under it: this is a heading, not contents.
|
||
output.append({"kind": BLOCK_BODY, "text": text, "substantive": False})
|
||
continue
|
||
output.append({"kind": block["kind"], "text": text})
|
||
# A block made only of headings carries navigation, not content.
|
||
for block in output:
|
||
if block["kind"] == BLOCK_BODY:
|
||
block["substantive"] = (any(_is_prose_line(line) for line in block["text"].splitlines())
|
||
or len(block["text"]) >= MIN_BODY_CHARS)
|
||
else:
|
||
block["substantive"] = False
|
||
return output
|
||
|
||
|
||
def classifier_signals(pages: list[dict]) -> list[dict]:
|
||
"""Per-page signals, shared by classification and by audit output."""
|
||
total = len(pages)
|
||
return [_signals(page, index=position, total=total) for position, page in enumerate(pages, start=1)]
|
||
|
||
|
||
def _is_cover(signal: dict) -> bool:
|
||
"""A cover page carries identity fields, not prose.
|
||
|
||
The weak rule requires a first page of a document that actually has a body, with
|
||
label-like lines and no sentence punctuation. Without those guards a short
|
||
single-page document would be mistaken for a cover page and lose all its content.
|
||
"""
|
||
if signal["cover_fields"] >= 2 and signal["body_chars"] < 600:
|
||
return True
|
||
return (signal["position"] == 1 and signal["total"] >= 3 and signal["chars"] <= 400
|
||
and signal["lines"] <= 8 and signal["toc_entries"] == 0 and signal["headings"] == 0
|
||
and not signal["has_table"] and not signal["prose_punctuation"])
|
||
|
||
|
||
def _is_toc_page(signal: dict, *, previous_is_toc: bool) -> bool:
|
||
if signal["toc_entries"] >= 4 and signal["toc_ratio"] >= 0.4:
|
||
return True
|
||
if _TOC_TITLE.match(signal["first_line"]) and signal["toc_entries"] >= 2:
|
||
return True
|
||
# Continuation page: the previous page is a table of contents and this page is
|
||
# still almost nothing but dotted entries.
|
||
return bool(previous_is_toc and signal["toc_entries"] >= 2 and signal["toc_ratio"] >= 0.5)
|
||
|
||
|
||
def _looks_heading_only(signal: dict, blocks: list[dict]) -> bool:
|
||
"""A page that carries no running prose at all, so it proves no content.
|
||
|
||
The test is structural, not a length test: a short page with one real sentence is
|
||
body text, a page with a table carries data, while a page of nothing but headings
|
||
or labels cannot support a judgement about what was done.
|
||
"""
|
||
if signal["has_table"]:
|
||
return False
|
||
if any(block["kind"] == BLOCK_BODY and block.get("substantive") for block in blocks):
|
||
return False
|
||
return not signal["prose_lines"]
|
||
|
||
|
||
def classify_document(pages: list[dict]) -> list[dict]:
|
||
"""Classify every page and extract real section paths from body headings only."""
|
||
repeated = strip_repeated_lines(pages)
|
||
# The document's own contents list is the authority on which numbered lines are
|
||
# real headings; this keeps enumerated list items and equations out of the path.
|
||
toc_titles = set()
|
||
for page in pages:
|
||
for line in _lines(str(page.get("text") or "")):
|
||
if is_toc_entry(line):
|
||
title = normalize_heading(line)
|
||
if title:
|
||
toc_titles.add(title)
|
||
toc_titles = toc_titles or None
|
||
signals = [_signals(page, index=position, total=len(pages), toc_titles=toc_titles, repeated=repeated)
|
||
for position, page in enumerate(pages, start=1)]
|
||
results: list[dict] = []
|
||
previous = None
|
||
chapter = ""
|
||
section = ""
|
||
for position, page in enumerate(pages, start=1):
|
||
signal = signals[position - 1]
|
||
lines = clean_page_lines(page, repeated)
|
||
blocks = split_blocks(lines)
|
||
toc_blocks = [block for block in blocks if block["kind"] == BLOCK_TOC]
|
||
body_blocks = [block for block in blocks if block["kind"] == BLOCK_BODY and block["substantive"]]
|
||
first = signal["first_line"]
|
||
|
||
appendix = signal["appendix_title"]
|
||
if _TOC_TITLE.match(first) and signal["toc_entries"] >= 1:
|
||
page_type, confidence = "toc", 0.95
|
||
elif _ABSTRACT_TITLE.match(first) or (previous == "abstract" and signal["footer"] in ("roman", "none")
|
||
and signal["headings"] == 0 and signal["chars"] >= 100):
|
||
page_type, confidence = "abstract", 0.9
|
||
elif _REFERENCE_TITLE.match(first) or _ACK_TITLE.match(first):
|
||
page_type, confidence = "references", 0.9
|
||
elif _is_toc_page(signal, previous_is_toc=(previous == "toc")):
|
||
page_type, confidence = "toc", 0.85 if signal["toc_ratio"] >= 0.4 else 0.75
|
||
elif previous == "references" and signal["headings"] == 0 and signal["toc_entries"] == 0:
|
||
page_type, confidence = "references", 0.7
|
||
elif signal["reference_page"] and signal["toc_entries"] == 0 and signal["chapter_headings"] <= 1:
|
||
page_type, confidence = "references", 0.8
|
||
elif appendix:
|
||
# An appendix carries substantive material, so it counts as content, but it
|
||
# is labelled separately so the interface can show where a quotation came from.
|
||
page_type, confidence = "appendix", 0.8
|
||
elif _is_cover(signal):
|
||
# Checked before the heading-only rule: a cover page carries identity fields
|
||
# and no prose, so it would otherwise be mistaken for an unsettled page.
|
||
page_type, confidence = "cover", 0.8
|
||
elif _looks_heading_only(signal, blocks) and signal["toc_entries"] == 0:
|
||
# A body page that carries only a section title cannot support a content
|
||
# judgement on its own, so it is not silently treated as ordinary body text.
|
||
page_type, confidence = "unknown", 0.5
|
||
else:
|
||
page_type, confidence = "body", 0.6
|
||
confidence = 0.85 if body_blocks and signal["toc_entries"] == 0 else confidence
|
||
|
||
mixed = page_type == "toc" and bool(body_blocks)
|
||
if mixed:
|
||
confidence = 0.7
|
||
if page_type in ("body", "appendix", "unknown") or mixed:
|
||
# Structure-extracted headings come from body text only, never from a TOC.
|
||
for block in body_blocks:
|
||
for line in block["text"].splitlines():
|
||
found = heading_level(line, toc_titles)
|
||
if not found:
|
||
continue
|
||
level, title = found
|
||
if level == 1:
|
||
chapter, section = title, title
|
||
else:
|
||
section = title
|
||
# A table of contents may only name the chapter path; the body supplies the
|
||
# section wording. Navigation pages therefore carry the chapter, not a section.
|
||
section_for_page = section if page_type in ("body", "unknown") or mixed else chapter
|
||
results.append({
|
||
"page": position,
|
||
"page_type": page_type,
|
||
"mixed": mixed,
|
||
"chapter": chapter,
|
||
"section": section_for_page,
|
||
"blocks": blocks,
|
||
"confidence": round(confidence, 2),
|
||
"signals": {**signal, "repeated_line_count": len(repeated), "toc_blocks": len(toc_blocks)},
|
||
"classifier_version": CLASSIFIER_VERSION,
|
||
})
|
||
previous = page_type
|
||
return results
|
||
|
||
|
||
def block_text(blocks: list[dict], *, kinds: tuple[str, ...] = (BLOCK_BODY,)) -> str:
|
||
return "\n".join(block["text"] for block in blocks if block["kind"] in kinds).strip()
|
||
|
||
|
||
def toc_entries(blocks: list[dict]) -> list[str]:
|
||
"""Heading-shaped entries read out of a table of contents page."""
|
||
entries = []
|
||
for block in blocks:
|
||
if block["kind"] != BLOCK_TOC:
|
||
continue
|
||
for line in block["text"].splitlines():
|
||
if is_toc_entry(line):
|
||
entries.append(line)
|
||
return entries
|
||
|
||
|
||
def searchable_pages(pages: list[dict]) -> list[dict]:
|
||
"""Pages prepared for indexing: cleaned lines, typed blocks and section paths."""
|
||
classification = classify_document(pages)
|
||
output = []
|
||
for page, item in zip(pages, classification):
|
||
body = block_text(item["blocks"])
|
||
output.append({
|
||
"page_number": page.get("page_number"),
|
||
"section": item["section"] or page.get("section", ""),
|
||
"chapter": item["chapter"],
|
||
"text": item.get("text", ""),
|
||
"tables": page.get("tables", []),
|
||
"rendered_image_path": page.get("rendered_image_path"),
|
||
"page_type": item["page_type"],
|
||
"blocks": item["blocks"],
|
||
"classification": item,
|
||
"index_text": body,
|
||
})
|
||
return output
|