fix: improve import, evidence indexing and material-scoped grading

Preserve document revisions and evidence, harden indexing and export behavior, add explicit material selection and direct-folder import, validate model JSON schemas with safe error messages, and retain student selection while loading scoring data.

backend/wisegrade/parsing.py

@@ -1,4 +1,5 @@
 import subprocess
+import re
 import tempfile
 from functools import lru_cache
 from importlib.metadata import version
@@ -15,6 +16,60 @@ from .config import get_settings
 _ocr_lock = Lock()
 
 
+def text_quality(text: str) -> dict:
+    unresolved = sum(text.count(char) for char in ("\ufffd", "\ufffe", "\uffff"))
+    return {"unresolved_characters": unresolved,
+            "needs_review": bool(unresolved),
+            "warning": "文字层含未解析字符，请核对原页；相关片段不用于自动内容评分" if unresolved else ""}
+
+
+def _extract_tables(page):
+    """Extract grids, then bounded three-line tables without treating prose as a grid."""
+    result = []
+    try:
+        for table in page.find_tables().tables:
+            result.append({"rows": table.extract(), "bbox": list(table.bbox), "method": "lines"})
+    except Exception:
+        pass
+    # A three-line table has aligned, long horizontal rules and no vertical rules.
+    # Group by x extent; do not infer unbounded tables from whole-page prose.
+    groups = {}
+    for drawing in page.get_drawings():
+        for item in drawing["items"]:
+            if item[0] != "l":
+                continue
+            a, b = item[1], item[2]
+            left, right = sorted((a.x, b.x))
+            if abs(a.y - b.y) > 1 or right - left < page.rect.width * .4:
+                continue
+            key = (round(left / 3), round(right / 3))
+            groups.setdefault(key, []).append((a.y, left, right))
+    for rules in groups.values():
+        rules = sorted(set(rules))
+        for offset in range(0, len(rules) - 2, 3):
+            top, header, bottom = rules[offset:offset + 3]
+            if not (5 < header[0] - top[0] < 60 and bottom[0] - header[0] > 15):
+                continue
+            clip = fitz.Rect(top[1], top[0], top[2], bottom[0])
+            header_text = page.get_text(clip=fitz.Rect(top[1], top[0], top[2], header[0]), sort=True)
+            if (any(marker in header_text for marker in ("//", "#include", "{", "}", ";"))
+                    or re.search(r"\bListing\s+\d+", page.get_text(clip=clip), re.I)):
+                continue  # Code-listing boxes are not tabular data.
+            if any(clip.intersects(fitz.Rect(t["bbox"])) for t in result):
+                continue
+            try:
+                found = page.find_tables(clip=clip, strategy="text", min_words_vertical=2)
+                for table in found.tables:
+                    rows = [row for row in table.extract() if any(str(cell or "").strip() for cell in row)]
+                    if len(rows) >= 2 and table.col_count >= 2:
+                        result.append({"rows": rows, "bbox": list(clip), "method": "three_line_text",
+                                       "text": page.get_text(clip=clip, sort=True),
+                                       "warning": "列边界由文字对齐推断，请核对表头及跨列内容"})
+            except Exception:
+                continue
+    return result
+
+
 def _render(page, output: Path):
     output.parent.mkdir(parents=True, exist_ok=True)
     page.get_pixmap(matrix=fitz.Matrix(1.5, 1.5), alpha=False).save(output)
@@ -78,15 +133,11 @@ def parse_pdf(path: Path, asset_dir: Path, *, ocr=True):
             text = page.get_text(sort=True).strip()
             if ocr and len(text) < 30:
                 text = _ocr(image)
-            tables = []
-            try:
-                tables = [{"rows": table.extract(), "bbox": list(table.bbox)} for table in page.find_tables().tables]
-            except Exception:
-                pass
+            tables = _extract_tables(page)
             images = [{"xref": item[0], "width": item[2], "height": item[3]} for item in page.get_images(full=True)]
             pages.append({"page_number": index + 1, "text": text, "tables": tables, "images": images,
                           "section": next((line.strip() for line in text.splitlines() if line.strip()), ""),
-                          "rendered_image_path": image})
+                          "rendered_image_path": image, "text_quality": text_quality(text)})
     return pages