"""Persist canonical knowledge page text and image locations. Revision ID: 0005 Revises: 0004 """ import hashlib import json import re from collections import defaultdict from pathlib import Path from alembic import op import sqlalchemy as sa from wisegrade.config import get_settings revision = "0005" down_revision = "0004" branch_labels = None depends_on = None def _json(value): return json.loads(value) if isinstance(value, str) else value or {} def _text(source): blocks = [str(source.get("text") or "").strip()] blocks.extend("表格: " + str(table.get("rows", table)) for table in source.get("tables", [])) return "\n".join(block for block in blocks if block) def _relative_image(root, value): if not value: return None path = Path(value).resolve() return str(path.relative_to(root)) if path.is_relative_to(root) and path.is_file() else None def _restore_chunk_text(parts): """Undo the 100-character overlap used by index version 2.""" if not parts: return "" blocks = [] current = previous = parts[0] for part in parts[1:]: if len(previous) == 650 and previous[-100:] == part[:100]: current += part[100:] else: blocks.append(current) current = part previous = part blocks.append(current) return "\n".join(blocks) _PAGE_URL = re.compile(r"/api/knowledge/documents/(\d+)/pages/(\d+)/?") _ASSET_URL = re.compile(r"/api/assets/(\d+)/(\d+)/?") def _migrate_references(db): """Rewrite saved citations once; API reads the stored canonical links as-is.""" pages = {(row.document_id, row.page): row for row in db.execute(sa.text( "SELECT document_id, page, text, image_path FROM knowledge_pages")).mappings()} evidence = defaultdict(list) for row in db.execute(sa.text( "SELECT submission_file_id, page, quote, document_id FROM score_evidences " "WHERE submission_file_id IS NOT NULL AND document_id IS NOT NULL")).mappings(): evidence[row.submission_file_id, row.page].append((row.quote, row.document_id)) def normalize(item): if not isinstance(item, dict): return item old_url = item.get("url") raw_page = item.get("page") identity = None if isinstance(item.get("document_id"), int) and isinstance(raw_page, int): identity = (item["document_id"], raw_page) for raw in (item.get("text_url"), old_url): match = _PAGE_URL.fullmatch(str(raw or "")) if match: from_url = (int(match[1]), int(match[2])) if identity and identity != from_url: raise RuntimeError("旧引用的文档 ID 与页面链接冲突") identity = from_url asset = next((match for raw in (old_url, item.get("image_url")) if (match := _ASSET_URL.fullmatch(str(raw or "")))), None) if not identity and asset and isinstance(item.get("quote"), str) and item["quote"]: file_id, page = int(asset[1]), int(asset[2]) quote = item["quote"] matches = {document_id for saved, document_id in evidence[file_id, page] if saved.startswith(quote) and (record := pages.get((document_id, page))) and quote in record.text} if len(matches) != 1: raise RuntimeError("旧评分引用无法唯一对应资料页，请先核对评分证据") identity = (matches.pop(), page) if not identity: return item if raw_page is not None and raw_page != identity[1]: raise RuntimeError("旧引用的页码与来源链接冲突") record = pages.get(identity) if not record: raise RuntimeError("旧引用对应的资料页不存在") base = f"/api/knowledge/documents/{identity[0]}/pages/{identity[1]}" value = {**item, "document_id": identity[0], "page": identity[1], "url": base, "image_url": base + "/image" if record.image_path else None} if isinstance(old_url, str) and old_url.startswith("/?page=scores"): value["score_url"] = old_url value.pop("text_url", None) return value def walk(value): if isinstance(value, list): return [walk(item) for item in value] if isinstance(value, dict): return normalize({key: walk(item) for key, item in value.items()}) return value for table, columns in (("chat_messages", ("citations", "tool_calls")), ("agent_runs", ("tool_calls", "events")), ("pending_actions", ("preview", "result"))): for row in db.execute(sa.text(f"SELECT id, {', '.join(columns)} FROM {table}")).mappings(): changed = {} for column in columns: old = row[column] new = walk(old) if new != old: changed[column] = new if changed: assignments = ", ".join(f"{column} = :{column}" for column in changed) statement = sa.text(f"UPDATE {table} SET {assignments} WHERE id = :id") for column in changed: statement = statement.bindparams(sa.bindparam(column, type_=sa.JSON())) db.execute(statement, {"id": row.id, **changed}) def upgrade(): if "knowledge_pages" not in sa.inspect(op.get_bind()).get_table_names(): op.create_table( "knowledge_pages", sa.Column("document_id", sa.Integer(), sa.ForeignKey("knowledge_documents.id", ondelete="CASCADE"), primary_key=True), sa.Column("page", sa.Integer(), primary_key=True), sa.Column("text", sa.Text(), nullable=False), sa.Column("image_path", sa.String(1200), nullable=True), ) db = op.get_bind() root = get_settings().processed_path.resolve() documents = db.execute(sa.text( "SELECT id, course_version_id, submission_file_id, title FROM knowledge_documents")).mappings().all() files = {row.id: _json(row.parsed) for row in db.execute(sa.text( "SELECT id, parsed FROM submission_files")).mappings()} templates = {row.id: _json(row.templates) for row in db.execute(sa.text( "SELECT id, templates FROM course_versions")).mappings()} chunks = defaultdict(list) for row in db.execute(sa.text( "SELECT document_id, page, text FROM knowledge_chunks ORDER BY document_id, page, id")).mappings(): chunks[row.document_id, row.page].append(row.text) insert = sa.text("INSERT INTO knowledge_pages (document_id, page, text, image_path) " "VALUES (:document_id, :page, :text, :image_path) ON CONFLICT DO NOTHING") pages_by_document = defaultdict(list) for (document_id, page), parts in chunks.items(): pages_by_document[document_id].append((page, parts)) for doc in documents: parsed_pages = {item.get("page_number"): item for item in files.get(doc.submission_file_id, {}).get("pages", [])} for page in parsed_pages: pages_by_document[doc.id].append((page, chunks.get((doc.id, page), []))) seen = set() for page, parts in pages_by_document[doc.id]: if page in seen: continue seen.add(page) source = parsed_pages.get(page) text = _text(source) if source else _restore_chunk_text(parts) image = _relative_image(root, source.get("rendered_image_path")) if source else None if not image and doc.submission_file_id is None and doc.title in templates.get(doc.course_version_id, []): folder = hashlib.sha256(doc.title.encode()).hexdigest()[:16] image = _relative_image(root, root / f"course-{doc.course_version_id}" / folder / f"page-{page}.png") db.execute(insert, {"document_id": doc.id, "page": page, "text": text, "image_path": image}) _migrate_references(db) def downgrade(): op.drop_table("knowledge_pages")