184 lines
7.9 KiB
Python
184 lines
7.9 KiB
Python
"""Persist canonical knowledge page text and image locations.
|
|
|
|
Revision ID: 0005
|
|
Revises: 0004
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
|
|
from wisegrade.config import get_settings
|
|
|
|
revision = "0005"
|
|
down_revision = "0004"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def _json(value):
|
|
return json.loads(value) if isinstance(value, str) else value or {}
|
|
|
|
|
|
def _text(source):
|
|
blocks = [str(source.get("text") or "").strip()]
|
|
blocks.extend("表格: " + str(table.get("rows", table)) for table in source.get("tables", []))
|
|
return "\n".join(block for block in blocks if block)
|
|
|
|
|
|
def _relative_image(root, value):
|
|
if not value:
|
|
return None
|
|
path = Path(value).resolve()
|
|
return str(path.relative_to(root)) if path.is_relative_to(root) and path.is_file() else None
|
|
|
|
|
|
def _restore_chunk_text(parts):
|
|
"""Undo the 100-character overlap used by index version 2."""
|
|
if not parts:
|
|
return ""
|
|
blocks = []
|
|
current = previous = parts[0]
|
|
for part in parts[1:]:
|
|
if len(previous) == 650 and previous[-100:] == part[:100]:
|
|
current += part[100:]
|
|
else:
|
|
blocks.append(current)
|
|
current = part
|
|
previous = part
|
|
blocks.append(current)
|
|
return "\n".join(blocks)
|
|
|
|
|
|
_PAGE_URL = re.compile(r"/api/knowledge/documents/(\d+)/pages/(\d+)/?")
|
|
_ASSET_URL = re.compile(r"/api/assets/(\d+)/(\d+)/?")
|
|
|
|
|
|
def _migrate_references(db):
|
|
"""Rewrite saved citations once; API reads the stored canonical links as-is."""
|
|
pages = {(row.document_id, row.page): row for row in db.execute(sa.text(
|
|
"SELECT document_id, page, text, image_path FROM knowledge_pages")).mappings()}
|
|
evidence = defaultdict(list)
|
|
for row in db.execute(sa.text(
|
|
"SELECT submission_file_id, page, quote, document_id FROM score_evidences "
|
|
"WHERE submission_file_id IS NOT NULL AND document_id IS NOT NULL")).mappings():
|
|
evidence[row.submission_file_id, row.page].append((row.quote, row.document_id))
|
|
|
|
def normalize(item):
|
|
if not isinstance(item, dict):
|
|
return item
|
|
old_url = item.get("url")
|
|
raw_page = item.get("page")
|
|
identity = None
|
|
if isinstance(item.get("document_id"), int) and isinstance(raw_page, int):
|
|
identity = (item["document_id"], raw_page)
|
|
for raw in (item.get("text_url"), old_url):
|
|
match = _PAGE_URL.fullmatch(str(raw or ""))
|
|
if match:
|
|
from_url = (int(match[1]), int(match[2]))
|
|
if identity and identity != from_url:
|
|
raise RuntimeError("旧引用的文档 ID 与页面链接冲突")
|
|
identity = from_url
|
|
asset = next((match for raw in (old_url, item.get("image_url"))
|
|
if (match := _ASSET_URL.fullmatch(str(raw or "")))), None)
|
|
if not identity and asset and isinstance(item.get("quote"), str) and item["quote"]:
|
|
file_id, page = int(asset[1]), int(asset[2])
|
|
quote = item["quote"]
|
|
matches = {document_id for saved, document_id in evidence[file_id, page]
|
|
if saved.startswith(quote) and (record := pages.get((document_id, page)))
|
|
and quote in record.text}
|
|
if len(matches) != 1:
|
|
raise RuntimeError("旧评分引用无法唯一对应资料页，请先核对评分证据")
|
|
identity = (matches.pop(), page)
|
|
if not identity:
|
|
return item
|
|
if raw_page is not None and raw_page != identity[1]:
|
|
raise RuntimeError("旧引用的页码与来源链接冲突")
|
|
record = pages.get(identity)
|
|
if not record:
|
|
raise RuntimeError("旧引用对应的资料页不存在")
|
|
base = f"/api/knowledge/documents/{identity[0]}/pages/{identity[1]}"
|
|
value = {**item, "document_id": identity[0], "page": identity[1], "url": base,
|
|
"image_url": base + "/image" if record.image_path else None}
|
|
if isinstance(old_url, str) and old_url.startswith("/?page=scores"):
|
|
value["score_url"] = old_url
|
|
value.pop("text_url", None)
|
|
return value
|
|
|
|
def walk(value):
|
|
if isinstance(value, list):
|
|
return [walk(item) for item in value]
|
|
if isinstance(value, dict):
|
|
return normalize({key: walk(item) for key, item in value.items()})
|
|
return value
|
|
|
|
for table, columns in (("chat_messages", ("citations", "tool_calls")),
|
|
("agent_runs", ("tool_calls", "events")),
|
|
("pending_actions", ("preview", "result"))):
|
|
for row in db.execute(sa.text(f"SELECT id, {', '.join(columns)} FROM {table}")).mappings():
|
|
changed = {}
|
|
for column in columns:
|
|
old = row[column]
|
|
new = walk(old)
|
|
if new != old:
|
|
changed[column] = new
|
|
if changed:
|
|
assignments = ", ".join(f"{column} = :{column}" for column in changed)
|
|
statement = sa.text(f"UPDATE {table} SET {assignments} WHERE id = :id")
|
|
for column in changed:
|
|
statement = statement.bindparams(sa.bindparam(column, type_=sa.JSON()))
|
|
db.execute(statement, {"id": row.id, **changed})
|
|
|
|
|
|
def upgrade():
|
|
if "knowledge_pages" not in sa.inspect(op.get_bind()).get_table_names():
|
|
op.create_table(
|
|
"knowledge_pages",
|
|
sa.Column("document_id", sa.Integer(), sa.ForeignKey("knowledge_documents.id", ondelete="CASCADE"), primary_key=True),
|
|
sa.Column("page", sa.Integer(), primary_key=True),
|
|
sa.Column("text", sa.Text(), nullable=False),
|
|
sa.Column("image_path", sa.String(1200), nullable=True),
|
|
)
|
|
db = op.get_bind()
|
|
root = get_settings().processed_path.resolve()
|
|
documents = db.execute(sa.text(
|
|
"SELECT id, course_version_id, submission_file_id, title FROM knowledge_documents")).mappings().all()
|
|
files = {row.id: _json(row.parsed) for row in db.execute(sa.text(
|
|
"SELECT id, parsed FROM submission_files")).mappings()}
|
|
templates = {row.id: _json(row.templates) for row in db.execute(sa.text(
|
|
"SELECT id, templates FROM course_versions")).mappings()}
|
|
chunks = defaultdict(list)
|
|
for row in db.execute(sa.text(
|
|
"SELECT document_id, page, text FROM knowledge_chunks ORDER BY document_id, page, id")).mappings():
|
|
chunks[row.document_id, row.page].append(row.text)
|
|
insert = sa.text("INSERT INTO knowledge_pages (document_id, page, text, image_path) "
|
|
"VALUES (:document_id, :page, :text, :image_path) ON CONFLICT DO NOTHING")
|
|
pages_by_document = defaultdict(list)
|
|
for (document_id, page), parts in chunks.items():
|
|
pages_by_document[document_id].append((page, parts))
|
|
for doc in documents:
|
|
parsed_pages = {item.get("page_number"): item for item in files.get(doc.submission_file_id, {}).get("pages", [])}
|
|
for page in parsed_pages:
|
|
pages_by_document[doc.id].append((page, chunks.get((doc.id, page), [])))
|
|
seen = set()
|
|
for page, parts in pages_by_document[doc.id]:
|
|
if page in seen:
|
|
continue
|
|
seen.add(page)
|
|
source = parsed_pages.get(page)
|
|
text = _text(source) if source else _restore_chunk_text(parts)
|
|
image = _relative_image(root, source.get("rendered_image_path")) if source else None
|
|
if not image and doc.submission_file_id is None and doc.title in templates.get(doc.course_version_id, []):
|
|
folder = hashlib.sha256(doc.title.encode()).hexdigest()[:16]
|
|
image = _relative_image(root, root / f"course-{doc.course_version_id}" / folder / f"page-{page}.png")
|
|
db.execute(insert, {"document_id": doc.id, "page": page, "text": text, "image_path": image})
|
|
_migrate_references(db)
|
|
|
|
|
|
def downgrade():
|
|
op.drop_table("knowledge_pages")
|