feat: unify scoring and chat evidence pages
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
"""Persist canonical knowledge page text and image locations.
|
||||
|
||||
Revision ID: 0005
|
||||
Revises: 0004
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
from wisegrade.config import get_settings
|
||||
|
||||
revision = "0005"
|
||||
down_revision = "0004"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _json(value):
|
||||
return json.loads(value) if isinstance(value, str) else value or {}
|
||||
|
||||
|
||||
def _text(source):
|
||||
blocks = [str(source.get("text") or "").strip()]
|
||||
blocks.extend("表格: " + str(table.get("rows", table)) for table in source.get("tables", []))
|
||||
return "\n".join(block for block in blocks if block)
|
||||
|
||||
|
||||
def _relative_image(root, value):
|
||||
if not value:
|
||||
return None
|
||||
path = Path(value).resolve()
|
||||
return str(path.relative_to(root)) if path.is_relative_to(root) and path.is_file() else None
|
||||
|
||||
|
||||
def _restore_chunk_text(parts):
|
||||
"""Undo the 100-character overlap used by index version 2."""
|
||||
if not parts:
|
||||
return ""
|
||||
blocks = []
|
||||
current = previous = parts[0]
|
||||
for part in parts[1:]:
|
||||
if len(previous) == 650 and previous[-100:] == part[:100]:
|
||||
current += part[100:]
|
||||
else:
|
||||
blocks.append(current)
|
||||
current = part
|
||||
previous = part
|
||||
blocks.append(current)
|
||||
return "\n".join(blocks)
|
||||
|
||||
|
||||
_PAGE_URL = re.compile(r"/api/knowledge/documents/(\d+)/pages/(\d+)/?")
|
||||
_ASSET_URL = re.compile(r"/api/assets/(\d+)/(\d+)/?")
|
||||
|
||||
|
||||
def _migrate_references(db):
|
||||
"""Rewrite saved citations once; API reads the stored canonical links as-is."""
|
||||
pages = {(row.document_id, row.page): row for row in db.execute(sa.text(
|
||||
"SELECT document_id, page, text, image_path FROM knowledge_pages")).mappings()}
|
||||
evidence = defaultdict(list)
|
||||
for row in db.execute(sa.text(
|
||||
"SELECT submission_file_id, page, quote, document_id FROM score_evidences "
|
||||
"WHERE submission_file_id IS NOT NULL AND document_id IS NOT NULL")).mappings():
|
||||
evidence[row.submission_file_id, row.page].append((row.quote, row.document_id))
|
||||
|
||||
def normalize(item):
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
old_url = item.get("url")
|
||||
raw_page = item.get("page")
|
||||
identity = None
|
||||
if isinstance(item.get("document_id"), int) and isinstance(raw_page, int):
|
||||
identity = (item["document_id"], raw_page)
|
||||
for raw in (item.get("text_url"), old_url):
|
||||
match = _PAGE_URL.fullmatch(str(raw or ""))
|
||||
if match:
|
||||
from_url = (int(match[1]), int(match[2]))
|
||||
if identity and identity != from_url:
|
||||
raise RuntimeError("旧引用的文档 ID 与页面链接冲突")
|
||||
identity = from_url
|
||||
asset = next((match for raw in (old_url, item.get("image_url"))
|
||||
if (match := _ASSET_URL.fullmatch(str(raw or "")))), None)
|
||||
if not identity and asset and isinstance(item.get("quote"), str) and item["quote"]:
|
||||
file_id, page = int(asset[1]), int(asset[2])
|
||||
quote = item["quote"]
|
||||
matches = {document_id for saved, document_id in evidence[file_id, page]
|
||||
if saved.startswith(quote) and (record := pages.get((document_id, page)))
|
||||
and quote in record.text}
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("旧评分引用无法唯一对应资料页，请先核对评分证据")
|
||||
identity = (matches.pop(), page)
|
||||
if not identity:
|
||||
return item
|
||||
if raw_page is not None and raw_page != identity[1]:
|
||||
raise RuntimeError("旧引用的页码与来源链接冲突")
|
||||
record = pages.get(identity)
|
||||
if not record:
|
||||
raise RuntimeError("旧引用对应的资料页不存在")
|
||||
base = f"/api/knowledge/documents/{identity[0]}/pages/{identity[1]}"
|
||||
value = {**item, "document_id": identity[0], "page": identity[1], "url": base,
|
||||
"image_url": base + "/image" if record.image_path else None}
|
||||
if isinstance(old_url, str) and old_url.startswith("/?page=scores"):
|
||||
value["score_url"] = old_url
|
||||
value.pop("text_url", None)
|
||||
return value
|
||||
|
||||
def walk(value):
|
||||
if isinstance(value, list):
|
||||
return [walk(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return normalize({key: walk(item) for key, item in value.items()})
|
||||
return value
|
||||
|
||||
for table, columns in (("chat_messages", ("citations", "tool_calls")),
|
||||
("agent_runs", ("tool_calls", "events")),
|
||||
("pending_actions", ("preview", "result"))):
|
||||
for row in db.execute(sa.text(f"SELECT id, {', '.join(columns)} FROM {table}")).mappings():
|
||||
changed = {}
|
||||
for column in columns:
|
||||
old = row[column]
|
||||
new = walk(old)
|
||||
if new != old:
|
||||
changed[column] = new
|
||||
if changed:
|
||||
assignments = ", ".join(f"{column} = :{column}" for column in changed)
|
||||
statement = sa.text(f"UPDATE {table} SET {assignments} WHERE id = :id")
|
||||
for column in changed:
|
||||
statement = statement.bindparams(sa.bindparam(column, type_=sa.JSON()))
|
||||
db.execute(statement, {"id": row.id, **changed})
|
||||
|
||||
|
||||
def upgrade():
|
||||
if "knowledge_pages" not in sa.inspect(op.get_bind()).get_table_names():
|
||||
op.create_table(
|
||||
"knowledge_pages",
|
||||
sa.Column("document_id", sa.Integer(), sa.ForeignKey("knowledge_documents.id", ondelete="CASCADE"), primary_key=True),
|
||||
sa.Column("page", sa.Integer(), primary_key=True),
|
||||
sa.Column("text", sa.Text(), nullable=False),
|
||||
sa.Column("image_path", sa.String(1200), nullable=True),
|
||||
)
|
||||
db = op.get_bind()
|
||||
root = get_settings().processed_path.resolve()
|
||||
documents = db.execute(sa.text(
|
||||
"SELECT id, course_version_id, submission_file_id, title FROM knowledge_documents")).mappings().all()
|
||||
files = {row.id: _json(row.parsed) for row in db.execute(sa.text(
|
||||
"SELECT id, parsed FROM submission_files")).mappings()}
|
||||
templates = {row.id: _json(row.templates) for row in db.execute(sa.text(
|
||||
"SELECT id, templates FROM course_versions")).mappings()}
|
||||
chunks = defaultdict(list)
|
||||
for row in db.execute(sa.text(
|
||||
"SELECT document_id, page, text FROM knowledge_chunks ORDER BY document_id, page, id")).mappings():
|
||||
chunks[row.document_id, row.page].append(row.text)
|
||||
insert = sa.text("INSERT INTO knowledge_pages (document_id, page, text, image_path) "
|
||||
"VALUES (:document_id, :page, :text, :image_path) ON CONFLICT DO NOTHING")
|
||||
pages_by_document = defaultdict(list)
|
||||
for (document_id, page), parts in chunks.items():
|
||||
pages_by_document[document_id].append((page, parts))
|
||||
for doc in documents:
|
||||
parsed_pages = {item.get("page_number"): item for item in files.get(doc.submission_file_id, {}).get("pages", [])}
|
||||
for page in parsed_pages:
|
||||
pages_by_document[doc.id].append((page, chunks.get((doc.id, page), [])))
|
||||
seen = set()
|
||||
for page, parts in pages_by_document[doc.id]:
|
||||
if page in seen:
|
||||
continue
|
||||
seen.add(page)
|
||||
source = parsed_pages.get(page)
|
||||
text = _text(source) if source else _restore_chunk_text(parts)
|
||||
image = _relative_image(root, source.get("rendered_image_path")) if source else None
|
||||
if not image and doc.submission_file_id is None and doc.title in templates.get(doc.course_version_id, []):
|
||||
folder = hashlib.sha256(doc.title.encode()).hexdigest()[:16]
|
||||
image = _relative_image(root, root / f"course-{doc.course_version_id}" / folder / f"page-{page}.png")
|
||||
db.execute(insert, {"document_id": doc.id, "page": page, "text": text, "image_path": image})
|
||||
_migrate_references(db)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table("knowledge_pages")
|
||||
Reference in New Issue
Block a user