feat: require offline PaddleOCR model assets
This commit is contained in:
@@ -8,6 +8,7 @@ WISEGRADE_SECRET_KEY=replace-with-a-random-32-character-secret
|
|||||||
EMBEDDING_MODEL_NAME=BAAI/bge-m3
|
EMBEDDING_MODEL_NAME=BAAI/bge-m3
|
||||||
EMBEDDING_MODEL_PATH=/models/bge-m3
|
EMBEDDING_MODEL_PATH=/models/bge-m3
|
||||||
EMBEDDING_MODEL_VERSION=local
|
EMBEDDING_MODEL_VERSION=local
|
||||||
|
OCR_MODEL_DIR=/models/ocr
|
||||||
MODEL_TIMEOUT_SECONDS=120
|
MODEL_TIMEOUT_SECONDS=120
|
||||||
MODEL_CONCURRENCY=2
|
MODEL_CONCURRENCY=2
|
||||||
MODEL_RETRIES=2
|
MODEL_RETRIES=2
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from docx import Document
|
|||||||
from pptx import Presentation
|
from pptx import Presentation
|
||||||
from pptx.util import Inches
|
from pptx.util import Inches
|
||||||
|
|
||||||
from wisegrade.parsing import parse_document
|
from wisegrade.parsing import _ocr_engine, parse_document
|
||||||
|
|
||||||
|
|
||||||
def test_pdf_pages_and_images(tmp_path):
|
def test_pdf_pages_and_images(tmp_path):
|
||||||
@@ -20,6 +20,11 @@ def test_pdf_pages_and_images(tmp_path):
|
|||||||
assert (tmp_path / "assets" / "page-1.png").exists()
|
assert (tmp_path / "assets" / "page-1.png").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_scanned_pdf_requires_offline_ocr_package(tmp_path):
|
||||||
|
with pytest.raises(RuntimeError, match="离线OCR模型目录"):
|
||||||
|
_ocr_engine(str(tmp_path / "missing-ocr"))
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(not shutil.which("libreoffice"), reason="LibreOffice is installed in the deployment image")
|
@pytest.mark.skipif(not shutil.which("libreoffice"), reason="LibreOffice is installed in the deployment image")
|
||||||
def test_docx_and_pptx_render(tmp_path):
|
def test_docx_and_pptx_render(tmp_path):
|
||||||
word = Document()
|
word = Document()
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ class Settings(BaseSettings):
|
|||||||
embedding_model_name: str = "BAAI/bge-m3"
|
embedding_model_name: str = "BAAI/bge-m3"
|
||||||
embedding_model_path: Path = Path("/models/bge-m3")
|
embedding_model_path: Path = Path("/models/bge-m3")
|
||||||
embedding_model_version: str = "local"
|
embedding_model_version: str = "local"
|
||||||
|
ocr_model_dir: Path = Path("/models/ocr")
|
||||||
model_timeout_seconds: int = 120
|
model_timeout_seconds: int = 120
|
||||||
model_concurrency: int = 2
|
model_concurrency: int = 2
|
||||||
model_retries: int = 2
|
model_retries: int = 2
|
||||||
|
|||||||
@@ -1,11 +1,18 @@
|
|||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
import pymupdf as fitz
|
import pymupdf as fitz
|
||||||
from docx import Document
|
from docx import Document
|
||||||
from pptx import Presentation
|
from pptx import Presentation
|
||||||
|
|
||||||
|
from .config import get_settings
|
||||||
|
|
||||||
|
|
||||||
|
_ocr_lock = Lock()
|
||||||
|
|
||||||
|
|
||||||
def _render(page, output: Path):
|
def _render(page, output: Path):
|
||||||
output.parent.mkdir(parents=True, exist_ok=True)
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -13,15 +20,30 @@ def _render(page, output: Path):
|
|||||||
return str(output)
|
return str(output)
|
||||||
|
|
||||||
|
|
||||||
def _ocr(image_path: str):
|
@lru_cache(maxsize=1)
|
||||||
|
def _ocr_engine(model_dir: str):
|
||||||
|
root = Path(model_dir)
|
||||||
|
detection, recognition = root / "detection", root / "recognition"
|
||||||
|
if not detection.is_dir() or not recognition.is_dir():
|
||||||
|
raise RuntimeError(f"扫描PDF需要离线OCR模型目录: {detection} 和 {recognition}")
|
||||||
try:
|
try:
|
||||||
from paddleocr import PaddleOCR
|
from paddleocr import PaddleOCR
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise RuntimeError("扫描PDF需要PaddleOCR;请安装OCR依赖或交由教师处理") from exc
|
raise RuntimeError("扫描PDF需要PaddleOCR;请安装OCR依赖或交由教师处理") from exc
|
||||||
try:
|
try:
|
||||||
engine = PaddleOCR(lang="ch", use_textline_orientation=True)
|
return PaddleOCR(lang="ch", text_detection_model_dir=str(detection), text_recognition_model_dir=str(recognition),
|
||||||
|
use_textline_orientation=False, use_doc_orientation_classify=False, use_doc_unwarping=False)
|
||||||
except TypeError:
|
except TypeError:
|
||||||
engine = PaddleOCR(use_angle_cls=True, lang="ch", show_log=False)
|
return PaddleOCR(lang="ch", det_model_dir=str(detection), rec_model_dir=str(recognition), use_angle_cls=False, show_log=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _ocr(image_path: str):
|
||||||
|
engine = _ocr_engine(str(get_settings().ocr_model_dir))
|
||||||
|
with _ocr_lock:
|
||||||
|
return _ocr_unlocked(engine, image_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _ocr_unlocked(engine, image_path: str):
|
||||||
if hasattr(engine, "predict"):
|
if hasattr(engine, "predict"):
|
||||||
result = engine.predict(image_path)
|
result = engine.predict(image_path)
|
||||||
lines = []
|
lines = []
|
||||||
|
|||||||
+1
-1
@@ -59,7 +59,7 @@ EMBEDDING_MODEL_VERSION=your-package-version
|
|||||||
|
|
||||||
`EMBEDDING_MODEL_VERSION` 会写入每个评分批次快照。若模型目录缺失或内容不完整,导入仍可完成,知识库入库和评分会显示错误。可离线分发 `docker save` 生成的镜像包及单独的 `models-bge-m3.tar.zst`;目标机执行 `docker load -i docker-images.tar` 后解压模型包,运行相同的 Compose 配置。
|
`EMBEDDING_MODEL_VERSION` 会写入每个评分批次快照。若模型目录缺失或内容不完整,导入仍可完成,知识库入库和评分会显示错误。可离线分发 `docker save` 生成的镜像包及单独的 `models-bge-m3.tar.zst`;目标机执行 `docker load -i docker-images.tar` 后解压模型包,运行相同的 Compose 配置。
|
||||||
|
|
||||||
扫描 PDF 的 OCR 使用 PaddleOCR;需在离线环境准备其运行所需的 OCR 模型缓存。没有 OCR 模型且 PDF 无可提取文字时,文件进入解析失败状态并显示原因。
|
扫描 PDF 的 OCR 使用 PaddleOCR。离线 OCR 包放在 `models/ocr/detection/` 和 `models/ocr/recognition/`,或修改 `OCR_MODEL_DIR` 指向包含这两个子目录的位置。没有 OCR 模型且 PDF 无可提取文字时,文件进入解析失败状态并显示原因;系统不会在解析时自动下载 OCR 模型。
|
||||||
|
|
||||||
## 教师操作
|
## 教师操作
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user