Initial commit: book-mcp 教辅处理 MCP server + Arco Vue 管理端
后端 (book_mcp_server.py, 335 行, 15 个 MCP 工具):
- create_book / add_document / mark_html_ready / mark_json_ready / mark_failed
- update_book_status / soft_delete_book / retry_failed
- get_book / list_books / get_document / list_documents / get_document_url
- upload_to_rustfs / health
- 走 JSON-RPC + stateless_http + json_response(前端可直接调)
- launchd 守护 / 6 秒拉起(macOS)
- systemd 守护脚本(Linux/codelab)
存储:
- MongoDB (mongo:8.0 容器) - book/book_document/document_file 三表
- RustFS 对象存储(fs.code-lab.cn/scan/)- book/<id>/docs/<doc_id>/{docx,html,json}
- 幂等设计: zip_sha256 唯一 + (book_id+original_path) 唯一
前端 (Vue 3 + TS + Arco Design):
- 列表页 /book/list - 搜索 + 状态过滤 + 进度
- 详情页 /book/:id - 文档折叠 + 文件下载
- MCP 客户端 src/api/mcp.ts (JSON-RPC over fetch)
部署:
- deploy/install.sh (codelab 一键装 backend + systemd)
- deploy/install-frontend.sh (vite dev / 生产构建)
- docs/deploy.md (运维手册)
测试:
- tests/test_basic.py (sanity 7 项)
- tests/test_e2e.py (完整 e2e: docx→HTML→入库→上传→校验)
This commit is contained in:
9
services/__init__.py
Normal file
9
services/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
"""services 子包 — 业务逻辑层
|
||||
|
||||
子模块:
|
||||
wordparse — docx → HTML 工具集(cleaner / post / render / convert)
|
||||
rustfs_service — 对象存储封装
|
||||
book_service — 教辅 book 业务
|
||||
document_service — 教辅文档 book_document 业务
|
||||
file_service — 文档文件 document_file 业务
|
||||
"""
|
||||
138
services/book_service.py
Normal file
138
services/book_service.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""教辅 (book) 业务层 — MongoDB 版本
|
||||
|
||||
幂等策略:
|
||||
- 同 zip_sha256 直接 return 旧 book
|
||||
- 同 (name, version, is_deleted) 唯一索引保护
|
||||
"""
|
||||
import re
|
||||
from pymongo.errors import DuplicateKeyError
|
||||
|
||||
from db import get_collection, COL_BOOK, COL_DOC, COL_FILE, next_id, Book, BookDocument, DocumentFile
|
||||
from db.models import _now
|
||||
|
||||
|
||||
def create_book(name: str, zip_path: str, zip_sha256: str, zip_size: int = 0,
|
||||
subject: str | None = None, grade: str | None = None,
|
||||
version: str | None = None, type_: str | None = None,
|
||||
created_by: str | None = None) -> dict:
|
||||
"""创建教辅。同 zip_sha256 幂等。"""
|
||||
col = get_collection(COL_BOOK)
|
||||
|
||||
# 幂等查
|
||||
existing = col.find_one({'zip_sha256': zip_sha256, 'is_deleted': 0})
|
||||
if existing:
|
||||
b = Book.from_mongo(existing)
|
||||
return {
|
||||
'ok': True, 'idempotent': True, 'book_id': b.id,
|
||||
'status': b.status, 'total_count': b.total_count,
|
||||
}
|
||||
|
||||
# 自增 book_id
|
||||
book_id = next_id('book_id')
|
||||
book = Book(
|
||||
id=book_id, name=name, zip_path=zip_path, zip_sha256=zip_sha256,
|
||||
zip_size=zip_size, subject=subject, grade=grade, version=version,
|
||||
type=type_, created_by=created_by, status='pending',
|
||||
)
|
||||
try:
|
||||
col.insert_one(book.to_mongo())
|
||||
except DuplicateKeyError:
|
||||
# uk_name_version 撞了,重查
|
||||
existing = col.find_one({'zip_sha256': zip_sha256, 'is_deleted': 0})
|
||||
if existing:
|
||||
b = Book.from_mongo(existing)
|
||||
return {
|
||||
'ok': True, 'idempotent': True, 'book_id': b.id,
|
||||
'status': b.status, 'total_count': b.total_count,
|
||||
}
|
||||
return {'ok': False, 'error': 'duplicate key (name+version)'}
|
||||
|
||||
return {
|
||||
'ok': True, 'idempotent': False, 'book_id': book_id,
|
||||
'status': 'pending', 'total_count': 0,
|
||||
}
|
||||
|
||||
|
||||
def get_book(book_id: int, include_documents: bool = True) -> dict | None:
|
||||
"""查教辅(含所有 doc + file)"""
|
||||
col = get_collection(COL_BOOK)
|
||||
doc = col.find_one({'_id': book_id, 'is_deleted': 0})
|
||||
if not doc:
|
||||
return None
|
||||
book = Book.from_mongo(doc).to_dict()
|
||||
|
||||
if include_documents:
|
||||
docs = list(get_collection(COL_DOC).find({'book_id': book_id})
|
||||
.sort('order_index', 1))
|
||||
book['documents'] = []
|
||||
for d in docs:
|
||||
bd = BookDocument.from_mongo(d).to_dict()
|
||||
files = list(get_collection(COL_FILE).find({'doc_id': bd['doc_id']}))
|
||||
bd['files'] = [DocumentFile.from_mongo(f).to_dict() for f in files]
|
||||
book['documents'].append(bd)
|
||||
|
||||
return book
|
||||
|
||||
|
||||
def list_books(status: str | None = None, page: int = 1, page_size: int = 20,
|
||||
keyword: str | None = None) -> dict:
|
||||
"""列表查询(分页)"""
|
||||
col = get_collection(COL_BOOK)
|
||||
q: dict = {'is_deleted': 0}
|
||||
if status:
|
||||
q['status'] = status
|
||||
if keyword:
|
||||
kw = re.escape(keyword)
|
||||
q['$or'] = [
|
||||
{'name': {'$regex': kw, '$options': 'i'}},
|
||||
{'subject': {'$regex': kw, '$options': 'i'}},
|
||||
{'grade': {'$regex': kw, '$options': 'i'}},
|
||||
{'version': {'$regex': kw, '$options': 'i'}},
|
||||
]
|
||||
total = col.count_documents(q)
|
||||
cursor = (col.find(q).sort('created_at', -1)
|
||||
.skip((page - 1) * page_size).limit(page_size))
|
||||
books = [Book.from_mongo(d).to_dict() for d in cursor]
|
||||
return {'books': books, 'total': total, 'page': page, 'page_size': page_size}
|
||||
|
||||
|
||||
def update_book_status(book_id: int, status: str, error_msg: str | None = None) -> bool:
|
||||
col = get_collection(COL_BOOK)
|
||||
upd = {'status': status, 'updated_at': _now()}
|
||||
if error_msg is not None:
|
||||
upd['error_msg'] = error_msg
|
||||
r = col.update_one({'_id': book_id, 'is_deleted': 0}, {'$set': upd})
|
||||
return r.matched_count > 0
|
||||
|
||||
|
||||
def recompute_book_stats(book_id: int) -> None:
|
||||
"""重算 total / success / failed,并触发状态机"""
|
||||
col_book = get_collection(COL_BOOK)
|
||||
col_doc = get_collection(COL_DOC)
|
||||
|
||||
docs = list(col_doc.find({'book_id': book_id}))
|
||||
total = len(docs)
|
||||
success = sum(1 for d in docs if d.get('status') in ('html_ready', 'json_ready'))
|
||||
failed = sum(1 for d in docs if d.get('status') == 'failed')
|
||||
|
||||
new_status = ('pending' if total == 0 else
|
||||
'done' if failed == 0 else
|
||||
'failed' if success == 0 else 'partial')
|
||||
|
||||
book = col_book.find_one({'_id': book_id})
|
||||
if not book:
|
||||
return
|
||||
upd = {
|
||||
'total_count': total, 'success_count': success,
|
||||
'failed_count': failed, 'updated_at': _now(),
|
||||
}
|
||||
if book.get('status') != 'processing':
|
||||
upd['status'] = new_status
|
||||
col_book.update_one({'_id': book_id}, {'$set': upd})
|
||||
|
||||
|
||||
def soft_delete_book(book_id: int) -> bool:
|
||||
col = get_collection(COL_BOOK)
|
||||
r = col.update_one({'_id': book_id, 'is_deleted': 0},
|
||||
{'$set': {'is_deleted': 1, 'updated_at': _now()}})
|
||||
return r.matched_count > 0
|
||||
151
services/document_service.py
Normal file
151
services/document_service.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""教辅文档 (book_document) 业务层 — MongoDB 版本"""
|
||||
import re
|
||||
from pymongo.errors import DuplicateKeyError
|
||||
|
||||
from db import get_collection, COL_BOOK, COL_DOC, next_id, BookDocument
|
||||
from db.models import _now
|
||||
|
||||
|
||||
def _infer_doc_type(filename: str) -> str:
|
||||
"""从文件名猜题型:课时作业 / 周末提升 / 阶段质量评估 / 综合评估"""
|
||||
n = filename
|
||||
if re.search(r'阶段质量评估', n):
|
||||
return '阶段质量评估'
|
||||
if re.search(r'综合.*评估|期末', n):
|
||||
return '综合评估'
|
||||
if re.search(r'周末提升', n):
|
||||
return '周末提升'
|
||||
if re.search(r'课时|梯级|高考真题集训', n):
|
||||
return '课时作业'
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def add_document(book_id: int, original_filename: str, original_path: str,
|
||||
title: str | None = None, doc_type: str | None = None,
|
||||
order_index: int | None = None) -> dict:
|
||||
"""添加文档。同 (book_id, original_path) 幂等。"""
|
||||
col_doc = get_collection(COL_DOC)
|
||||
col_book = get_collection(COL_BOOK)
|
||||
|
||||
# book 存在性
|
||||
if not col_book.find_one({'_id': book_id, 'is_deleted': 0}):
|
||||
return {'ok': False, 'error': f'book {book_id} not found'}
|
||||
|
||||
# 1. 算 doc_id(稳定哈希)
|
||||
doc_id = BookDocument.make_doc_id(book_id, original_path)
|
||||
|
||||
# 2. 幂等查
|
||||
existing = col_doc.find_one({'_id': doc_id})
|
||||
if existing:
|
||||
bd = BookDocument.from_mongo(existing)
|
||||
return {
|
||||
'ok': True, 'idempotent': True, 'doc_id': bd.doc_id,
|
||||
'order_index': bd.order_index, 'status': bd.status,
|
||||
'doc_type': bd.doc_type,
|
||||
}
|
||||
|
||||
# 3. order_index(自动递增)
|
||||
if order_index is None:
|
||||
cur_max = col_doc.find({'book_id': book_id}) \
|
||||
.sort('order_index', -1).limit(1)
|
||||
cur_max_list = list(cur_max)
|
||||
order_index = (cur_max_list[0]['order_index'] + 1) if cur_max_list else 1
|
||||
|
||||
# 4. doc_type 推断
|
||||
dt = doc_type or _infer_doc_type(original_filename)
|
||||
|
||||
# 5. 入库
|
||||
doc = BookDocument(
|
||||
book_id=book_id, doc_id=doc_id,
|
||||
original_filename=original_filename, original_path=original_path,
|
||||
title=title, doc_type=dt, order_index=order_index,
|
||||
)
|
||||
try:
|
||||
col_doc.insert_one(doc.to_mongo())
|
||||
except DuplicateKeyError:
|
||||
# 竞争:重查
|
||||
existing = col_doc.find_one({'_id': doc_id})
|
||||
bd = BookDocument.from_mongo(existing)
|
||||
return {
|
||||
'ok': True, 'idempotent': True, 'doc_id': bd.doc_id,
|
||||
'order_index': bd.order_index, 'status': bd.status,
|
||||
'doc_type': bd.doc_type,
|
||||
}
|
||||
return {
|
||||
'ok': True, 'idempotent': False, 'doc_id': doc_id,
|
||||
'order_index': order_index, 'status': 'pending', 'doc_type': dt,
|
||||
}
|
||||
|
||||
|
||||
def get_document(doc_id: str, include_files: bool = True) -> dict | None:
|
||||
col = get_collection(COL_DOC)
|
||||
d = col.find_one({'_id': doc_id})
|
||||
if not d:
|
||||
return None
|
||||
bd = BookDocument.from_mongo(d).to_dict()
|
||||
if include_files:
|
||||
from .file_service import get_files
|
||||
bd['files'] = get_files(doc_id)
|
||||
return bd
|
||||
|
||||
|
||||
def list_documents(book_id: int, status: str | None = None,
|
||||
page: int = 1, page_size: int = 50) -> dict:
|
||||
col = get_collection(COL_DOC)
|
||||
q: dict = {'book_id': book_id}
|
||||
if status:
|
||||
q['status'] = status
|
||||
total = col.count_documents(q)
|
||||
cursor = col.find(q).sort('order_index', 1) \
|
||||
.skip((page - 1) * page_size).limit(page_size)
|
||||
docs = [BookDocument.from_mongo(d).to_dict() for d in cursor]
|
||||
return {'documents': docs, 'total': total, 'page': page, 'page_size': page_size}
|
||||
|
||||
|
||||
def _mark_file_ready(doc_id: str, file_type: str, url: str, size: int,
|
||||
meta: dict | None) -> dict:
|
||||
"""mark_html/json_ready 通用实现"""
|
||||
from .file_service import upsert_file
|
||||
from db import COL_FILE, COL_DOC, COL_BOOK
|
||||
from db.database import load_config
|
||||
from services.rustfs_service import build_key
|
||||
|
||||
col_doc = get_collection(COL_DOC)
|
||||
doc = col_doc.find_one({'_id': doc_id})
|
||||
if not doc:
|
||||
return {'ok': False, 'error': f'doc {doc_id} not found'}
|
||||
|
||||
bucket = load_config()['rustfs']['default_bucket']
|
||||
key = build_key(doc['book_id'], doc_id, file_type)
|
||||
f = upsert_file(
|
||||
doc_id=doc_id, file_type=file_type, object_key=key, object_bucket=bucket,
|
||||
size=size, url=url, meta=meta,
|
||||
)
|
||||
new_status = 'html_ready' if file_type == 'html' else 'json_ready'
|
||||
col_doc.update_one({'_id': doc_id}, {'$set': {
|
||||
'status': new_status, 'updated_at': _now(), 'error_msg': None,
|
||||
}})
|
||||
return {'ok': True, 'file_id': f['id'], 'object_key': key, 'url': url}
|
||||
|
||||
|
||||
def mark_html_ready(doc_id: str, html_url: str, html_size: int,
|
||||
stats: dict | None = None) -> dict:
|
||||
"""标记文档 html 已就绪"""
|
||||
return _mark_file_ready(doc_id, 'html', html_url, html_size, stats)
|
||||
|
||||
|
||||
def mark_json_ready(doc_id: str, json_url: str, json_size: int,
|
||||
schema_version: str | None = None) -> dict:
|
||||
"""标记文档 json 已就绪"""
|
||||
meta = {'schema_version': schema_version} if schema_version else None
|
||||
return _mark_file_ready(doc_id, 'json', json_url, json_size, meta)
|
||||
|
||||
|
||||
def mark_failed(doc_id: str, error_msg: str, stage: str = 'html') -> dict:
|
||||
"""标记文档失败"""
|
||||
col = get_collection(COL_DOC)
|
||||
r = col.update_one({'_id': doc_id}, {'$set': {
|
||||
'status': 'failed', 'error_msg': f'[{stage}] {error_msg}'[:1000],
|
||||
'updated_at': _now(),
|
||||
}})
|
||||
return {'ok': r.matched_count > 0}
|
||||
50
services/file_service.py
Normal file
50
services/file_service.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""文档文件 (document_file) 业务层 — MongoDB 版本
|
||||
|
||||
upsert 策略:(doc_id, file_type) 唯一索引 → update_one / insert_one
|
||||
"""
|
||||
from pymongo.errors import DuplicateKeyError
|
||||
|
||||
from db import get_collection, COL_FILE, next_id, DocumentFile
|
||||
from db.models import _now
|
||||
|
||||
|
||||
def upsert_file(doc_id: str, file_type: str, object_key: str, object_bucket: str,
|
||||
size: int = 0, checksum: str | None = None, url: str | None = None,
|
||||
meta: dict | None = None) -> dict:
|
||||
"""插入或更新 (doc_id, file_type) 唯一键"""
|
||||
col = get_collection(COL_FILE)
|
||||
filt = {'doc_id': doc_id, 'file_type': file_type}
|
||||
upd = {'$set': {
|
||||
'object_key': object_key, 'object_bucket': object_bucket,
|
||||
'size': size, 'checksum': checksum, 'url': url, 'meta': meta,
|
||||
}}
|
||||
r = col.update_one(filt, upd)
|
||||
if r.matched_count > 0:
|
||||
f = col.find_one(filt)
|
||||
return {'id': f['_id'], 'updated': True}
|
||||
f = DocumentFile(
|
||||
id=next_id('file_id'),
|
||||
doc_id=doc_id, file_type=file_type, object_key=object_key,
|
||||
object_bucket=object_bucket, size=size, checksum=checksum,
|
||||
url=url, meta=meta,
|
||||
)
|
||||
try:
|
||||
col.insert_one(f.to_mongo())
|
||||
except DuplicateKeyError:
|
||||
# 极小概率竞争,重试
|
||||
f2 = col.find_one(filt)
|
||||
col.update_one(filt, upd)
|
||||
return {'id': f2['_id'], 'updated': True}
|
||||
return {'id': f.id, 'updated': False}
|
||||
|
||||
|
||||
def get_files(doc_id: str) -> list[dict]:
|
||||
col = get_collection(COL_FILE)
|
||||
return [DocumentFile.from_mongo(d).to_dict()
|
||||
for d in col.find({'doc_id': doc_id})]
|
||||
|
||||
|
||||
def get_file_by_type(doc_id: str, file_type: str) -> dict | None:
|
||||
col = get_collection(COL_FILE)
|
||||
d = col.find_one({'doc_id': doc_id, 'file_type': file_type})
|
||||
return DocumentFile.from_mongo(d).to_dict() if d else None
|
||||
94
services/rustfs_service.py
Normal file
94
services/rustfs_service.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""RustFS 对象存储封装(基于 boto3)"""
|
||||
import os
|
||||
import hashlib
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
import boto3
|
||||
import botocore
|
||||
|
||||
from db.database import load_config
|
||||
|
||||
|
||||
def _s3():
|
||||
cfg = load_config()['rustfs']
|
||||
return boto3.client(
|
||||
's3',
|
||||
endpoint_url=cfg['endpoint'],
|
||||
aws_access_key_id=cfg['access_key'],
|
||||
aws_secret_access_key=cfg['secret_key'],
|
||||
region_name='us-east-1',
|
||||
config=botocore.config.Config(
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'},
|
||||
),
|
||||
verify=cfg.get('verify_ssl', False),
|
||||
)
|
||||
|
||||
|
||||
def _content_type(path: str) -> str:
|
||||
p = path.lower()
|
||||
if p.endswith('.html'):
|
||||
return 'text/html; charset=utf-8'
|
||||
if p.endswith('.js'):
|
||||
return 'application/javascript'
|
||||
if p.endswith('.json'):
|
||||
return 'application/json'
|
||||
if p.endswith('.docx'):
|
||||
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
if p.endswith('.png'):
|
||||
return 'image/png'
|
||||
if p.endswith('.log'):
|
||||
return 'text/plain; charset=utf-8'
|
||||
if p.endswith('.zip'):
|
||||
return 'application/zip'
|
||||
return 'application/octet-stream'
|
||||
|
||||
|
||||
def upload(local_path: str, key: str, bucket: str | None = None) -> dict:
|
||||
"""上传文件 → 返回 {url, key, bucket, size, checksum}"""
|
||||
cfg = load_config()['rustfs']
|
||||
bucket = bucket or cfg['default_bucket']
|
||||
|
||||
size = os.path.getsize(local_path)
|
||||
# sha256
|
||||
h = hashlib.sha256()
|
||||
with open(local_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(65536), b''):
|
||||
h.update(chunk)
|
||||
checksum = h.hexdigest()
|
||||
|
||||
_s3().upload_file(
|
||||
local_path, bucket, key,
|
||||
ExtraArgs={'ContentType': _content_type(local_path)},
|
||||
)
|
||||
url = f"{cfg['public_base']}/{bucket}/" + urllib.parse.quote(key, safe='/')
|
||||
return {
|
||||
'url': url,
|
||||
'key': key,
|
||||
'bucket': bucket,
|
||||
'size': size,
|
||||
'checksum': checksum,
|
||||
}
|
||||
|
||||
|
||||
def delete_prefix(prefix: str, bucket: str | None = None) -> int:
|
||||
"""批量删一个前缀下的所有对象,返回删除条数"""
|
||||
cfg = load_config()['rustfs']
|
||||
bucket = bucket or cfg['default_bucket']
|
||||
s3 = _s3()
|
||||
n = 0
|
||||
paginator = s3.get_paginator('list_objects_v2')
|
||||
for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
|
||||
keys = [{'Key': o['Key']} for o in page.get('Contents', [])]
|
||||
if keys:
|
||||
s3.delete_objects(Bucket=bucket, Delete={'Objects': keys})
|
||||
n += len(keys)
|
||||
return n
|
||||
|
||||
|
||||
def build_key(book_id: int, doc_id: str, file_type: str) -> str:
|
||||
"""根据 book_id / doc_id / file_type 生成对象存储 key"""
|
||||
cfg = load_config()['storage']
|
||||
file_naming = cfg['file_naming']
|
||||
filename = file_naming.get(file_type, f'{file_type}.bin')
|
||||
return f"{cfg['book_prefix']}/{book_id}/{cfg['docs_subdir']}/{doc_id}/{filename}"
|
||||
18
services/wordparse/__init__.py
Normal file
18
services/wordparse/__init__.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""wordparse 子包:docx → HTML 工具集
|
||||
|
||||
子模块:
|
||||
cleaner — 修脏 docx XML
|
||||
post — HTTP 调 docx4j
|
||||
render — HTML 美化(MathJax / 样式 / 导航 / 统计)
|
||||
convert — 高级入口(convert_one / batch_convert)
|
||||
"""
|
||||
from .cleaner import clean_docx
|
||||
from .post import post_docx, health
|
||||
from .render import clean_html, stats, copy_mathjax_to
|
||||
from .convert import convert_one, batch_convert
|
||||
|
||||
__all__ = [
|
||||
'clean_docx', 'post_docx', 'health',
|
||||
'clean_html', 'stats', 'copy_mathjax_to',
|
||||
'convert_one', 'batch_convert',
|
||||
]
|
||||
76
services/wordparse/cleaner.py
Normal file
76
services/wordparse/cleaner.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""docx XML 清洗:去脏值(单位 / NaN / 浮点)以避开 docx4j 8.3.9 JAXB 解析失败。
|
||||
|
||||
参考 wordparse_server.py 的 _clean_docx_xml,独立成模块方便测试。
|
||||
"""
|
||||
import re
|
||||
import os
|
||||
import zipfile
|
||||
|
||||
XML_DECL = re.compile(r'<\?xml[^?]*\?>')
|
||||
|
||||
|
||||
def _fix_text(txt: str) -> str:
|
||||
"""修三类问题:
|
||||
1. 带单位的数字属性:"210mm" / "25.4cm" → "0"
|
||||
2. NaN / Inf / -Inf / 空字符串 → "0"
|
||||
3. 浮点属性 → 整数(JAXB BigInteger 不接小数)
|
||||
"""
|
||||
decl = XML_DECL.search(txt)
|
||||
decl_str = decl.group(0) if decl else '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
|
||||
if decl:
|
||||
txt = XML_DECL.sub('__XMLDECL__', txt, count=1)
|
||||
|
||||
# 1. 单位
|
||||
txt = re.sub(r'="(-?[0-9]*\.?[0-9]+)\s*(mm|cm|pt|px|in|em|%)"', r'="0"', txt, flags=re.I)
|
||||
# 2. NaN / Inf
|
||||
txt = re.sub(r'="(NaN|nan|Inf|-Inf|inf|-inf)"', '="0"', txt, flags=re.I)
|
||||
# 3. 空
|
||||
txt = re.sub(r'=""', '="0"', txt)
|
||||
# 4. 浮点 → 整数
|
||||
def _num(m):
|
||||
try:
|
||||
f = float(m.group(1))
|
||||
if f != f or f == float('inf') or f == float('-inf'):
|
||||
return '="0"'
|
||||
return f'="{int(f)}"'
|
||||
except Exception:
|
||||
return m.group(0)
|
||||
|
||||
txt = re.sub(r'="(-?[0-9]+\.[0-9]+(?:[eE][-+]?[0-9]+)?)"', _num, txt)
|
||||
|
||||
if decl:
|
||||
txt = txt.replace('__XMLDECL__', decl_str, 1)
|
||||
return txt
|
||||
|
||||
|
||||
def clean_docx(path: str) -> str:
|
||||
"""清洗 docx 里的 word/document.xml,幂等。失败回退原文件。"""
|
||||
marker = path + '.cleaned'
|
||||
if os.path.exists(marker):
|
||||
return path
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(path, 'r') as zin:
|
||||
entries = []
|
||||
changed = False
|
||||
for item in zin.infolist():
|
||||
data = zin.read(item.filename)
|
||||
if item.filename == 'word/document.xml':
|
||||
raw = data.decode('utf-8', errors='replace')
|
||||
new = _fix_text(raw)
|
||||
if new != raw:
|
||||
data = new.encode('utf-8')
|
||||
changed = True
|
||||
entries.append((item, data))
|
||||
if not changed:
|
||||
open(marker, 'w').close()
|
||||
return path
|
||||
tmp = path + '.tmp'
|
||||
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED) as zout:
|
||||
for item, data in entries:
|
||||
zout.writestr(item, data)
|
||||
os.replace(tmp, path)
|
||||
open(marker, 'w').close()
|
||||
return path
|
||||
except Exception:
|
||||
return path
|
||||
91
services/wordparse/convert.py
Normal file
91
services/wordparse/convert.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""docx → HTML 高层入口(集成清洗 + 转码 + 美化)"""
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from typing import Callable
|
||||
|
||||
from .cleaner import clean_docx
|
||||
from .post import post_docx
|
||||
from .render import clean_html, stats as render_stats, copy_mathjax_to
|
||||
|
||||
|
||||
def _zip_name(info) -> str:
|
||||
"""zip 条目名:带 UTF-8 标志位(0x800)直接用;否则按 cp437→GBK 还原"""
|
||||
if info.flag_bits // 2048 % 2:
|
||||
return info.filename
|
||||
try:
|
||||
return info.filename.encode('cp437').decode('gbk')
|
||||
except Exception:
|
||||
return info.filename
|
||||
|
||||
|
||||
def convert_one(docx_path: str, auto_clean: bool = True) -> dict:
|
||||
"""单份 docx → HTML 字符串 + stats
|
||||
|
||||
Returns: {ok, html, stats, error?}
|
||||
"""
|
||||
try:
|
||||
if auto_clean:
|
||||
docx_path = clean_docx(docx_path)
|
||||
raw = post_docx(docx_path)
|
||||
html = clean_html(raw, mathjax=True)
|
||||
return {'ok': True, 'html': html, 'stats': render_stats(html)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e)[:300]}
|
||||
|
||||
|
||||
def batch_convert(path: str, out_dir: str, auto_clean: bool = True,
|
||||
progress_cb: Callable | None = None) -> dict:
|
||||
"""目录或 zip → 每份 docx 转 HTML 落 out_dir
|
||||
|
||||
Returns: {count, errors[{file, error}], files[{name, group, stats}]}
|
||||
"""
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
copy_mathjax_to(out_dir)
|
||||
|
||||
items: list[tuple[str, str]] = []
|
||||
if os.path.isdir(path):
|
||||
for f in sorted(os.listdir(path)):
|
||||
if f.lower().endswith('.docx'):
|
||||
items.append((f, os.path.join(path, f)))
|
||||
else:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
for i in z.infolist():
|
||||
if i.is_dir() or i.filename.startswith('__MACOSX/') or not i.filename.lower().endswith('.docx'):
|
||||
continue
|
||||
parts = [p for p in _zip_name(i).split('/') if p]
|
||||
rel = '/'.join(parts[1:]) if len(parts) > 1 else parts[-1]
|
||||
# 不解压到磁盘,直接 post bytes
|
||||
items.append((rel, zipfile.ZipFile(path).read(i)))
|
||||
|
||||
files, errors = [], []
|
||||
for rel, src in items:
|
||||
try:
|
||||
if isinstance(src, bytes):
|
||||
# 写临时文件
|
||||
tmp = f'/tmp/_wpc_{abs(hash(rel)) & 0xFFFFFFFF:x}.docx'
|
||||
with open(tmp, 'wb') as f:
|
||||
f.write(src)
|
||||
src = tmp
|
||||
r = convert_one(src, auto_clean=auto_clean)
|
||||
if r['ok']:
|
||||
html = r['html']
|
||||
rel_html = re.sub(r'\.docx$', '', rel, flags=re.I) + '.html'
|
||||
out = os.path.join(out_dir, rel_html)
|
||||
os.makedirs(os.path.dirname(out) or out_dir, exist_ok=True)
|
||||
with open(out, 'w', encoding='utf-8') as f:
|
||||
f.write(html)
|
||||
files.append({
|
||||
'name': rel_html,
|
||||
'group': os.path.dirname(rel_html),
|
||||
**r['stats'],
|
||||
})
|
||||
else:
|
||||
errors.append({'file': rel, 'error': r.get('error', '?')})
|
||||
except Exception as e:
|
||||
errors.append({'file': rel, 'error': str(e)[:200]})
|
||||
|
||||
if progress_cb:
|
||||
progress_cb(len(files) + len(errors), len(items))
|
||||
|
||||
return {'count': len(files), 'errors': errors, 'files': files}
|
||||
46
services/wordparse/post.py
Normal file
46
services/wordparse/post.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""docx4j HTTP 客户端 — POST 一份 docx,返回 HTML 字符串"""
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import urllib.request
|
||||
from db.database import load_config
|
||||
|
||||
|
||||
def _config():
|
||||
return load_config()['wordparse']
|
||||
|
||||
|
||||
def post_docx(path: str, timeout: int | None = None) -> str:
|
||||
"""返回 HTML 字符串(原始,未做 MathJax/导航注入)"""
|
||||
cfg = _config()
|
||||
url = cfg['url']
|
||||
timeout = timeout or cfg.get('timeout', 600)
|
||||
|
||||
boundary = '----WP' + uuid.uuid4().hex
|
||||
with open(path, 'rb') as f:
|
||||
data = f.read()
|
||||
fn = os.path.basename(path)
|
||||
body = (
|
||||
f'--{boundary}\r\n'.encode()
|
||||
+ f'Content-Disposition: form-data; name="file"; filename="{fn}"\r\n'.encode('utf-8')
|
||||
+ b'Content-Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document\r\n\r\n'
|
||||
+ data
|
||||
+ b'\r\n' + f'--{boundary}--\r\n'.encode()
|
||||
)
|
||||
req = urllib.request.Request(url, data=body, headers={
|
||||
'Content-Type': f'multipart/form-data; boundary={boundary}',
|
||||
'Content-Length': str(len(body)),
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
resp = json.loads(r.read().decode('utf-8'))
|
||||
return resp.get('content') or ''
|
||||
|
||||
|
||||
def health() -> bool:
|
||||
cfg = _config()
|
||||
try:
|
||||
with urllib.request.urlopen(cfg['health_url'], timeout=5) as r:
|
||||
j = json.loads(r.read().decode('utf-8'))
|
||||
return j.get('status') == 'ok'
|
||||
except Exception:
|
||||
return False
|
||||
70
services/wordparse/render.py
Normal file
70
services/wordparse/render.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""HTML 美化:注入 MathJax / 样式 / 导航 / 统计"""
|
||||
import re
|
||||
import os
|
||||
import shutil
|
||||
import urllib.request
|
||||
|
||||
# --- 常量 ---
|
||||
NAV = ('<div class="nav" style="position:sticky;top:0;background:#f8fafc;border-bottom:1px solid #e2e8f0;'
|
||||
'padding:8px 0;margin:0 0 16px;font-size:14px"><a href="index.html" '
|
||||
'style="color:#2563eb;text-decoration:none">← 返回索引</a></div>')
|
||||
|
||||
STYLE = ('<meta name="viewport" content="width=device-width, initial-scale=1">'
|
||||
'<style>body{max-width:900px;margin:0 auto;padding:24px 18px;font-family:-apple-system,'
|
||||
'"PingFang SC","Microsoft YaHei",serif;font-size:16px;line-height:1.9;color:#1a1a1a;background:#fff}'
|
||||
'img{max-width:100%;height:auto}table{border-collapse:collapse}td,th{border:1px solid #ccc;padding:4px 6px}'
|
||||
'mjx-container{overflow-x:auto;overflow-y:hidden;max-width:100%}</style>')
|
||||
|
||||
MATHJAX_TAG = ('<script>window.MathJax={tex:{inlineMath:[["\\\\(","\\\\)"]],displayMath:[["\\\\[","\\\\]"]]},'
|
||||
'svg:{fontCache:"local"},options:{enableMenu:false}};</script>'
|
||||
'<script src="tex-svg.js"></script>')
|
||||
|
||||
ASSETS_DIR = os.path.expanduser('~/.hermes/mcp/assets')
|
||||
MATHJAX_FILE = os.path.join(ASSETS_DIR, 'tex-svg.js')
|
||||
MATHJAX_MIRROR = 'https://registry.npmmirror.com/mathjax/3.2.2/files/es5/tex-svg.js'
|
||||
|
||||
|
||||
def _fix_math(m):
|
||||
inner = (m.group(1).replace(' ', ' ').replace('&', '&')
|
||||
.replace('<', '<').replace('>', '>').replace(' ', ' '))
|
||||
return r'\(' + re.sub(r'[ \t]{2,}', ' ', inner).strip() + r'\)'
|
||||
|
||||
|
||||
def clean_html(content: str, mathjax: bool = True) -> str:
|
||||
"""原始 HTML → 加 MathJax + 导航 + 样式"""
|
||||
c = re.sub(r'^<\?xml[^>]*\?>\s*', '', content)
|
||||
c = re.sub(r'<!DOCTYPE[^>]*>', '<!DOCTYPE html>', c, count=1)
|
||||
c = re.sub(r'\\\((.*?)\\\)', _fix_math, c, flags=re.S)
|
||||
inject = STYLE + (MATHJAX_TAG if mathjax else '')
|
||||
c = c.replace('</head>', inject + '</head>', 1) if '</head>' in c else inject + c
|
||||
m = re.search(r'<body[^>]*>', c, re.I)
|
||||
c = (c[:m.end()] + NAV + c[m.end():]) if m else (NAV + c)
|
||||
return c
|
||||
|
||||
|
||||
def stats(html: str) -> dict:
|
||||
"""HTML 统计:字符 / 公式 / 图片 / 未包裹 LaTeX"""
|
||||
stripped = re.sub(r'\\\(.*?\\\)', '', html, flags=re.S)
|
||||
return {
|
||||
'chars': len(html),
|
||||
'formulas': len(re.findall(r'\\\(', html)),
|
||||
'images': html.count('<img'),
|
||||
'unwrapped_latex': len(re.findall(
|
||||
r'\\(?:dfrac|frac|sqrt|left|begin\{array)', stripped)),
|
||||
}
|
||||
|
||||
|
||||
def ensure_mathjax() -> str:
|
||||
"""下载/复用 MathJax 离线文件 → 返回路径"""
|
||||
os.makedirs(ASSETS_DIR, exist_ok=True)
|
||||
if not os.path.exists(MATHJAX_FILE) or os.path.getsize(MATHJAX_FILE) < 500_000:
|
||||
urllib.request.urlretrieve(MATHJAX_MIRROR, MATHJAX_FILE)
|
||||
return MATHJAX_FILE
|
||||
|
||||
|
||||
def copy_mathjax_to(out_dir: str) -> str:
|
||||
"""把 MathJax 复制到 out_dir,返回目标路径"""
|
||||
src = ensure_mathjax()
|
||||
dst = os.path.join(out_dir, 'tex-svg.js')
|
||||
shutil.copy(src, dst)
|
||||
return dst
|
||||
Reference in New Issue
Block a user