Initial commit: book-mcp 教辅处理 MCP server + Arco Vue 管理端
后端 (book_mcp_server.py, 335 行, 15 个 MCP 工具):
- create_book / add_document / mark_html_ready / mark_json_ready / mark_failed
- update_book_status / soft_delete_book / retry_failed
- get_book / list_books / get_document / list_documents / get_document_url
- upload_to_rustfs / health
- 走 JSON-RPC + stateless_http + json_response（前端可直接调）
- launchd 守护 / 6 秒拉起（macOS）
- systemd 守护脚本（Linux/codelab）
存储:
- MongoDB (mongo:8.0 容器) - book/book_document/document_file 三表
- RustFS 对象存储（fs.code-lab.cn/scan/）- book/<id>/docs/<doc_id>/{docx,html,json}
- 幂等设计: zip_sha256 唯一 + (book_id+original_path) 唯一
前端 (Vue 3 + TS + Arco Design):
- 列表页 /book/list - 搜索 + 状态过滤 + 进度
- 详情页 /book/:id - 文档折叠 + 文件下载
- MCP 客户端 src/api/mcp.ts (JSON-RPC over fetch)
部署:
- deploy/install.sh (codelab 一键装 backend + systemd)
- deploy/install-frontend.sh (vite dev / 生产构建)
- docs/deploy.md (运维手册)
测试:
- tests/test_basic.py (sanity 7 项)
- tests/test_e2e.py (完整 e2e: docx→HTML→入库→上传→校验)
This commit is contained in:
19
db/__init__.py
Normal file
19
db/__init__.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""db 子包 — MongoDB 文档模型 + 连接管理
|
||||
|
||||
子模块:
|
||||
models — Book / BookDocument / DocumentFile helper class
|
||||
database — MongoClient + collection + init_db
|
||||
"""
|
||||
from .models import Book, BookDocument, DocumentFile
|
||||
from .database import (
|
||||
load_config, get_client, get_db, get_collection,
|
||||
init_db, health, next_id,
|
||||
COL_BOOK, COL_DOC, COL_FILE, COL_COUNTER,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'Book', 'BookDocument', 'DocumentFile',
|
||||
'load_config', 'get_client', 'get_db', 'get_collection',
|
||||
'init_db', 'health', 'next_id',
|
||||
'COL_BOOK', 'COL_DOC', 'COL_FILE', 'COL_COUNTER',
|
||||
]
|
||||
98
db/database.py
Normal file
98
db/database.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""MongoDB 连接 + collection 管理
|
||||
|
||||
3 个 collection:
|
||||
book — 教辅
|
||||
book_document — 教辅文档
|
||||
document_file — 文档文件
|
||||
|
||||
设计思路：
|
||||
- _id 用业务 ID（doc_id 16 字符 sha1 / book_id 自增字符串 / file_id 自增字符串）
|
||||
- book_id 用 INT 自增（mongo 中用 find_one_and_update + $inc 实现）
|
||||
- 不强制 schema（mongo 灵活），但用 helper 函数做字段约束
|
||||
- 索引在 init_db() 时一次性建好
|
||||
"""
|
||||
import os
|
||||
import yaml
|
||||
from pymongo import MongoClient, ASCENDING, DESCENDING, ReturnDocument
|
||||
from pymongo.database import Database
|
||||
|
||||
_CONFIG_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'config.yaml')
|
||||
|
||||
_client: MongoClient | None = None
|
||||
|
||||
|
||||
def load_config() -> dict:
|
||||
with open(_CONFIG_PATH) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def get_client() -> MongoClient:
|
||||
global _client
|
||||
if _client is None:
|
||||
cfg = load_config()['database']
|
||||
_client = MongoClient(cfg['url'], serverSelectionTimeoutMS=5000)
|
||||
return _client
|
||||
|
||||
|
||||
def get_db() -> Database:
|
||||
cfg = load_config()['database']
|
||||
return get_client()[cfg['db_name']]
|
||||
|
||||
|
||||
def get_collection(name: str):
|
||||
return get_db()[name]
|
||||
|
||||
|
||||
# ---- collection 名常量 ----
|
||||
COL_BOOK = 'book'
|
||||
COL_DOC = 'book_document'
|
||||
COL_FILE = 'document_file'
|
||||
|
||||
# ---- counters collection（book_id / file_id 自增用）----
|
||||
COL_COUNTER = 'counters'
|
||||
|
||||
|
||||
# ---- 索引初始化 ----
|
||||
def init_db():
|
||||
"""建所有索引（幂等）"""
|
||||
db = get_db()
|
||||
|
||||
# book: zip_sha256 唯一 + (name, version, is_deleted) 唯一
|
||||
db[COL_BOOK].create_index([('zip_sha256', ASCENDING)], unique=True, name='uk_book_zip_sha')
|
||||
db[COL_BOOK].create_index([('name', ASCENDING), ('version', ASCENDING), ('is_deleted', ASCENDING)],
|
||||
unique=True, name='uk_book_name_version')
|
||||
db[COL_BOOK].create_index([('status', ASCENDING)], name='idx_book_status')
|
||||
db[COL_BOOK].create_index([('created_at', DESCENDING)], name='idx_book_created')
|
||||
|
||||
# book_document: _id 已唯一（doc_id 即 _id）；额外索引仅用于查询加速
|
||||
db[COL_DOC].create_index([('book_id', ASCENDING), ('order_index', ASCENDING)], name='idx_book_order')
|
||||
db[COL_DOC].create_index([('status', ASCENDING)], name='idx_doc_status')
|
||||
|
||||
# document_file: (doc_id, file_type) 唯一 + (bucket, key)
|
||||
db[COL_FILE].create_index([('doc_id', ASCENDING), ('file_type', ASCENDING)],
|
||||
unique=True, name='uk_doc_type')
|
||||
db[COL_FILE].create_index([('object_bucket', ASCENDING), ('object_key', ASCENDING)],
|
||||
name='idx_bucket_key')
|
||||
|
||||
print(f"✓ MongoDB 已就绪（db: {db.name}, collections: {[COL_BOOK, COL_DOC, COL_FILE]}）")
|
||||
print(f" ✓ 8 个索引创建完成")
|
||||
|
||||
|
||||
def health() -> bool:
|
||||
try:
|
||||
get_client().admin.command('ping')
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def next_id(name: str) -> int:
|
||||
"""原子自增 ID（mongo 没有 AUTO_INCREMENT，用 counters collection 模拟）"""
|
||||
db = get_db()
|
||||
r = db[COL_COUNTER].find_one_and_update(
|
||||
{'_id': name},
|
||||
{'$inc': {'seq': 1}},
|
||||
upsert=True,
|
||||
return_document=ReturnDocument.AFTER,
|
||||
)
|
||||
return int(r['seq'])
|
||||
226
db/models.py
Normal file
226
db/models.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""MongoDB 文档模型 — helper class（非 ORM，纯字典封装）
|
||||
|
||||
3 个 collection：
|
||||
Book → book
|
||||
BookDocument → book_document
|
||||
DocumentFile → document_file
|
||||
|
||||
每个 class 有 to_mongo / from_mongo / to_dict 等方法。
|
||||
"""
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Book — 教辅
|
||||
# ============================================================
|
||||
class Book:
|
||||
"""教辅文档。_id = book_id（int）"""
|
||||
COLLECTION = 'book'
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.id: Optional[int] = kwargs.get('id')
|
||||
self.name: str = kwargs['name']
|
||||
self.subject: Optional[str] = kwargs.get('subject')
|
||||
self.grade: Optional[str] = kwargs.get('grade')
|
||||
self.version: Optional[str] = kwargs.get('version')
|
||||
self.type: Optional[str] = kwargs.get('type')
|
||||
self.zip_path: str = kwargs['zip_path']
|
||||
self.zip_sha256: str = kwargs['zip_sha256']
|
||||
self.zip_size: int = kwargs.get('zip_size', 0)
|
||||
self.status: str = kwargs.get('status', 'pending')
|
||||
self.total_count: int = kwargs.get('total_count', 0)
|
||||
self.success_count: int = kwargs.get('success_count', 0)
|
||||
self.failed_count: int = kwargs.get('failed_count', 0)
|
||||
self.error_msg: Optional[str] = kwargs.get('error_msg')
|
||||
self.created_at: datetime = kwargs.get('created_at', _now())
|
||||
self.updated_at: datetime = kwargs.get('updated_at', _now())
|
||||
self.created_by: Optional[str] = kwargs.get('created_by')
|
||||
self.is_deleted: int = kwargs.get('is_deleted', 0)
|
||||
|
||||
def to_mongo(self) -> dict:
|
||||
d = {
|
||||
'name': self.name,
|
||||
'subject': self.subject,
|
||||
'grade': self.grade,
|
||||
'version': self.version,
|
||||
'type': self.type,
|
||||
'zip_path': self.zip_path,
|
||||
'zip_sha256': self.zip_sha256,
|
||||
'zip_size': self.zip_size,
|
||||
'status': self.status,
|
||||
'total_count': self.total_count,
|
||||
'success_count': self.success_count,
|
||||
'failed_count': self.failed_count,
|
||||
'error_msg': self.error_msg,
|
||||
'created_at': self.created_at,
|
||||
'updated_at': self.updated_at,
|
||||
'created_by': self.created_by,
|
||||
'is_deleted': self.is_deleted,
|
||||
}
|
||||
if self.id is not None:
|
||||
d['_id'] = self.id
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_mongo(cls, doc: dict) -> 'Book':
|
||||
if not doc:
|
||||
return None
|
||||
d = dict(doc)
|
||||
d['id'] = d.pop('_id', None)
|
||||
return cls(**d)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'name': self.name,
|
||||
'subject': self.subject,
|
||||
'grade': self.grade,
|
||||
'version': self.version,
|
||||
'type': self.type,
|
||||
'zip_path': self.zip_path,
|
||||
'zip_sha256': self.zip_sha256,
|
||||
'zip_size': self.zip_size,
|
||||
'status': self.status,
|
||||
'total_count': self.total_count,
|
||||
'success_count': self.success_count,
|
||||
'failed_count': self.failed_count,
|
||||
'error_msg': self.error_msg,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||||
'updated_at': self.updated_at.isoformat() if self.updated_at else None,
|
||||
'created_by': self.created_by,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# BookDocument — 教辅文档
|
||||
# ============================================================
|
||||
class BookDocument:
|
||||
"""教辅文档。_id = doc_id（16 字符 sha1 前缀）"""
|
||||
COLLECTION = 'book_document'
|
||||
|
||||
DOC_TYPES = ('课时作业', '周末提升', '阶段质量评估', '综合评估', 'unknown')
|
||||
STATUSES = ('pending', 'html_ready', 'json_ready', 'failed')
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.book_id: int = kwargs['book_id']
|
||||
self.original_filename: str = kwargs['original_filename']
|
||||
self.original_path: str = kwargs['original_path']
|
||||
self.doc_id: str = kwargs.get('doc_id') or self.make_doc_id(
|
||||
kwargs['book_id'], kwargs['original_path'])
|
||||
self.title: Optional[str] = kwargs.get('title')
|
||||
self.doc_type: str = kwargs.get('doc_type', 'unknown')
|
||||
self.order_index: int = kwargs.get('order_index', 0)
|
||||
self.status: str = kwargs.get('status', 'pending')
|
||||
self.error_msg: Optional[str] = kwargs.get('error_msg')
|
||||
self.created_at: datetime = kwargs.get('created_at', _now())
|
||||
self.updated_at: datetime = kwargs.get('updated_at', _now())
|
||||
|
||||
@staticmethod
|
||||
def make_doc_id(book_id: int, original_path: str) -> str:
|
||||
"""稳定 doc_id：sha1(book_id|path).hexdigest()[:16]"""
|
||||
h = hashlib.sha1(f"{book_id}|{original_path}".encode('utf-8')).hexdigest()
|
||||
return h[:16]
|
||||
|
||||
def to_mongo(self) -> dict:
|
||||
return {
|
||||
'_id': self.doc_id,
|
||||
'book_id': self.book_id,
|
||||
'original_filename': self.original_filename,
|
||||
'original_path': self.original_path,
|
||||
'title': self.title,
|
||||
'doc_type': self.doc_type,
|
||||
'order_index': self.order_index,
|
||||
'status': self.status,
|
||||
'error_msg': self.error_msg,
|
||||
'created_at': self.created_at,
|
||||
'updated_at': self.updated_at,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_mongo(cls, doc: dict) -> 'BookDocument':
|
||||
if not doc:
|
||||
return None
|
||||
d = dict(doc)
|
||||
d['doc_id'] = d.pop('_id')
|
||||
return cls(**d)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'book_id': self.book_id,
|
||||
'doc_id': self.doc_id,
|
||||
'original_filename': self.original_filename,
|
||||
'original_path': self.original_path,
|
||||
'title': self.title,
|
||||
'doc_type': self.doc_type,
|
||||
'order_index': self.order_index,
|
||||
'status': self.status,
|
||||
'error_msg': self.error_msg,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||||
'updated_at': self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# DocumentFile — 文档文件
|
||||
# ============================================================
|
||||
class DocumentFile:
|
||||
"""文档文件。_id = 自增 int（next_id('file')）"""
|
||||
COLLECTION = 'document_file'
|
||||
|
||||
FILE_TYPES = ('docx', 'html', 'json', 'preview_png', 'error_log')
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.id: Optional[int] = kwargs.get('id')
|
||||
self.doc_id: str = kwargs['doc_id']
|
||||
self.file_type: str = kwargs['file_type']
|
||||
self.object_key: str = kwargs['object_key']
|
||||
self.object_bucket: str = kwargs['object_bucket']
|
||||
self.size: int = kwargs.get('size', 0)
|
||||
self.checksum: Optional[str] = kwargs.get('checksum')
|
||||
self.url: Optional[str] = kwargs.get('url')
|
||||
self.meta: Optional[dict] = kwargs.get('meta')
|
||||
self.created_at: datetime = kwargs.get('created_at', _now())
|
||||
|
||||
def to_mongo(self) -> dict:
|
||||
d = {
|
||||
'doc_id': self.doc_id,
|
||||
'file_type': self.file_type,
|
||||
'object_key': self.object_key,
|
||||
'object_bucket': self.object_bucket,
|
||||
'size': self.size,
|
||||
'checksum': self.checksum,
|
||||
'url': self.url,
|
||||
'meta': self.meta,
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
if self.id is not None:
|
||||
d['_id'] = self.id
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_mongo(cls, doc: dict) -> 'DocumentFile':
|
||||
if not doc:
|
||||
return None
|
||||
d = dict(doc)
|
||||
d['id'] = d.pop('_id')
|
||||
return cls(**d)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'doc_id': self.doc_id,
|
||||
'file_type': self.file_type,
|
||||
'object_key': self.object_key,
|
||||
'object_bucket': self.object_bucket,
|
||||
'size': self.size,
|
||||
'checksum': self.checksum,
|
||||
'url': self.url,
|
||||
'meta': self.meta,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||||
}
|
||||
77
db/schema_doc.md
Normal file
77
db/schema_doc.md
Normal file
@@ -0,0 +1,77 @@
|
||||
# MongoDB Schema — book-mcp
|
||||
|
||||
3 个 collection（无固定 schema，字段约束在 `db/models.py`）：
|
||||
|
||||
## book（教辅）
|
||||
|
||||
| 字段 | 类型 | 索引 |
|
||||
|---|---|---|
|
||||
| `_id` | int | PK（自增，counters.book_id）|
|
||||
| `name` | str | uk_name_version 一部分 |
|
||||
| `subject` | str \| null | |
|
||||
| `grade` | str \| null | |
|
||||
| `version` | str \| null | uk_name_version 一部分 |
|
||||
| `type` | str \| null | |
|
||||
| `zip_path` | str | |
|
||||
| `zip_sha256` | str | **uk_book_zip_sha 唯一** |
|
||||
| `zip_size` | int | |
|
||||
| `status` | enum | idx_book_status |
|
||||
| `total_count` | int | |
|
||||
| `success_count` | int | |
|
||||
| `failed_count` | int | |
|
||||
| `error_msg` | str \| null | |
|
||||
| `created_at` | datetime | idx_book_created |
|
||||
| `updated_at` | datetime | |
|
||||
| `created_by` | str \| null | |
|
||||
| `is_deleted` | int (0/1) | uk_name_version 一部分 |
|
||||
|
||||
## book_document（教辅文档）
|
||||
|
||||
| 字段 | 类型 | 索引 |
|
||||
|---|---|---|
|
||||
| `_id` | str (16 字符 sha1) | **uk_doc_id 唯一** |
|
||||
| `book_id` | int | idx_book_order |
|
||||
| `original_filename` | str | |
|
||||
| `original_path` | str | |
|
||||
| `title` | str \| null | |
|
||||
| `doc_type` | enum | |
|
||||
| `order_index` | int | idx_book_order |
|
||||
| `status` | enum | idx_doc_status |
|
||||
| `error_msg` | str \| null | |
|
||||
| `created_at` | datetime | |
|
||||
| `updated_at` | datetime | |
|
||||
|
||||
## document_file（文档文件）
|
||||
|
||||
| 字段 | 类型 | 索引 |
|
||||
|---|---|---|
|
||||
| `_id` | int | PK（自增，counters.file_id）|
|
||||
| `doc_id` | str | **uk_doc_type 联合唯一** + idx_bucket_key |
|
||||
| `file_type` | enum | uk_doc_type 联合唯一 |
|
||||
| `object_key` | str | idx_bucket_key |
|
||||
| `object_bucket` | str | idx_bucket_key |
|
||||
| `size` | int | |
|
||||
| `checksum` | str \| null | |
|
||||
| `url` | str \| null | |
|
||||
| `meta` | object \| null | |
|
||||
| `created_at` | datetime | |
|
||||
|
||||
## counters（自增 ID 模拟）
|
||||
|
||||
| _id (name) | seq |
|
||||
|---|---|
|
||||
| `book_id` | int |
|
||||
| `file_id` | int |
|
||||
|
||||
## 初始化
|
||||
|
||||
```bash
|
||||
# 自动建所有索引（幂等）
|
||||
/tmp/wordparse_venv/bin/python book_mcp_server.py --init-db
|
||||
```
|
||||
|
||||
输出：
|
||||
```
|
||||
✓ MongoDB 已就绪（db: book_mcp, collections: ['book', 'book_document', 'document_file']）
|
||||
✓ 8 个索引创建完成
|
||||
```
|
||||
Reference in New Issue
Block a user