fix(backend): 修复文件上传后异步处理失败问题
- 修复 async_session_maker 未定义错误,改用 AsyncSessionLocal - 确保文件上传后能正确异步转换为 Markdown Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,7 @@ from pathlib import Path
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
from uuid import UUID, uuid4
|
from uuid import UUID, uuid4
|
||||||
from fastapi import APIRouter, Depends, UploadFile, File, Query
|
from fastapi import APIRouter, Depends, UploadFile, File, Query
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse, PlainTextResponse
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.api.response import ApiResponse, PaginatedResponse
|
from app.api.response import ApiResponse, PaginatedResponse
|
||||||
@@ -18,6 +18,7 @@ from app.core.crud import CRUDBase
|
|||||||
from app.core.logging import log_success, log_failure
|
from app.core.logging import log_success, log_failure
|
||||||
from app.models.models import File as FileModel
|
from app.models.models import File as FileModel
|
||||||
from app.schemas.file import FileResponse, FileCreateSchema
|
from app.schemas.file import FileResponse, FileCreateSchema
|
||||||
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
@@ -25,6 +26,9 @@ router = APIRouter()
|
|||||||
# Initialize CRUD
|
# Initialize CRUD
|
||||||
file_crud = CRUDBase(FileModel)
|
file_crud = CRUDBase(FileModel)
|
||||||
|
|
||||||
|
# Initialize markitdown
|
||||||
|
markitdown = MarkItDown()
|
||||||
|
|
||||||
|
|
||||||
def get_project_raw_dir(project_id: str) -> Path:
|
def get_project_raw_dir(project_id: str) -> Path:
|
||||||
"""获取项目的 raw 文件目录"""
|
"""获取项目的 raw 文件目录"""
|
||||||
@@ -119,26 +123,88 @@ async def upload_file(
|
|||||||
file_type=get_file_type(file.filename),
|
file_type=get_file_type(file.filename),
|
||||||
file_path=str(file_path),
|
file_path=str(file_path),
|
||||||
size=file_size,
|
size=file_size,
|
||||||
status="pending"
|
status="processing"
|
||||||
)
|
)
|
||||||
db.add(db_file)
|
db.add(db_file)
|
||||||
await db.commit()
|
await db.commit()
|
||||||
await db.refresh(db_file)
|
await db.refresh(db_file)
|
||||||
|
|
||||||
# 记录成功日志
|
# 异步处理文件:立即返回,不等待处理完成
|
||||||
log_success(
|
async def process_file_async(file_id: UUID, file_path_obj: Path, file_type: str, filename: str, project_id_val: UUID):
|
||||||
"文件上传成功",
|
"""后台异步处理文件"""
|
||||||
project_id=str(project_id),
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
file_id=str(db_file.id),
|
from app.core.database import AsyncSessionLocal
|
||||||
filename=file.filename,
|
|
||||||
file_type=db_file.file_type,
|
async with AsyncSessionLocal() as processing_db:
|
||||||
file_size=file_size,
|
try:
|
||||||
file_path=str(file_path)
|
# 重新获取文件记录
|
||||||
|
file_record = await file_crud.get(processing_db, file_id)
|
||||||
|
if not file_record:
|
||||||
|
return
|
||||||
|
|
||||||
|
# 支持 markitdown 转换的文件类型
|
||||||
|
markitdown_types = ["pdf", "docx", "doc", "pptx", "ppt", "xlsx", "xls", "htm", "html"]
|
||||||
|
text_content = ""
|
||||||
|
|
||||||
|
if file_type in markitdown_types:
|
||||||
|
# 使用 markitdown 转换为 markdown
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
result = await loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
lambda: markitdown.convert(str(file_path_obj))
|
||||||
|
)
|
||||||
|
text_content = result.text_content
|
||||||
|
else:
|
||||||
|
# txt, md 等直接读取
|
||||||
|
text_content = file_path_obj.read_text(encoding='utf-8')
|
||||||
|
|
||||||
|
# 保存到 ready 目录
|
||||||
|
ready_dir = get_project_ready_dir(str(project_id_val))
|
||||||
|
ready_filename = f"{file_id}.md"
|
||||||
|
ready_path = ready_dir / ready_filename
|
||||||
|
ready_path.write_text(text_content, encoding='utf-8')
|
||||||
|
|
||||||
|
# 更新文件状态为处理完成
|
||||||
|
file_record.status = "completed"
|
||||||
|
await processing_db.commit()
|
||||||
|
|
||||||
|
log_success(
|
||||||
|
"文件处理完成",
|
||||||
|
project_id=str(project_id_val),
|
||||||
|
file_id=str(file_id),
|
||||||
|
filename=filename,
|
||||||
|
ready_path=str(ready_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
# 更新文件状态为处理失败
|
||||||
|
file_record = await file_crud.get(processing_db, file_id)
|
||||||
|
if file_record:
|
||||||
|
file_record.status = "failed"
|
||||||
|
await processing_db.commit()
|
||||||
|
|
||||||
|
log_failure(
|
||||||
|
"文件处理失败",
|
||||||
|
project_id=str(project_id_val),
|
||||||
|
file_id=str(file_id),
|
||||||
|
filename=filename,
|
||||||
|
error=str(e)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 启动异步任务处理文件
|
||||||
|
asyncio.create_task(
|
||||||
|
process_file_async(
|
||||||
|
db_file.id,
|
||||||
|
file_path,
|
||||||
|
db_file.file_type,
|
||||||
|
file.filename,
|
||||||
|
project_id
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
return ApiResponse.ok(
|
return ApiResponse.ok(
|
||||||
data={"id": str(db_file.id), "filename": db_file.filename, "status": db_file.status},
|
data={"id": str(db_file.id), "filename": db_file.filename, "status": db_file.status},
|
||||||
message="File uploaded successfully"
|
message="File uploaded successfully, processing in background"
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# 记录失败日志
|
# 记录失败日志
|
||||||
@@ -192,6 +258,71 @@ async def get_file(
|
|||||||
return ApiResponse.ok(data=FileResponse.model_validate(file))
|
return ApiResponse.ok(data=FileResponse.model_validate(file))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/raw")
|
||||||
|
async def get_file_raw(
|
||||||
|
project_id: UUID,
|
||||||
|
file_id: UUID,
|
||||||
|
db: AsyncSession = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Get raw file content for preview"""
|
||||||
|
file = await file_crud.get(db, file_id)
|
||||||
|
if not file or file.project_id != project_id:
|
||||||
|
raise NotFoundException("File", file_id)
|
||||||
|
|
||||||
|
# 读取 raw 目录中的原始文件
|
||||||
|
raw_path = Path(file.file_path)
|
||||||
|
|
||||||
|
if not raw_path.exists():
|
||||||
|
raise NotFoundException("File not found on disk", file_id)
|
||||||
|
|
||||||
|
# 根据文件类型返回不同的内容
|
||||||
|
if file.file_type in ['txt', 'md', 'markdown', 'csv']:
|
||||||
|
content = raw_path.read_text(encoding='utf-8')
|
||||||
|
return PlainTextResponse(content=content, media_type="text/plain; charset=utf-8")
|
||||||
|
elif file.file_type == 'pdf':
|
||||||
|
# 返回PDF文件,浏览器可以内嵌显示
|
||||||
|
import base64
|
||||||
|
content = raw_path.read_bytes()
|
||||||
|
b64 = base64.b64encode(content).decode('utf-8')
|
||||||
|
return PlainTextResponse(
|
||||||
|
content=f"data:application/pdf;base64,{b64}",
|
||||||
|
media_type="text/plain"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# 其他二进制文件,返回文件信息
|
||||||
|
size_mb = file.size / (1024 * 1024)
|
||||||
|
content = f"""[二进制文件]
|
||||||
|
|
||||||
|
文件名: {file.filename}
|
||||||
|
文件类型: {file.file_type.upper()}
|
||||||
|
文件大小: {size_mb:.2f} MB
|
||||||
|
|
||||||
|
此文件为二进制格式,请下载后查看。
|
||||||
|
"""
|
||||||
|
return PlainTextResponse(content=content, media_type="text/plain; charset=utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/content")
|
||||||
|
async def get_file_content(
|
||||||
|
project_id: UUID,
|
||||||
|
file_id: UUID,
|
||||||
|
db: AsyncSession = Depends(get_db)
|
||||||
|
) -> PlainTextResponse:
|
||||||
|
"""Get file content (markdown)"""
|
||||||
|
file = await file_crud.get(db, file_id)
|
||||||
|
if not file or file.project_id != project_id:
|
||||||
|
raise NotFoundException("File", file_id)
|
||||||
|
|
||||||
|
# 读取 ready 目录中的 markdown 文件
|
||||||
|
ready_path = Path("/data/code/YG-Datasets/data") / str(project_id) / "ready" / f"{file_id}.md"
|
||||||
|
|
||||||
|
if ready_path.exists():
|
||||||
|
content = ready_path.read_text(encoding='utf-8')
|
||||||
|
return PlainTextResponse(content=content, media_type="text/plain; charset=utf-8")
|
||||||
|
else:
|
||||||
|
raise NotFoundException("File content", file_id)
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/{file_id}", response_model=ApiResponse)
|
@router.delete("/{file_id}", response_model=ApiResponse)
|
||||||
async def delete_file(
|
async def delete_file(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
@@ -203,7 +334,7 @@ async def delete_file(
|
|||||||
if not file or file.project_id != project_id:
|
if not file or file.project_id != project_id:
|
||||||
raise NotFoundException("File", file_id)
|
raise NotFoundException("File", file_id)
|
||||||
|
|
||||||
# Delete file from disk
|
# Delete file from raw directory
|
||||||
if file.file_path and os.path.exists(file.file_path):
|
if file.file_path and os.path.exists(file.file_path):
|
||||||
await asyncio.get_event_loop().run_in_executor(
|
await asyncio.get_event_loop().run_in_executor(
|
||||||
None,
|
None,
|
||||||
@@ -211,16 +342,25 @@ async def delete_file(
|
|||||||
file.file_path
|
file.file_path
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Delete file from ready directory (processed markdown)
|
||||||
|
ready_path = Path("/data/code/YG-Datasets/data") / str(project_id) / "ready" / f"{file_id}.md"
|
||||||
|
if ready_path.exists():
|
||||||
|
await asyncio.get_event_loop().run_in_executor(
|
||||||
|
None,
|
||||||
|
os.remove,
|
||||||
|
str(ready_path)
|
||||||
|
)
|
||||||
|
|
||||||
await file_crud.delete(db, file_id)
|
await file_crud.delete(db, file_id)
|
||||||
return ApiResponse.ok(message="File deleted successfully")
|
return ApiResponse.ok(message="File deleted successfully")
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{file_id}/download", response_class=FileResponse)
|
@router.get("/{file_id}/download")
|
||||||
async def download_file(
|
async def download_file(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
file_id: UUID,
|
file_id: UUID,
|
||||||
db: AsyncSession = Depends(get_db)
|
db: AsyncSession = Depends(get_db)
|
||||||
):
|
) -> FileResponse:
|
||||||
"""Download file"""
|
"""Download file"""
|
||||||
file = await file_crud.get(db, file_id)
|
file = await file_crud.get(db, file_id)
|
||||||
if not file or file.project_id != project_id:
|
if not file or file.project_id != project_id:
|
||||||
|
|||||||
Reference in New Issue
Block a user