This commit is contained in:
wangjiming
2026-08-03 09:34:08 +08:00
parent b975de02da
commit 15c4223f2c
43 changed files with 4498 additions and 234 deletions

View File

@@ -34,6 +34,7 @@ from fastapi import (
from fastapi.responses import StreamingResponse
from psycopg.rows import dict_row
from app.core.auth import filter_accessible_resource_ids, get_current_user, is_admin
from app.modules.data_process.algorithms import (
ParsedText,
canonical_record_json,
@@ -815,18 +816,33 @@ def list_tasks(
keyword: str | None = Query(default=None),
status: DataProcessStatus | None = Query(default=None),
process_type: ProcessType | None = Query(default=None),
tenant_id: str | None = Query(default=None),
project_id: str | None = Query(default=None),
store: DataProcessStore = Depends(get_data_process_store),
current_user: dict = Depends(get_current_user),
) -> dict[str, Any]:
with api_errors():
return ok(
store.list_tasks(
page=page,
page_size=page_size,
keyword=keyword,
status=status,
process_type=process_type,
)
tasks = store.list_tasks(
page=page,
page_size=page_size,
keyword=keyword,
status=status,
process_type=process_type,
tenant_id=tenant_id,
project_id=project_id,
)
# #4 资源 ACL 过滤admin 放行,普通用户只看到自己被授权的数据处理任务
items = tasks.get("items", [])
if not is_admin(current_user) and items:
accessible_ids = set(
filter_accessible_resource_ids(
"data-process", [t["id"] for t in items], current_user
)
)
items = [t for t in items if t["id"] in accessible_ids]
tasks["items"] = items
tasks["total"] = len(items)
return ok(tasks)
@router.post("")
@@ -2282,13 +2298,11 @@ def regenerate_results_batch(
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
)
# httpx.Client 支持跨线程复用,批次内共享连接池可减少重复建连开销。
with (
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor,
):
with httpx.Client(timeout=model_timeout, limits=model_limits) as model_client, \
ThreadPoolExecutor(
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
thread_name_prefix="data-result-regeneration",
) as executor:
futures = {
executor.submit(
_regenerate_result_in_place,