2455 lines
91 KiB
Python
2455 lines
91 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import socket
|
|
import time
|
|
from collections.abc import Iterator, Mapping
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from contextlib import contextmanager
|
|
from copy import deepcopy
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
from threading import BoundedSemaphore, Lock
|
|
from typing import Any, Literal
|
|
from urllib.parse import quote, urlsplit
|
|
|
|
import httpx
|
|
import psycopg
|
|
from fastapi import (
|
|
APIRouter,
|
|
BackgroundTasks,
|
|
Body,
|
|
Depends,
|
|
File,
|
|
Header,
|
|
HTTPException,
|
|
Query,
|
|
UploadFile,
|
|
)
|
|
from fastapi.responses import StreamingResponse
|
|
from psycopg.rows import dict_row
|
|
|
|
from app.modules.data_process.algorithms import (
|
|
ParsedText,
|
|
canonical_record_json,
|
|
content_quality_flags,
|
|
desensitize_pii,
|
|
desensitize_structured_record,
|
|
detect_pdf_document_noise,
|
|
estimate_token_count,
|
|
extract_pdf_page_texts,
|
|
generate_standard_records,
|
|
is_near_duplicate,
|
|
near_duplicate_fingerprint,
|
|
parse_text_content,
|
|
preprocess_structured_records_with_lineage,
|
|
remove_document_noise,
|
|
score_quality,
|
|
structured_json_dumps,
|
|
)
|
|
from app.modules.data_process.document_chunking import (
|
|
DocumentChunk,
|
|
chunk_fixed_text,
|
|
chunk_layout_document,
|
|
chunk_semantic_text,
|
|
merge_short_chunks,
|
|
)
|
|
from app.modules.data_process.generation import generate_model_records
|
|
from app.modules.data_process.office_preview import (
|
|
MAX_XLSX_PREVIEW_ROWS,
|
|
build_docx_preview,
|
|
build_xlsx_preview,
|
|
)
|
|
from app.modules.data_process.storage import (
|
|
LocalDataProcessStorage,
|
|
StagedSourceObject,
|
|
get_data_process_storage,
|
|
)
|
|
from app.modules.data_process.store import (
|
|
ConflictError,
|
|
DataProcessStore,
|
|
DataProcessStoreError,
|
|
InvalidStateError,
|
|
NotFoundError,
|
|
get_data_process_store,
|
|
new_id,
|
|
repeat_task_id,
|
|
)
|
|
from app.schemas.data_process import (
|
|
DataProcessRegenerateRequest,
|
|
DataProcessRepeatRequest,
|
|
DataProcessStatus,
|
|
DataProcessTaskCreate,
|
|
DataProcessTaskUpdate,
|
|
DataProcessWorkflowStepUpdate,
|
|
ExternalPullRequest,
|
|
ExternalSourceRequest,
|
|
GenerateRequest,
|
|
PreviewBuildRequest,
|
|
PreviewItemCreate,
|
|
PreviewItemUpdate,
|
|
ProcessType,
|
|
PublishRequest,
|
|
ResultBatchRegenerateRequest,
|
|
ResultRegenerateRequest,
|
|
ResultUpdate,
|
|
)
|
|
|
|
router = APIRouter(prefix="/data-process")
|
|
logger = logging.getLogger(__name__)
|
|
MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
|
MAX_SOURCE_FILE_COUNT = 20
|
|
MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
|
MAX_EXTERNAL_PULL_BYTES = 50 * 1024 * 1024
|
|
STRUCTURED_SOURCE_SUFFIXES = frozenset(
|
|
{".json", ".jsonl", ".ndjson", ".csv", ".tsv", ".xlsx"}
|
|
)
|
|
UNSTRUCTURED_SOURCE_SUFFIXES = frozenset(
|
|
{
|
|
".txt",
|
|
".md",
|
|
".markdown",
|
|
".pdf",
|
|
".docx",
|
|
".pptx",
|
|
".json",
|
|
".jsonl",
|
|
".ndjson",
|
|
}
|
|
)
|
|
SUPPORTED_SOURCE_SUFFIXES = STRUCTURED_SOURCE_SUFFIXES | UNSTRUCTURED_SOURCE_SUFFIXES
|
|
LEGACY_OFFICE_CONVERSIONS = {
|
|
".doc": ".docx",
|
|
".xls": ".xlsx",
|
|
".ppt": ".pptx",
|
|
}
|
|
RAW_INLINE_PREVIEW_MEDIA_TYPES = {
|
|
"pdf": "application/pdf",
|
|
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
}
|
|
RESULT_REGENERATION_CONCURRENCY = 4
|
|
RESULT_REGENERATION_RETRIES = 0
|
|
RESULT_REGENERATION_TIMEOUT_SECONDS = 60.0
|
|
_result_regeneration_slots = BoundedSemaphore(RESULT_REGENERATION_CONCURRENCY)
|
|
_result_regeneration_claims_lock = Lock()
|
|
_active_result_regenerations: set[tuple[str, str]] = set()
|
|
|
|
|
|
def ok(data: Any = None, message: str = "ok") -> dict[str, Any]:
|
|
return {"code": 0, "message": message, "data": data}
|
|
|
|
|
|
def fail(status_code: int, message: str) -> HTTPException:
|
|
return HTTPException(
|
|
status_code=status_code,
|
|
detail={"code": status_code, "message": message, "data": None},
|
|
)
|
|
|
|
|
|
@contextmanager
|
|
def api_errors() -> Iterator[None]:
|
|
try:
|
|
yield
|
|
except NotFoundError as exc:
|
|
raise fail(404, str(exc)) from exc
|
|
except ConflictError as exc:
|
|
raise fail(409, str(exc)) from exc
|
|
except InvalidStateError as exc:
|
|
raise fail(409, str(exc)) from exc
|
|
except (DataProcessStoreError, ValueError) as exc:
|
|
raise fail(400, str(exc)) from exc
|
|
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn) as exc:
|
|
raise fail(
|
|
503,
|
|
"data process schema is missing or out of date; run schema_cli --check",
|
|
) from exc
|
|
except psycopg.OperationalError as exc:
|
|
raise fail(503, "data process database is unavailable") from exc
|
|
|
|
|
|
def _safe_file_name(value: str | None, fallback: str) -> str:
|
|
name = Path((value or "").replace("\\", "/")).name.replace("\x00", "").strip()
|
|
return name if name not in {"", ".", ".."} else fallback
|
|
|
|
|
|
def _range_not_satisfiable(size: int) -> HTTPException:
|
|
return HTTPException(
|
|
status_code=416,
|
|
detail={"code": 416, "message": "invalid source byte range", "data": None},
|
|
headers={"Content-Range": f"bytes */{size}"},
|
|
)
|
|
|
|
|
|
def _source_byte_range(value: str | None, size: int) -> tuple[int, int] | None:
|
|
if value is None:
|
|
return None
|
|
match = re.fullmatch(r"bytes=(\d*)-(\d*)", value.strip())
|
|
if match is None or size <= 0:
|
|
raise _range_not_satisfiable(size)
|
|
start_text, end_text = match.groups()
|
|
if not start_text and not end_text:
|
|
raise _range_not_satisfiable(size)
|
|
if start_text:
|
|
start = int(start_text)
|
|
end = int(end_text) if end_text else size - 1
|
|
if start >= size or end < start:
|
|
raise _range_not_satisfiable(size)
|
|
else:
|
|
suffix_length = int(end_text)
|
|
if suffix_length <= 0:
|
|
raise _range_not_satisfiable(size)
|
|
start = max(0, size - suffix_length)
|
|
end = size - 1
|
|
return start, min(end, size - 1)
|
|
|
|
|
|
def _commit_source_batch(
|
|
store: DataProcessStore,
|
|
storage: LocalDataProcessStorage,
|
|
task_id: str,
|
|
prepared: list[dict[str, Any]],
|
|
staged: list[StagedSourceObject],
|
|
) -> list[dict[str, Any]]:
|
|
storage.publish(staged)
|
|
try:
|
|
return store.add_source_files(task_id, prepared)
|
|
except Exception:
|
|
for item in staged:
|
|
try:
|
|
storage.delete(item.reference)
|
|
except Exception:
|
|
# 文件系统回滚失败不能覆盖数据库抛出的根因，并继续清理其余对象。
|
|
logger.exception(
|
|
"failed to roll back data process source object task_id=%s",
|
|
task_id,
|
|
)
|
|
raise
|
|
|
|
|
|
def _value(config: dict[str, Any], snake_name: str, camel_name: str, default: Any) -> Any:
|
|
if snake_name in config:
|
|
return config[snake_name]
|
|
return config.get(camel_name, default)
|
|
|
|
|
|
def _preprocess_options(config: dict[str, Any]) -> set[str]:
|
|
values = _value(config, "preprocess_options", "preprocessOptions", [])
|
|
return {str(item) for item in values} if isinstance(values, list) else set()
|
|
|
|
|
|
def _preview_quality(content: str, config: dict[str, Any]) -> dict[str, Any]:
|
|
records = generate_standard_records(
|
|
[{"id": "quality-preview", "edited_content": content}],
|
|
split={"train": 100, "validation": 0, "test": 0},
|
|
)
|
|
record = records[0] if records else {"instruction": "", "input": "", "output": ""}
|
|
minimum = int(_value(config, "min_output_length", "minOutputLength", 20) or 20)
|
|
return asdict(
|
|
score_quality(
|
|
record,
|
|
min_output_length=max(1, minimum),
|
|
source_content=content,
|
|
)
|
|
)
|
|
|
|
|
|
def _parse_stored_source(source: dict[str, Any]) -> ParsedText:
|
|
"""重新读取已入库的规范化正文，避免把二进制格式当二进制重复解析。"""
|
|
|
|
content = str(source.get("content") or "")
|
|
file_format = str(source.get("file_format") or "").lower()
|
|
if file_format == "xlsx":
|
|
raw_content = source.get("raw_content")
|
|
if isinstance(raw_content, bytes):
|
|
return parse_text_content(
|
|
raw_content,
|
|
filename=str(source.get("name") or "source.xlsx"),
|
|
file_format="xlsx",
|
|
)
|
|
# 兼容原始对象已缺失的历史文件：退化为上传阶段生成的 JSONL。
|
|
return parse_text_content(content, file_format="jsonl")
|
|
if file_format in {"pdf", "docx", "pptx"}:
|
|
# 文档上传阶段已抽取文本，预览阶段只需要对正文切片。
|
|
return ParsedText(format=file_format, text=content, records=())
|
|
return parse_text_content(
|
|
content,
|
|
filename=source.get("name"),
|
|
file_format=file_format or None,
|
|
)
|
|
|
|
|
|
def _chunk_source_text(
|
|
source: dict[str, Any],
|
|
config: dict[str, Any],
|
|
preprocess_options: set[str],
|
|
) -> list[DocumentChunk]:
|
|
"""根据任务配置调用真实的 Docling/LlamaIndex 切分器。"""
|
|
|
|
method = str(_value(config, "chunk_method", "chunkMethod", "layout_hybrid"))
|
|
preserve_context = "preserve_context" in preprocess_options
|
|
chunk_size = int(_value(config, "chunk_size", "chunkSize", 800))
|
|
overlap = (
|
|
int(_value(config, "chunk_overlap", "chunkOverlap", 100))
|
|
if preserve_context
|
|
else 0
|
|
)
|
|
text = str(source.get("content") or "")
|
|
if method == "layout_hybrid":
|
|
raw = source.get("raw_content")
|
|
if not isinstance(raw, bytes):
|
|
raise InvalidStateError("版面结构混合切分需要原始文件，请重新上传后再处理")
|
|
chunks = chunk_layout_document(
|
|
raw,
|
|
filename=str(source.get("name") or "document.pdf"),
|
|
source_text=text,
|
|
chunk_size=chunk_size,
|
|
)
|
|
elif method == "semantic":
|
|
chunks = chunk_semantic_text(
|
|
text,
|
|
chunk_size=chunk_size,
|
|
chunk_overlap=overlap,
|
|
breakpoint_percentile_threshold=int(
|
|
_value(
|
|
config,
|
|
"semantic_breakpoint_percentile",
|
|
"semanticBreakpointPercentile",
|
|
95,
|
|
)
|
|
),
|
|
)
|
|
elif method == "fixed":
|
|
chunks = chunk_fixed_text(text, chunk_size=chunk_size, chunk_overlap=overlap)
|
|
else:
|
|
raise ValueError(f"unsupported chunk method: {method}")
|
|
if "merge_short_content" in preprocess_options:
|
|
chunks = merge_short_chunks(
|
|
chunks,
|
|
source_text=text,
|
|
min_token_count=int(_value(config, "min_chunk_size", "minChunkSize", 100)),
|
|
max_token_count=chunk_size,
|
|
)
|
|
return chunks
|
|
|
|
|
|
_NEGATION_MARKERS = frozenset({"不", "无", "未", "否", "没有", "并非", "not", "no", "never"})
|
|
|
|
|
|
def _safe_near_duplicate(left: str, right: str) -> bool:
|
|
"""保守判断近重复，数字或否定含义变化时始终保留。"""
|
|
|
|
if min(estimate_token_count(left), estimate_token_count(right)) < 20:
|
|
return False
|
|
if re.findall(r"\d+(?:\.\d+)?", left) != re.findall(r"\d+(?:\.\d+)?", right):
|
|
return False
|
|
left_lower = left.casefold()
|
|
right_lower = right.casefold()
|
|
left_negations = {marker for marker in _NEGATION_MARKERS if marker in left_lower}
|
|
right_negations = {marker for marker in _NEGATION_MARKERS if marker in right_lower}
|
|
if left_negations != right_negations:
|
|
return False
|
|
return is_near_duplicate(
|
|
left,
|
|
right,
|
|
similarity_threshold=0.92,
|
|
max_hamming_distance=2,
|
|
)
|
|
|
|
|
|
def _near_duplicate_band_keys(content: str) -> tuple[tuple[int, int], ...]:
|
|
"""将 64 位 SimHash 分为三段，汉明距离不超过 2 时至少命中一段。"""
|
|
|
|
fingerprint = int(near_duplicate_fingerprint(content), 16)
|
|
widths = (22, 21, 21)
|
|
shift = 0
|
|
keys: list[tuple[int, int]] = []
|
|
for index, width in enumerate(widths):
|
|
keys.append((index, (fingerprint >> shift) & ((1 << width) - 1)))
|
|
shift += width
|
|
return tuple(keys)
|
|
|
|
|
|
def _build_preview_items(
|
|
task: dict[str, Any], source_files: list[dict[str, Any]]
|
|
) -> list[dict[str, Any]]:
|
|
config = task.get("config") or {}
|
|
process_type = task["process_type"]
|
|
preprocess_options = _preprocess_options(config)
|
|
should_desensitize = "desensitize" in preprocess_options
|
|
should_clean_invalid = bool(
|
|
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
|
)
|
|
should_deduplicate = bool(
|
|
preprocess_options & {"deduplicate", "deduplicate_content"}
|
|
)
|
|
seen_content_hashes: set[str] = set()
|
|
seen_near_duplicate_bands: dict[tuple[int, int], list[str]] = {}
|
|
items: list[dict[str, Any]] = []
|
|
|
|
def append_item(item: dict[str, Any], *, dedup_content: str) -> None:
|
|
content = str(item.get("edited_content") or "").strip()
|
|
if should_clean_invalid and not content:
|
|
return
|
|
# 去重必须基于脱敏前内容，否则不同原文可能在替换 PII 后被错误合并。
|
|
content_hash = hashlib.sha256(dedup_content.strip().encode("utf-8")).hexdigest()
|
|
if should_deduplicate and content_hash in seen_content_hashes:
|
|
return
|
|
seen_content_hashes.add(content_hash)
|
|
if not content:
|
|
item["status"] = "invalid"
|
|
items.append(item)
|
|
|
|
for source in source_files:
|
|
parsed = _parse_stored_source(source)
|
|
if process_type == "unstructured":
|
|
document_noise_spans = tuple(source.get("document_noise_spans") or ())
|
|
chunks = _chunk_source_text(source, config, preprocess_options)
|
|
for chunk in chunks:
|
|
content = (
|
|
remove_document_noise(
|
|
chunk.contextualized_content,
|
|
document_noise_spans,
|
|
source_offset=chunk.source_start or 0,
|
|
)
|
|
if (
|
|
should_clean_invalid
|
|
and document_noise_spans
|
|
and chunk.source_start is not None
|
|
)
|
|
else chunk.contextualized_content
|
|
)
|
|
preprocess_flags = content_quality_flags(
|
|
content,
|
|
min_chars=0,
|
|
min_tokens=0,
|
|
)
|
|
if content != chunk.contextualized_content:
|
|
preprocess_flags = (*preprocess_flags, "document_noise_removed")
|
|
flag_set = set(preprocess_flags)
|
|
if "clean_invalid_content" in preprocess_options and flag_set & {
|
|
"empty_content",
|
|
"low_printable_ratio",
|
|
"repetitive_content",
|
|
}:
|
|
continue
|
|
if "filter_low_quality" in preprocess_options and flag_set & {
|
|
"content_too_long",
|
|
"mojibake",
|
|
"low_printable_ratio",
|
|
"repetitive_content",
|
|
}:
|
|
continue
|
|
if "deduplicate_content" in preprocess_options:
|
|
band_keys = _near_duplicate_band_keys(content)
|
|
candidates = {
|
|
previous
|
|
for key in band_keys
|
|
for previous in seen_near_duplicate_bands.get(key, ())
|
|
}
|
|
if any(
|
|
_safe_near_duplicate(content, previous)
|
|
for previous in candidates
|
|
):
|
|
continue
|
|
for key in band_keys:
|
|
seen_near_duplicate_bands.setdefault(key, []).append(content)
|
|
dedup_content = content
|
|
pii_counts: dict[str, int] = {}
|
|
if should_desensitize:
|
|
content, pii_counts = desensitize_pii(content)
|
|
quality = _preview_quality(content, config)
|
|
quality["pii_replacements"] = pii_counts
|
|
quality["preprocess_flags"] = list(preprocess_flags)
|
|
quality["chunk_method"] = str(
|
|
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
|
|
)
|
|
quality["heading_path"] = list(chunk.heading_path)
|
|
quality["source_pages"] = list(chunk.source_pages)
|
|
quality["doc_item_refs"] = list(chunk.doc_item_refs)
|
|
quality["source_bboxes"] = list(chunk.source_bboxes)
|
|
append_item(
|
|
{
|
|
"source_file_id": source["id"],
|
|
"original_content": chunk.original_content,
|
|
"edited_content": content,
|
|
"source_start": chunk.source_start,
|
|
"source_end": chunk.source_end,
|
|
"source_start_line": chunk.source_start_line,
|
|
"source_end_line": chunk.source_end_line,
|
|
"token_count": estimate_token_count(content),
|
|
"status": (
|
|
"modified"
|
|
if content != chunk.original_content
|
|
else "original"
|
|
),
|
|
"quality_score": quality,
|
|
},
|
|
dedup_content=dedup_content,
|
|
)
|
|
continue
|
|
|
|
structured_options = preprocess_options & {
|
|
"clean_invalid",
|
|
"detect_structure",
|
|
"deduplicate",
|
|
"normalize_format",
|
|
"filter_anomaly",
|
|
}
|
|
source_records = list(parsed.records)
|
|
processed_records = preprocess_structured_records_with_lineage(
|
|
source_records,
|
|
structured_options,
|
|
)
|
|
for processed in processed_records:
|
|
source_index = processed.source_index
|
|
record = processed.record
|
|
original_record = (
|
|
source_records[source_index]
|
|
if source_index < len(source_records)
|
|
else record
|
|
)
|
|
source_locator = (
|
|
deepcopy(parsed.record_locators[source_index])
|
|
if source_index < len(parsed.record_locators)
|
|
else None
|
|
)
|
|
original_content = structured_json_dumps(original_record)
|
|
pii_counts: dict[str, int] = {}
|
|
edited_record = record
|
|
dedup_content = (
|
|
canonical_record_json(record)
|
|
if "normalize_format" in preprocess_options
|
|
else structured_json_dumps(record)
|
|
)
|
|
if should_desensitize:
|
|
edited_record, pii_counts = desensitize_structured_record(record)
|
|
content = (
|
|
canonical_record_json(edited_record)
|
|
if "normalize_format" in preprocess_options
|
|
else structured_json_dumps(edited_record)
|
|
)
|
|
quality = _preview_quality(content, config)
|
|
quality["pii_replacements"] = pii_counts
|
|
if source_locator is not None:
|
|
quality["source_locator"] = source_locator
|
|
source_start = (
|
|
source_locator.get("source_start")
|
|
if source_locator is not None
|
|
else None
|
|
)
|
|
source_end = (
|
|
source_locator.get("source_end")
|
|
if source_locator is not None
|
|
else None
|
|
)
|
|
source_start_line = (
|
|
source_locator.get("start_line")
|
|
if source_locator is not None
|
|
else None
|
|
)
|
|
source_end_line = (
|
|
source_locator.get("end_line")
|
|
if source_locator is not None
|
|
else None
|
|
)
|
|
append_item(
|
|
{
|
|
"source_file_id": source["id"],
|
|
"original_content": original_content,
|
|
"edited_content": content,
|
|
"source_start": source_start,
|
|
"source_end": source_end,
|
|
"source_start_line": source_start_line,
|
|
"source_end_line": source_end_line,
|
|
"token_count": estimate_token_count(content),
|
|
"status": "modified" if content != original_content else "original",
|
|
"quality_score": quality,
|
|
},
|
|
dedup_content=dedup_content,
|
|
)
|
|
return items
|
|
|
|
|
|
def _all_preview_items(store: DataProcessStore, task_id: str) -> list[dict[str, Any]]:
|
|
"""分页读取全部预览项，避免固定上限静默截断任务。"""
|
|
|
|
items: list[dict[str, Any]] = []
|
|
page = 1
|
|
page_size = 5_000
|
|
while True:
|
|
result = store.list_preview_items(task_id, page=page, page_size=page_size)
|
|
batch = result["items"]
|
|
items.extend(batch)
|
|
if len(items) >= int(result["total"]) or not batch:
|
|
return items
|
|
page += 1
|
|
|
|
|
|
def _run_generation(
|
|
store: DataProcessStore, task_id: str, generation_run_id: str
|
|
) -> None:
|
|
started_at = time.perf_counter()
|
|
logger.info(
|
|
"data process generation worker started task_id=%s generation_run_id=%s",
|
|
task_id,
|
|
generation_run_id,
|
|
)
|
|
try:
|
|
task = store.get_task(task_id)
|
|
if not store.generation_is_running(task_id, generation_run_id):
|
|
logger.info(
|
|
"data process generation worker skipped inactive run task_id=%s "
|
|
"generation_run_id=%s",
|
|
task_id,
|
|
generation_run_id,
|
|
)
|
|
return
|
|
all_preview_items = _all_preview_items(store, task_id)
|
|
preview_items = [
|
|
item
|
|
for item in all_preview_items
|
|
if item.get("status") != "invalid"
|
|
and str(item.get("edited_content") or item.get("original_content") or "").strip()
|
|
]
|
|
pre_filtered_count = len(all_preview_items) - len(preview_items)
|
|
config = task.get("config") or {}
|
|
model_id = _value(config, "generation_model_id", "generationModelId", None)
|
|
generation_model: dict[str, Any] | None = None
|
|
if model_id:
|
|
generation_model = store.get_generation_model(str(model_id))
|
|
task = store.save_generation_model_snapshot(
|
|
task_id,
|
|
generation_model,
|
|
generation_run_id=generation_run_id,
|
|
)
|
|
config = task.get("config") or config
|
|
split = _value(
|
|
config,
|
|
"dataset_split",
|
|
"datasetSplit",
|
|
{"train": 80, "validation": 10, "test": 10},
|
|
)
|
|
pairs = (
|
|
_value(config, "qa_pairs_per_chunk", "qaPairsPerChunk", 1)
|
|
if task["process_type"] == "unstructured"
|
|
else _value(config, "qa_pairs_per_row", "qaPairsPerRow", 1)
|
|
)
|
|
output_type = str(
|
|
_value(config, "output_type", "outputType", "standard")
|
|
).strip().lower()
|
|
if generation_model:
|
|
runtime_config = {
|
|
**config,
|
|
"generation_prompt": _value(
|
|
config, "generation_prompt", "generationPrompt", ""
|
|
),
|
|
"output_type": output_type,
|
|
"reasoning_detail": _value(
|
|
config, "reasoning_detail", "reasoningDetail", "normal"
|
|
),
|
|
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
|
|
"json_mode": _value(config, "json_mode", "jsonMode", False),
|
|
}
|
|
def report_progress(processed_count: int, total_count: int) -> None:
|
|
if not store.update_generation_progress(
|
|
task_id,
|
|
generation_run_id,
|
|
processed_count,
|
|
total_count,
|
|
):
|
|
raise InvalidStateError("generation run is no longer active")
|
|
|
|
generated = generate_model_records(
|
|
preview_items,
|
|
model=generation_model,
|
|
config=runtime_config,
|
|
task_id=task_id,
|
|
split=split,
|
|
qa_pairs_per_item=int(pairs or 1),
|
|
on_progress=report_progress,
|
|
)
|
|
elif output_type == "reasoning":
|
|
raise InvalidStateError("思维链输出必须配置可用的数据生成模型")
|
|
else:
|
|
generated = generate_standard_records(
|
|
preview_items,
|
|
qa_pairs_per_item=int(pairs or 1),
|
|
semantic_enrichment=bool(
|
|
_value(config, "semantic_enrichment", "semanticEnrichment", False)
|
|
),
|
|
split=split,
|
|
split_seed=task_id,
|
|
)
|
|
if not store.update_generation_progress(
|
|
task_id,
|
|
generation_run_id,
|
|
len(preview_items),
|
|
len(preview_items),
|
|
):
|
|
logger.info(
|
|
"data process generation stopped before completion task_id=%s "
|
|
"generation_run_id=%s",
|
|
task_id,
|
|
generation_run_id,
|
|
)
|
|
return
|
|
|
|
known_fingerprints: set[str] = set()
|
|
accepted: list[dict[str, Any]] = []
|
|
filtered_count = pre_filtered_count
|
|
duplicate_count = 0
|
|
error_count = 0
|
|
quality_filter = bool(
|
|
_value(config, "quality_filter_enabled", "qualityFilterEnabled", False)
|
|
)
|
|
filter_low = bool(_value(config, "filter_low_quality", "filterLowQuality", True))
|
|
filter_short = bool(
|
|
_value(config, "filter_short_content", "filterShortContent", True)
|
|
)
|
|
deduplicate = bool(
|
|
_preprocess_options(config)
|
|
& {"deduplicate", "deduplicate_content"}
|
|
)
|
|
minimum = max(1, int(_value(config, "min_output_length", "minOutputLength", 20) or 20))
|
|
preview_sources = {
|
|
str(item["id"]): str(
|
|
item.get("edited_content") or item.get("original_content") or ""
|
|
)
|
|
for item in preview_items
|
|
}
|
|
|
|
for record in generated:
|
|
quality = score_quality(
|
|
record,
|
|
min_output_length=minimum,
|
|
source_content=preview_sources.get(str(record.get("preview_item_id") or ""), ""),
|
|
known_fingerprints=known_fingerprints,
|
|
)
|
|
if "duplicate_record" in quality.flags:
|
|
duplicate_count += 1
|
|
else:
|
|
# 即使首条随后因短文本/低质量被过滤，也要阻止同批后续重复结果。
|
|
known_fingerprints.add(quality.fingerprint)
|
|
should_filter = (
|
|
(deduplicate and "duplicate_record" in quality.flags)
|
|
or (
|
|
quality_filter
|
|
and filter_short
|
|
and "output_too_short" in quality.flags
|
|
)
|
|
or (quality_filter and filter_low and not quality.is_valid)
|
|
)
|
|
if not quality.is_valid:
|
|
error_count += 1
|
|
record["status"] = "invalid"
|
|
record["error"] = ", ".join(quality.flags) or "quality validation failed"
|
|
if should_filter:
|
|
filtered_count += 1
|
|
continue
|
|
record["quality_score"] = asdict(quality)
|
|
accepted.append(record)
|
|
|
|
# stop 请求可能在纯函数计算期间到达，最终写入前再次检查状态。
|
|
if store.generation_is_running(task_id, generation_run_id):
|
|
completed = store.complete_generation(
|
|
task_id,
|
|
accepted,
|
|
generation_run_id=generation_run_id,
|
|
filtered_count=filtered_count,
|
|
duplicate_count=duplicate_count,
|
|
error_count=error_count,
|
|
)
|
|
logger.info(
|
|
"data process generation completed task_id=%s generation_run_id=%s "
|
|
"output_count=%s filtered_count=%s duplicate_count=%s error_count=%s "
|
|
"duration_ms=%.2f",
|
|
task_id,
|
|
generation_run_id,
|
|
completed.get("output_count", len(accepted)),
|
|
completed.get("filtered_count", filtered_count),
|
|
completed.get("duplicate_count", duplicate_count),
|
|
completed.get("error_count", error_count),
|
|
(time.perf_counter() - started_at) * 1000,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"data process generation stopped before result persistence task_id=%s "
|
|
"generation_run_id=%s",
|
|
task_id,
|
|
generation_run_id,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception(
|
|
"data process generation failed task_id=%s generation_run_id=%s duration_ms=%.2f",
|
|
task_id,
|
|
generation_run_id,
|
|
(time.perf_counter() - started_at) * 1000,
|
|
)
|
|
try:
|
|
if store.generation_is_running(task_id, generation_run_id):
|
|
store.mark_failed(
|
|
task_id,
|
|
str(exc),
|
|
generation_run_id=generation_run_id,
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"failed to persist data process generation failure task_id=%s "
|
|
"generation_run_id=%s",
|
|
task_id,
|
|
generation_run_id,
|
|
)
|
|
|
|
|
|
@router.get("")
|
|
def list_tasks(
|
|
page: int = Query(default=1, ge=1),
|
|
page_size: int = Query(default=20, ge=1, le=200),
|
|
keyword: str | None = Query(default=None),
|
|
status: DataProcessStatus | None = Query(default=None),
|
|
process_type: ProcessType | None = Query(default=None),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(
|
|
store.list_tasks(
|
|
page=page,
|
|
page_size=page_size,
|
|
keyword=keyword,
|
|
status=status,
|
|
process_type=process_type,
|
|
)
|
|
)
|
|
|
|
|
|
@router.post("")
|
|
def create_task(
|
|
payload: DataProcessTaskCreate,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.create_task(payload.model_dump(mode="json"))
|
|
return ok(task, "data process task created")
|
|
|
|
|
|
@router.get("/{task_id}")
|
|
def task_detail(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
# 兼容旧版曾在第五步前清空结果的异常任务；严格特征匹配且幂等。
|
|
store.recover_legacy_aborted_regeneration(task_id)
|
|
task = store.get_task(task_id)
|
|
source_files = store.list_source_files(task_id)
|
|
task["source_files"] = source_files
|
|
task["source_file_count"] = len(source_files)
|
|
return ok(task)
|
|
|
|
|
|
@router.put("/{task_id}")
|
|
def update_task(
|
|
task_id: str,
|
|
payload: DataProcessTaskUpdate,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(
|
|
store.update_task(task_id, payload.model_dump(exclude_unset=True, mode="json")),
|
|
"data process task updated",
|
|
)
|
|
|
|
|
|
@router.put("/{task_id}/workflow-step")
|
|
def update_workflow_step(
|
|
task_id: str,
|
|
payload: DataProcessWorkflowStepUpdate,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(
|
|
store.update_workflow_step(task_id, payload.workflow_step.value),
|
|
"data process workflow step updated",
|
|
)
|
|
|
|
|
|
@router.post("/{task_id}/regenerate")
|
|
def prepare_regeneration(
|
|
task_id: str,
|
|
payload: DataProcessRegenerateRequest,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(
|
|
store.prepare_regeneration(task_id, payload.model_dump(mode="json")),
|
|
"data process task prepared for regeneration",
|
|
)
|
|
|
|
|
|
def _repeat_file_copies(
|
|
store: DataProcessStore,
|
|
storage: LocalDataProcessStorage,
|
|
source_task_id: str,
|
|
request_id: str,
|
|
) -> tuple[dict[str, dict[str, str]], list[StagedSourceObject]]:
|
|
"""为新任务创建独立的源文件引用，避免删除任一任务时互相影响。"""
|
|
|
|
target_task_id = repeat_task_id(source_task_id, request_id)
|
|
copies: dict[str, dict[str, str]] = {}
|
|
staged: list[StagedSourceObject] = []
|
|
batch_id = storage.new_batch_id()
|
|
for summary in store.list_source_files(source_task_id):
|
|
old_file_id = str(summary["id"])
|
|
source = store.get_source_file(source_task_id, old_file_id, include_content=True)
|
|
new_file_id = new_id("dpsf")
|
|
old_reference = str(source.get("storage_object_id") or "")
|
|
if old_reference.startswith("local://data-process/"):
|
|
staged_object = storage.stage_copy(
|
|
batch_id=batch_id,
|
|
source_reference=old_reference,
|
|
expected_source_task_id=source_task_id,
|
|
expected_source_file_id=old_file_id,
|
|
task_id=target_task_id,
|
|
source_file_id=new_file_id,
|
|
version=1,
|
|
name=str(source["name"]),
|
|
)
|
|
staged.append(staged_object)
|
|
new_reference = staged_object.reference
|
|
elif old_reference.startswith("db://data-process/") or not old_reference:
|
|
new_reference = f"db://data-process/{target_task_id}/{new_file_id}/v1"
|
|
else:
|
|
raise ValueError("源任务包含不受支持的文件存储引用")
|
|
copies[old_file_id] = {
|
|
"id": new_file_id,
|
|
"storage_object_id": new_reference,
|
|
}
|
|
return copies, staged
|
|
|
|
|
|
def _remove_repeated_storage_objects(
|
|
storage: LocalDataProcessStorage,
|
|
task_id: str,
|
|
staged: list[StagedSourceObject],
|
|
copies: dict[str, dict[str, str]],
|
|
) -> None:
|
|
source_file_ids = {
|
|
str(copy["storage_object_id"]): str(copy["id"])
|
|
for copy in copies.values()
|
|
}
|
|
for item in staged:
|
|
try:
|
|
storage.delete(
|
|
item.reference,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=source_file_ids[item.reference],
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"failed to roll back repeated data process source object task_id=%s",
|
|
task_id,
|
|
)
|
|
|
|
|
|
@router.post("/{task_id}/repeat", status_code=202)
|
|
def repeat_generation(
|
|
task_id: str,
|
|
payload: DataProcessRepeatRequest,
|
|
background_tasks: BackgroundTasks,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
"""按原任务快照创建独立任务，并立即在后台开始新一批生成。"""
|
|
|
|
with api_errors():
|
|
repeated = store.find_repeated_task(task_id, payload.request_id)
|
|
staged: list[StagedSourceObject] = []
|
|
target_task_id = repeat_task_id(task_id, payload.request_id)
|
|
if repeated is None:
|
|
copies, staged = _repeat_file_copies(
|
|
store,
|
|
storage,
|
|
task_id,
|
|
payload.request_id,
|
|
)
|
|
storage.publish(staged)
|
|
try:
|
|
repeated = store.repeat_task(
|
|
task_id,
|
|
expected_updated_at=payload.expected_updated_at,
|
|
request_id=payload.request_id,
|
|
file_copies=copies,
|
|
)
|
|
except Exception:
|
|
_remove_repeated_storage_objects(
|
|
storage,
|
|
target_task_id,
|
|
staged,
|
|
copies,
|
|
)
|
|
raise
|
|
if not repeated["created"]:
|
|
_remove_repeated_storage_objects(
|
|
storage,
|
|
target_task_id,
|
|
staged,
|
|
copies,
|
|
)
|
|
|
|
repeated_task = repeated["task"]
|
|
if repeated_task.get("status") == "pending":
|
|
try:
|
|
started = store.start_generation(target_task_id, replace_existing=True)
|
|
background_tasks.add_task(
|
|
_run_generation,
|
|
store,
|
|
target_task_id,
|
|
str(started["generation_run_id"]),
|
|
)
|
|
except ConflictError:
|
|
latest = store.get_task(target_task_id)
|
|
if latest.get("status") != "running":
|
|
raise
|
|
repeated["task"] = store.get_task(target_task_id)
|
|
repeated["progress"] = store.progress(target_task_id)
|
|
return ok(repeated, "已按原配置创建新任务并开始后台生成")
|
|
|
|
|
|
@router.delete("/{task_id}")
|
|
def delete_task(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
store.delete_task(task_id)
|
|
return ok({"deleted": task_id}, "data process task deleted")
|
|
|
|
|
|
@router.get("/{task_id}/source-files")
|
|
def source_files(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok({"files": store.list_source_files(task_id)})
|
|
|
|
|
|
@router.post("/{task_id}/source-files")
|
|
async def upload_source_files(
|
|
task_id: str,
|
|
files: list[UploadFile] = File(...),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
if not files:
|
|
raise fail(400, "at least one source file is required")
|
|
if len(files) > MAX_SOURCE_FILE_COUNT:
|
|
raise fail(413, f"a source batch may contain at most {MAX_SOURCE_FILE_COUNT} files")
|
|
prepared: list[dict[str, Any]] = []
|
|
staged: list[StagedSourceObject] = []
|
|
batch_id = storage.new_batch_id()
|
|
batch_size = 0
|
|
commit_attempted = False
|
|
try:
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
process_type = str(task["process_type"])
|
|
if process_type == "external":
|
|
raise InvalidStateError(
|
|
"external tasks must import data through the external source endpoint"
|
|
)
|
|
allowed_suffixes = (
|
|
UNSTRUCTURED_SOURCE_SUFFIXES
|
|
if process_type == "unstructured"
|
|
else STRUCTURED_SOURCE_SUFFIXES
|
|
)
|
|
for upload in files:
|
|
raw = await upload.read(MAX_SOURCE_FILE_BYTES + 1)
|
|
if len(raw) > MAX_SOURCE_FILE_BYTES:
|
|
raise fail(413, f"source file exceeds {MAX_SOURCE_FILE_BYTES} bytes")
|
|
name = _safe_file_name(upload.filename, "source.txt")
|
|
suffix = Path(name).suffix.lower()
|
|
if suffix in LEGACY_OFFICE_CONVERSIONS:
|
|
replacement = LEGACY_OFFICE_CONVERSIONS[suffix]
|
|
raise fail(
|
|
415,
|
|
f"legacy {suffix} format is not supported; "
|
|
f"convert the file to {replacement} and upload again",
|
|
)
|
|
if suffix not in SUPPORTED_SOURCE_SUFFIXES:
|
|
raise fail(415, f"unsupported source file format: {suffix or 'none'}")
|
|
if suffix not in allowed_suffixes:
|
|
raise fail(
|
|
415,
|
|
f"{suffix} is not supported for {process_type} data processing",
|
|
)
|
|
parsed = parse_text_content(raw, filename=name)
|
|
if not parsed.text.strip():
|
|
raise fail(400, f"source file is empty: {name}")
|
|
batch_size += len(raw)
|
|
if batch_size > MAX_SOURCE_BATCH_BYTES:
|
|
raise fail(413, f"source batch exceeds {MAX_SOURCE_BATCH_BYTES} bytes")
|
|
source_file_id = new_id("dpsf")
|
|
staged_object = storage.stage_bytes(
|
|
batch_id=batch_id,
|
|
task_id=task_id,
|
|
source_file_id=source_file_id,
|
|
version=1,
|
|
name=name,
|
|
content=raw,
|
|
)
|
|
staged.append(staged_object)
|
|
record_count = (
|
|
len(parsed.records)
|
|
if process_type == "structured"
|
|
else (1 if parsed.text else 0)
|
|
)
|
|
prepared.append(
|
|
{
|
|
"id": source_file_id,
|
|
"storage_object_id": staged_object.reference,
|
|
"name": name,
|
|
"content": parsed.text,
|
|
"raw_size": len(raw),
|
|
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
|
"file_format": parsed.format,
|
|
"record_count": record_count,
|
|
"metadata": {
|
|
"content_type": upload.content_type or "text/plain",
|
|
"original_size_bytes": len(raw),
|
|
"original_checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
|
},
|
|
"created_by": None,
|
|
}
|
|
)
|
|
# publish 无论成功或失败都会消费并清理暂存对象，外层不能再次 discard。
|
|
commit_attempted = True
|
|
created = _commit_source_batch(store, storage, task_id, prepared, staged)
|
|
finally:
|
|
if not commit_attempted:
|
|
storage.discard(staged)
|
|
return ok({"files": created}, "source files uploaded")
|
|
|
|
|
|
@router.get("/{task_id}/source-files/{file_id}/content")
|
|
def source_file_content(
|
|
task_id: str,
|
|
file_id: str,
|
|
start_line: int | None = Query(default=None, ge=1),
|
|
line_count: int = Query(default=200, ge=1, le=10_000),
|
|
offset: int = Query(default=0, ge=0),
|
|
limit: int = Query(default=100_000, ge=1, le=1_000_000),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
if start_line is not None:
|
|
return ok(store.source_content_lines(task_id, file_id, start_line, line_count))
|
|
return ok(store.source_content_window(task_id, file_id, offset, limit))
|
|
|
|
|
|
@router.get("/{task_id}/source-files/{file_id}/raw")
|
|
def source_file_raw(
|
|
task_id: str,
|
|
file_id: str,
|
|
range_header: str | None = Header(default=None, alias="Range"),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> StreamingResponse:
|
|
with api_errors():
|
|
source = store.get_source_file(task_id, file_id, include_content=False)
|
|
file_format = str(source.get("file_format") or "").lower()
|
|
media_type = RAW_INLINE_PREVIEW_MEDIA_TYPES.get(file_format)
|
|
if media_type is None:
|
|
raise fail(
|
|
415,
|
|
"raw inline preview is only available for PDF and modern Office source files",
|
|
)
|
|
storage_object_id = str(source.get("storage_object_id") or "")
|
|
actual_size = storage.file_size(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
)
|
|
if actual_size is None:
|
|
raise fail(410, "the original file is unavailable for this legacy source file")
|
|
expected_size = int(source.get("size_bytes") or 0)
|
|
if actual_size != expected_size:
|
|
raise ValueError("source object size does not match metadata")
|
|
selected_range = _source_byte_range(range_header, actual_size)
|
|
start, end = selected_range or (0, actual_size - 1)
|
|
length = end - start + 1
|
|
default_name = f"source.{file_format}"
|
|
name = _safe_file_name(str(source.get("name") or default_name), default_name)
|
|
headers = {
|
|
"Accept-Ranges": "bytes",
|
|
"Cache-Control": "private, no-store",
|
|
"Content-Disposition": f"inline; filename*=UTF-8''{quote(name, safe='')}",
|
|
"Content-Length": str(length),
|
|
"X-Accel-Buffering": "no",
|
|
"X-Content-Type-Options": "nosniff",
|
|
}
|
|
checksum = str(source.get("checksum_sha256") or "")
|
|
if checksum:
|
|
headers["ETag"] = f'"{checksum}"'
|
|
if selected_range is not None:
|
|
headers["Content-Range"] = f"bytes {start}-{end}/{actual_size}"
|
|
body = storage.iter_bytes(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
expected_size=actual_size,
|
|
start=start,
|
|
length=length,
|
|
)
|
|
return StreamingResponse(
|
|
body,
|
|
status_code=206 if selected_range is not None else 200,
|
|
media_type=media_type,
|
|
headers=headers,
|
|
)
|
|
|
|
|
|
@router.get("/{task_id}/source-files/{file_id}/office-preview")
|
|
def source_file_office_preview(
|
|
task_id: str,
|
|
file_id: str,
|
|
sheet_index: int = Query(default=0, ge=0),
|
|
offset: int = Query(default=0, ge=0),
|
|
limit: int = Query(default=100, ge=1, le=MAX_XLSX_PREVIEW_ROWS),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
"""返回 Word 版式块或 Excel 工作表网格，不把二进制内容下发给组件解析。"""
|
|
|
|
with api_errors():
|
|
source = store.get_source_file(task_id, file_id, include_content=False)
|
|
file_format = str(source.get("file_format") or "").lower()
|
|
if file_format not in {"docx", "xlsx"}:
|
|
raise fail(415, "Office preview is only available for DOCX and XLSX source files")
|
|
storage_object_id = str(source.get("storage_object_id") or "")
|
|
actual_size = storage.file_size(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
)
|
|
if actual_size is None:
|
|
raise fail(410, "the original Office file is unavailable for this legacy source file")
|
|
expected_size = int(source.get("size_bytes") or 0)
|
|
if actual_size != expected_size:
|
|
raise ValueError("source object size does not match metadata")
|
|
raw = b"".join(
|
|
storage.iter_bytes(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
expected_size=actual_size,
|
|
)
|
|
)
|
|
preview = (
|
|
build_docx_preview(raw)
|
|
if file_format == "docx"
|
|
else build_xlsx_preview(
|
|
raw,
|
|
sheet_index=sheet_index,
|
|
offset=offset,
|
|
limit=limit,
|
|
)
|
|
)
|
|
preview["file_name"] = str(source.get("name") or f"source.{file_format}")
|
|
return ok(preview)
|
|
|
|
|
|
@router.get("/{task_id}/source-files/{file_id}/pdf-pages")
|
|
def source_file_pdf_pages(
|
|
task_id: str,
|
|
file_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
"""返回切片全文偏移对应的 PDF 物理页范围。"""
|
|
|
|
with api_errors():
|
|
source = store.get_source_file(task_id, file_id, include_content=False)
|
|
if str(source.get("file_format") or "").lower() != "pdf":
|
|
raise fail(415, "PDF page mapping is only available for PDF source files")
|
|
storage_object_id = str(source.get("storage_object_id") or "")
|
|
actual_size = storage.file_size(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
)
|
|
if actual_size is None:
|
|
raise fail(410, "the original PDF is unavailable for this legacy source file")
|
|
expected_size = int(source.get("size_bytes") or 0)
|
|
if actual_size != expected_size:
|
|
raise ValueError("source object size does not match metadata")
|
|
raw = b"".join(
|
|
storage.iter_bytes(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
expected_size=actual_size,
|
|
)
|
|
)
|
|
pages = extract_pdf_page_texts(raw)
|
|
return ok(
|
|
{
|
|
"page_count": len(pages),
|
|
"pages": [
|
|
{
|
|
"page_number": page.page_number,
|
|
"source_start": page.source_start,
|
|
"source_end": page.source_end,
|
|
}
|
|
for page in pages
|
|
],
|
|
}
|
|
)
|
|
|
|
|
|
@router.delete("/{task_id}/source-files/{file_id}")
|
|
def delete_source_file(
|
|
task_id: str,
|
|
file_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
source = store.get_source_file(task_id, file_id, include_content=False)
|
|
storage_object_id = str(source.get("storage_object_id") or "")
|
|
storage.validate_owner(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
)
|
|
store.delete_source_file(task_id, file_id)
|
|
cleanup_pending = False
|
|
try:
|
|
storage.delete(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=file_id,
|
|
)
|
|
except Exception:
|
|
# 数据库软删除已经提交，不能再向客户端返回可重试的失败；保留逻辑引用，
|
|
# 由后续存储清理任务重试物理删除。
|
|
cleanup_pending = True
|
|
logger.exception(
|
|
"failed to remove data process source object after soft deletion",
|
|
extra={"task_id": task_id, "source_file_id": file_id},
|
|
)
|
|
return ok(
|
|
{"deleted": file_id, "storage_cleanup_pending": cleanup_pending},
|
|
"source file removed",
|
|
)
|
|
|
|
|
|
def _external_postgres_connection(payload: ExternalSourceRequest) -> psycopg.Connection[Any]:
|
|
kind = payload.type.strip().lower()
|
|
parsed_url = urlsplit(payload.url)
|
|
scheme = parsed_url.scheme.lower()
|
|
if kind not in {"postgres", "postgresql"} or scheme not in {"postgres", "postgresql"}:
|
|
raise fail(501, f"external data source type is not supported: {payload.type}")
|
|
if parsed_url.username or parsed_url.password:
|
|
raise fail(400, "database credentials must use the account and password fields")
|
|
if payload.auth_mode not in {"none", "basic"}:
|
|
raise fail(400, "PostgreSQL supports only none or basic authentication")
|
|
if payload.auth_mode == "basic" and not payload.username:
|
|
raise fail(400, "database username is required for basic authentication")
|
|
hostname = parsed_url.hostname
|
|
if not hostname:
|
|
raise fail(400, "external PostgreSQL URL must include a hostname")
|
|
allow_private = os.getenv("DATA_PROCESS_ALLOW_PRIVATE_EXTERNAL_DB", "").lower() in {
|
|
"1",
|
|
"true",
|
|
"yes",
|
|
}
|
|
if not allow_private:
|
|
try:
|
|
addresses = {
|
|
item[4][0]
|
|
for item in socket.getaddrinfo(
|
|
hostname,
|
|
parsed_url.port or 5432,
|
|
type=socket.SOCK_STREAM,
|
|
)
|
|
}
|
|
except socket.gaierror as exc:
|
|
raise fail(400, "external PostgreSQL hostname cannot be resolved") from exc
|
|
if any(
|
|
(address := ipaddress.ip_address(value)).is_private
|
|
or address.is_loopback
|
|
or address.is_link_local
|
|
or address.is_reserved
|
|
or address.is_unspecified
|
|
for value in addresses
|
|
):
|
|
raise fail(
|
|
403,
|
|
"private or local database addresses are disabled; "
|
|
"set DATA_PROCESS_ALLOW_PRIVATE_EXTERNAL_DB=true only in a trusted deployment",
|
|
)
|
|
kwargs: dict[str, Any] = {
|
|
"connect_timeout": 5,
|
|
"row_factory": dict_row,
|
|
"application_name": "yg-ft-data-process-readonly",
|
|
"options": "-c default_transaction_read_only=on -c statement_timeout=30000",
|
|
}
|
|
if payload.auth_mode == "basic" and payload.username:
|
|
kwargs["user"] = payload.username
|
|
if payload.auth_mode == "basic" and payload.password:
|
|
kwargs["password"] = payload.password
|
|
return psycopg.connect(payload.url, **kwargs)
|
|
|
|
|
|
@router.post("/{task_id}/external/test")
|
|
def test_external_source(
|
|
task_id: str,
|
|
payload: ExternalSourceRequest,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
if str(task.get("process_type")) != "external":
|
|
raise InvalidStateError(
|
|
"external source access requires an external data processing task"
|
|
)
|
|
try:
|
|
with _external_postgres_connection(payload) as conn:
|
|
conn.execute("SELECT 1 AS ok").fetchone()
|
|
except psycopg.Error as exc:
|
|
raise fail(502, "external PostgreSQL connection test failed") from exc
|
|
return ok({"connected": True, "type": payload.type})
|
|
|
|
|
|
@router.post("/{task_id}/external/pull")
|
|
def pull_external_source(
|
|
task_id: str,
|
|
payload: ExternalPullRequest,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
query = (payload.query or "").strip()
|
|
if query.endswith(";"):
|
|
query = query[:-1].rstrip()
|
|
if ";" in query:
|
|
raise fail(400, "external pull accepts exactly one read-only query")
|
|
first_token = query.split(maxsplit=1)[0].lower() if query else ""
|
|
if first_token not in {"select", "with"}:
|
|
raise fail(400, "a read-only SELECT or WITH query is required for external pull")
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
if str(task.get("process_type")) != "external":
|
|
raise InvalidStateError("external pull requires an external data processing task")
|
|
try:
|
|
with _external_postgres_connection(payload) as conn:
|
|
conn.execute("SET TRANSACTION READ ONLY")
|
|
conn.execute("SET LOCAL statement_timeout = '30s'")
|
|
cursor = conn.execute(query)
|
|
rows: list[dict[str, Any]] = []
|
|
content_parts: list[str] = []
|
|
content_size = 0
|
|
while len(rows) < payload.limit:
|
|
batch = cursor.fetchmany(min(1_000, payload.limit - len(rows)))
|
|
if not batch:
|
|
break
|
|
for row in batch:
|
|
line = json.dumps(row, ensure_ascii=False, default=str) + "\n"
|
|
content_size += len(line.encode("utf-8"))
|
|
if content_size > MAX_EXTERNAL_PULL_BYTES:
|
|
raise fail(413, "external pull result exceeds the 50 MiB safety limit")
|
|
rows.append(row)
|
|
content_parts.append(line)
|
|
conn.rollback()
|
|
except psycopg.Error as exc:
|
|
raise fail(502, "external PostgreSQL query failed") from exc
|
|
if not rows:
|
|
raise fail(400, "external query returned no rows")
|
|
content = "".join(content_parts)
|
|
raw = content.encode("utf-8")
|
|
name = _safe_file_name(payload.file_name, "external-data.jsonl")
|
|
source_file_id = new_id("dpsf")
|
|
staged = storage.stage_bytes(
|
|
batch_id=storage.new_batch_id(),
|
|
task_id=task_id,
|
|
source_file_id=source_file_id,
|
|
version=1,
|
|
name=name,
|
|
content=raw,
|
|
)
|
|
sources = _commit_source_batch(
|
|
store,
|
|
storage,
|
|
task_id,
|
|
[
|
|
{
|
|
"id": source_file_id,
|
|
"storage_object_id": staged.reference,
|
|
"name": name,
|
|
"content": content,
|
|
"raw_size": len(raw),
|
|
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
|
"file_format": "jsonl",
|
|
"record_count": len(rows),
|
|
"metadata": {
|
|
"external_type": payload.type,
|
|
"external_host": urlsplit(payload.url).hostname,
|
|
"external_limit": payload.limit,
|
|
},
|
|
}
|
|
],
|
|
[staged],
|
|
)
|
|
source = sources[0]
|
|
return ok({"files": [source]}, "external source pulled")
|
|
|
|
|
|
def _prepare_preview_items(
|
|
task_id: str,
|
|
store: DataProcessStore,
|
|
storage: LocalDataProcessStorage,
|
|
source_file_ids: list[str] | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
task = store.get_task(task_id)
|
|
source_summaries = store.list_source_files(task_id)
|
|
if source_file_ids is not None:
|
|
requested = set(source_file_ids)
|
|
source_summaries = [item for item in source_summaries if item["id"] in requested]
|
|
found = {item["id"] for item in source_summaries}
|
|
missing = requested - found
|
|
if missing:
|
|
raise NotFoundError(f"source files not found: {', '.join(sorted(missing))}")
|
|
sources = [
|
|
store.get_source_file(task_id, item["id"], include_content=True)
|
|
for item in source_summaries
|
|
]
|
|
if not sources:
|
|
raise InvalidStateError("at least one source file is required")
|
|
config = task.get("config") or {}
|
|
preprocess_options = _preprocess_options(config)
|
|
chunk_method = str(
|
|
_value(config, "chunk_method", "chunkMethod", "layout_hybrid")
|
|
)
|
|
is_unstructured = task.get("process_type") == "unstructured"
|
|
needs_unstructured_raw = is_unstructured and (
|
|
chunk_method == "layout_hybrid"
|
|
or preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
|
)
|
|
has_structured_xlsx = not is_unstructured and any(
|
|
str(source.get("file_format") or "").lower() == "xlsx"
|
|
for source in sources
|
|
)
|
|
if needs_unstructured_raw or has_structured_xlsx:
|
|
for index, source in enumerate(sources):
|
|
source_format = str(source.get("file_format") or "").lower()
|
|
needs_structured_xlsx = not is_unstructured and source_format == "xlsx"
|
|
needs_layout_raw = is_unstructured and chunk_method == "layout_hybrid"
|
|
needs_pdf_noise = (
|
|
is_unstructured
|
|
and not needs_layout_raw
|
|
and source_format == "pdf"
|
|
and bool(
|
|
preprocess_options & {"clean_invalid", "clean_invalid_content"}
|
|
)
|
|
)
|
|
if not (needs_structured_xlsx or needs_layout_raw or needs_pdf_noise):
|
|
continue
|
|
storage_object_id = str(source.get("storage_object_id") or "")
|
|
actual_size = storage.file_size(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=str(source["id"]),
|
|
)
|
|
if actual_size is None:
|
|
if needs_layout_raw:
|
|
raise InvalidStateError(
|
|
"版面结构混合切分无法读取原始文件，请重新上传后再处理"
|
|
)
|
|
continue
|
|
expected_size = int(source.get("size_bytes") or 0)
|
|
if expected_size and actual_size != expected_size:
|
|
raise ValueError("source object size does not match metadata")
|
|
raw = b"".join(
|
|
storage.iter_bytes(
|
|
storage_object_id,
|
|
expected_task_id=task_id,
|
|
expected_source_file_id=str(source["id"]),
|
|
expected_size=actual_size,
|
|
)
|
|
)
|
|
enriched = dict(source)
|
|
if needs_structured_xlsx or needs_layout_raw:
|
|
enriched["raw_content"] = raw
|
|
sources[index] = enriched
|
|
continue
|
|
pages = extract_pdf_page_texts(raw)
|
|
extracted_text = "\n\n".join(page.text for page in pages if page.text)
|
|
if extracted_text != str(source.get("content") or ""):
|
|
logger.warning(
|
|
"skip PDF document noise detection because stored offsets differ for %s",
|
|
source["id"],
|
|
)
|
|
continue
|
|
enriched["document_noise_spans"] = detect_pdf_document_noise(pages)
|
|
sources[index] = enriched
|
|
items = _build_preview_items(task, sources)
|
|
if not items and source_file_ids is None and is_unstructured:
|
|
raise InvalidStateError("source files did not produce preview items")
|
|
return items
|
|
|
|
|
|
def _run_preview(
|
|
store: DataProcessStore,
|
|
storage: LocalDataProcessStorage,
|
|
task_id: str,
|
|
preview_run_id: str,
|
|
source_file_ids: list[str],
|
|
) -> None:
|
|
"""后台逐文件切分；所有写入均由 preview_run_id 保护。"""
|
|
|
|
started_at = time.perf_counter()
|
|
logger.info(
|
|
"data process preview started task_id=%s preview_run_id=%s total_files=%s",
|
|
task_id,
|
|
preview_run_id,
|
|
len(source_file_ids),
|
|
)
|
|
try:
|
|
is_unstructured = store.get_task(task_id).get("process_type") == "unstructured"
|
|
if not store.mark_preview_running(task_id, preview_run_id):
|
|
logger.info(
|
|
"data process preview skipped inactive run task_id=%s preview_run_id=%s",
|
|
task_id,
|
|
preview_run_id,
|
|
)
|
|
return
|
|
total_files = len(source_file_ids)
|
|
total_items = 0
|
|
for completed_files, source_file_id in enumerate(source_file_ids, start=1):
|
|
if not store.preview_is_running(task_id, preview_run_id):
|
|
logger.info(
|
|
"data process preview cancelled task_id=%s preview_run_id=%s "
|
|
"completed_files=%s total_files=%s",
|
|
task_id,
|
|
preview_run_id,
|
|
completed_files - 1,
|
|
total_files,
|
|
)
|
|
return
|
|
items = _prepare_preview_items(
|
|
task_id,
|
|
store,
|
|
storage,
|
|
[source_file_id],
|
|
)
|
|
if not items and is_unstructured:
|
|
raise InvalidStateError(
|
|
f"source file did not produce preview items: {source_file_id}"
|
|
)
|
|
created = store.replace_preview_items(
|
|
task_id,
|
|
items,
|
|
source_file_ids=[source_file_id],
|
|
preview_run_id=preview_run_id,
|
|
)
|
|
total_items += len(created)
|
|
if not store.update_preview_progress(
|
|
task_id,
|
|
preview_run_id,
|
|
completed_files,
|
|
total_files,
|
|
):
|
|
logger.info(
|
|
"data process preview stopped before progress update task_id=%s "
|
|
"preview_run_id=%s completed_files=%s total_files=%s",
|
|
task_id,
|
|
preview_run_id,
|
|
completed_files,
|
|
total_files,
|
|
)
|
|
return
|
|
if store.complete_preview(task_id, preview_run_id):
|
|
logger.info(
|
|
"data process preview completed task_id=%s preview_run_id=%s "
|
|
"total_files=%s total_items=%s duration_ms=%.2f",
|
|
task_id,
|
|
preview_run_id,
|
|
total_files,
|
|
total_items,
|
|
(time.perf_counter() - started_at) * 1000,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"data process preview completion ignored for inactive run task_id=%s "
|
|
"preview_run_id=%s",
|
|
task_id,
|
|
preview_run_id,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception(
|
|
"data process preview failed task_id=%s preview_run_id=%s duration_ms=%.2f",
|
|
task_id,
|
|
preview_run_id,
|
|
(time.perf_counter() - started_at) * 1000,
|
|
)
|
|
try:
|
|
if store.preview_is_running(task_id, preview_run_id):
|
|
store.mark_preview_failed(
|
|
task_id,
|
|
str(exc),
|
|
preview_run_id=preview_run_id,
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"failed to persist data process preview failure task_id=%s "
|
|
"preview_run_id=%s",
|
|
task_id,
|
|
preview_run_id,
|
|
)
|
|
|
|
|
|
@router.post("/{task_id}/preview/start", status_code=202)
|
|
def start_preview(
|
|
task_id: str,
|
|
background_tasks: BackgroundTasks,
|
|
payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task, selected_ids = store.start_preview(
|
|
task_id,
|
|
source_file_ids=payload.source_file_ids,
|
|
)
|
|
preview_run_id = str(task["preview_run_id"])
|
|
progress = store.preview_progress(task_id)
|
|
background_tasks.add_task(
|
|
_run_preview,
|
|
store,
|
|
storage,
|
|
task_id,
|
|
preview_run_id,
|
|
selected_ids,
|
|
)
|
|
return ok(progress, "data process preview started")
|
|
|
|
|
|
@router.get("/{task_id}/preview/progress")
|
|
def preview_progress(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(store.preview_progress(task_id))
|
|
|
|
|
|
@router.post("/{task_id}/preview/build")
|
|
def build_preview(
|
|
task_id: str,
|
|
payload: PreviewBuildRequest = Body(default_factory=PreviewBuildRequest),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
selected_ids = payload.source_file_ids
|
|
items = _prepare_preview_items(task_id, store, storage, selected_ids)
|
|
created = store.replace_preview_items(
|
|
task_id,
|
|
items,
|
|
source_file_ids=selected_ids,
|
|
)
|
|
target_ids = selected_ids or [
|
|
str(source["id"]) for source in store.list_source_files(task_id)
|
|
]
|
|
file_counts = dict.fromkeys(target_ids, 0)
|
|
for item in created:
|
|
source_file_id = str(item.get("source_file_id") or "")
|
|
if source_file_id in file_counts:
|
|
file_counts[source_file_id] += 1
|
|
files = [
|
|
{
|
|
"source_file_id": source_file_id,
|
|
"preview_count": count,
|
|
"status": "completed" if count else "empty",
|
|
}
|
|
for source_file_id, count in file_counts.items()
|
|
]
|
|
return ok(
|
|
{
|
|
"items": created,
|
|
"total": len(created),
|
|
"page": 1,
|
|
"page_size": len(created),
|
|
"file_counts": file_counts,
|
|
"files": files,
|
|
},
|
|
"preview built",
|
|
)
|
|
|
|
|
|
@router.get("/{task_id}/preview")
|
|
def preview_items(
|
|
task_id: str,
|
|
source_file_id: str | None = Query(default=None),
|
|
page: int = Query(default=1, ge=1),
|
|
page_size: int = Query(default=200, ge=1, le=1000),
|
|
keyword: str | None = Query(default=None),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(
|
|
store.list_preview_items(
|
|
task_id,
|
|
source_file_id=source_file_id,
|
|
page=page,
|
|
page_size=page_size,
|
|
keyword=keyword,
|
|
)
|
|
)
|
|
|
|
|
|
@router.post("/{task_id}/preview")
|
|
def create_preview_item(
|
|
task_id: str,
|
|
payload: PreviewItemCreate,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
item_payload = payload.model_dump(mode="json")
|
|
content = payload.edited_content
|
|
item_payload["token_count"] = estimate_token_count(content)
|
|
item_payload["quality_score"] = _preview_quality(
|
|
content,
|
|
task.get("config") or {},
|
|
)
|
|
if not content.strip():
|
|
item_payload["status"] = "invalid"
|
|
item = store.create_preview_item(task_id, item_payload)
|
|
return ok(item, "preview item created")
|
|
|
|
|
|
@router.put("/{task_id}/preview/{preview_id}")
|
|
def update_preview_item(
|
|
task_id: str,
|
|
preview_id: str,
|
|
payload: PreviewItemUpdate,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
existing = store.get_preview_item(task_id, preview_id)
|
|
update = payload.model_dump(exclude_unset=True, mode="json")
|
|
quality = _preview_quality(payload.edited_content, task.get("config") or {})
|
|
source_locator = (existing.get("quality_score") or {}).get("source_locator")
|
|
if isinstance(source_locator, Mapping):
|
|
quality["source_locator"] = deepcopy(dict(source_locator))
|
|
update["quality_score"] = quality
|
|
item = store.update_preview_item(
|
|
task_id,
|
|
preview_id,
|
|
update,
|
|
)
|
|
return ok(item, "preview item updated")
|
|
|
|
|
|
@router.delete("/{task_id}/preview/{preview_id}")
|
|
def delete_preview_item(
|
|
task_id: str,
|
|
preview_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
store.delete_preview_item(task_id, preview_id)
|
|
return ok({"deleted": preview_id}, "preview item deleted")
|
|
|
|
|
|
def _start_generation(
|
|
task_id: str,
|
|
payload: GenerateRequest,
|
|
background_tasks: BackgroundTasks,
|
|
store: DataProcessStore,
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.start_generation(task_id, replace_existing=payload.replace_existing)
|
|
background_tasks.add_task(
|
|
_run_generation,
|
|
store,
|
|
task_id,
|
|
str(task["generation_run_id"]),
|
|
)
|
|
return ok(store.progress(task_id), "data process generation started")
|
|
|
|
|
|
@router.post("/{task_id}/generate")
|
|
def generate(
|
|
task_id: str,
|
|
background_tasks: BackgroundTasks,
|
|
payload: GenerateRequest = Body(default_factory=GenerateRequest),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
return _start_generation(task_id, payload, background_tasks, store)
|
|
|
|
|
|
@router.post("/{task_id}/start")
|
|
def start(
|
|
task_id: str,
|
|
background_tasks: BackgroundTasks,
|
|
payload: GenerateRequest = Body(default_factory=GenerateRequest),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
storage: LocalDataProcessStorage = Depends(get_data_process_storage),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
items = _prepare_preview_items(task_id, store, storage)
|
|
store.replace_preview_items(task_id, items)
|
|
return _start_generation(task_id, payload, background_tasks, store)
|
|
|
|
|
|
@router.post("/{task_id}/stop")
|
|
def stop(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
store.stop_task(task_id)
|
|
return ok(store.progress(task_id), "data process task stopped")
|
|
|
|
|
|
@router.get("/{task_id}/progress")
|
|
def progress(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(store.progress(task_id))
|
|
|
|
|
|
@router.get("/{task_id}/results")
|
|
def results(
|
|
task_id: str,
|
|
page: int = Query(default=1, ge=1),
|
|
page_size: int = Query(default=100, ge=1, le=1000),
|
|
status: Literal["valid", "modified", "invalid"] | None = Query(default=None),
|
|
split: Literal["train", "validation", "test"] | None = Query(default=None),
|
|
keyword: str | None = Query(default=None),
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(
|
|
store.list_results(
|
|
task_id,
|
|
page=page,
|
|
page_size=page_size,
|
|
status=status,
|
|
split=split,
|
|
keyword=keyword,
|
|
)
|
|
)
|
|
|
|
|
|
@router.post("/{task_id}/confirm-results")
|
|
def confirm_results(
|
|
task_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
return ok(store.confirm_results(task_id), "data process results confirmed")
|
|
|
|
|
|
@router.put("/{task_id}/results/{result_id}")
|
|
def update_result(
|
|
task_id: str,
|
|
result_id: str,
|
|
payload: ResultUpdate,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
current = store.get_result(task_id, result_id)
|
|
update = payload.model_dump(exclude_unset=True, mode="json")
|
|
merged = {**current, **update}
|
|
preview_id = current.get("preview_item_id")
|
|
source_content = ""
|
|
if preview_id:
|
|
preview = store.get_preview_item(task_id, str(preview_id))
|
|
source_content = str(
|
|
preview.get("edited_content") or preview.get("original_content") or ""
|
|
)
|
|
minimum = max(
|
|
1,
|
|
int(
|
|
_value(
|
|
task.get("config") or {},
|
|
"min_output_length",
|
|
"minOutputLength",
|
|
20,
|
|
)
|
|
or 20
|
|
),
|
|
)
|
|
quality = score_quality(
|
|
merged,
|
|
min_output_length=minimum,
|
|
source_content=source_content,
|
|
)
|
|
update["quality_score"] = asdict(quality)
|
|
result = store.update_result(
|
|
task_id,
|
|
result_id,
|
|
update,
|
|
)
|
|
return ok(result, "data process result updated")
|
|
|
|
|
|
@router.post("/{task_id}/results/{result_id}/restore")
|
|
def restore_result(
|
|
task_id: str,
|
|
result_id: str,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
current = store.get_result(task_id, result_id)
|
|
restored = {
|
|
**current,
|
|
"instruction": current.get("original_instruction") or current.get("instruction") or "",
|
|
"input": current.get("original_input") or current.get("input") or "",
|
|
"output": current.get("original_output") or current.get("output") or "",
|
|
}
|
|
preview_id = current.get("preview_item_id")
|
|
source_content = ""
|
|
if preview_id:
|
|
preview = store.get_preview_item(task_id, str(preview_id))
|
|
source_content = str(
|
|
preview.get("edited_content") or preview.get("original_content") or ""
|
|
)
|
|
minimum = max(
|
|
1,
|
|
int(
|
|
_value(
|
|
task.get("config") or {},
|
|
"min_output_length",
|
|
"minOutputLength",
|
|
20,
|
|
)
|
|
or 20
|
|
),
|
|
)
|
|
quality = score_quality(
|
|
restored,
|
|
min_output_length=minimum,
|
|
source_content=source_content,
|
|
)
|
|
restored = store.update_result(
|
|
task_id,
|
|
result_id,
|
|
{
|
|
"instruction": restored["instruction"],
|
|
"input": restored["input"],
|
|
"output": restored["output"],
|
|
"quality_score": asdict(quality),
|
|
"expected_updated_at": current.get("updated_at"),
|
|
},
|
|
)
|
|
return ok(restored, "data process result restored")
|
|
|
|
|
|
class _ResultRegenerationFailed(InvalidStateError):
|
|
"""模型返回或质量校验失败，原失败结果必须保持不变。"""
|
|
|
|
|
|
def _assert_result_regeneration_allowed(task: dict[str, Any]) -> None:
|
|
if task.get("status") != "completed" or task.get("workflow_step") != "results":
|
|
raise InvalidStateError("task is not editing generation results")
|
|
if task.get("results_confirmed"):
|
|
raise InvalidStateError("confirmed results cannot be regenerated")
|
|
if task.get("output_dataset_id"):
|
|
raise InvalidStateError("published results cannot be regenerated")
|
|
|
|
|
|
def _result_regeneration_model(
|
|
task: dict[str, Any],
|
|
store: DataProcessStore,
|
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
config = task.get("config") or {}
|
|
model_id = _value(config, "generation_model_id", "generationModelId", None)
|
|
if not model_id:
|
|
raise InvalidStateError("task does not have a generation model")
|
|
return config, store.get_generation_model(str(model_id))
|
|
|
|
|
|
def _result_regeneration_timeout(config: dict[str, Any]) -> float:
|
|
configured = float(
|
|
_value(config, "request_timeout_seconds", "requestTimeoutSeconds", 60)
|
|
)
|
|
return max(1.0, min(RESULT_REGENERATION_TIMEOUT_SECONDS, configured))
|
|
|
|
|
|
@contextmanager
|
|
def _claim_result_regeneration(task_id: str, result_id: str) -> Iterator[None]:
|
|
key = (task_id, result_id)
|
|
with _result_regeneration_claims_lock:
|
|
if key in _active_result_regenerations:
|
|
raise ConflictError("data process result regeneration is already running")
|
|
_active_result_regenerations.add(key)
|
|
try:
|
|
yield
|
|
finally:
|
|
with _result_regeneration_claims_lock:
|
|
_active_result_regenerations.discard(key)
|
|
|
|
|
|
def _generate_result_replacement(
|
|
task_id: str,
|
|
current: dict[str, Any],
|
|
preview: dict[str, Any],
|
|
config: dict[str, Any],
|
|
generation_model: dict[str, Any],
|
|
model_client: httpx.Client | None = None,
|
|
) -> dict[str, Any]:
|
|
source_content = str(
|
|
preview.get("edited_content") or preview.get("original_content") or ""
|
|
).strip()
|
|
if not source_content or preview.get("status") == "invalid":
|
|
raise InvalidStateError("result source preview item is invalid or empty")
|
|
|
|
output_type = str(
|
|
_value(config, "output_type", "outputType", "standard")
|
|
).strip().lower()
|
|
previous_instruction = str(current.get("instruction") or "")[:1000]
|
|
previous_output = str(current.get("output") or "")[:1000]
|
|
base_prompt = str(
|
|
_value(config, "generation_prompt", "generationPrompt", "") or ""
|
|
)
|
|
regeneration_instruction = (
|
|
"这是一次失败结果的重新生成。请使用新的提问角度和表达，"
|
|
"不要复述旧结果。旧问题："
|
|
f"{previous_instruction or '无'}；旧答案：{previous_output or '无'}。"
|
|
)
|
|
runtime_config = {
|
|
**config,
|
|
"generation_prompt": f"{base_prompt}\n{regeneration_instruction}".strip(),
|
|
"output_type": output_type,
|
|
"reasoning_detail": _value(
|
|
config, "reasoning_detail", "reasoningDetail", "normal"
|
|
),
|
|
"max_tokens": _value(config, "max_tokens", "maxTokens", 1024),
|
|
"json_mode": _value(config, "json_mode", "jsonMode", False),
|
|
# 交互式重新生成只做一次新尝试，避免继承整任务的重试配置后长时间等待。
|
|
"generation_retries": RESULT_REGENERATION_RETRIES,
|
|
"request_timeout_seconds": _result_regeneration_timeout(config),
|
|
}
|
|
with _result_regeneration_slots:
|
|
generated = generate_model_records(
|
|
[preview],
|
|
model=generation_model,
|
|
config=runtime_config,
|
|
task_id=task_id,
|
|
split={"train": 100, "validation": 0, "test": 0},
|
|
qa_pairs_per_item=1,
|
|
client=model_client,
|
|
)
|
|
if not generated or generated[0].get("status") == "invalid":
|
|
reason = str(
|
|
(generated[0] if generated else {}).get("error")
|
|
or "model generation failed"
|
|
)
|
|
raise _ResultRegenerationFailed(f"重新生成结果仍无效：{reason}")
|
|
|
|
minimum = max(
|
|
1,
|
|
int(_value(config, "min_output_length", "minOutputLength", 20) or 20),
|
|
)
|
|
replacement = generated[0]
|
|
quality = score_quality(
|
|
replacement,
|
|
min_output_length=minimum,
|
|
source_content=source_content,
|
|
)
|
|
if not quality.is_valid:
|
|
reason = ", ".join(quality.flags) or "quality validation failed"
|
|
raise _ResultRegenerationFailed(f"重新生成结果未通过质量校验：{reason}")
|
|
replacement["quality_score"] = asdict(quality)
|
|
replacement["status"] = "valid"
|
|
replacement["error"] = None
|
|
return replacement
|
|
|
|
|
|
def _regenerate_result_in_place(
|
|
task_id: str,
|
|
current: dict[str, Any],
|
|
preview: dict[str, Any],
|
|
config: dict[str, Any],
|
|
generation_model: dict[str, Any],
|
|
store: DataProcessStore,
|
|
*,
|
|
expected_updated_at: str,
|
|
model_client: httpx.Client | None = None,
|
|
) -> dict[str, Any]:
|
|
result_id = str(current["id"])
|
|
with _claim_result_regeneration(task_id, result_id):
|
|
replacement = _generate_result_replacement(
|
|
task_id,
|
|
current,
|
|
preview,
|
|
config,
|
|
generation_model,
|
|
model_client,
|
|
)
|
|
return store.replace_generated_result(
|
|
task_id,
|
|
result_id,
|
|
replacement,
|
|
expected_updated_at=expected_updated_at,
|
|
)
|
|
|
|
|
|
def _safe_regeneration_error(exc: Exception) -> str:
|
|
return re.sub(r"\s+", " ", str(exc)).strip()[:500] or "result regeneration failed"
|
|
|
|
|
|
@router.post("/{task_id}/results/regenerate-batch")
|
|
def regenerate_results_batch(
|
|
task_id: str,
|
|
payload: ResultBatchRegenerateRequest,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
"""并发重新生成一批失败结果；每条独立提交并允许部分成功。"""
|
|
|
|
started_at = time.perf_counter()
|
|
batch_id = new_id("dprb")
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
_assert_result_regeneration_allowed(task)
|
|
config, generation_model = _result_regeneration_model(task, store)
|
|
prepared: list[tuple[int, dict[str, Any], dict[str, Any], str]] = []
|
|
failures: list[tuple[int, dict[str, str]]] = []
|
|
|
|
for index, requested in enumerate(payload.items):
|
|
try:
|
|
current = store.get_result(task_id, requested.result_id)
|
|
if current.get("status") != "invalid":
|
|
raise InvalidStateError("only an invalid result can be regenerated")
|
|
if requested.expected_updated_at != str(current.get("updated_at") or ""):
|
|
raise ConflictError("data process result was modified by another request")
|
|
preview_id = current.get("preview_item_id")
|
|
if not preview_id:
|
|
raise InvalidStateError(
|
|
"result is not associated with a source preview item"
|
|
)
|
|
preview = store.get_preview_item(task_id, str(preview_id))
|
|
prepared.append(
|
|
(index, current, preview, requested.expected_updated_at)
|
|
)
|
|
except ConflictError as exc:
|
|
failures.append((index, {
|
|
"result_id": requested.result_id,
|
|
"code": "conflict",
|
|
"message": _safe_regeneration_error(exc),
|
|
}))
|
|
except (NotFoundError, InvalidStateError) as exc:
|
|
failures.append((index, {
|
|
"result_id": requested.result_id,
|
|
"code": "skipped",
|
|
"message": _safe_regeneration_error(exc),
|
|
}))
|
|
|
|
logger.info(
|
|
"data process result batch regeneration started batch_id=%s task_id=%s "
|
|
"requested=%s prepared=%s concurrency=%s",
|
|
batch_id,
|
|
task_id,
|
|
len(payload.items),
|
|
len(prepared),
|
|
min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
|
)
|
|
successes: list[tuple[int, dict[str, Any]]] = []
|
|
if prepared:
|
|
request_timeout = _result_regeneration_timeout(config)
|
|
model_timeout = httpx.Timeout(
|
|
request_timeout,
|
|
connect=min(10.0, request_timeout),
|
|
)
|
|
model_limits = httpx.Limits(
|
|
max_connections=RESULT_REGENERATION_CONCURRENCY,
|
|
max_keepalive_connections=RESULT_REGENERATION_CONCURRENCY,
|
|
)
|
|
# httpx.Client 支持跨线程复用，批次内共享连接池可减少重复建连开销。
|
|
with (
|
|
httpx.Client(timeout=model_timeout, limits=model_limits) as model_client,
|
|
ThreadPoolExecutor(
|
|
max_workers=min(RESULT_REGENERATION_CONCURRENCY, len(prepared)),
|
|
thread_name_prefix="data-result-regeneration",
|
|
) as executor,
|
|
):
|
|
futures = {
|
|
executor.submit(
|
|
_regenerate_result_in_place,
|
|
task_id,
|
|
current,
|
|
preview,
|
|
config,
|
|
generation_model,
|
|
store,
|
|
expected_updated_at=expected_updated_at,
|
|
model_client=model_client,
|
|
): (index, str(current["id"]), time.perf_counter())
|
|
for index, current, preview, expected_updated_at in prepared
|
|
}
|
|
for future in as_completed(futures):
|
|
index, result_id, item_started_at = futures[future]
|
|
try:
|
|
regenerated = future.result()
|
|
successes.append((index, regenerated))
|
|
outcome = "succeeded"
|
|
except ConflictError as exc:
|
|
outcome = "conflict"
|
|
failures.append((index, {
|
|
"result_id": result_id,
|
|
"code": outcome,
|
|
"message": _safe_regeneration_error(exc),
|
|
}))
|
|
except _ResultRegenerationFailed as exc:
|
|
outcome = "generation_failed"
|
|
failures.append((index, {
|
|
"result_id": result_id,
|
|
"code": outcome,
|
|
"message": _safe_regeneration_error(exc),
|
|
}))
|
|
except (NotFoundError, InvalidStateError) as exc:
|
|
outcome = "skipped"
|
|
failures.append((index, {
|
|
"result_id": result_id,
|
|
"code": outcome,
|
|
"message": _safe_regeneration_error(exc),
|
|
}))
|
|
except Exception as exc: # pragma: no cover - defensive boundary
|
|
outcome = "internal_error"
|
|
logger.exception(
|
|
"data process result batch regeneration crashed "
|
|
"batch_id=%s task_id=%s result_id=%s",
|
|
batch_id,
|
|
task_id,
|
|
result_id,
|
|
)
|
|
failures.append((index, {
|
|
"result_id": result_id,
|
|
"code": outcome,
|
|
"message": _safe_regeneration_error(exc),
|
|
}))
|
|
logger.info(
|
|
"data process result batch item finished batch_id=%s task_id=%s "
|
|
"result_id=%s outcome=%s duration_ms=%.2f",
|
|
batch_id,
|
|
task_id,
|
|
result_id,
|
|
outcome,
|
|
(time.perf_counter() - item_started_at) * 1000,
|
|
)
|
|
|
|
success_items = [item for _, item in sorted(successes, key=lambda pair: pair[0])]
|
|
failure_items = [item for _, item in sorted(failures, key=lambda pair: pair[0])]
|
|
remaining_invalid_count = int(
|
|
store.list_results(
|
|
task_id,
|
|
page=1,
|
|
page_size=1,
|
|
status="invalid",
|
|
)["total"]
|
|
)
|
|
duration_ms = (time.perf_counter() - started_at) * 1000
|
|
logger.info(
|
|
"data process result batch regeneration completed batch_id=%s task_id=%s "
|
|
"succeeded=%s failed=%s remaining_invalid=%s duration_ms=%.2f",
|
|
batch_id,
|
|
task_id,
|
|
len(success_items),
|
|
len(failure_items),
|
|
remaining_invalid_count,
|
|
duration_ms,
|
|
)
|
|
return ok(
|
|
{
|
|
"batch_id": batch_id,
|
|
"total": len(payload.items),
|
|
"succeeded": len(success_items),
|
|
"failed": len(failure_items),
|
|
"remaining_invalid_count": remaining_invalid_count,
|
|
"duration_ms": round(duration_ms, 2),
|
|
"items": success_items,
|
|
"failures": failure_items,
|
|
},
|
|
"data process results regenerated",
|
|
)
|
|
|
|
|
|
@router.post("/{task_id}/results/{result_id}/regenerate")
|
|
def regenerate_result(
|
|
task_id: str,
|
|
result_id: str,
|
|
payload: ResultRegenerateRequest,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
"""只重新生成一个失败结果，成功后原位替换且不影响其他结果。"""
|
|
|
|
started_at = time.perf_counter()
|
|
with api_errors():
|
|
task = store.get_task(task_id)
|
|
_assert_result_regeneration_allowed(task)
|
|
current = store.get_result(task_id, result_id)
|
|
if current.get("status") != "invalid":
|
|
raise InvalidStateError("only an invalid result can be regenerated")
|
|
if payload.expected_updated_at != str(current.get("updated_at") or ""):
|
|
raise ConflictError("data process result was modified by another request")
|
|
preview_id = current.get("preview_item_id")
|
|
if not preview_id:
|
|
raise InvalidStateError("result is not associated with a source preview item")
|
|
preview = store.get_preview_item(task_id, str(preview_id))
|
|
config, generation_model = _result_regeneration_model(task, store)
|
|
try:
|
|
result = _regenerate_result_in_place(
|
|
task_id,
|
|
current,
|
|
preview,
|
|
config,
|
|
generation_model,
|
|
store,
|
|
expected_updated_at=payload.expected_updated_at,
|
|
)
|
|
except _ResultRegenerationFailed as exc:
|
|
logger.warning(
|
|
"data process result regeneration failed task_id=%s result_id=%s "
|
|
"duration_ms=%.2f reason=%s",
|
|
task_id,
|
|
result_id,
|
|
(time.perf_counter() - started_at) * 1000,
|
|
_safe_regeneration_error(exc),
|
|
)
|
|
raise
|
|
logger.info(
|
|
"data process result regenerated task_id=%s result_id=%s duration_ms=%.2f",
|
|
task_id,
|
|
result_id,
|
|
(time.perf_counter() - started_at) * 1000,
|
|
)
|
|
return ok(result, "data process result regenerated")
|
|
|
|
|
|
@router.post("/{task_id}/publish")
|
|
def publish(
|
|
task_id: str,
|
|
payload: PublishRequest,
|
|
store: DataProcessStore = Depends(get_data_process_store),
|
|
) -> dict[str, Any]:
|
|
with api_errors():
|
|
result = store.publish(task_id, payload.model_dump(mode="json"))
|
|
message = "dataset published" if result["created"] else "dataset already published"
|
|
return ok(result, message)
|