feat(data-process): 完善文件解析与切分存储链路

This commit is contained in:
caoxiaozhu
2026-07-24 11:27:51 +08:00
parent 6d4bf85284
commit 33d0ed2e01
12 changed files with 4813 additions and 213 deletions

View File

@@ -13,6 +13,24 @@ def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, defa
def _validate_process_config(config: dict[str, Any]) -> None:
chunk_method = _config_value(config, "chunk_method", "chunkMethod", "structure")
if not isinstance(chunk_method, str) or chunk_method not in {
"structure",
"fixed",
"custom",
}:
raise ValueError("chunk_method must be one of: structure, fixed, custom")
custom_delimiter = _config_value(
config,
"custom_delimiter",
"customDelimiter",
"",
)
if chunk_method == "custom" and (
not isinstance(custom_delimiter, str) or not custom_delimiter
):
raise ValueError("custom_delimiter is required for custom chunking")
split = _config_value(config, "dataset_split", "datasetSplit", None)
if split is not None:
if not isinstance(split, dict) or set(split) != {"train", "validation", "test"}:
@@ -140,6 +158,23 @@ class PreviewBuildRequest(BaseModel):
replace_existing: Literal[True] = True
source_file_ids: list[str] | None = None
source_file_id: str | None = None
@model_validator(mode="after")
def validate_source_file_selection(self) -> "PreviewBuildRequest":
if self.source_file_ids is not None and self.source_file_id is not None:
raise ValueError("source_file_id and source_file_ids cannot be used together")
values = self.source_file_ids
if values is None and self.source_file_id is not None:
values = [self.source_file_id]
if values is None:
return self
normalized = list(dict.fromkeys(str(value).strip() for value in values))
if not normalized or any(not value for value in normalized):
raise ValueError("at least one non-empty source file id is required")
self.source_file_ids = normalized
self.source_file_id = None
return self
class PreviewItemCreate(BaseModel):