feat(data-process): 完善文件解析与切分存储链路
This commit is contained in:
@@ -13,6 +13,24 @@ def _config_value(config: dict[str, Any], snake_name: str, camel_name: str, defa
|
||||
|
||||
|
||||
def _validate_process_config(config: dict[str, Any]) -> None:
|
||||
chunk_method = _config_value(config, "chunk_method", "chunkMethod", "structure")
|
||||
if not isinstance(chunk_method, str) or chunk_method not in {
|
||||
"structure",
|
||||
"fixed",
|
||||
"custom",
|
||||
}:
|
||||
raise ValueError("chunk_method must be one of: structure, fixed, custom")
|
||||
custom_delimiter = _config_value(
|
||||
config,
|
||||
"custom_delimiter",
|
||||
"customDelimiter",
|
||||
"",
|
||||
)
|
||||
if chunk_method == "custom" and (
|
||||
not isinstance(custom_delimiter, str) or not custom_delimiter
|
||||
):
|
||||
raise ValueError("custom_delimiter is required for custom chunking")
|
||||
|
||||
split = _config_value(config, "dataset_split", "datasetSplit", None)
|
||||
if split is not None:
|
||||
if not isinstance(split, dict) or set(split) != {"train", "validation", "test"}:
|
||||
@@ -140,6 +158,23 @@ class PreviewBuildRequest(BaseModel):
|
||||
|
||||
replace_existing: Literal[True] = True
|
||||
source_file_ids: list[str] | None = None
|
||||
source_file_id: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_source_file_selection(self) -> "PreviewBuildRequest":
|
||||
if self.source_file_ids is not None and self.source_file_id is not None:
|
||||
raise ValueError("source_file_id and source_file_ids cannot be used together")
|
||||
values = self.source_file_ids
|
||||
if values is None and self.source_file_id is not None:
|
||||
values = [self.source_file_id]
|
||||
if values is None:
|
||||
return self
|
||||
normalized = list(dict.fromkeys(str(value).strip() for value in values))
|
||||
if not normalized or any(not value for value in normalized):
|
||||
raise ValueError("at least one non-empty source file id is required")
|
||||
self.source_file_ids = normalized
|
||||
self.source_file_id = None
|
||||
return self
|
||||
|
||||
|
||||
class PreviewItemCreate(BaseModel):
|
||||
|
||||
Reference in New Issue
Block a user