fix: 完善数据预处理与 JSON 上传链路

This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions

View File

@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
export function createDefaultStructuredOptions(): StructuredProcessOptions {
return {
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
preprocessOptions: [],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
return {
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
preprocessOptions: [],
chunkMethod: 'layout_hybrid',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
semanticBreakpointPercentile: 95,
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
const defaults = createDefaultStructuredOptions()
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
const supportedPreprocessOptions = new Set<PreprocessOption>([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
return {
...defaults,
...generationOptionsFromConfig(config, defaults),
preprocessOptions: Array.isArray(preprocessOptions)
? preprocessOptions.map(String) as PreprocessOption[]
? Array.from(new Set(preprocessOptions.map(String).filter(
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
)))
: defaults.preprocessOptions,
semanticEnrichment: Boolean(configValue(
config,