fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
|
||||
|
||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
preprocessOptions: [],
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
|
||||
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
preprocessOptions: [],
|
||||
chunkMethod: 'layout_hybrid',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
semanticBreakpointPercentile: 95,
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
|
||||
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
|
||||
const defaults = createDefaultStructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
const supportedPreprocessOptions = new Set<PreprocessOption>([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as PreprocessOption[]
|
||||
? Array.from(new Set(preprocessOptions.map(String).filter(
|
||||
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
|
||||
)))
|
||||
: defaults.preprocessOptions,
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
|
||||
Reference in New Issue
Block a user