feat: 数据处理向导配置体系扩展
新增结构化与非结构化处理选项类型，TaskSetupStep 增加预处理、切分方法、数据集划分等配置 UI，previewModel 实现语义切分与受保护区间算法，CreateView 接入配置状态与草稿持久化并替换为 AppConfirmDialog，回归脚本扩充配置与弹窗断言。
This commit is contained in:
caoxiaozhu
@@ -1,4 +1,11 @@
|
||||
import type { PreviewItem, ProcessType, ResultItem, SourceLine } from './types'
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
SourceLine,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
|
||||
export const DEFAULT_SOURCE_TEXT = [
|
||||
'问：如何看待当前的通货膨胀风险？',
|
||||
@@ -39,8 +46,417 @@ export function sourceLines(sourceText: string): SourceLine[] {
|
||||
})
|
||||
}
|
||||
|
||||
export function buildPreviewItems(sourceText: string, processType: ProcessType, sourceFileId = 'default-source'): PreviewItem[] {
|
||||
const meaningfulLines = sourceLines(sourceText).filter((line) => line.content.trim())
|
||||
interface SourceRange {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
interface ProtectedRange extends SourceRange {
|
||||
kind: 'code' | 'table' | 'list'
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNK_SIZE = 800
|
||||
const DEFAULT_CHUNK_OVERLAP = 100
|
||||
const DEFAULT_MIN_CHUNK_SIZE = 100
|
||||
|
||||
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
|
||||
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
|
||||
}
|
||||
|
||||
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
|
||||
let nextStart = Math.max(0, start)
|
||||
let nextEnd = Math.min(sourceText.length, end)
|
||||
|
||||
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
|
||||
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
|
||||
|
||||
return { start: nextStart, end: nextEnd }
|
||||
}
|
||||
|
||||
function normalizeDelimiter(delimiter: string | undefined): string {
|
||||
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
|
||||
}
|
||||
|
||||
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
|
||||
return line.start < range.end && line.end > range.start
|
||||
}
|
||||
|
||||
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
|
||||
return ranges.some((range) => overlapsRange(line, range))
|
||||
}
|
||||
|
||||
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
let openFence: { start: number; marker: string; length: number } | null = null
|
||||
|
||||
for (const line of lines) {
|
||||
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
|
||||
if (!fence) continue
|
||||
|
||||
const marker = fence[1][0]
|
||||
if (!openFence) {
|
||||
openFence = { start: line.start, marker, length: fence[1].length }
|
||||
continue
|
||||
}
|
||||
|
||||
if (marker === openFence.marker && fence[1].length >= openFence.length) {
|
||||
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
|
||||
openFence = null
|
||||
}
|
||||
}
|
||||
|
||||
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
|
||||
return ranges
|
||||
}
|
||||
|
||||
function isTableSeparator(content: string): boolean {
|
||||
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
|
||||
const cells = normalized.split('|').map((cell) => cell.trim())
|
||||
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
|
||||
}
|
||||
|
||||
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
|
||||
for (let index = 0; index < lines.length - 1; index += 1) {
|
||||
const header = lines[index]
|
||||
const separator = lines[index + 1]
|
||||
if (
|
||||
isLineProtected(header, codeRanges)
|
||||
|| isLineProtected(separator, codeRanges)
|
||||
|| !header.content.includes('|')
|
||||
|| !isTableSeparator(separator.content)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
let endIndex = index + 1
|
||||
while (
|
||||
endIndex + 1 < lines.length
|
||||
&& !isLineProtected(lines[endIndex + 1], codeRanges)
|
||||
&& lines[endIndex + 1].content.trim()
|
||||
&& lines[endIndex + 1].content.includes('|')
|
||||
) {
|
||||
endIndex += 1
|
||||
}
|
||||
|
||||
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
|
||||
index = endIndex
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function isListItem(content: string): boolean {
|
||||
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
|
||||
}
|
||||
|
||||
function isListContinuation(content: string): boolean {
|
||||
return /^\s{2,}\S/.test(content)
|
||||
}
|
||||
|
||||
function detectListRanges(
|
||||
lines: SourceLine[],
|
||||
excludedRanges: SourceRange[],
|
||||
): ProtectedRange[] {
|
||||
const ranges: ProtectedRange[] = []
|
||||
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
|
||||
|
||||
let endIndex = index
|
||||
let itemCount = 1
|
||||
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
|
||||
const nextContent = lines[endIndex + 1].content
|
||||
if (isListItem(nextContent)) {
|
||||
itemCount += 1
|
||||
endIndex += 1
|
||||
continue
|
||||
}
|
||||
if (isListContinuation(nextContent)) {
|
||||
endIndex += 1
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if (itemCount >= 2) {
|
||||
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
|
||||
index = endIndex
|
||||
}
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
|
||||
return ranges
|
||||
.sort((left, right) => left.start - right.start || left.end - right.end)
|
||||
.reduce<ProtectedRange[]>((merged, range) => {
|
||||
const previous = merged[merged.length - 1]
|
||||
if (previous && range.start < previous.end) {
|
||||
previous.end = Math.max(previous.end, range.end)
|
||||
return merged
|
||||
}
|
||||
merged.push({ ...range })
|
||||
return merged
|
||||
}, [])
|
||||
}
|
||||
|
||||
function protectedRangesForOptions(
|
||||
sourceText: string,
|
||||
options?: UnstructuredProcessOptions,
|
||||
): ProtectedRange[] {
|
||||
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
|
||||
|
||||
const lines = sourceLines(sourceText)
|
||||
const codeRanges = detectCodeBlockRanges(sourceText, lines)
|
||||
const tableRanges = detectTableRanges(lines, codeRanges)
|
||||
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
|
||||
const enabledRanges = [
|
||||
...(options?.preserveCodeBlocks ? codeRanges : []),
|
||||
...(options?.preserveTables ? tableRanges : []),
|
||||
...(options?.preserveLists ? listRanges : []),
|
||||
]
|
||||
|
||||
return mergeProtectedRanges(enabledRanges)
|
||||
}
|
||||
|
||||
function protectedRangeContaining(
|
||||
ranges: ProtectedRange[],
|
||||
offset: number,
|
||||
): ProtectedRange | undefined {
|
||||
return ranges.find((range) => range.start < offset && offset < range.end)
|
||||
}
|
||||
|
||||
function normalizeChunkStart(
|
||||
sourceText: string,
|
||||
cursor: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): number {
|
||||
let start = Math.max(0, Math.min(cursor, sourceText.length))
|
||||
const overlapBlock = protectedRangeContaining(protectedRanges, start)
|
||||
if (overlapBlock) start = overlapBlock.end
|
||||
|
||||
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
|
||||
|
||||
// 去除块前空白时可能进入缩进代码块/列表；此时恢复到完整块起点。
|
||||
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
|
||||
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
|
||||
|
||||
return start
|
||||
}
|
||||
|
||||
function protectChunkEnd(
|
||||
proposedEnd: number,
|
||||
start: number,
|
||||
minimumEnd: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): number {
|
||||
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
|
||||
if (!splitBlock) return proposedEnd
|
||||
|
||||
// 优先在块前结束；块前不足最小切片长度时，将整个块收入当前切片。
|
||||
return splitBlock.start > start && splitBlock.start >= minimumEnd
|
||||
? splitBlock.start
|
||||
: splitBlock.end
|
||||
}
|
||||
|
||||
function restoreProtectedEdges(
|
||||
range: SourceRange,
|
||||
rawStart: number,
|
||||
rawEnd: number,
|
||||
protectedRanges: ProtectedRange[],
|
||||
): SourceRange {
|
||||
const nextRange = { ...range }
|
||||
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
|
||||
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
|
||||
|
||||
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
|
||||
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
|
||||
return nextRange
|
||||
}
|
||||
|
||||
function lastBoundaryInRange(
|
||||
sourceText: string,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
): number | null {
|
||||
const candidates: number[] = []
|
||||
const boundaryTokens = ['\n\n', '\n', '。', '！', '？', '；', '.', '!', '?', ';']
|
||||
|
||||
boundaryTokens.forEach((token) => {
|
||||
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
|
||||
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
|
||||
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
|
||||
})
|
||||
|
||||
return candidates.length ? Math.max(...candidates) : null
|
||||
}
|
||||
|
||||
function lastHeadingBoundary(
|
||||
sourceText: string,
|
||||
start: number,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
): number | null {
|
||||
const section = sourceText.slice(start, idealEnd)
|
||||
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
|
||||
let boundary: number | null = null
|
||||
let match: RegExpExecArray | null
|
||||
|
||||
while ((match = headingPattern.exec(section))) {
|
||||
const absoluteStart = start + match.index
|
||||
if (absoluteStart >= minimumEnd) boundary = absoluteStart
|
||||
}
|
||||
|
||||
return boundary
|
||||
}
|
||||
|
||||
function resolveChunkEnd(
|
||||
sourceText: string,
|
||||
start: number,
|
||||
idealEnd: number,
|
||||
minimumEnd: number,
|
||||
options: UnstructuredProcessOptions | undefined,
|
||||
): number {
|
||||
const method = options?.chunkMethod ?? 'semantic'
|
||||
|
||||
if (method === 'fixed') return idealEnd
|
||||
|
||||
if (method === 'custom') {
|
||||
const delimiter = normalizeDelimiter(options?.customDelimiter)
|
||||
if (!delimiter) return idealEnd
|
||||
|
||||
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
|
||||
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
|
||||
return boundary >= minimumEnd ? boundary : idealEnd
|
||||
}
|
||||
|
||||
if (method === 'heading') {
|
||||
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
|
||||
if (headingBoundary !== null) return headingBoundary
|
||||
}
|
||||
|
||||
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
|
||||
}
|
||||
|
||||
function buildUnstructuredRanges(
|
||||
sourceText: string,
|
||||
options?: UnstructuredProcessOptions,
|
||||
): SourceRange[] {
|
||||
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算，避免引入分词器依赖。
|
||||
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
|
||||
const minimumCharacters = Math.min(
|
||||
targetCharacters,
|
||||
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
|
||||
)
|
||||
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
|
||||
const protectedRanges = protectedRangesForOptions(sourceText, options)
|
||||
const ranges: SourceRange[] = []
|
||||
let cursor = 0
|
||||
|
||||
while (cursor < sourceText.length) {
|
||||
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
|
||||
if (start >= sourceText.length) break
|
||||
|
||||
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
|
||||
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
|
||||
let end = idealEnd === sourceText.length
|
||||
? idealEnd
|
||||
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
|
||||
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
|
||||
|
||||
// 所有自定义边界都必须向前推进；异常配置回退到固定长度切分。
|
||||
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
|
||||
|
||||
let range = trimSourceRange(sourceText, start, end)
|
||||
range = restoreProtectedEdges(range, start, end, protectedRanges)
|
||||
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
|
||||
range.end = Math.min(end, range.start + minimumCharacters)
|
||||
}
|
||||
if (range.end <= range.start) {
|
||||
cursor = Math.max(cursor + 1, end)
|
||||
continue
|
||||
}
|
||||
|
||||
const isLastRange = end >= sourceText.length
|
||||
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
|
||||
ranges[ranges.length - 1].end = range.end
|
||||
break
|
||||
}
|
||||
|
||||
ranges.push(range)
|
||||
if (isLastRange) break
|
||||
|
||||
// overlap 是允许的最大重叠量；按当前切片动态收缩，保证每轮至少推进最小切片长度。
|
||||
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
|
||||
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
|
||||
const nextCursor = range.end - actualOverlap
|
||||
cursor = nextCursor > start ? nextCursor : range.end
|
||||
}
|
||||
|
||||
return ranges
|
||||
}
|
||||
|
||||
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
|
||||
if (!lines.length) return null
|
||||
|
||||
let low = 0
|
||||
let high = lines.length - 1
|
||||
let result = 0
|
||||
|
||||
while (low <= high) {
|
||||
const middle = Math.floor((low + high) / 2)
|
||||
if (lines[middle].start <= offset) {
|
||||
result = middle
|
||||
low = middle + 1
|
||||
} else {
|
||||
high = middle - 1
|
||||
}
|
||||
}
|
||||
|
||||
return lines[result].number
|
||||
}
|
||||
|
||||
function previewItemFromRange(
|
||||
sourceText: string,
|
||||
lines: SourceLine[],
|
||||
range: SourceRange,
|
||||
sourceFileId: string,
|
||||
index: number,
|
||||
): PreviewItem {
|
||||
const content = sourceText.slice(range.start, range.end)
|
||||
|
||||
return {
|
||||
id: `preview-${sourceFileId}-${index + 1}`,
|
||||
sourceFileId,
|
||||
originalContent: content,
|
||||
editedContent: content,
|
||||
sourceStart: range.start,
|
||||
sourceEnd: range.end,
|
||||
sourceStartLine: lineNumberAtOffset(lines, range.start),
|
||||
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
|
||||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||||
status: 'original',
|
||||
}
|
||||
}
|
||||
|
||||
export function buildPreviewItems(
|
||||
sourceText: string,
|
||||
processType: ProcessType,
|
||||
sourceFileId = 'default-source',
|
||||
unstructuredOptions?: UnstructuredProcessOptions,
|
||||
): PreviewItem[] {
|
||||
const lines = sourceLines(sourceText)
|
||||
|
||||
if (processType === 'unstructured') {
|
||||
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
|
||||
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
|
||||
))
|
||||
}
|
||||
|
||||
const meaningfulLines = lines.filter((line) => line.content.trim())
|
||||
const groupSize = processType === 'structured' ? 1 : 3
|
||||
const items: PreviewItem[] = []
|
||||
|
||||
@@ -69,21 +485,44 @@ export function buildPreviewItems(sourceText: string, processType: ProcessType,
|
||||
return items
|
||||
}
|
||||
|
||||
export function createResults(items: PreviewItem[]): ResultItem[] {
|
||||
return items.slice(0, 12).map((item, index) => {
|
||||
const SEMANTIC_PREFIXES = [
|
||||
'请结合实际情况，说明一下：',
|
||||
'如果方便的话，请详细解答：',
|
||||
'请用通俗易懂的方式说明：',
|
||||
'请从实际应用角度说明：',
|
||||
'请简洁、自然地说明：',
|
||||
]
|
||||
|
||||
export function createResults(
|
||||
items: PreviewItem[],
|
||||
options?: StructuredProcessOptions | UnstructuredProcessOptions,
|
||||
): ResultItem[] {
|
||||
const resultCount = options && 'qaPairsPerChunk' in options
|
||||
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
|
||||
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
|
||||
|
||||
return items.flatMap((item, index) => {
|
||||
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
||||
const output = rest.join('\n').trim() || item.editedContent.trim()
|
||||
const instruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||||
const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||||
|
||||
return {
|
||||
id: `result-${index + 1}`,
|
||||
instruction,
|
||||
input: '',
|
||||
output,
|
||||
originalInstruction: instruction,
|
||||
originalInput: '',
|
||||
originalOutput: output,
|
||||
status: 'valid',
|
||||
}
|
||||
return Array.from({ length: resultCount }, (_, variantIndex) => {
|
||||
const instruction = options?.semanticEnrichment
|
||||
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
|
||||
: variantIndex === 0
|
||||
? baseInstruction
|
||||
: `${baseInstruction}（问法 ${variantIndex + 1}）`
|
||||
|
||||
return {
|
||||
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
|
||||
instruction,
|
||||
input: '',
|
||||
output,
|
||||
originalInstruction: instruction,
|
||||
originalInput: '',
|
||||
originalOutput: output,
|
||||
status: 'valid' as const,
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user