5c469a1778
提取 GenerationOptionsPanel 组件统一管理生成模型、温度、最大长度、JSON 模式与质量过滤配置，StructuredProcessOptions 与 UnstructuredProcessOptions 继承 GenerationControlOptions，CreateView、TaskSetupStep 及各子步骤同步接入，详情/列表小幅调整，回归脚本扩充生成选项断言。
541 lines
18 KiB
TypeScript
541 lines
18 KiB
TypeScript
import type {
|
||
PreviewItem,
|
||
ProcessType,
|
||
ResultItem,
|
||
SourceLine,
|
||
StructuredProcessOptions,
|
||
UnstructuredProcessOptions,
|
||
} from './types'
|
||
|
||
export const DEFAULT_SOURCE_TEXT = [
|
||
'问：如何看待当前的通货膨胀风险？',
|
||
'答：当前通胀水平总体可控，但仍需关注能源价格与供给扰动。',
|
||
'问：美联储下一次议息会议何时召开？',
|
||
'答：会议时间以美联储官方日历为准，市场会重点关注利率路径指引。',
|
||
'问：人民币汇率未来走势如何？',
|
||
'答：人民币汇率取决于中美利差、经济基本面与政策预期。',
|
||
'问：银行理财产品收益率为何持续走低？',
|
||
'答：主要与市场利率下行、资产端收益下降以及风险偏好变化有关。',
|
||
'问：什么是复利？',
|
||
'答：复利是指在计算利息时，将上一期利息加入本金，再计算下一期利息。',
|
||
'问：如何评估股票的投资价值？',
|
||
'答：评估股票投资价值可以从以下几个方面进行：',
|
||
'1. 公司基本面：分析公司的财务状况、盈利能力、成长性等。',
|
||
'2. 行业前景：考察公司所处行业的发展趋势和竞争格局。',
|
||
'3. 估值水平：通过市盈率、市净率等指标判断估值是否合理。',
|
||
'4. 财务健康：关注公司的负债情况、现金流状况等。',
|
||
'5. 管理团队：评估管理层的能力和过往业绩。',
|
||
'此外，还需要关注宏观经济环境、政策变化等因素对股票市场的影响。',
|
||
'问：债券和股票的主要区别是什么？',
|
||
'答：债券收益相对稳定但上行有限，股票波动更大且承担更高风险。',
|
||
'问：什么是市盈率？',
|
||
'答：市盈率是股票价格与每股收益的比值，常用于衡量估值水平。',
|
||
'问：如何进行资产配置？',
|
||
'答：应根据投资目标、风险承受能力和市场环境合理分配资产。',
|
||
].join('\n')
|
||
|
||
export function sourceLines(sourceText: string): SourceLine[] {
|
||
const rawLines = sourceText.split('\n')
|
||
let cursor = 0
|
||
|
||
return rawLines.map((content, index) => {
|
||
const start = cursor
|
||
const end = start + content.length
|
||
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
|
||
return { number: index + 1, content, start, end }
|
||
})
|
||
}
|
||
|
||
interface SourceRange {
|
||
start: number
|
||
end: number
|
||
}
|
||
|
||
interface ProtectedRange extends SourceRange {
|
||
kind: 'code' | 'table' | 'list'
|
||
}
|
||
|
||
const DEFAULT_CHUNK_SIZE = 800
|
||
const DEFAULT_CHUNK_OVERLAP = 100
|
||
const DEFAULT_MIN_CHUNK_SIZE = 100
|
||
|
||
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
|
||
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
|
||
}
|
||
|
||
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
|
||
let nextStart = Math.max(0, start)
|
||
let nextEnd = Math.min(sourceText.length, end)
|
||
|
||
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
|
||
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
|
||
|
||
return { start: nextStart, end: nextEnd }
|
||
}
|
||
|
||
function normalizeDelimiter(delimiter: string | undefined): string {
|
||
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
|
||
}
|
||
|
||
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
|
||
return line.start < range.end && line.end > range.start
|
||
}
|
||
|
||
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
|
||
return ranges.some((range) => overlapsRange(line, range))
|
||
}
|
||
|
||
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
|
||
const ranges: ProtectedRange[] = []
|
||
let openFence: { start: number; marker: string; length: number } | null = null
|
||
|
||
for (const line of lines) {
|
||
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
|
||
if (!fence) continue
|
||
|
||
const marker = fence[1][0]
|
||
if (!openFence) {
|
||
openFence = { start: line.start, marker, length: fence[1].length }
|
||
continue
|
||
}
|
||
|
||
if (marker === openFence.marker && fence[1].length >= openFence.length) {
|
||
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
|
||
openFence = null
|
||
}
|
||
}
|
||
|
||
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
|
||
return ranges
|
||
}
|
||
|
||
function isTableSeparator(content: string): boolean {
|
||
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
|
||
const cells = normalized.split('|').map((cell) => cell.trim())
|
||
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
|
||
}
|
||
|
||
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
|
||
const ranges: ProtectedRange[] = []
|
||
|
||
for (let index = 0; index < lines.length - 1; index += 1) {
|
||
const header = lines[index]
|
||
const separator = lines[index + 1]
|
||
if (
|
||
isLineProtected(header, codeRanges)
|
||
|| isLineProtected(separator, codeRanges)
|
||
|| !header.content.includes('|')
|
||
|| !isTableSeparator(separator.content)
|
||
) {
|
||
continue
|
||
}
|
||
|
||
let endIndex = index + 1
|
||
while (
|
||
endIndex + 1 < lines.length
|
||
&& !isLineProtected(lines[endIndex + 1], codeRanges)
|
||
&& lines[endIndex + 1].content.trim()
|
||
&& lines[endIndex + 1].content.includes('|')
|
||
) {
|
||
endIndex += 1
|
||
}
|
||
|
||
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
|
||
index = endIndex
|
||
}
|
||
|
||
return ranges
|
||
}
|
||
|
||
function isListItem(content: string): boolean {
|
||
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
|
||
}
|
||
|
||
function isListContinuation(content: string): boolean {
|
||
return /^\s{2,}\S/.test(content)
|
||
}
|
||
|
||
function detectListRanges(
|
||
lines: SourceLine[],
|
||
excludedRanges: SourceRange[],
|
||
): ProtectedRange[] {
|
||
const ranges: ProtectedRange[] = []
|
||
|
||
for (let index = 0; index < lines.length; index += 1) {
|
||
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
|
||
|
||
let endIndex = index
|
||
let itemCount = 1
|
||
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
|
||
const nextContent = lines[endIndex + 1].content
|
||
if (isListItem(nextContent)) {
|
||
itemCount += 1
|
||
endIndex += 1
|
||
continue
|
||
}
|
||
if (isListContinuation(nextContent)) {
|
||
endIndex += 1
|
||
continue
|
||
}
|
||
break
|
||
}
|
||
|
||
if (itemCount >= 2) {
|
||
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
|
||
index = endIndex
|
||
}
|
||
}
|
||
|
||
return ranges
|
||
}
|
||
|
||
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
|
||
return ranges
|
||
.sort((left, right) => left.start - right.start || left.end - right.end)
|
||
.reduce<ProtectedRange[]>((merged, range) => {
|
||
const previous = merged[merged.length - 1]
|
||
if (previous && range.start < previous.end) {
|
||
previous.end = Math.max(previous.end, range.end)
|
||
return merged
|
||
}
|
||
merged.push({ ...range })
|
||
return merged
|
||
}, [])
|
||
}
|
||
|
||
function protectedRangesForOptions(
|
||
sourceText: string,
|
||
options?: UnstructuredProcessOptions,
|
||
): ProtectedRange[] {
|
||
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
|
||
|
||
const lines = sourceLines(sourceText)
|
||
const codeRanges = detectCodeBlockRanges(sourceText, lines)
|
||
const tableRanges = detectTableRanges(lines, codeRanges)
|
||
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
|
||
const enabledRanges = [
|
||
...(options?.preserveCodeBlocks ? codeRanges : []),
|
||
...(options?.preserveTables ? tableRanges : []),
|
||
...(options?.preserveLists ? listRanges : []),
|
||
]
|
||
|
||
return mergeProtectedRanges(enabledRanges)
|
||
}
|
||
|
||
function protectedRangeContaining(
|
||
ranges: ProtectedRange[],
|
||
offset: number,
|
||
): ProtectedRange | undefined {
|
||
return ranges.find((range) => range.start < offset && offset < range.end)
|
||
}
|
||
|
||
function normalizeChunkStart(
|
||
sourceText: string,
|
||
cursor: number,
|
||
protectedRanges: ProtectedRange[],
|
||
): number {
|
||
let start = Math.max(0, Math.min(cursor, sourceText.length))
|
||
const overlapBlock = protectedRangeContaining(protectedRanges, start)
|
||
if (overlapBlock) start = overlapBlock.end
|
||
|
||
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
|
||
|
||
// 去除块前空白时可能进入缩进代码块/列表；此时恢复到完整块起点。
|
||
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
|
||
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
|
||
|
||
return start
|
||
}
|
||
|
||
function protectChunkEnd(
|
||
proposedEnd: number,
|
||
start: number,
|
||
minimumEnd: number,
|
||
protectedRanges: ProtectedRange[],
|
||
): number {
|
||
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
|
||
if (!splitBlock) return proposedEnd
|
||
|
||
// 优先在块前结束；块前不足最小切片长度时，将整个块收入当前切片。
|
||
return splitBlock.start > start && splitBlock.start >= minimumEnd
|
||
? splitBlock.start
|
||
: splitBlock.end
|
||
}
|
||
|
||
function restoreProtectedEdges(
|
||
range: SourceRange,
|
||
rawStart: number,
|
||
rawEnd: number,
|
||
protectedRanges: ProtectedRange[],
|
||
): SourceRange {
|
||
const nextRange = { ...range }
|
||
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
|
||
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
|
||
|
||
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
|
||
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
|
||
return nextRange
|
||
}
|
||
|
||
function lastBoundaryInRange(
|
||
sourceText: string,
|
||
idealEnd: number,
|
||
minimumEnd: number,
|
||
): number | null {
|
||
const candidates: number[] = []
|
||
const boundaryTokens = ['\n\n', '\n', '。', '！', '？', '；', '.', '!', '?', ';']
|
||
|
||
boundaryTokens.forEach((token) => {
|
||
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
|
||
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
|
||
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
|
||
})
|
||
|
||
return candidates.length ? Math.max(...candidates) : null
|
||
}
|
||
|
||
function lastHeadingBoundary(
|
||
sourceText: string,
|
||
start: number,
|
||
idealEnd: number,
|
||
minimumEnd: number,
|
||
): number | null {
|
||
const section = sourceText.slice(start, idealEnd)
|
||
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
|
||
let boundary: number | null = null
|
||
let match: RegExpExecArray | null
|
||
|
||
while ((match = headingPattern.exec(section))) {
|
||
const absoluteStart = start + match.index
|
||
if (absoluteStart >= minimumEnd) boundary = absoluteStart
|
||
}
|
||
|
||
return boundary
|
||
}
|
||
|
||
function resolveChunkEnd(
|
||
sourceText: string,
|
||
start: number,
|
||
idealEnd: number,
|
||
minimumEnd: number,
|
||
options: UnstructuredProcessOptions | undefined,
|
||
): number {
|
||
const method = options?.chunkMethod ?? 'semantic'
|
||
|
||
if (method === 'fixed') return idealEnd
|
||
|
||
if (method === 'custom') {
|
||
const delimiter = normalizeDelimiter(options?.customDelimiter)
|
||
if (!delimiter) return idealEnd
|
||
|
||
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
|
||
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
|
||
return boundary >= minimumEnd ? boundary : idealEnd
|
||
}
|
||
|
||
if (method === 'heading') {
|
||
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
|
||
if (headingBoundary !== null) return headingBoundary
|
||
}
|
||
|
||
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
|
||
}
|
||
|
||
function buildUnstructuredRanges(
|
||
sourceText: string,
|
||
options?: UnstructuredProcessOptions,
|
||
): SourceRange[] {
|
||
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算，避免引入分词器依赖。
|
||
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
|
||
const minimumCharacters = Math.min(
|
||
targetCharacters,
|
||
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
|
||
)
|
||
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
|
||
const protectedRanges = protectedRangesForOptions(sourceText, options)
|
||
const ranges: SourceRange[] = []
|
||
let cursor = 0
|
||
|
||
while (cursor < sourceText.length) {
|
||
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
|
||
if (start >= sourceText.length) break
|
||
|
||
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
|
||
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
|
||
let end = idealEnd === sourceText.length
|
||
? idealEnd
|
||
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
|
||
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
|
||
|
||
// 所有自定义边界都必须向前推进；异常配置回退到固定长度切分。
|
||
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
|
||
|
||
let range = trimSourceRange(sourceText, start, end)
|
||
range = restoreProtectedEdges(range, start, end, protectedRanges)
|
||
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
|
||
range.end = Math.min(end, range.start + minimumCharacters)
|
||
}
|
||
if (range.end <= range.start) {
|
||
cursor = Math.max(cursor + 1, end)
|
||
continue
|
||
}
|
||
|
||
const isLastRange = end >= sourceText.length
|
||
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
|
||
ranges[ranges.length - 1].end = range.end
|
||
break
|
||
}
|
||
|
||
ranges.push(range)
|
||
if (isLastRange) break
|
||
|
||
// overlap 是允许的最大重叠量；按当前切片动态收缩，保证每轮至少推进最小切片长度。
|
||
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
|
||
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
|
||
const nextCursor = range.end - actualOverlap
|
||
cursor = nextCursor > start ? nextCursor : range.end
|
||
}
|
||
|
||
return ranges
|
||
}
|
||
|
||
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
|
||
if (!lines.length) return null
|
||
|
||
let low = 0
|
||
let high = lines.length - 1
|
||
let result = 0
|
||
|
||
while (low <= high) {
|
||
const middle = Math.floor((low + high) / 2)
|
||
if (lines[middle].start <= offset) {
|
||
result = middle
|
||
low = middle + 1
|
||
} else {
|
||
high = middle - 1
|
||
}
|
||
}
|
||
|
||
return lines[result].number
|
||
}
|
||
|
||
function previewItemFromRange(
|
||
sourceText: string,
|
||
lines: SourceLine[],
|
||
range: SourceRange,
|
||
sourceFileId: string,
|
||
index: number,
|
||
): PreviewItem {
|
||
const content = sourceText.slice(range.start, range.end)
|
||
|
||
return {
|
||
id: `preview-${sourceFileId}-${index + 1}`,
|
||
sourceFileId,
|
||
originalContent: content,
|
||
editedContent: content,
|
||
sourceStart: range.start,
|
||
sourceEnd: range.end,
|
||
sourceStartLine: lineNumberAtOffset(lines, range.start),
|
||
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
|
||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||
status: 'original',
|
||
}
|
||
}
|
||
|
||
export function buildPreviewItems(
|
||
sourceText: string,
|
||
processType: ProcessType,
|
||
sourceFileId = 'default-source',
|
||
unstructuredOptions?: UnstructuredProcessOptions,
|
||
): PreviewItem[] {
|
||
const lines = sourceLines(sourceText)
|
||
|
||
if (processType === 'unstructured') {
|
||
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
|
||
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
|
||
))
|
||
}
|
||
|
||
const meaningfulLines = lines.filter((line) => line.content.trim())
|
||
const groupSize = processType === 'structured' ? 1 : 3
|
||
const items: PreviewItem[] = []
|
||
|
||
for (let index = 0; index < meaningfulLines.length; index += groupSize) {
|
||
const group = meaningfulLines.slice(index, index + groupSize)
|
||
if (!group.length) continue
|
||
|
||
const sourceStart = group[0].start
|
||
const sourceEnd = group[group.length - 1].end
|
||
const content = sourceText.slice(sourceStart, sourceEnd)
|
||
|
||
items.push({
|
||
id: `preview-${sourceFileId}-${items.length + 1}`,
|
||
sourceFileId,
|
||
originalContent: content,
|
||
editedContent: content,
|
||
sourceStart,
|
||
sourceEnd,
|
||
sourceStartLine: group[0].number,
|
||
sourceEndLine: group[group.length - 1].number,
|
||
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
|
||
status: 'original',
|
||
})
|
||
}
|
||
|
||
return items
|
||
}
|
||
|
||
const SEMANTIC_PREFIXES = [
|
||
'请结合实际情况，说明一下：',
|
||
'如果方便的话，请详细解答：',
|
||
'请用通俗易懂的方式说明：',
|
||
'请从实际应用角度说明：',
|
||
'请简洁、自然地说明：',
|
||
]
|
||
|
||
export function createResults(
|
||
items: PreviewItem[],
|
||
options?: StructuredProcessOptions | UnstructuredProcessOptions,
|
||
): ResultItem[] {
|
||
const resultCount = options && 'qaPairsPerChunk' in options
|
||
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
|
||
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
|
||
|
||
const generatedResults = items.flatMap((item, index) => {
|
||
if (options?.qualityFilterEnabled && options.filterLowQuality) {
|
||
if (item.status === 'invalid' || !item.editedContent.trim()) return []
|
||
}
|
||
|
||
const [firstLine = '', ...rest] = item.editedContent.split('\n')
|
||
const output = rest.join('\n').trim() || item.editedContent.trim()
|
||
const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
|
||
|
||
return Array.from({ length: resultCount }, (_, variantIndex) => {
|
||
const instruction = options?.semanticEnrichment
|
||
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
|
||
: variantIndex === 0
|
||
? baseInstruction
|
||
: `${baseInstruction}（问法 ${variantIndex + 1}）`
|
||
|
||
return {
|
||
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
|
||
instruction,
|
||
input: '',
|
||
output,
|
||
originalInstruction: instruction,
|
||
originalInput: '',
|
||
originalOutput: output,
|
||
status: 'valid' as const,
|
||
}
|
||
})
|
||
})
|
||
|
||
if (!options?.qualityFilterEnabled) return generatedResults
|
||
|
||
return generatedResults.filter((result) => {
|
||
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
|
||
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
|
||
return true
|
||
})
|
||
}
|