import type { PreviewItem, ProcessType, ResultItem, SourceLine, StructuredProcessOptions, UnstructuredProcessOptions, } from './types' export const DEFAULT_SOURCE_TEXT = [ '问:如何看待当前的通货膨胀风险?', '答:当前通胀水平总体可控,但仍需关注能源价格与供给扰动。', '问:美联储下一次议息会议何时召开?', '答:会议时间以美联储官方日历为准,市场会重点关注利率路径指引。', '问:人民币汇率未来走势如何?', '答:人民币汇率取决于中美利差、经济基本面与政策预期。', '问:银行理财产品收益率为何持续走低?', '答:主要与市场利率下行、资产端收益下降以及风险偏好变化有关。', '问:什么是复利?', '答:复利是指在计算利息时,将上一期利息加入本金,再计算下一期利息。', '问:如何评估股票的投资价值?', '答:评估股票投资价值可以从以下几个方面进行:', '1. 公司基本面:分析公司的财务状况、盈利能力、成长性等。', '2. 行业前景:考察公司所处行业的发展趋势和竞争格局。', '3. 估值水平:通过市盈率、市净率等指标判断估值是否合理。', '4. 财务健康:关注公司的负债情况、现金流状况等。', '5. 管理团队:评估管理层的能力和过往业绩。', '此外,还需要关注宏观经济环境、政策变化等因素对股票市场的影响。', '问:债券和股票的主要区别是什么?', '答:债券收益相对稳定但上行有限,股票波动更大且承担更高风险。', '问:什么是市盈率?', '答:市盈率是股票价格与每股收益的比值,常用于衡量估值水平。', '问:如何进行资产配置?', '答:应根据投资目标、风险承受能力和市场环境合理分配资产。', ].join('\n') export function sourceLines(sourceText: string): SourceLine[] { const rawLines = sourceText.split('\n') let cursor = 0 return rawLines.map((content, index) => { const start = cursor const end = start + content.length cursor = end + (index < rawLines.length - 1 ? 1 : 0) return { number: index + 1, content, start, end } }) } interface SourceRange { start: number end: number } interface ProtectedRange extends SourceRange { kind: 'code' | 'table' | 'list' } const DEFAULT_CHUNK_SIZE = 800 const DEFAULT_CHUNK_OVERLAP = 100 const DEFAULT_MIN_CHUNK_SIZE = 100 function finiteInteger(value: number | undefined, fallback: number, min: number): number { return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback } function trimSourceRange(sourceText: string, start: number, end: number): SourceRange { let nextStart = Math.max(0, start) let nextEnd = Math.min(sourceText.length, end) while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1 while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1 return { start: nextStart, end: nextEnd } } function normalizeDelimiter(delimiter: string | undefined): string { return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t') } function overlapsRange(line: SourceLine, range: SourceRange): boolean { return line.start < range.end && line.end > range.start } function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean { return ranges.some((range) => overlapsRange(line, range)) } function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] { const ranges: ProtectedRange[] = [] let openFence: { start: number; marker: string; length: number } | null = null for (const line of lines) { const fence = line.content.match(/^\s*(`{3,}|~{3,})/) if (!fence) continue const marker = fence[1][0] if (!openFence) { openFence = { start: line.start, marker, length: fence[1].length } continue } if (marker === openFence.marker && fence[1].length >= openFence.length) { ranges.push({ start: openFence.start, end: line.end, kind: 'code' }) openFence = null } } if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' }) return ranges } function isTableSeparator(content: string): boolean { const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '') const cells = normalized.split('|').map((cell) => cell.trim()) return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell)) } function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] { const ranges: ProtectedRange[] = [] for (let index = 0; index < lines.length - 1; index += 1) { const header = lines[index] const separator = lines[index + 1] if ( isLineProtected(header, codeRanges) || isLineProtected(separator, codeRanges) || !header.content.includes('|') || !isTableSeparator(separator.content) ) { continue } let endIndex = index + 1 while ( endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], codeRanges) && lines[endIndex + 1].content.trim() && lines[endIndex + 1].content.includes('|') ) { endIndex += 1 } ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' }) index = endIndex } return ranges } function isListItem(content: string): boolean { return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content) } function isListContinuation(content: string): boolean { return /^\s{2,}\S/.test(content) } function detectListRanges( lines: SourceLine[], excludedRanges: SourceRange[], ): ProtectedRange[] { const ranges: ProtectedRange[] = [] for (let index = 0; index < lines.length; index += 1) { if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue let endIndex = index let itemCount = 1 while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) { const nextContent = lines[endIndex + 1].content if (isListItem(nextContent)) { itemCount += 1 endIndex += 1 continue } if (isListContinuation(nextContent)) { endIndex += 1 continue } break } if (itemCount >= 2) { ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' }) index = endIndex } } return ranges } function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] { return ranges .sort((left, right) => left.start - right.start || left.end - right.end) .reduce((merged, range) => { const previous = merged[merged.length - 1] if (previous && range.start < previous.end) { previous.end = Math.max(previous.end, range.end) return merged } merged.push({ ...range }) return merged }, []) } function protectedRangesForOptions( sourceText: string, options?: UnstructuredProcessOptions, ): ProtectedRange[] { if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return [] const lines = sourceLines(sourceText) const codeRanges = detectCodeBlockRanges(sourceText, lines) const tableRanges = detectTableRanges(lines, codeRanges) const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges]) const enabledRanges = [ ...(options?.preserveCodeBlocks ? codeRanges : []), ...(options?.preserveTables ? tableRanges : []), ...(options?.preserveLists ? listRanges : []), ] return mergeProtectedRanges(enabledRanges) } function protectedRangeContaining( ranges: ProtectedRange[], offset: number, ): ProtectedRange | undefined { return ranges.find((range) => range.start < offset && offset < range.end) } function normalizeChunkStart( sourceText: string, cursor: number, protectedRanges: ProtectedRange[], ): number { let start = Math.max(0, Math.min(cursor, sourceText.length)) const overlapBlock = protectedRangeContaining(protectedRanges, start) if (overlapBlock) start = overlapBlock.end while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1 // 去除块前空白时可能进入缩进代码块/列表;此时恢复到完整块起点。 const blockAfterTrim = protectedRangeContaining(protectedRanges, start) if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end return start } function protectChunkEnd( proposedEnd: number, start: number, minimumEnd: number, protectedRanges: ProtectedRange[], ): number { const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd) if (!splitBlock) return proposedEnd // 优先在块前结束;块前不足最小切片长度时,将整个块收入当前切片。 return splitBlock.start > start && splitBlock.start >= minimumEnd ? splitBlock.start : splitBlock.end } function restoreProtectedEdges( range: SourceRange, rawStart: number, rawEnd: number, protectedRanges: ProtectedRange[], ): SourceRange { const nextRange = { ...range } const startBlock = protectedRangeContaining(protectedRanges, nextRange.start) if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start const endBlock = protectedRangeContaining(protectedRanges, nextRange.end) if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end return nextRange } function lastBoundaryInRange( sourceText: string, idealEnd: number, minimumEnd: number, ): number | null { const candidates: number[] = [] const boundaryTokens = ['\n\n', '\n', '。', '!', '?', ';', '.', '!', '?', ';'] boundaryTokens.forEach((token) => { const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length) const boundary = tokenStart === -1 ? -1 : tokenStart + token.length if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary) }) return candidates.length ? Math.max(...candidates) : null } function lastHeadingBoundary( sourceText: string, start: number, idealEnd: number, minimumEnd: number, ): number | null { const section = sourceText.slice(start, idealEnd) const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm let boundary: number | null = null let match: RegExpExecArray | null while ((match = headingPattern.exec(section))) { const absoluteStart = start + match.index if (absoluteStart >= minimumEnd) boundary = absoluteStart } return boundary } function resolveChunkEnd( sourceText: string, start: number, idealEnd: number, minimumEnd: number, options: UnstructuredProcessOptions | undefined, ): number { const method = options?.chunkMethod ?? 'semantic' if (method === 'fixed') return idealEnd if (method === 'custom') { const delimiter = normalizeDelimiter(options?.customDelimiter) if (!delimiter) return idealEnd const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length) const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length return boundary >= minimumEnd ? boundary : idealEnd } if (method === 'heading') { const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd) if (headingBoundary !== null) return headingBoundary } return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd } function buildUnstructuredRanges( sourceText: string, options?: UnstructuredProcessOptions, ): SourceRange[] { // 预览统一沿用“约 2 个字符 = 1 token”的轻量估算,避免引入分词器依赖。 const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2 const minimumCharacters = Math.min( targetCharacters, finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2, ) const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2 const protectedRanges = protectedRangesForOptions(sourceText, options) const ranges: SourceRange[] = [] let cursor = 0 while (cursor < sourceText.length) { const start = normalizeChunkStart(sourceText, cursor, protectedRanges) if (start >= sourceText.length) break const idealEnd = Math.min(sourceText.length, start + targetCharacters) const minimumEnd = Math.min(idealEnd, start + minimumCharacters) let end = idealEnd === sourceText.length ? idealEnd : resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options) end = protectChunkEnd(end, start, minimumEnd, protectedRanges) // 所有自定义边界都必须向前推进;异常配置回退到固定长度切分。 if (end <= start) end = Math.min(sourceText.length, start + targetCharacters) let range = trimSourceRange(sourceText, start, end) range = restoreProtectedEdges(range, start, end, protectedRanges) if (end < sourceText.length && range.end - range.start < minimumCharacters) { range.end = Math.min(end, range.start + minimumCharacters) } if (range.end <= range.start) { cursor = Math.max(cursor + 1, end) continue } const isLastRange = end >= sourceText.length if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) { ranges[ranges.length - 1].end = range.end break } ranges.push(range) if (isLastRange) break // overlap 是允许的最大重叠量;按当前切片动态收缩,保证每轮至少推进最小切片长度。 const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters) const actualOverlap = Math.min(requestedOverlap, maximumOverlap) const nextCursor = range.end - actualOverlap cursor = nextCursor > start ? nextCursor : range.end } return ranges } function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null { if (!lines.length) return null let low = 0 let high = lines.length - 1 let result = 0 while (low <= high) { const middle = Math.floor((low + high) / 2) if (lines[middle].start <= offset) { result = middle low = middle + 1 } else { high = middle - 1 } } return lines[result].number } function previewItemFromRange( sourceText: string, lines: SourceLine[], range: SourceRange, sourceFileId: string, index: number, ): PreviewItem { const content = sourceText.slice(range.start, range.end) return { id: `preview-${sourceFileId}-${index + 1}`, sourceFileId, originalContent: content, editedContent: content, sourceStart: range.start, sourceEnd: range.end, sourceStartLine: lineNumberAtOffset(lines, range.start), sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)), tokenCount: Math.max(1, Math.ceil(content.length / 2)), status: 'original', } } export function buildPreviewItems( sourceText: string, processType: ProcessType, sourceFileId = 'default-source', unstructuredOptions?: UnstructuredProcessOptions, ): PreviewItem[] { const lines = sourceLines(sourceText) if (processType === 'unstructured') { return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => ( previewItemFromRange(sourceText, lines, range, sourceFileId, index) )) } const meaningfulLines = lines.filter((line) => line.content.trim()) const groupSize = processType === 'structured' ? 1 : 3 const items: PreviewItem[] = [] for (let index = 0; index < meaningfulLines.length; index += groupSize) { const group = meaningfulLines.slice(index, index + groupSize) if (!group.length) continue const sourceStart = group[0].start const sourceEnd = group[group.length - 1].end const content = sourceText.slice(sourceStart, sourceEnd) items.push({ id: `preview-${sourceFileId}-${items.length + 1}`, sourceFileId, originalContent: content, editedContent: content, sourceStart, sourceEnd, sourceStartLine: group[0].number, sourceEndLine: group[group.length - 1].number, tokenCount: Math.max(1, Math.ceil(content.length / 2)), status: 'original', }) } return items } const SEMANTIC_PREFIXES = [ '请结合实际情况,说明一下:', '如果方便的话,请详细解答:', '请用通俗易懂的方式说明:', '请从实际应用角度说明:', '请简洁、自然地说明:', ] export function createResults( items: PreviewItem[], options?: StructuredProcessOptions | UnstructuredProcessOptions, ): ResultItem[] { const resultCount = options && 'qaPairsPerChunk' in options ? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1)) : Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1)) return items.flatMap((item, index) => { const [firstLine = '', ...rest] = item.editedContent.split('\n') const output = rest.join('\n').trim() || item.editedContent.trim() const baseInstruction = firstLine.replace(/^问[::]\s*/, '').trim() || `数据条目 ${index + 1}` return Array.from({ length: resultCount }, (_, variantIndex) => { const instruction = options?.semanticEnrichment ? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}` : variantIndex === 0 ? baseInstruction : `${baseInstruction}(问法 ${variantIndex + 1})` return { id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`, instruction, input: '', output, originalInstruction: instruction, originalInput: '', originalOutput: output, status: 'valid' as const, } }) }) }