YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完成数据处理接口与前端接入

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-23 15:10:13 +08:00
parent f453234057
commit f04dc479bb
29 changed files with 7126 additions and 1144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

525
frontend/src/views/data-process/create/previewModel.ts
View File

@@ -1,12 +1,6 @@
import type {
PreviewItem,
ProcessType,
ResultItem,
SourceLine,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
import type { SourceLine } from './types'
/** 仅用于“使用示例”上传；正式预览和切片全部由后端生成。 */
export const DEFAULT_SOURCE_TEXT = [
'问：如何看待当前的通货膨胀风险？',
'答：当前通胀水平总体可控，但仍需关注能源价格与供给扰动。',
@@ -14,26 +8,13 @@ export const DEFAULT_SOURCE_TEXT = [
'答：会议时间以美联储官方日历为准，市场会重点关注利率路径指引。',
'问：人民币汇率未来走势如何？',
'答：人民币汇率取决于中美利差、经济基本面与政策预期。',
'问：银行理财产品收益率为何持续走低？',
'答：主要与市场利率下行、资产端收益下降以及风险偏好变化有关。',
'问：什么是复利？',
'答：复利是指在计算利息时，将上一期利息加入本金，再计算下一期利息。',
'问：如何评估股票的投资价值？',
'答：评估股票投资价值可以从以下几个方面进行：',
'1. 公司基本面：分析公司的财务状况、盈利能力、成长性等。',
'2. 行业前景：考察公司所处行业的发展趋势和竞争格局。',
'3. 估值水平：通过市盈率、市净率等指标判断估值是否合理。',
'4. 财务健康：关注公司的负债情况、现金流状况等。',
'5. 管理团队：评估管理层的能力和过往业绩。',
'此外，还需要关注宏观经济环境、政策变化等因素对股票市场的影响。',
'问：债券和股票的主要区别是什么？',
'答：债券收益相对稳定但上行有限，股票波动更大且承担更高风险。',
'问：什么是市盈率？',
'答：市盈率是股票价格与每股收益的比值，常用于衡量估值水平。',
'问：如何进行资产配置？',
'答：应根据投资目标、风险承受能力和市场环境合理分配资产。',
'答：复利是将上一期利息加入本金，再计算下一期利息。',
].join('\n')
/**
* 把后端返回的字符偏移映射为源文件行，仅负责界面高亮，不参与切片。
*/
export function sourceLines(sourceText: string): SourceLine[] {
const rawLines = sourceText.split('\n')
let cursor = 0
@@ -46,495 +27,7 @@ export function sourceLines(sourceText: string): SourceLine[] {
})
}
interface SourceRange {
start: number
end: number
}
interface ProtectedRange extends SourceRange {
kind: 'code' | 'table' | 'list'
}
const DEFAULT_CHUNK_SIZE = 800
const DEFAULT_CHUNK_OVERLAP = 100
const DEFAULT_MIN_CHUNK_SIZE = 100
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
}
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
let nextStart = Math.max(0, start)
let nextEnd = Math.min(sourceText.length, end)
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
return { start: nextStart, end: nextEnd }
}
function normalizeDelimiter(delimiter: string | undefined): string {
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
}
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
return line.start < range.end && line.end > range.start
}
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
return ranges.some((range) => overlapsRange(line, range))
}
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
const ranges: ProtectedRange[] = []
let openFence: { start: number; marker: string; length: number } | null = null
for (const line of lines) {
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
if (!fence) continue
const marker = fence[1][0]
if (!openFence) {
openFence = { start: line.start, marker, length: fence[1].length }
continue
}
if (marker === openFence.marker && fence[1].length >= openFence.length) {
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
openFence = null
}
}
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
return ranges
}
function isTableSeparator(content: string): boolean {
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
const cells = normalized.split('|').map((cell) => cell.trim())
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
}
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
const ranges: ProtectedRange[] = []
for (let index = 0; index < lines.length - 1; index += 1) {
const header = lines[index]
const separator = lines[index + 1]
if (
isLineProtected(header, codeRanges)
|| isLineProtected(separator, codeRanges)
|| !header.content.includes('|')
|| !isTableSeparator(separator.content)
) {
continue
}
let endIndex = index + 1
while (
endIndex + 1 < lines.length
&& !isLineProtected(lines[endIndex + 1], codeRanges)
&& lines[endIndex + 1].content.trim()
&& lines[endIndex + 1].content.includes('|')
) {
endIndex += 1
}
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
index = endIndex
}
return ranges
}
function isListItem(content: string): boolean {
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
}
function isListContinuation(content: string): boolean {
return /^\s{2,}\S/.test(content)
}
function detectListRanges(
lines: SourceLine[],
excludedRanges: SourceRange[],
): ProtectedRange[] {
const ranges: ProtectedRange[] = []
for (let index = 0; index < lines.length; index += 1) {
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
let endIndex = index
let itemCount = 1
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
const nextContent = lines[endIndex + 1].content
if (isListItem(nextContent)) {
itemCount += 1
endIndex += 1
continue
}
if (isListContinuation(nextContent)) {
endIndex += 1
continue
}
break
}
if (itemCount >= 2) {
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
index = endIndex
}
}
return ranges
}
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
return ranges
.sort((left, right) => left.start - right.start || left.end - right.end)
.reduce<ProtectedRange[]>((merged, range) => {
const previous = merged[merged.length - 1]
if (previous && range.start < previous.end) {
previous.end = Math.max(previous.end, range.end)
return merged
}
merged.push({ ...range })
return merged
}, [])
}
function protectedRangesForOptions(
sourceText: string,
options?: UnstructuredProcessOptions,
): ProtectedRange[] {
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
const lines = sourceLines(sourceText)
const codeRanges = detectCodeBlockRanges(sourceText, lines)
const tableRanges = detectTableRanges(lines, codeRanges)
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
const enabledRanges = [
...(options?.preserveCodeBlocks ? codeRanges : []),
...(options?.preserveTables ? tableRanges : []),
...(options?.preserveLists ? listRanges : []),
]
return mergeProtectedRanges(enabledRanges)
}
function protectedRangeContaining(
ranges: ProtectedRange[],
offset: number,
): ProtectedRange | undefined {
return ranges.find((range) => range.start < offset && offset < range.end)
}
function normalizeChunkStart(
sourceText: string,
cursor: number,
protectedRanges: ProtectedRange[],
): number {
let start = Math.max(0, Math.min(cursor, sourceText.length))
const overlapBlock = protectedRangeContaining(protectedRanges, start)
if (overlapBlock) start = overlapBlock.end
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
// 去除块前空白时可能进入缩进代码块/列表；此时恢复到完整块起点。
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
return start
}
function protectChunkEnd(
proposedEnd: number,
start: number,
minimumEnd: number,
protectedRanges: ProtectedRange[],
): number {
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
if (!splitBlock) return proposedEnd
// 优先在块前结束；块前不足最小切片长度时，将整个块收入当前切片。
return splitBlock.start > start && splitBlock.start >= minimumEnd
? splitBlock.start
: splitBlock.end
}
function restoreProtectedEdges(
range: SourceRange,
rawStart: number,
rawEnd: number,
protectedRanges: ProtectedRange[],
): SourceRange {
const nextRange = { ...range }
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
return nextRange
}
function lastBoundaryInRange(
sourceText: string,
idealEnd: number,
minimumEnd: number,
): number | null {
const candidates: number[] = []
const boundaryTokens = ['\n\n', '\n', '。', '！', '？', '；', '.', '!', '?', ';']
boundaryTokens.forEach((token) => {
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
})
return candidates.length ? Math.max(...candidates) : null
}
function lastHeadingBoundary(
sourceText: string,
start: number,
idealEnd: number,
minimumEnd: number,
): number | null {
const section = sourceText.slice(start, idealEnd)
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
let boundary: number | null = null
let match: RegExpExecArray | null
while ((match = headingPattern.exec(section))) {
const absoluteStart = start + match.index
if (absoluteStart >= minimumEnd) boundary = absoluteStart
}
return boundary
}
function resolveChunkEnd(
sourceText: string,
start: number,
idealEnd: number,
minimumEnd: number,
options: UnstructuredProcessOptions | undefined,
): number {
const method = options?.chunkMethod ?? 'semantic'
if (method === 'fixed') return idealEnd
if (method === 'custom') {
const delimiter = normalizeDelimiter(options?.customDelimiter)
if (!delimiter) return idealEnd
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
return boundary >= minimumEnd ? boundary : idealEnd
}
if (method === 'heading') {
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
if (headingBoundary !== null) return headingBoundary
}
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
}
function buildUnstructuredRanges(
sourceText: string,
options?: UnstructuredProcessOptions,
): SourceRange[] {
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算，避免引入分词器依赖。
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
const minimumCharacters = Math.min(
targetCharacters,
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
)
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
const protectedRanges = protectedRangesForOptions(sourceText, options)
const ranges: SourceRange[] = []
let cursor = 0
while (cursor < sourceText.length) {
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
if (start >= sourceText.length) break
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
let end = idealEnd === sourceText.length
? idealEnd
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
// 所有自定义边界都必须向前推进；异常配置回退到固定长度切分。
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
let range = trimSourceRange(sourceText, start, end)
range = restoreProtectedEdges(range, start, end, protectedRanges)
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
range.end = Math.min(end, range.start + minimumCharacters)
}
if (range.end <= range.start) {
cursor = Math.max(cursor + 1, end)
continue
}
const isLastRange = end >= sourceText.length
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
ranges[ranges.length - 1].end = range.end
break
}
ranges.push(range)
if (isLastRange) break
// overlap 是允许的最大重叠量；按当前切片动态收缩，保证每轮至少推进最小切片长度。
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
const nextCursor = range.end - actualOverlap
cursor = nextCursor > start ? nextCursor : range.end
}
return ranges
}
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
if (!lines.length) return null
let low = 0
let high = lines.length - 1
let result = 0
while (low <= high) {
const middle = Math.floor((low + high) / 2)
if (lines[middle].start <= offset) {
result = middle
low = middle + 1
} else {
high = middle - 1
}
}
return lines[result].number
}
function previewItemFromRange(
sourceText: string,
lines: SourceLine[],
range: SourceRange,
sourceFileId: string,
index: number,
): PreviewItem {
const content = sourceText.slice(range.start, range.end)
return {
id: `preview-${sourceFileId}-${index + 1}`,
sourceFileId,
originalContent: content,
editedContent: content,
sourceStart: range.start,
sourceEnd: range.end,
sourceStartLine: lineNumberAtOffset(lines, range.start),
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
status: 'original',
}
}
export function buildPreviewItems(
sourceText: string,
processType: ProcessType,
sourceFileId = 'default-source',
unstructuredOptions?: UnstructuredProcessOptions,
): PreviewItem[] {
const lines = sourceLines(sourceText)
if (processType === 'unstructured') {
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
))
}
const meaningfulLines = lines.filter((line) => line.content.trim())
const groupSize = processType === 'structured' ? 1 : 3
const items: PreviewItem[] = []
for (let index = 0; index < meaningfulLines.length; index += groupSize) {
const group = meaningfulLines.slice(index, index + groupSize)
if (!group.length) continue
const sourceStart = group[0].start
const sourceEnd = group[group.length - 1].end
const content = sourceText.slice(sourceStart, sourceEnd)
items.push({
id: `preview-${sourceFileId}-${items.length + 1}`,
sourceFileId,
originalContent: content,
editedContent: content,
sourceStart,
sourceEnd,
sourceStartLine: group[0].number,
sourceEndLine: group[group.length - 1].number,
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
status: 'original',
})
}
return items
}
const SEMANTIC_PREFIXES = [
'请结合实际情况，说明一下：',
'如果方便的话，请详细解答：',
'请用通俗易懂的方式说明：',
'请从实际应用角度说明：',
'请简洁、自然地说明：',
]
export function createResults(
items: PreviewItem[],
options?: StructuredProcessOptions | UnstructuredProcessOptions,
): ResultItem[] {
const resultCount = options && 'qaPairsPerChunk' in options
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
const generatedResults = items.flatMap((item, index) => {
if (options?.qualityFilterEnabled && options.filterLowQuality) {
if (item.status === 'invalid' || !item.editedContent.trim()) return []
}
const [firstLine = '', ...rest] = item.editedContent.split('\n')
const output = rest.join('\n').trim() || item.editedContent.trim()
const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
return Array.from({ length: resultCount }, (_, variantIndex) => {
const instruction = options?.semanticEnrichment
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
: variantIndex === 0
? baseInstruction
: `${baseInstruction}（问法 ${variantIndex + 1}）`
return {
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
instruction,
input: '',
output,
originalInstruction: instruction,
originalInput: '',
originalOutput: output,
status: 'valid' as const,
}
})
})
if (!options?.qualityFilterEnabled) return generatedResults
return generatedResults.filter((result) => {
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
return true
})
/** 与后端预览 token 估算规则一致，仅用于编辑中的即时计数。 */
export function estimateTokenCount(text: string): number {
return text.match(/[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]/gu)?.length ?? 0
}