YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完成数据处理接口与前端接入

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-23 15:10:13 +08:00
parent f453234057
commit f04dc479bb
29 changed files with 7126 additions and 1144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
frontend/src/views/data-process/create/PreviewCompareStep.vue
View File

@@ -17,6 +17,8 @@ const emit = defineEmits<{
'update:selectedId': [value: string]
'update:selectedFileId': [value: string]
'update:item-content': [id: string, value: string]
'restore:item': [id: string]
'add:item': []
'remove:item': [id: string]
}>()
@@ -71,6 +73,12 @@ function saveEditor() {
closeEditor()
}
function restoreItem() {
if (!editingItem.value) return
emit('restore:item', editingItem.value.id)
closeEditor()
}
function removeItem(item: PreviewItem) {
selectItem(item.id)
emit('remove:item', item.id)
@@ -169,7 +177,12 @@ function lineRange(item: PreviewItem) {
<div class="preview-pane">
<div class="pane-header">
<strong>{{ processType === 'unstructured' ? '切片内容' : '记录内容' }}</strong>
<span>共 {{ items.length.toLocaleString() }} 条</span>
<div class="pane-header-actions">
<span>共 {{ items.length.toLocaleString() }} 条</span>
<el-button link type="primary" @click="emit('add:item')">
<i class="fa fa-plus" /> 手动新增
</el-button>
</div>
</div>
<template v-if="!editingItem">
@@ -234,6 +247,13 @@ function lineRange(item: PreviewItem) {
resize="none"
/>
<div class="editor-actions">
<el-button
v-if="editingItem.sourceStart != null"
link
@click="restoreItem"
>
<i class="fa fa-undo" /> 恢复原始内容
</el-button>
<div>
<el-button @click="closeEditor">取消</el-button>
<el-button type="primary" @click="saveEditor">保存修改</el-button>
@@ -288,6 +308,17 @@ function lineRange(item: PreviewItem) {
white-space: nowrap;
}
.pane-header-actions {
display: flex;
align-items: center;
gap: 10px;
> span {
color: #8a93a3;
font-size: 11px;
}
}
.file-option {
display: grid;
grid-template-columns: minmax(0, 1fr) auto auto;

21
frontend/src/views/data-process/create/ResultEditorStep.vue
View File

@@ -69,6 +69,12 @@ function selectRelative(offset: number) {
<div>
<strong>结果 #{{ String(selectedIndex + 1).padStart(3, '0') }}</strong>
<span v-if="selectedItem.status === 'modified'" class="modified-label">已修改</span>
<el-tag v-if="selectedItem.split" size="small" effect="plain">{{ selectedItem.split }}</el-tag>
<el-tag
v-if="selectedItem.qualityScore != null"
size="small"
:type="selectedItem.qualityScore >= 80 ? 'success' : selectedItem.qualityScore >= 60 ? 'warning' : 'danger'"
>质量 {{ selectedItem.qualityScore.toFixed(1) }}</el-tag>
</div>
<el-button link @click="emit('restore:item', selectedItem.id)"><i class="fa fa-undo" /> 恢复生成结果</el-button>
</div>
@@ -106,6 +112,15 @@ function selectRelative(offset: number) {
<div v-else class="validation-success">
<i class="fa fa-check-circle" /> 字段校验通过
</div>
<div v-if="selectedItem.qualityFlags?.length" class="quality-flags">
<el-tag
v-for="flag in selectedItem.qualityFlags"
:key="flag"
size="small"
type="warning"
effect="plain"
>{{ flag }}</el-tag>
</div>
<div class="editor-pagination">
<el-button :disabled="selectedIndex <= 0" @click="selectRelative(-1)">上一条</el-button>
<span>{{ selectedIndex + 1 }} / {{ items.length }}</span>
@@ -275,6 +290,12 @@ function selectRelative(offset: number) {
border-radius: 6px;
}
.quality-flags {
display: flex;
flex-wrap: wrap;
gap: 6px;
}
.validation-error {
color: #b45309;
background: #fff7e8;

36
frontend/src/views/data-process/create/SourceUploadStep.vue
View File

@@ -21,16 +21,12 @@ const emit = defineEmits<{
}>()
const DATA_SOURCE_TYPES = [
{ value: 'mysql', label: 'MySQL' },
{ value: 'postgresql', label: 'PostgreSQL' },
{ value: 'mongodb', label: 'MongoDB' },
{ value: 'api', label: 'REST API' },
]
const AUTH_MODES = [
{ value: 'none', label: '免鉴权' },
{ value: 'basic', label: '账号密码' },
{ value: 'token', label: 'Token' },
]
const FILE_PAGE_SIZE = 10
@@ -38,9 +34,11 @@ const currentFilePage = ref(1)
const isExternal = computed(() => props.processType === 'external')
// 后端首版严格支持这些可验证的文本格式；不要把无法解析的二进制文档
// 静默替换成示例正文。
const uploadAccept = computed(() => props.processType === 'unstructured'
? '.txt,.md,.pdf,.docx,.doc,.json,.jsonl'
: '.json,.jsonl,.csv,.xlsx,.xls')
? '.txt,.md,.json,.jsonl'
: '.json,.jsonl,.csv,.txt,.md')
const pagedUploadedFiles = computed(() => {
const start = (currentFilePage.value - 1) * FILE_PAGE_SIZE
@@ -101,7 +99,7 @@ function formatSize(size: number) {
<el-form-item label="地址 / URL">
<el-input
:model-value="externalSource.url"
placeholder="例如：mysql://host:3306/db 或 https://api.example.com/data"
placeholder="例如：postgresql://db.example.com:5432/my_database"
aria-label="数据源地址或 URL"
@update:model-value="updateExternalField('url', $event)"
/>
@@ -140,17 +138,6 @@ function formatSize(size: number) {
@update:model-value="updateExternalField('password', $event)"
/>
</el-form-item>
<el-form-item v-if="externalSource.authMode === 'token'" label="Token">
<el-input
:model-value="externalSource.token"
type="password"
show-password
autocomplete="off"
placeholder="请输入访问 Token"
aria-label="数据源访问 Token"
@update:model-value="updateExternalField('token', $event)"
/>
</el-form-item>
<el-form-item label="拉取条数">
<el-input-number
:model-value="externalSource.limit"
@@ -162,6 +149,19 @@ function formatSize(size: number) {
@update:model-value="updateExternalField('limit', Number($event) || 0)"
/>
</el-form-item>
<el-form-item label="只读查询语句" class="external-query-field">
<el-input
:model-value="externalSource.query"
type="textarea"
:rows="4"
maxlength="20000"
show-word-limit
placeholder="例如：SELECT question, answer FROM qa_data ORDER BY id"
aria-label="外部数据源只读查询语句"
@update:model-value="updateExternalField('query', $event)"
/>
<small>只允许单条 SELECT 或 WITH 查询；后端会拒绝写入、DDL 和多语句。</small>
</el-form-item>
</el-form>
<div class="external-actions">

2
frontend/src/views/data-process/create/StructuredOptionsPanel.vue
View File

@@ -25,7 +25,7 @@ const PREPROCESS_OPTIONS: Array<{
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
{ value: 'normalize_format', label: '数据格式标准化', description: '统一编码、空白、字段名和 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
]

525
frontend/src/views/data-process/create/previewModel.ts
View File

@@ -1,12 +1,6 @@
import type {
PreviewItem,
ProcessType,
ResultItem,
SourceLine,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
import type { SourceLine } from './types'
/** 仅用于“使用示例”上传；正式预览和切片全部由后端生成。 */
export const DEFAULT_SOURCE_TEXT = [
'问：如何看待当前的通货膨胀风险？',
'答：当前通胀水平总体可控，但仍需关注能源价格与供给扰动。',
@@ -14,26 +8,13 @@ export const DEFAULT_SOURCE_TEXT = [
'答：会议时间以美联储官方日历为准，市场会重点关注利率路径指引。',
'问：人民币汇率未来走势如何？',
'答：人民币汇率取决于中美利差、经济基本面与政策预期。',
'问：银行理财产品收益率为何持续走低？',
'答：主要与市场利率下行、资产端收益下降以及风险偏好变化有关。',
'问：什么是复利？',
'答：复利是指在计算利息时，将上一期利息加入本金，再计算下一期利息。',
'问：如何评估股票的投资价值？',
'答：评估股票投资价值可以从以下几个方面进行：',
'1. 公司基本面：分析公司的财务状况、盈利能力、成长性等。',
'2. 行业前景：考察公司所处行业的发展趋势和竞争格局。',
'3. 估值水平：通过市盈率、市净率等指标判断估值是否合理。',
'4. 财务健康：关注公司的负债情况、现金流状况等。',
'5. 管理团队：评估管理层的能力和过往业绩。',
'此外，还需要关注宏观经济环境、政策变化等因素对股票市场的影响。',
'问：债券和股票的主要区别是什么？',
'答：债券收益相对稳定但上行有限，股票波动更大且承担更高风险。',
'问：什么是市盈率？',
'答：市盈率是股票价格与每股收益的比值，常用于衡量估值水平。',
'问：如何进行资产配置？',
'答：应根据投资目标、风险承受能力和市场环境合理分配资产。',
'答：复利是将上一期利息加入本金，再计算下一期利息。',
].join('\n')
/**
* 把后端返回的字符偏移映射为源文件行，仅负责界面高亮，不参与切片。
*/
export function sourceLines(sourceText: string): SourceLine[] {
const rawLines = sourceText.split('\n')
let cursor = 0
@@ -46,495 +27,7 @@ export function sourceLines(sourceText: string): SourceLine[] {
})
}
interface SourceRange {
start: number
end: number
}
interface ProtectedRange extends SourceRange {
kind: 'code' | 'table' | 'list'
}
const DEFAULT_CHUNK_SIZE = 800
const DEFAULT_CHUNK_OVERLAP = 100
const DEFAULT_MIN_CHUNK_SIZE = 100
function finiteInteger(value: number | undefined, fallback: number, min: number): number {
return Number.isFinite(value) ? Math.max(min, Math.round(value as number)) : fallback
}
function trimSourceRange(sourceText: string, start: number, end: number): SourceRange {
let nextStart = Math.max(0, start)
let nextEnd = Math.min(sourceText.length, end)
while (nextStart < nextEnd && /\s/.test(sourceText[nextStart])) nextStart += 1
while (nextEnd > nextStart && /\s/.test(sourceText[nextEnd - 1])) nextEnd -= 1
return { start: nextStart, end: nextEnd }
}
function normalizeDelimiter(delimiter: string | undefined): string {
return (delimiter ?? '').replace(/\\n/g, '\n').replace(/\\t/g, '\t')
}
function overlapsRange(line: SourceLine, range: SourceRange): boolean {
return line.start < range.end && line.end > range.start
}
function isLineProtected(line: SourceLine, ranges: SourceRange[]): boolean {
return ranges.some((range) => overlapsRange(line, range))
}
function detectCodeBlockRanges(sourceText: string, lines: SourceLine[]): ProtectedRange[] {
const ranges: ProtectedRange[] = []
let openFence: { start: number; marker: string; length: number } | null = null
for (const line of lines) {
const fence = line.content.match(/^\s*(`{3,}|~{3,})/)
if (!fence) continue
const marker = fence[1][0]
if (!openFence) {
openFence = { start: line.start, marker, length: fence[1].length }
continue
}
if (marker === openFence.marker && fence[1].length >= openFence.length) {
ranges.push({ start: openFence.start, end: line.end, kind: 'code' })
openFence = null
}
}
if (openFence) ranges.push({ start: openFence.start, end: sourceText.length, kind: 'code' })
return ranges
}
function isTableSeparator(content: string): boolean {
const normalized = content.trim().replace(/^\|/, '').replace(/\|$/, '')
const cells = normalized.split('|').map((cell) => cell.trim())
return cells.length >= 2 && cells.every((cell) => /^:?-{3,}:?$/.test(cell))
}
function detectTableRanges(lines: SourceLine[], codeRanges: SourceRange[]): ProtectedRange[] {
const ranges: ProtectedRange[] = []
for (let index = 0; index < lines.length - 1; index += 1) {
const header = lines[index]
const separator = lines[index + 1]
if (
isLineProtected(header, codeRanges)
|| isLineProtected(separator, codeRanges)
|| !header.content.includes('|')
|| !isTableSeparator(separator.content)
) {
continue
}
let endIndex = index + 1
while (
endIndex + 1 < lines.length
&& !isLineProtected(lines[endIndex + 1], codeRanges)
&& lines[endIndex + 1].content.trim()
&& lines[endIndex + 1].content.includes('|')
) {
endIndex += 1
}
ranges.push({ start: header.start, end: lines[endIndex].end, kind: 'table' })
index = endIndex
}
return ranges
}
function isListItem(content: string): boolean {
return /^\s*(?:[-+*]|\d+[.)])\s+\S/.test(content)
}
function isListContinuation(content: string): boolean {
return /^\s{2,}\S/.test(content)
}
function detectListRanges(
lines: SourceLine[],
excludedRanges: SourceRange[],
): ProtectedRange[] {
const ranges: ProtectedRange[] = []
for (let index = 0; index < lines.length; index += 1) {
if (isLineProtected(lines[index], excludedRanges) || !isListItem(lines[index].content)) continue
let endIndex = index
let itemCount = 1
while (endIndex + 1 < lines.length && !isLineProtected(lines[endIndex + 1], excludedRanges)) {
const nextContent = lines[endIndex + 1].content
if (isListItem(nextContent)) {
itemCount += 1
endIndex += 1
continue
}
if (isListContinuation(nextContent)) {
endIndex += 1
continue
}
break
}
if (itemCount >= 2) {
ranges.push({ start: lines[index].start, end: lines[endIndex].end, kind: 'list' })
index = endIndex
}
}
return ranges
}
function mergeProtectedRanges(ranges: ProtectedRange[]): ProtectedRange[] {
return ranges
.sort((left, right) => left.start - right.start || left.end - right.end)
.reduce<ProtectedRange[]>((merged, range) => {
const previous = merged[merged.length - 1]
if (previous && range.start < previous.end) {
previous.end = Math.max(previous.end, range.end)
return merged
}
merged.push({ ...range })
return merged
}, [])
}
function protectedRangesForOptions(
sourceText: string,
options?: UnstructuredProcessOptions,
): ProtectedRange[] {
if (!options?.preserveCodeBlocks && !options?.preserveTables && !options?.preserveLists) return []
const lines = sourceLines(sourceText)
const codeRanges = detectCodeBlockRanges(sourceText, lines)
const tableRanges = detectTableRanges(lines, codeRanges)
const listRanges = detectListRanges(lines, [...codeRanges, ...tableRanges])
const enabledRanges = [
...(options?.preserveCodeBlocks ? codeRanges : []),
...(options?.preserveTables ? tableRanges : []),
...(options?.preserveLists ? listRanges : []),
]
return mergeProtectedRanges(enabledRanges)
}
function protectedRangeContaining(
ranges: ProtectedRange[],
offset: number,
): ProtectedRange | undefined {
return ranges.find((range) => range.start < offset && offset < range.end)
}
function normalizeChunkStart(
sourceText: string,
cursor: number,
protectedRanges: ProtectedRange[],
): number {
let start = Math.max(0, Math.min(cursor, sourceText.length))
const overlapBlock = protectedRangeContaining(protectedRanges, start)
if (overlapBlock) start = overlapBlock.end
while (start < sourceText.length && /\s/.test(sourceText[start])) start += 1
// 去除块前空白时可能进入缩进代码块/列表；此时恢复到完整块起点。
const blockAfterTrim = protectedRangeContaining(protectedRanges, start)
if (blockAfterTrim) return cursor <= blockAfterTrim.start ? blockAfterTrim.start : blockAfterTrim.end
return start
}
function protectChunkEnd(
proposedEnd: number,
start: number,
minimumEnd: number,
protectedRanges: ProtectedRange[],
): number {
const splitBlock = protectedRangeContaining(protectedRanges, proposedEnd)
if (!splitBlock) return proposedEnd
// 优先在块前结束；块前不足最小切片长度时，将整个块收入当前切片。
return splitBlock.start > start && splitBlock.start >= minimumEnd
? splitBlock.start
: splitBlock.end
}
function restoreProtectedEdges(
range: SourceRange,
rawStart: number,
rawEnd: number,
protectedRanges: ProtectedRange[],
): SourceRange {
const nextRange = { ...range }
const startBlock = protectedRangeContaining(protectedRanges, nextRange.start)
if (startBlock && rawStart <= startBlock.start) nextRange.start = startBlock.start
const endBlock = protectedRangeContaining(protectedRanges, nextRange.end)
if (endBlock && rawEnd >= endBlock.end) nextRange.end = endBlock.end
return nextRange
}
function lastBoundaryInRange(
sourceText: string,
idealEnd: number,
minimumEnd: number,
): number | null {
const candidates: number[] = []
const boundaryTokens = ['\n\n', '\n', '。', '！', '？', '；', '.', '!', '?', ';']
boundaryTokens.forEach((token) => {
const tokenStart = sourceText.lastIndexOf(token, idealEnd - token.length)
const boundary = tokenStart === -1 ? -1 : tokenStart + token.length
if (boundary >= minimumEnd && boundary <= idealEnd) candidates.push(boundary)
})
return candidates.length ? Math.max(...candidates) : null
}
function lastHeadingBoundary(
sourceText: string,
start: number,
idealEnd: number,
minimumEnd: number,
): number | null {
const section = sourceText.slice(start, idealEnd)
const headingPattern = /^(?:#{1,6}\s+|第[一二三四五六七八九十百]+[章节篇部分]|\d+(?:\.\d+)*[、.\s])/gm
let boundary: number | null = null
let match: RegExpExecArray | null
while ((match = headingPattern.exec(section))) {
const absoluteStart = start + match.index
if (absoluteStart >= minimumEnd) boundary = absoluteStart
}
return boundary
}
function resolveChunkEnd(
sourceText: string,
start: number,
idealEnd: number,
minimumEnd: number,
options: UnstructuredProcessOptions | undefined,
): number {
const method = options?.chunkMethod ?? 'semantic'
if (method === 'fixed') return idealEnd
if (method === 'custom') {
const delimiter = normalizeDelimiter(options?.customDelimiter)
if (!delimiter) return idealEnd
const delimiterStart = sourceText.lastIndexOf(delimiter, idealEnd - delimiter.length)
const boundary = delimiterStart === -1 ? -1 : delimiterStart + delimiter.length
return boundary >= minimumEnd ? boundary : idealEnd
}
if (method === 'heading') {
const headingBoundary = lastHeadingBoundary(sourceText, start, idealEnd, minimumEnd)
if (headingBoundary !== null) return headingBoundary
}
return lastBoundaryInRange(sourceText, idealEnd, minimumEnd) ?? idealEnd
}
function buildUnstructuredRanges(
sourceText: string,
options?: UnstructuredProcessOptions,
): SourceRange[] {
// 预览统一沿用“约 2 个字符 = 1 token”的轻量估算，避免引入分词器依赖。
const targetCharacters = finiteInteger(options?.chunkSize, DEFAULT_CHUNK_SIZE, 1) * 2
const minimumCharacters = Math.min(
targetCharacters,
finiteInteger(options?.minChunkSize, DEFAULT_MIN_CHUNK_SIZE, 1) * 2,
)
const requestedOverlap = finiteInteger(options?.chunkOverlap, DEFAULT_CHUNK_OVERLAP, 0) * 2
const protectedRanges = protectedRangesForOptions(sourceText, options)
const ranges: SourceRange[] = []
let cursor = 0
while (cursor < sourceText.length) {
const start = normalizeChunkStart(sourceText, cursor, protectedRanges)
if (start >= sourceText.length) break
const idealEnd = Math.min(sourceText.length, start + targetCharacters)
const minimumEnd = Math.min(idealEnd, start + minimumCharacters)
let end = idealEnd === sourceText.length
? idealEnd
: resolveChunkEnd(sourceText, start, idealEnd, minimumEnd, options)
end = protectChunkEnd(end, start, minimumEnd, protectedRanges)
// 所有自定义边界都必须向前推进；异常配置回退到固定长度切分。
if (end <= start) end = Math.min(sourceText.length, start + targetCharacters)
let range = trimSourceRange(sourceText, start, end)
range = restoreProtectedEdges(range, start, end, protectedRanges)
if (end < sourceText.length && range.end - range.start < minimumCharacters) {
range.end = Math.min(end, range.start + minimumCharacters)
}
if (range.end <= range.start) {
cursor = Math.max(cursor + 1, end)
continue
}
const isLastRange = end >= sourceText.length
if (isLastRange && range.end - range.start < minimumCharacters && ranges.length) {
ranges[ranges.length - 1].end = range.end
break
}
ranges.push(range)
if (isLastRange) break
// overlap 是允许的最大重叠量；按当前切片动态收缩，保证每轮至少推进最小切片长度。
const maximumOverlap = Math.max(0, range.end - range.start - minimumCharacters)
const actualOverlap = Math.min(requestedOverlap, maximumOverlap)
const nextCursor = range.end - actualOverlap
cursor = nextCursor > start ? nextCursor : range.end
}
return ranges
}
function lineNumberAtOffset(lines: SourceLine[], offset: number): number | null {
if (!lines.length) return null
let low = 0
let high = lines.length - 1
let result = 0
while (low <= high) {
const middle = Math.floor((low + high) / 2)
if (lines[middle].start <= offset) {
result = middle
low = middle + 1
} else {
high = middle - 1
}
}
return lines[result].number
}
function previewItemFromRange(
sourceText: string,
lines: SourceLine[],
range: SourceRange,
sourceFileId: string,
index: number,
): PreviewItem {
const content = sourceText.slice(range.start, range.end)
return {
id: `preview-${sourceFileId}-${index + 1}`,
sourceFileId,
originalContent: content,
editedContent: content,
sourceStart: range.start,
sourceEnd: range.end,
sourceStartLine: lineNumberAtOffset(lines, range.start),
sourceEndLine: lineNumberAtOffset(lines, Math.max(range.start, range.end - 1)),
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
status: 'original',
}
}
export function buildPreviewItems(
sourceText: string,
processType: ProcessType,
sourceFileId = 'default-source',
unstructuredOptions?: UnstructuredProcessOptions,
): PreviewItem[] {
const lines = sourceLines(sourceText)
if (processType === 'unstructured') {
return buildUnstructuredRanges(sourceText, unstructuredOptions).map((range, index) => (
previewItemFromRange(sourceText, lines, range, sourceFileId, index)
))
}
const meaningfulLines = lines.filter((line) => line.content.trim())
const groupSize = processType === 'structured' ? 1 : 3
const items: PreviewItem[] = []
for (let index = 0; index < meaningfulLines.length; index += groupSize) {
const group = meaningfulLines.slice(index, index + groupSize)
if (!group.length) continue
const sourceStart = group[0].start
const sourceEnd = group[group.length - 1].end
const content = sourceText.slice(sourceStart, sourceEnd)
items.push({
id: `preview-${sourceFileId}-${items.length + 1}`,
sourceFileId,
originalContent: content,
editedContent: content,
sourceStart,
sourceEnd,
sourceStartLine: group[0].number,
sourceEndLine: group[group.length - 1].number,
tokenCount: Math.max(1, Math.ceil(content.length / 2)),
status: 'original',
})
}
return items
}
const SEMANTIC_PREFIXES = [
'请结合实际情况，说明一下：',
'如果方便的话，请详细解答：',
'请用通俗易懂的方式说明：',
'请从实际应用角度说明：',
'请简洁、自然地说明：',
]
export function createResults(
items: PreviewItem[],
options?: StructuredProcessOptions | UnstructuredProcessOptions,
): ResultItem[] {
const resultCount = options && 'qaPairsPerChunk' in options
? Math.min(3, finiteInteger(options.qaPairsPerChunk, 1, 1))
: Math.min(5, finiteInteger(options?.qaPairsPerRow, 1, 1))
const generatedResults = items.flatMap((item, index) => {
if (options?.qualityFilterEnabled && options.filterLowQuality) {
if (item.status === 'invalid' || !item.editedContent.trim()) return []
}
const [firstLine = '', ...rest] = item.editedContent.split('\n')
const output = rest.join('\n').trim() || item.editedContent.trim()
const baseInstruction = firstLine.replace(/^问[:：]\s*/, '').trim() || `数据条目 ${index + 1}`
return Array.from({ length: resultCount }, (_, variantIndex) => {
const instruction = options?.semanticEnrichment
? `${SEMANTIC_PREFIXES[variantIndex]}${baseInstruction}`
: variantIndex === 0
? baseInstruction
: `${baseInstruction}（问法 ${variantIndex + 1}）`
return {
id: resultCount === 1 ? `result-${index + 1}` : `result-${index + 1}-${variantIndex + 1}`,
instruction,
input: '',
output,
originalInstruction: instruction,
originalInput: '',
originalOutput: output,
status: 'valid' as const,
}
})
})
if (!options?.qualityFilterEnabled) return generatedResults
return generatedResults.filter((result) => {
if (options.filterLowQuality && (!result.instruction.trim() || !result.output.trim())) return false
if (options.filterShortContent && result.output.trim().length < options.minOutputLength) return false
return true
})
/** 与后端预览 token 估算规则一致，仅用于编辑中的即时计数。 */
export function estimateTokenCount(text: string): number {
return text.match(/[\u3400-\u4dbf\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]/gu)?.length ?? 0
}

19
frontend/src/views/data-process/create/types.ts
View File

@@ -64,19 +64,25 @@ export interface UnstructuredProcessOptions extends GenerationControlOptions {
export interface ExternalDataSource {
type: string
url: string
authMode: string
authMode: 'none' | 'basic'
username?: string
password?: string
token?: string
limit: number
query?: string
fileName?: string
}
export interface UploadedDataFile {
uid: string | number
sourceFileId?: string
name: string
size: number
count: number
content: string
fileFormat?: string
checksumSha256?: string
status?: 'uploading' | 'ready' | 'failed'
error?: string
}
export interface SourceLine {
@@ -97,6 +103,10 @@ export interface PreviewItem {
sourceEndLine: number | null
tokenCount: number
status: 'original' | 'modified' | 'manual' | 'invalid'
qualityScore?: number
qualityDetails?: Record<string, number>
piiStats?: Record<string, number>
updatedAt?: string
}
export interface GenerationState {
@@ -115,4 +125,9 @@ export interface ResultItem {
originalOutput: string
status: 'valid' | 'modified' | 'invalid'
error?: string
split?: 'train' | 'validation' | 'test'
qualityScore?: number
qualityDetails?: Record<string, number>
qualityFlags?: string[]
updatedAt?: string
}

15
frontend/src/views/data-process/create/useDataProcessDraft.ts
View File

@@ -9,10 +9,11 @@ import type {
} from './types'
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 7
interface DraftSnapshot {
schemaVersion?: number
taskId?: string
currentStepId?: StepId
task?: { name?: string; description?: string }
processType?: ProcessType
@@ -22,6 +23,7 @@ interface DraftSnapshot {
}
interface DraftBindings {
taskId: Ref<string | null>
currentStepId: Readonly<Ref<StepId>>
task: Reactive<{ name: string; description: string }>
processType: Ref<ProcessType>
@@ -35,11 +37,13 @@ interface DraftBindings {
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
return {
type: typeof source.type === 'string' ? source.type : 'mysql',
type: typeof source.type === 'string' ? source.type : 'postgresql',
url: typeof source.url === 'string' ? source.url : '',
authMode: typeof source.authMode === 'string' ? source.authMode : 'none',
authMode: source.authMode === 'basic' ? 'basic' as const : 'none' as const,
username: typeof source.username === 'string' ? source.username : '',
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
query: typeof source.query === 'string' ? source.query : '',
fileName: typeof source.fileName === 'string' ? source.fileName : 'external-data.jsonl',
}
}
@@ -54,6 +58,7 @@ export function useDataProcessDraft(bindings: DraftBindings) {
function draftSnapshot(): DraftSnapshot {
return {
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
taskId: bindings.taskId.value || undefined,
currentStepId: bindings.currentStepId.value,
task: { ...bindings.task },
processType: bindings.processType.value,
@@ -92,6 +97,7 @@ export function useDataProcessDraft(bindings: DraftBindings) {
bindings.restoringDraft.value = true
bindings.goToStep('create')
bindings.taskId.value = typeof snapshot.taskId === 'string' ? snapshot.taskId : null
bindings.task.name = snapshot.task?.name || ''
bindings.task.description = snapshot.task?.description || ''
bindings.processType.value = snapshot.processType === 'unstructured' || snapshot.processType === 'external'
@@ -128,7 +134,6 @@ export function useDataProcessDraft(bindings: DraftBindings) {
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
password: '',
token: '',
})
bindings.dirty.value = false
@@ -137,7 +142,7 @@ export function useDataProcessDraft(bindings: DraftBindings) {
// 立即覆盖 v5 及更早草稿，清除其中可能存在的敏感值和大段正文。
writeDraft(false)
})
ElMessage.info('已恢复上次的任务配置，请重新上传或拉取源数据')
ElMessage.info('已恢复上次的任务配置')
} catch {
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
}

232
frontend/src/views/data-process/create/useDataProcessGeneration.ts
View File

@@ -1,21 +1,42 @@
import { reactive, ref, type Ref } from 'vue'
import { ElMessage } from 'element-plus'
import { createResults } from './previewModel'
import type {
GenerationState,
PreviewItem,
ProcessType,
ResultItem,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
import {
generateDataProcess,
getDataProcessProgress,
getDataProcessResults,
restoreDataProcessResult,
stopDataProcess,
updateDataProcessResult,
type DataProcessProgress,
type DataProcessResult,
} from '@/api/modules/dataProcess'
import type { GenerationState, ResultItem } from './types'
interface GenerationBindings {
previewItems: Ref<PreviewItem[]>
processType: Ref<ProcessType>
structuredOptions: Ref<StructuredProcessOptions>
unstructuredOptions: Ref<UnstructuredProcessOptions>
taskId: Ref<string | null>
dirty: Ref<boolean>
beforeGenerate?: () => Promise<void>
}
const RESULT_PAGE_SIZE = 500
const POLL_INTERVAL_MS = 1500
function mapResult(item: DataProcessResult): ResultItem {
return {
id: String(item.id),
instruction: item.instruction,
input: item.input || '',
output: item.output,
originalInstruction: item.original_instruction ?? item.instruction,
originalInput: item.original_input ?? item.input ?? '',
originalOutput: item.original_output ?? item.output,
status: item.status,
error: item.error || undefined,
split: item.split || undefined,
qualityScore: item.quality_score?.overall,
qualityFlags: item.quality_score?.flags || [],
updatedAt: item.updated_at,
}
}
export function useDataProcessGeneration(bindings: GenerationBindings) {
@@ -26,10 +47,13 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
progress: 0,
message: '确认摘要后即可开始生成，过程中可查看实时进度。',
})
let generationTimer: ReturnType<typeof setInterval> | null = null
let generationTimer: ReturnType<typeof setTimeout> | null = null
let generationRun = 0
let pollFailureCount = 0
function stopGenerationTimer() {
if (generationTimer) clearInterval(generationTimer)
generationRun += 1
if (generationTimer) clearTimeout(generationTimer)
generationTimer = null
}
@@ -42,35 +66,123 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
selectedResultId.value = null
}
function startGeneration() {
stopGenerationTimer()
generation.status = 'running'
generation.progress = 0
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
generationTimer = setInterval(() => {
generation.progress = Math.min(100, generation.progress + 8)
if (generation.progress < 100) return
stopGenerationTimer()
generation.status = 'success'
results.value = createResults(
bindings.previewItems.value,
bindings.processType.value === 'structured'
? bindings.structuredOptions.value
: bindings.processType.value === 'unstructured'
? bindings.unstructuredOptions.value
: undefined,
)
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
selectedResultId.value = results.value[0]?.id ?? null
bindings.dirty.value = true
ElMessage.success('数据处理完成')
}, 180)
function applyProgress(progress: DataProcessProgress) {
generation.progress = Math.max(0, Math.min(100, Number(progress.progress) || 0))
generation.message = progress.message || (
progress.status === 'running'
? '后端正在生成标准化结果并进行质量评分。'
: progress.status === 'completed'
? '数据处理已完成。'
: progress.failure_reason || '任务已停止。'
)
}
function stopGeneration() {
async function loadAllResults(taskId: string) {
const first = await getDataProcessResults(taskId, { page: 1, page_size: RESULT_PAGE_SIZE })
const items = [...first.items]
const pages = Math.ceil(first.total / first.page_size)
for (let page = 2; page <= pages; page += 1) {
const next = await getDataProcessResults(taskId, { page, page_size: RESULT_PAGE_SIZE })
items.push(...next.items)
}
results.value = items.map(mapResult)
selectedResultId.value = results.value[0]?.id ?? null
}
async function finishFromProgress(progress: DataProcessProgress) {
pollFailureCount = 0
applyProgress(progress)
if (progress.status === 'completed') {
const taskId = bindings.taskId.value
if (!taskId) return
await loadAllResults(taskId)
generation.status = 'success'
generation.progress = 100
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
bindings.dirty.value = true
ElMessage.success('数据处理完成')
return
}
if (progress.status === 'failed' || progress.status === 'stopped') {
generation.status = 'failed'
generation.message = progress.failure_reason || progress.message || (
progress.status === 'stopped' ? '任务已停止，可以重新生成。' : '数据处理失败，请检查配置后重试。'
)
}
}
async function pollGeneration(runId: number) {
const taskId = bindings.taskId.value
if (!taskId || runId !== generationRun || generation.status !== 'running') return
try {
const progress = await getDataProcessProgress(taskId)
if (runId !== generationRun) return
pollFailureCount = 0
if (progress.status === 'running' || progress.status === 'pending') {
applyProgress(progress)
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
return
}
await finishFromProgress(progress)
} catch (error) {
if (runId !== generationRun) return
pollFailureCount += 1
if (pollFailureCount <= 3) {
generation.message = `进度查询暂时失败，正在重试（${pollFailureCount}/3）…`
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
return
}
generation.status = 'failed'
generation.message = error instanceof Error ? error.message : '查询任务进度失败，请重试。'
}
}
async function startGeneration() {
const taskId = bindings.taskId.value
if (!taskId) {
ElMessage.error('任务尚未创建，请返回上一步重试')
return
}
stopGenerationTimer()
const runId = generationRun
generation.status = 'running'
pollFailureCount = 0
generation.progress = 0
generation.message = '正在同步预览修改并启动后端处理，请稍候。'
try {
await bindings.beforeGenerate?.()
const progress = await generateDataProcess(taskId)
if (runId !== generationRun) return
if (progress.status === 'completed' || progress.status === 'failed' || progress.status === 'stopped') {
await finishFromProgress(progress)
return
}
applyProgress(progress)
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
} catch (error) {
if (runId !== generationRun) return
generation.status = 'failed'
generation.message = error instanceof Error ? error.message : '启动数据处理失败，请重试。'
}
}
async function stopGeneration() {
const taskId = bindings.taskId.value
if (!taskId) return
stopGenerationTimer()
try {
const progress = await stopDataProcess(taskId)
applyProgress(progress)
} catch {
generation.status = 'running'
generation.message = '停止请求失败，继续查询后端任务状态。'
const runId = generationRun
generationTimer = setTimeout(() => void pollGeneration(runId), POLL_INTERVAL_MS)
return
}
generation.status = 'failed'
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
}
@@ -88,22 +200,41 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
bindings.dirty.value = true
}
function restoreResult(id: string) {
async function restoreResult(id: string) {
const taskId = bindings.taskId.value
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item.instruction = item.originalInstruction
item.input = item.originalInput
item.output = item.originalOutput
item.error = undefined
item.status = 'valid'
if (!taskId || !item) return
const restored = await restoreDataProcessResult(taskId, id)
const index = results.value.indexOf(item)
results.value[index] = mapResult(restored)
bindings.dirty.value = true
}
async function persistResultChanges() {
const taskId = bindings.taskId.value
if (!taskId) throw new Error('任务尚未创建')
const changed = results.value.filter((item) => (
item.instruction !== item.originalInstruction
|| item.input !== item.originalInput
|| item.output !== item.originalOutput
))
for (const item of changed) {
const saved = await updateDataProcessResult(taskId, item.id, {
instruction: item.instruction,
input: item.input,
output: item.output,
expected_updated_at: item.updatedAt,
})
const index = results.value.findIndex((entry) => entry.id === item.id)
if (index >= 0) results.value[index] = mapResult(saved)
}
}
function validateResults() {
let firstInvalidId: string | null = null
for (const item of results.value) {
if (!item.instruction.trim() || !item.output.trim()) {
item.error = 'Instruction 和 Output 不能为空'
if (!item.instruction.trim() || !item.output.trim() || item.status === 'invalid') {
item.error ||= '结果未通过后端质量校验，请修改后重新保存'
item.status = 'invalid'
firstInvalidId ??= item.id
}
@@ -116,6 +247,7 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
generation,
results,
selectedResultId,
persistResultChanges,
resetDownstream,
restoreResult,
startGeneration,