first-update

2026-03-17 14:36:31 +08:00
parent 72f08aee7c
commit 4eddf05e79
516 changed files with 115270 additions and 1 deletions
--- a/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/copy-to-eval/route.js
+++ b/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/copy-to-eval/route.js
@@ -0,0 +1,77 @@
+import { NextResponse } from 'next/server';
+import { db } from '@/lib/db';
+
+export async function POST(req, { params }) {
+  try {
+    const { projectId, datasetId } = params;
+
+    // 1. 获取数据集详情
+    const dataset = await db.datasets.findUnique({
+      where: { id: datasetId, projectId }
+    });
+
+    if (!dataset) {
+      return NextResponse.json({ error: 'Dataset not found' }, { status: 404 });
+    }
+
+    // 2. 尝试通过 questionId 查找关联的 chunkId
+    let chunkId = null;
+    if (dataset.questionId) {
+      const question = await db.questions.findUnique({
+        where: { id: dataset.questionId }
+      });
+      if (question) {
+        chunkId = question.chunkId;
+      }
+    }
+
+    // 3. 创建评估数据集记录
+    // 默认使用 open_ended 类型，因为通常数据集是问答对，适合作为评估
+    let evalTags = [];
+    try {
+      evalTags = JSON.parse(dataset.tags || '[]');
+      if (!Array.isArray(evalTags)) evalTags = [];
+    } catch (e) {
+      evalTags = [];
+    }
+
+    // 排除 'Eval' 标签，并将数组转为逗号分隔的字符串
+    const evalTagsString = evalTags.filter(tag => tag !== 'Eval').join(',');
+
+    const evalDataset = await db.evalDatasets.create({
+      data: {
+        projectId,
+        question: dataset.question,
+        questionType: 'open_ended',
+        correctAnswer: dataset.answer,
+        tags: evalTagsString,
+        note: dataset.note,
+        chunkId: chunkId,
+        options: '' // 开放题不需要选项
+      }
+    });
+
+    // 4. 更新原数据集，添加 'Eval' 标签
+    let currentTags = [];
+    try {
+      currentTags = JSON.parse(dataset.tags || '[]');
+    } catch (e) {
+      // ignore error
+    }
+
+    if (!currentTags.includes('Eval')) {
+      currentTags.push('Eval');
+      await db.datasets.update({
+        where: { id: datasetId },
+        data: {
+          tags: JSON.stringify(currentTags)
+        }
+      });
+    }
+
+    return NextResponse.json({ success: true, evalDataset });
+  } catch (error) {
+    console.error('Failed to copy dataset to eval:', error);
+    return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
+  }
+}
--- a/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/evaluate/route.js
+++ b/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/evaluate/route.js
@@ -0,0 +1,36 @@
+import { NextResponse } from 'next/server';
+import { evaluateDataset } from '@/lib/services/datasets/evaluation';
+
+/**
+ * 评估单个数据集的质量
+ */
+export async function POST(request, { params }) {
+  try {
+    const { projectId, datasetId } = params;
+    const { model, language = 'zh-CN' } = await request.json();
+
+    if (!projectId || !datasetId) {
+      return NextResponse.json({ success: false, message: '项目ID和数据集ID不能为空' }, { status: 400 });
+    }
+
+    if (!model) {
+      return NextResponse.json({ success: false, message: '模型配置不能为空' }, { status: 400 });
+    }
+
+    // 使用评估服务进行数据集评估
+    const result = await evaluateDataset(projectId, datasetId, model, language);
+
+    if (!result.success) {
+      return NextResponse.json({ success: false, message: result.error }, { status: 500 });
+    }
+
+    return NextResponse.json({
+      success: true,
+      message: '数据集评估完成',
+      data: result.data
+    });
+  } catch (error) {
+    console.error('数据集评估失败:', error);
+    return NextResponse.json({ success: false, message: `评估失败: ${error.message}` }, { status: 500 });
+  }
+}
--- a/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/route.js
+++ b/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/route.js
@@ -0,0 +1,82 @@
+import { NextResponse } from 'next/server';
+import { getDatasetsById, getDatasetsCounts, getNavigationItems, updateDatasetMetadata } from '@/lib/db/datasets';
+
+/**
+ * 获取项目的所有数据集
+ */
+export async function GET(request, { params }) {
+  try {
+    const { projectId, datasetId } = params;
+    // 验证项目ID
+    if (!projectId) {
+      return NextResponse.json({ error: '项目ID不能为空' }, { status: 400 });
+    }
+    if (!datasetId) {
+      return NextResponse.json({ error: '数据集ID不能为空' }, { status: 400 });
+    }
+    const { searchParams } = new URL(request.url);
+    const operateType = searchParams.get('operateType');
+    if (operateType !== null) {
+      const data = await getNavigationItems(projectId, datasetId, operateType);
+      return NextResponse.json(data);
+    }
+    const datasets = await getDatasetsById(datasetId);
+    let counts = await getDatasetsCounts(projectId);
+
+    return NextResponse.json({ datasets, ...counts });
+  } catch (error) {
+    console.error('获取数据集详情失败:', String(error));
+    return NextResponse.json(
+      {
+        error: error.message || '获取数据集详情失败'
+      },
+      { status: 500 }
+    );
+  }
+}
+
+/**
+ * 更新数据集元数据（评分、标签、备注）
+ */
+export async function PATCH(request, { params }) {
+  try {
+    const { projectId, datasetId } = params;
+
+    // 验证参数
+    if (!projectId) {
+      return NextResponse.json({ error: '项目ID不能为空' }, { status: 400 });
+    }
+    if (!datasetId) {
+      return NextResponse.json({ error: '数据集ID不能为空' }, { status: 400 });
+    }
+
+    const body = await request.json();
+    const { score, tags, note } = body;
+
+    // 验证评分范围
+    if (score !== undefined && (score < 0 || score > 5)) {
+      return NextResponse.json({ error: '评分必须在0-5之间' }, { status: 400 });
+    }
+
+    // 验证标签格式
+    if (tags !== undefined && !Array.isArray(tags)) {
+      return NextResponse.json({ error: '标签必须是数组格式' }, { status: 400 });
+    }
+
+    // 更新数据集元数据
+    const updatedDataset = await updateDatasetMetadata(datasetId, { score, tags, note });
+
+    return NextResponse.json({
+      success: true,
+      dataset: updatedDataset
+    });
+  } catch (error) {
+    console.error('更新数据集元数据失败:', String(error));
+    return NextResponse.json(
+      {
+        error: error.message || '更新数据集元数据失败'
+      },
+      { status: 500 }
+    );
+  }
+}
--- a/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/token-count/route.js
+++ b/easy-dataset-main/app/api/projects/[projectId]/datasets/[datasetId]/token-count/route.js
@@ -0,0 +1,52 @@
+import { NextResponse } from 'next/server';
+import { getDatasetsById } from '@/lib/db/datasets';
+import { getEncoding } from '@langchain/core/utils/tiktoken';
+
+/**
+ * 异步计算数据集文本的Token数量
+ */
+export async function GET(request, { params }) {
+  try {
+    const { projectId, datasetId } = params;
+
+    if (!datasetId) {
+      return NextResponse.json({ error: '数据集ID不能为空' }, { status: 400 });
+    }
+
+    const datasets = await getDatasetsById(datasetId);
+    const tokenCounts = {
+      answerTokens: 0,
+      cotTokens: 0
+    };
+
+    try {
+      if (datasets.answer || datasets.cot) {
+        // 使用 cl100k_base 编码，适用于 gpt-3.5-turbo 和 gpt-4
+        const encoding = await getEncoding('cl100k_base');
+
+        if (datasets.answer) {
+          const tokens = encoding.encode(datasets.answer);
+          tokenCounts.answerTokens = tokens.length;
+        }
+
+        if (datasets.cot) {
+          const tokens = encoding.encode(datasets.cot);
+          tokenCounts.cotTokens = tokens.length;
+        }
+      }
+    } catch (error) {
+      console.error('计算Token数量失败:', String(error));
+      return NextResponse.json({ error: '计算Token数量失败' }, { status: 500 });
+    }
+
+    return NextResponse.json(tokenCounts);
+  } catch (error) {
+    console.error('获取Token计数失败:', String(error));
+    return NextResponse.json(
+      {
+        error: error.message || '获取Token计数失败'
+      },
+      { status: 500 }
+    );
+  }
+}