Merge branch 'ft_wyt' of http://www.caoxiaozhu.com:13001/YG-Soft/YG_FT into ft_wyt
This commit is contained in:
@@ -5,12 +5,15 @@ import asyncio
|
|||||||
import hashlib
|
import hashlib
|
||||||
import uuid
|
import uuid
|
||||||
import time
|
import time
|
||||||
|
from io import BytesIO
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
from urllib.parse import quote
|
||||||
|
from zipfile import ZIP_DEFLATED, ZipFile
|
||||||
|
|
||||||
from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
|
from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Query, Request, UploadFile
|
||||||
from fastapi.responses import PlainTextResponse, StreamingResponse
|
from fastapi.responses import PlainTextResponse, Response, StreamingResponse
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
@@ -365,9 +368,16 @@ async def _fine_tune_preflight_payload(
|
|||||||
store: Any,
|
store: Any,
|
||||||
payload: dict[str, Any],
|
payload: dict[str, Any],
|
||||||
validate: bool = True,
|
validate: bool = True,
|
||||||
|
sync_resources: bool = True,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
node, job_payload = store.prepare_compute_job_payload_from_payload(payload)
|
node, job_payload = store.prepare_compute_job_payload_from_payload(payload)
|
||||||
return await _fine_tune_preflight_with_job_payload(node, job_payload, validate=validate, sync_resources=False, store=store)
|
return await _fine_tune_preflight_with_job_payload(
|
||||||
|
node,
|
||||||
|
job_payload,
|
||||||
|
validate=validate,
|
||||||
|
sync_resources=sync_resources,
|
||||||
|
store=store,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _fine_tune_preflight_with_job_payload(
|
async def _fine_tune_preflight_with_job_payload(
|
||||||
@@ -1147,13 +1157,42 @@ async def _sync_training_dataset_to_compute_node(
|
|||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
if get_settings().minio_enabled:
|
if get_settings().minio_enabled:
|
||||||
files = store.training_dataset_files(dataset_id)
|
files = store.training_dataset_files(dataset_id)
|
||||||
objects = store.storage_objects_for_resource("dataset", dataset_id)
|
object_by_resource_name: dict[tuple[str, str], dict[str, Any]] = {}
|
||||||
object_by_name = {Path(str(item.get("file_name") or item.get("object_key") or "")).name: item for item in objects}
|
resource_ids = {str(dataset_id)} | {
|
||||||
|
str(item.get("dataset_id"))
|
||||||
|
for item in files
|
||||||
|
if item.get("dataset_id")
|
||||||
|
}
|
||||||
|
for resource_id in resource_ids:
|
||||||
|
for obj in store.storage_objects_for_resource("dataset", resource_id):
|
||||||
|
file_name = Path(str(obj.get("file_name") or obj.get("object_key") or "")).name
|
||||||
|
object_by_resource_name[(resource_id, file_name)] = obj
|
||||||
results: list[dict[str, Any]] = []
|
results: list[dict[str, Any]] = []
|
||||||
client = ComputeNodeClient(node["api_base_url"])
|
client = ComputeNodeClient(node["api_base_url"])
|
||||||
for item in files:
|
for item in files:
|
||||||
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
|
target_name = Path(str(item.get("name") or f"{item['id']}.jsonl")).name
|
||||||
obj = object_by_name.get(target_name)
|
item_dataset_id = str(item.get("dataset_id") or dataset_id)
|
||||||
|
obj = object_by_resource_name.get((item_dataset_id, target_name))
|
||||||
|
if not obj and item.get("content"):
|
||||||
|
# 兼容 MinIO 接入前已经发布的数据处理数据集:
|
||||||
|
# 预检时用数据库正文补建对象,避免要求用户重新处理数据集。
|
||||||
|
raw = str(item.get("content") or "").encode("utf-8")
|
||||||
|
version_id = str(item.get("active_version_id") or item["id"])
|
||||||
|
object_key = f"datasets/{item_dataset_id}/versions/{version_id}/{target_name}"
|
||||||
|
uploaded = get_object_storage().put_bytes(object_key, raw, "application/jsonl")
|
||||||
|
obj = store.create_storage_object({
|
||||||
|
"resource_type": "dataset",
|
||||||
|
"resource_id": item_dataset_id,
|
||||||
|
"version_id": version_id,
|
||||||
|
"bucket": uploaded["bucket"],
|
||||||
|
"object_key": object_key,
|
||||||
|
"file_name": target_name,
|
||||||
|
"content_type": "application/jsonl",
|
||||||
|
"byte_size": len(raw),
|
||||||
|
"checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
||||||
|
"status": "available",
|
||||||
|
})
|
||||||
|
store.link_dataset_file_storage_object(str(item["id"]), obj["id"])
|
||||||
if not obj:
|
if not obj:
|
||||||
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
|
raise RuntimeError(f"dataset file is not available in MinIO: {target_name}")
|
||||||
url = get_object_storage().presigned_get(obj["object_key"])
|
url = get_object_storage().presigned_get(obj["object_key"])
|
||||||
@@ -1165,6 +1204,12 @@ async def _sync_training_dataset_to_compute_node(
|
|||||||
"byte_size": obj.get("byte_size") or 0,
|
"byte_size": obj.get("byte_size") or 0,
|
||||||
"relative_path": f"datasets/{dataset_id}/{target_name}",
|
"relative_path": f"datasets/{dataset_id}/{target_name}",
|
||||||
})
|
})
|
||||||
|
store.upsert_resource_replica(
|
||||||
|
node["id"],
|
||||||
|
"dataset",
|
||||||
|
dataset_id,
|
||||||
|
str(result.get("local_path") or ""),
|
||||||
|
)
|
||||||
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
|
results.append({**result, "file_id": item.get("id"), "name": target_name, "node_id": node["id"]})
|
||||||
return results
|
return results
|
||||||
if not dataset_id:
|
if not dataset_id:
|
||||||
@@ -1233,7 +1278,7 @@ async def upload_dataset_files(
|
|||||||
if get_settings().minio_enabled:
|
if get_settings().minio_enabled:
|
||||||
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
|
object_key = f"datasets/{dataset_id}/versions/{created_file.get('active_version_id') or created_file['id']}/{Path(created_file['name']).name}"
|
||||||
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
|
uploaded = get_object_storage().put_bytes(object_key, raw, file.content_type or "application/octet-stream")
|
||||||
get_platform_store().create_storage_object({
|
storage_object = get_platform_store().create_storage_object({
|
||||||
"resource_type": "dataset", "resource_id": dataset_id,
|
"resource_type": "dataset", "resource_id": dataset_id,
|
||||||
"version_id": created_file.get("active_version_id") or created_file["id"],
|
"version_id": created_file.get("active_version_id") or created_file["id"],
|
||||||
"bucket": uploaded["bucket"], "object_key": object_key,
|
"bucket": uploaded["bucket"], "object_key": object_key,
|
||||||
@@ -1241,6 +1286,7 @@ async def upload_dataset_files(
|
|||||||
"byte_size": len(raw), "checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
"byte_size": len(raw), "checksum_sha256": hashlib.sha256(raw).hexdigest(),
|
||||||
"status": "available",
|
"status": "available",
|
||||||
})
|
})
|
||||||
|
store.link_dataset_file_storage_object(created_file["id"], storage_object["id"])
|
||||||
if sync_to_compute:
|
if sync_to_compute:
|
||||||
for file_id, file_name, raw in pending_sync:
|
for file_id, file_name, raw in pending_sync:
|
||||||
compute_sync.extend(
|
compute_sync.extend(
|
||||||
@@ -1256,10 +1302,60 @@ async def upload_dataset_files(
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/dataset-manage/download/{dataset_id}")
|
@router.get("/dataset-manage/download/{dataset_id}")
|
||||||
async def download_dataset(dataset_id: str) -> PlainTextResponse:
|
async def download_dataset(dataset_id: str, current_user: dict = Depends(get_current_user)) -> Response:
|
||||||
dataset = get_platform_store().dataset(dataset_id)
|
store = get_platform_store()
|
||||||
content = "\n".join([f"{file['name']}" for file in dataset.get("files", [])])
|
try:
|
||||||
return PlainTextResponse(content, media_type="text/plain")
|
dataset = store.dataset(dataset_id)
|
||||||
|
except KeyError:
|
||||||
|
raise fail(404, "dataset not found")
|
||||||
|
if not has_resource_access("dataset", dataset_id, current_user, "read"):
|
||||||
|
raise fail(403, "no permission to access this dataset")
|
||||||
|
|
||||||
|
files = []
|
||||||
|
for item in dataset.get("files", []):
|
||||||
|
if item.get("deleted_at"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
full_file = store.dataset_file(str(item["id"]))
|
||||||
|
except KeyError:
|
||||||
|
continue
|
||||||
|
files.append({**item, "content": full_file.get("content") or ""})
|
||||||
|
if not files:
|
||||||
|
raise fail(404, "dataset has no downloadable files")
|
||||||
|
|
||||||
|
if len(files) == 1:
|
||||||
|
item = files[0]
|
||||||
|
filename = Path(str(item.get("name") or f"{dataset_id}.jsonl")).name
|
||||||
|
encoded_name = quote(filename, safe="")
|
||||||
|
return Response(
|
||||||
|
content=str(item.get("content") or "").encode("utf-8"),
|
||||||
|
media_type="application/octet-stream",
|
||||||
|
headers={
|
||||||
|
"Content-Disposition": f"attachment; filename=dataset-file; filename*=UTF-8''{encoded_name}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
archive = BytesIO()
|
||||||
|
with ZipFile(archive, "w", compression=ZIP_DEFLATED) as bundle:
|
||||||
|
used_names: set[str] = set()
|
||||||
|
for index, item in enumerate(files, start=1):
|
||||||
|
filename = Path(str(item.get("name") or f"file-{index}.jsonl")).name
|
||||||
|
unique_name = filename
|
||||||
|
if unique_name in used_names:
|
||||||
|
stem = Path(filename).stem
|
||||||
|
suffix = Path(filename).suffix
|
||||||
|
unique_name = f"{stem}-{index}{suffix}"
|
||||||
|
used_names.add(unique_name)
|
||||||
|
bundle.writestr(unique_name, str(item.get("content") or ""))
|
||||||
|
archive.seek(0)
|
||||||
|
archive_name = quote(f"{dataset.get('name') or dataset_id}.zip", safe="")
|
||||||
|
return StreamingResponse(
|
||||||
|
archive,
|
||||||
|
media_type="application/zip",
|
||||||
|
headers={
|
||||||
|
"Content-Disposition": f"attachment; filename=dataset.zip; filename*=UTF-8''{archive_name}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
|
@router.get("/dataset-manage/download/{dataset_id}/{file_id}")
|
||||||
@@ -1289,8 +1385,11 @@ async def dataset_list(current_user: dict = Depends(get_current_user)) -> dict[s
|
|||||||
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
|
@op_log(module=OpModule.DATASET, action=OpAction.CREATE, target_type="dataset", target_name_param="name")
|
||||||
async def create_dataset(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
async def create_dataset(payload: dict[str, Any] = Body(...), current_user: dict = Depends(get_current_user)) -> dict[str, Any]:
|
||||||
payload.setdefault("created_by", current_user.get("id"))
|
payload.setdefault("created_by", current_user.get("id"))
|
||||||
dataset = get_platform_store().create_dataset(payload)
|
try:
|
||||||
return ok({"id": dataset["id"]})
|
dataset = get_platform_store().create_dataset(payload)
|
||||||
|
return ok({"id": dataset["id"]})
|
||||||
|
except ValueError as exc:
|
||||||
|
raise fail(409, str(exc))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/dataset-manage/{dataset_id}")
|
@router.get("/dataset-manage/{dataset_id}")
|
||||||
@@ -1423,7 +1522,7 @@ async def start_fine_tune(
|
|||||||
@router.post("/fine-tune/preflight")
|
@router.post("/fine-tune/preflight")
|
||||||
async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True))
|
return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=True, sync_resources=True))
|
||||||
except RuntimeError as exc:
|
except RuntimeError as exc:
|
||||||
return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
|
return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
|
||||||
except Exception as exc: # noqa: BLE001 - expose compute validation errors to training create page
|
except Exception as exc: # noqa: BLE001 - expose compute validation errors to training create page
|
||||||
@@ -1433,7 +1532,7 @@ async def fine_tune_create_preflight(payload: dict[str, Any] = Body(...)) -> dic
|
|||||||
@router.post("/fine-tune/command-preview")
|
@router.post("/fine-tune/command-preview")
|
||||||
async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
async def fine_tune_create_command_preview(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False))
|
return ok(await _fine_tune_preflight_payload(get_platform_store(), payload, validate=False, sync_resources=False))
|
||||||
except RuntimeError as exc:
|
except RuntimeError as exc:
|
||||||
return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
|
return ok({"valid": False, "errors": [str(exc)], "warnings": [], "diagnostics": _training_diagnostics([str(exc)])})
|
||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc: # noqa: BLE001
|
||||||
|
|||||||
@@ -1634,7 +1634,24 @@ class PlatformStore:
|
|||||||
|
|
||||||
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
|
def create_dataset(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
dataset_id = payload.get("id") or new_id("ds")
|
dataset_id = payload.get("id") or new_id("ds")
|
||||||
|
name = str(payload.get("name") or "").strip()
|
||||||
|
if not name:
|
||||||
|
raise ValueError("dataset name is required")
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
|
existing = conn.execute(
|
||||||
|
"SELECT id, deleted_at FROM datasets WHERE name=?",
|
||||||
|
(name,),
|
||||||
|
).fetchone()
|
||||||
|
if existing and not existing.get("deleted_at"):
|
||||||
|
raise ValueError(f"dataset name already exists: {name}")
|
||||||
|
# Soft-deleted records remain in the database for audit/history and
|
||||||
|
# still participate in the legacy unique constraint. Free the name
|
||||||
|
# while retaining a traceable tombstone before creating the new row.
|
||||||
|
if existing:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE datasets SET name=? WHERE id=?",
|
||||||
|
(f"{name}__deleted__{existing['id']}", existing["id"]),
|
||||||
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO datasets
|
INSERT INTO datasets
|
||||||
@@ -1643,7 +1660,7 @@ class PlatformStore:
|
|||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
dataset_id,
|
dataset_id,
|
||||||
payload["name"],
|
name,
|
||||||
payload.get("type", "train"),
|
payload.get("type", "train"),
|
||||||
payload.get("storage_type", "local"),
|
payload.get("storage_type", "local"),
|
||||||
payload.get("source", "upload"),
|
payload.get("source", "upload"),
|
||||||
@@ -2992,6 +3009,27 @@ class PlatformStore:
|
|||||||
).fetchone()
|
).fetchone()
|
||||||
return dict(row)
|
return dict(row)
|
||||||
|
|
||||||
|
def link_dataset_file_storage_object(self, file_id: str, storage_object_id: str) -> None:
|
||||||
|
"""Link an uploaded dataset file to its canonical MinIO object."""
|
||||||
|
with self.connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE dataset_files
|
||||||
|
SET storage_object_id=?
|
||||||
|
WHERE id=?
|
||||||
|
""",
|
||||||
|
(storage_object_id, file_id),
|
||||||
|
)
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT dataset_id FROM dataset_files WHERE id=?",
|
||||||
|
(file_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE datasets SET storage_type='minio' WHERE id=?",
|
||||||
|
(row["dataset_id"],),
|
||||||
|
)
|
||||||
|
|
||||||
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
def storage_objects_for_resource(self, resource_type: str, resource_id: str) -> list[dict[str, Any]]:
|
||||||
with self.connect() as conn:
|
with self.connect() as conn:
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
|
|||||||
@@ -506,6 +506,13 @@ CREATE INDEX IF NOT EXISTS idx_approval_instances_applicant ON approval_instance
|
|||||||
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
|
CREATE INDEX IF NOT EXISTS idx_approval_instances_resource ON approval_instances(resource_type, resource_id, status);
|
||||||
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
|
CREATE INDEX IF NOT EXISTS idx_approval_steps_approver ON approval_steps(approver_id, status);
|
||||||
|
|
||||||
|
-- Compatibility for databases created from an older approval_steps definition.
|
||||||
|
ALTER TABLE approval_steps ADD COLUMN IF NOT EXISTS id TEXT;
|
||||||
|
UPDATE approval_steps
|
||||||
|
SET id = 'astep_' || md5(concat_ws(':', instance_id, step_index, coalesce(approver_id, ''), coalesce(time, '')))
|
||||||
|
WHERE id IS NULL;
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_approval_steps_id ON approval_steps(id);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
tenant_id TEXT,
|
tenant_id TEXT,
|
||||||
@@ -535,6 +542,38 @@ CREATE TABLE IF NOT EXISTS retention_policies (
|
|||||||
updated_at TEXT
|
updated_at TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Compatibility for the earlier retention policy schema
|
||||||
|
-- (resource_type/retention_days). Keep legacy columns if they exist.
|
||||||
|
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS scope TEXT;
|
||||||
|
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS rule TEXT;
|
||||||
|
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'active';
|
||||||
|
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS create_by TEXT;
|
||||||
|
ALTER TABLE retention_policies ADD COLUMN IF NOT EXISTS updated_at TEXT;
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (
|
||||||
|
SELECT 1 FROM information_schema.columns
|
||||||
|
WHERE table_schema='public' AND table_name='retention_policies' AND column_name='resource_type'
|
||||||
|
) AND EXISTS (
|
||||||
|
SELECT 1 FROM information_schema.columns
|
||||||
|
WHERE table_schema='public' AND table_name='retention_policies' AND column_name='retention_days'
|
||||||
|
) THEN
|
||||||
|
EXECUTE $migration$
|
||||||
|
UPDATE retention_policies
|
||||||
|
SET scope = COALESCE(scope, resource_type),
|
||||||
|
rule = COALESCE(rule, retention_days::text),
|
||||||
|
status = COALESCE(status, 'active'),
|
||||||
|
updated_at = COALESCE(updated_at, create_time)
|
||||||
|
WHERE scope IS NULL OR rule IS NULL OR status IS NULL OR updated_at IS NULL
|
||||||
|
$migration$;
|
||||||
|
ELSE
|
||||||
|
UPDATE retention_policies
|
||||||
|
SET status = COALESCE(status, 'active'),
|
||||||
|
updated_at = COALESCE(updated_at, create_time)
|
||||||
|
WHERE status IS NULL OR updated_at IS NULL;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
-- 三、租户配额扩展(来源:003_tenant_quota.sql)
|
-- 三、租户配额扩展(来源:003_tenant_quota.sql)
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
@@ -723,9 +762,13 @@ CREATE TABLE IF NOT EXISTS data_process_results (
|
|||||||
instruction TEXT NOT NULL,
|
instruction TEXT NOT NULL,
|
||||||
input TEXT NOT NULL DEFAULT '',
|
input TEXT NOT NULL DEFAULT '',
|
||||||
output TEXT NOT NULL,
|
output TEXT NOT NULL,
|
||||||
|
chosen TEXT NOT NULL DEFAULT '',
|
||||||
|
rejected TEXT NOT NULL DEFAULT '',
|
||||||
original_instruction TEXT,
|
original_instruction TEXT,
|
||||||
original_input TEXT,
|
original_input TEXT,
|
||||||
original_output TEXT,
|
original_output TEXT,
|
||||||
|
original_chosen TEXT,
|
||||||
|
original_rejected TEXT,
|
||||||
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
status VARCHAR(20) NOT NULL DEFAULT 'valid'
|
||||||
CHECK (status IN ('valid', 'modified', 'invalid')),
|
CHECK (status IN ('valid', 'modified', 'invalid')),
|
||||||
error TEXT,
|
error TEXT,
|
||||||
@@ -735,6 +778,12 @@ CREATE TABLE IF NOT EXISTS data_process_results (
|
|||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Keep existing databases compatible with the current generation result model.
|
||||||
|
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS chosen TEXT NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS rejected TEXT NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_chosen TEXT;
|
||||||
|
ALTER TABLE data_process_results ADD COLUMN IF NOT EXISTS original_rejected TEXT;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_status
|
||||||
ON data_process_results(task_id, status, id);
|
ON data_process_results(task_id, status, id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
CREATE INDEX IF NOT EXISTS idx_data_process_results_task_split
|
||||||
|
|||||||
@@ -28,6 +28,9 @@ class StagedSourceObject:
|
|||||||
|
|
||||||
|
|
||||||
def _default_storage_root() -> Path:
|
def _default_storage_root() -> Path:
|
||||||
|
data_root = os.getenv("YG_FT_DATA_ROOT", "").strip()
|
||||||
|
if data_root:
|
||||||
|
return Path(data_root).expanduser() / "data-process"
|
||||||
return Path(__file__).resolve().parents[3] / "storage" / "data-process"
|
return Path(__file__).resolve().parents[3] / "storage" / "data-process"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ import hashlib
|
|||||||
|
|
||||||
import psycopg
|
import psycopg
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.modules.storage.minio_store import get_object_storage
|
||||||
|
|
||||||
from .base import (
|
from .base import (
|
||||||
StoreBase,
|
StoreBase,
|
||||||
utcnow,
|
utcnow,
|
||||||
@@ -165,6 +168,7 @@ class DatasetsMixin:
|
|||||||
split_name: assignments.count(split_name) for split_name in split_order
|
split_name: assignments.count(split_name) for split_name in split_order
|
||||||
}
|
}
|
||||||
split_specs: list[dict[str, Any]] = []
|
split_specs: list[dict[str, Any]] = []
|
||||||
|
use_minio = bool(get_settings().minio_enabled)
|
||||||
for split_name in split_order:
|
for split_name in split_order:
|
||||||
split_records = [
|
split_records = [
|
||||||
(source_row, record)
|
(source_row, record)
|
||||||
@@ -194,7 +198,7 @@ class DatasetsMixin:
|
|||||||
source_result_ids = [row["id"] for row in rows]
|
source_result_ids = [row["id"] for row in rows]
|
||||||
common_metadata = {
|
common_metadata = {
|
||||||
"source": "data_process",
|
"source": "data_process",
|
||||||
"storage_backend": "database",
|
"storage_backend": "minio" if use_minio else "database",
|
||||||
"source_task_id": task_id,
|
"source_task_id": task_id,
|
||||||
"output_type": _task_output_type(task),
|
"output_type": _task_output_type(task),
|
||||||
"reasoning_detail": _task_reasoning_detail(task),
|
"reasoning_detail": _task_reasoning_detail(task),
|
||||||
@@ -268,6 +272,53 @@ class DatasetsMixin:
|
|||||||
for spec in split_specs:
|
for spec in split_specs:
|
||||||
split_name = str(spec["split"])
|
split_name = str(spec["split"])
|
||||||
dataset_id = dataset_ids[split_name]
|
dataset_id = dataset_ids[split_name]
|
||||||
|
storage_object_id = str(spec["storage_object_id"])
|
||||||
|
if use_minio:
|
||||||
|
file_name = f"{base_dataset_name}.{split_name}.jsonl"
|
||||||
|
object_key = f"datasets/{dataset_id}/versions/{spec['version_id']}/{file_name}"
|
||||||
|
uploaded = get_object_storage().put_bytes(
|
||||||
|
object_key,
|
||||||
|
spec["raw"],
|
||||||
|
"application/jsonl",
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO storage_objects
|
||||||
|
(id, resource_type, resource_id, version_id, bucket, object_key,
|
||||||
|
file_name, content_type, checksum_sha256, byte_size, status,
|
||||||
|
created_by, create_time)
|
||||||
|
VALUES (%s, 'dataset', %s, %s, %s, %s, %s, %s, %s, %s,
|
||||||
|
'available', %s, %s)
|
||||||
|
ON CONFLICT (resource_type, resource_id, version_id, object_key)
|
||||||
|
DO UPDATE SET file_name=EXCLUDED.file_name,
|
||||||
|
content_type=EXCLUDED.content_type,
|
||||||
|
checksum_sha256=EXCLUDED.checksum_sha256,
|
||||||
|
byte_size=EXCLUDED.byte_size,
|
||||||
|
status='available',
|
||||||
|
created_by=EXCLUDED.created_by
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
new_id("object"),
|
||||||
|
dataset_id,
|
||||||
|
spec["version_id"],
|
||||||
|
uploaded["bucket"],
|
||||||
|
object_key,
|
||||||
|
file_name,
|
||||||
|
"application/jsonl",
|
||||||
|
spec["checksum"],
|
||||||
|
len(spec["raw"]),
|
||||||
|
payload.get("created_by") or task.get("created_by"),
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
storage_object_id = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id FROM storage_objects
|
||||||
|
WHERE resource_type='dataset' AND resource_id=%s
|
||||||
|
AND version_id=%s AND object_key=%s
|
||||||
|
""",
|
||||||
|
(dataset_id, spec["version_id"], object_key),
|
||||||
|
).fetchone()["id"]
|
||||||
existing_dataset = existing_by_split.get(split_name)
|
existing_dataset = existing_by_split.get(split_name)
|
||||||
dataset_metadata = {
|
dataset_metadata = {
|
||||||
**common_metadata,
|
**common_metadata,
|
||||||
@@ -305,7 +356,7 @@ class DatasetsMixin:
|
|||||||
(
|
(
|
||||||
dataset_name,
|
dataset_name,
|
||||||
dataset_types[split_name],
|
dataset_types[split_name],
|
||||||
payload.get("storage_type") or "local",
|
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||||
f"{len(spec['raw'])} B",
|
f"{len(spec['raw'])} B",
|
||||||
len(spec["raw"]),
|
len(spec["raw"]),
|
||||||
len(spec["records"]),
|
len(spec["records"]),
|
||||||
@@ -335,7 +386,7 @@ class DatasetsMixin:
|
|||||||
dataset_id,
|
dataset_id,
|
||||||
dataset_name,
|
dataset_name,
|
||||||
dataset_types[split_name],
|
dataset_types[split_name],
|
||||||
payload.get("storage_type") or "local",
|
"minio" if use_minio else (payload.get("storage_type") or "local"),
|
||||||
task_id,
|
task_id,
|
||||||
task_id,
|
task_id,
|
||||||
f"{len(spec['raw'])} B",
|
f"{len(spec['raw'])} B",
|
||||||
@@ -366,7 +417,7 @@ class DatasetsMixin:
|
|||||||
"created_at": now,
|
"created_at": now,
|
||||||
"create_time": now,
|
"create_time": now,
|
||||||
"source_task_id": task_id,
|
"source_task_id": task_id,
|
||||||
"storage_object_id": spec["storage_object_id"],
|
"storage_object_id": storage_object_id,
|
||||||
}
|
}
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
@@ -383,7 +434,7 @@ class DatasetsMixin:
|
|||||||
spec["file_id"],
|
spec["file_id"],
|
||||||
dataset_id,
|
dataset_id,
|
||||||
f"{base_dataset_name}.{split_name}.jsonl",
|
f"{base_dataset_name}.{split_name}.jsonl",
|
||||||
spec["storage_object_id"],
|
storage_object_id,
|
||||||
f"{len(spec['raw'])} B",
|
f"{len(spec['raw'])} B",
|
||||||
spec["content"],
|
spec["content"],
|
||||||
spec["version_id"],
|
spec["version_id"],
|
||||||
@@ -414,7 +465,7 @@ class DatasetsMixin:
|
|||||||
(
|
(
|
||||||
spec["version_id"],
|
spec["version_id"],
|
||||||
spec["file_id"],
|
spec["file_id"],
|
||||||
spec["storage_object_id"],
|
storage_object_id,
|
||||||
spec["content"][:2000],
|
spec["content"][:2000],
|
||||||
f"data process {split_name} publish",
|
f"data process {split_name} publish",
|
||||||
len(spec["raw"]),
|
len(spec["raw"]),
|
||||||
|
|||||||
@@ -72,6 +72,8 @@ services:
|
|||||||
MINIO_SECURE: ${MINIO_SECURE:-false}
|
MINIO_SECURE: ${MINIO_SECURE:-false}
|
||||||
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
|
STORAGE_WAIT_SECONDS: ${STORAGE_WAIT_SECONDS:-300}
|
||||||
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
|
STORAGE_CHECK_INTERVAL_SECONDS: ${STORAGE_CHECK_INTERVAL_SECONDS:-10}
|
||||||
|
DATA_PROCESS_STORAGE_DIR: ${DATA_PROCESS_STORAGE_DIR:-/data/yg-ft/data-process}
|
||||||
|
YG_FT_DATA_ROOT: ${YG_FT_DATA_ROOT:-/data/yg-ft}
|
||||||
PYTHONPATH: /app
|
PYTHONPATH: /app
|
||||||
volumes:
|
volumes:
|
||||||
- ../../backend:/app:ro
|
- ../../backend:/app:ro
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import type {
|
|||||||
DatasetVersionList,
|
DatasetVersionList,
|
||||||
} from '@/types'
|
} from '@/types'
|
||||||
|
|
||||||
|
export type { DatasetItem }
|
||||||
|
|
||||||
/** 数据集列表 */
|
/** 数据集列表 */
|
||||||
export const getDatasetList = () => get<DatasetItem[]>('/dataset-manage')
|
export const getDatasetList = () => get<DatasetItem[]>('/dataset-manage')
|
||||||
|
|
||||||
@@ -101,10 +103,14 @@ export const downloadFileUrl = (
|
|||||||
fileId: string | number,
|
fileId: string | number,
|
||||||
versionId?: string,
|
versionId?: string,
|
||||||
) => {
|
) => {
|
||||||
const baseUrl = `/modelTF/dataset-manage/download/${datasetId}/${fileId}`
|
const baseUrl = `/dataset-manage/download/${datasetId}/${fileId}`
|
||||||
return versionId ? `${baseUrl}?version_id=${encodeURIComponent(versionId)}` : baseUrl
|
return versionId ? `${baseUrl}?version_id=${encodeURIComponent(versionId)}` : baseUrl
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 打包下载数据集 URL */
|
/** 打包下载数据集 URL */
|
||||||
export const downloadDatasetUrl = (datasetId: string | number) =>
|
export const downloadDatasetUrl = (datasetId: string | number) =>
|
||||||
`/modelTF/dataset-manage/download/${datasetId}`
|
`/dataset-manage/download/${datasetId}`
|
||||||
|
|
||||||
|
/** 通过统一请求客户端下载数据集,确保携带登录 Token。 */
|
||||||
|
export const downloadDataset = (datasetId: string | number) =>
|
||||||
|
get<unknown>(downloadDatasetUrl(datasetId), undefined, { responseType: 'blob' })
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { ref, onMounted, computed, watch } from 'vue'
|
|||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||||
import DataTablePage from '@/components/DataTablePage.vue'
|
import DataTablePage from '@/components/DataTablePage.vue'
|
||||||
import { getDatasetList, deleteDataset, downloadDatasetUrl } from '@/api/modules/dataset'
|
import { getDatasetList, deleteDataset, downloadDataset } from '@/api/modules/dataset'
|
||||||
import { DATASET_TYPE_MAP, STORAGE_MAP } from '@/constants'
|
import { DATASET_TYPE_MAP, STORAGE_MAP } from '@/constants'
|
||||||
import { formatMegabytes } from '@/utils/fileSize'
|
import { formatMegabytes } from '@/utils/fileSize'
|
||||||
import type { DatasetItem, DatasetSource } from '@/types'
|
import type { DatasetItem, DatasetSource } from '@/types'
|
||||||
@@ -104,8 +104,29 @@ function handlePreview(row: any) {
|
|||||||
router.push(`/dataset/${row.id}/preview`)
|
router.push(`/dataset/${row.id}/preview`)
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleDownload(row: any) {
|
async function handleDownload(row: any) {
|
||||||
window.open(downloadDatasetUrl(row.id), '_blank')
|
try {
|
||||||
|
const response = await downloadDataset(row.id) as {
|
||||||
|
data?: Blob
|
||||||
|
headers?: Record<string, string>
|
||||||
|
}
|
||||||
|
const blob = response.data instanceof Blob ? response.data : new Blob()
|
||||||
|
const disposition = response.headers?.['content-disposition'] || ''
|
||||||
|
const encodedName = disposition.match(/filename\*=UTF-8''([^;]+)/i)?.[1]
|
||||||
|
const filename = encodedName
|
||||||
|
? decodeURIComponent(encodedName)
|
||||||
|
: `${row.name || row.id}.zip`
|
||||||
|
const url = URL.createObjectURL(blob)
|
||||||
|
const anchor = document.createElement('a')
|
||||||
|
anchor.href = url
|
||||||
|
anchor.download = filename
|
||||||
|
document.body.appendChild(anchor)
|
||||||
|
anchor.click()
|
||||||
|
anchor.remove()
|
||||||
|
URL.revokeObjectURL(url)
|
||||||
|
} catch {
|
||||||
|
// 统一请求层已展示后端返回的失败原因。
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatSizeMb(row: DatasetItem) {
|
function formatSizeMb(row: DatasetItem) {
|
||||||
|
|||||||
@@ -63,12 +63,12 @@ async function loadAll() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 获取当前资源类型的列表
|
// 获取当前资源类型的列表
|
||||||
function currentResourceList() {
|
function currentResourceList(): Array<{ id: string; name: string }> {
|
||||||
if (form.resourceType === 'dataset') {
|
if (form.resourceType === 'dataset') {
|
||||||
return datasets.value.map((d) => ({ id: d.id, name: d.name || d.id }))
|
return datasets.value.map((d: DatasetItem) => ({ id: String(d.id), name: d.name || String(d.id) }))
|
||||||
}
|
}
|
||||||
if (form.resourceType === 'trained_model') {
|
if (form.resourceType === 'trained_model') {
|
||||||
return models.value.map((m) => ({ id: String(m.id || m.name || ''), name: m.name || String(m.id) || '' }))
|
return models.value.map((m: TrainedModel) => ({ id: String(m.id || m.name || ''), name: m.name || String(m.id) || '' }))
|
||||||
}
|
}
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
@@ -176,9 +176,10 @@ async function saveAcl() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 显示名称
|
// 显示名称
|
||||||
function principalName(entry: AclEntry) {
|
function principalName(entry: unknown) {
|
||||||
const user = users.value.find((u) => u.id === entry.principal_id)
|
const aclEntry = entry as AclEntry
|
||||||
return user ? `${user.display_name || user.username}` : entry.principal_id
|
const user = users.value.find((u) => u.id === aclEntry.principal_id)
|
||||||
|
return user ? `${user.display_name || user.username}` : aclEntry.principal_id
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(loadAll)
|
onMounted(loadAll)
|
||||||
@@ -250,7 +251,7 @@ onMounted(loadAll)
|
|||||||
v-for="p in row.permissions"
|
v-for="p in row.permissions"
|
||||||
:key="p"
|
:key="p"
|
||||||
size="small"
|
size="small"
|
||||||
:type="p === 'admin' ? 'danger' : p === 'delete' || p === 'write' ? 'warning' : ''"
|
:type="p === 'admin' ? 'danger' : p === 'delete' || p === 'write' ? 'warning' : undefined"
|
||||||
>{{ p }}</el-tag>
|
>{{ p }}</el-tag>
|
||||||
</el-space>
|
</el-space>
|
||||||
</template>
|
</template>
|
||||||
|
|||||||
@@ -128,7 +128,14 @@ export function buildMetricChartOption(
|
|||||||
color: string,
|
color: string,
|
||||||
logScale = false,
|
logScale = false,
|
||||||
): EChartsOption {
|
): EChartsOption {
|
||||||
const visibleData = data.map((value) => (Number.isFinite(value) ? value : null))
|
// 每条曲线独立过滤缺失采样点,避免某个指标缺失导致三个图共用的 step 轴出现空洞。
|
||||||
|
const points = data
|
||||||
|
.map((value, index) => ({ value, step: steps[index] }))
|
||||||
|
.filter((point) => Number.isFinite(point.value))
|
||||||
|
const visibleData = points.map((point) => point.value)
|
||||||
|
const visibleSteps = points.map((point, index) => (
|
||||||
|
Number.isFinite(point.step) ? String(point.step) : String(index + 1)
|
||||||
|
))
|
||||||
return {
|
return {
|
||||||
grid: { top: 24, right: 20, bottom: 56, left: 56 },
|
grid: { top: 24, right: 20, bottom: 56, left: 56 },
|
||||||
graphic: visibleData.some((value) => value != null)
|
graphic: visibleData.some((value) => value != null)
|
||||||
@@ -150,7 +157,7 @@ export function buildMetricChartOption(
|
|||||||
},
|
},
|
||||||
xAxis: {
|
xAxis: {
|
||||||
type: 'category',
|
type: 'category',
|
||||||
data: steps.map((step, index) => (Number.isFinite(step) ? String(step) : String(index + 1))),
|
data: visibleSteps,
|
||||||
boundaryGap: false,
|
boundaryGap: false,
|
||||||
name: 'Step',
|
name: 'Step',
|
||||||
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
|
nameTextStyle: { color: '#94a3b8', fontSize: 11 },
|
||||||
@@ -167,7 +174,7 @@ export function buildMetricChartOption(
|
|||||||
axisLabel: { color: '#94a3b8', fontSize: 11 },
|
axisLabel: { color: '#94a3b8', fontSize: 11 },
|
||||||
splitLine: { lineStyle: { color: '#f1f5f9' } },
|
splitLine: { lineStyle: { color: '#f1f5f9' } },
|
||||||
},
|
},
|
||||||
dataZoom: data.length > 30
|
dataZoom: visibleData.length > 30
|
||||||
? [
|
? [
|
||||||
{ type: 'inside', start: 0, end: 100 },
|
{ type: 'inside', start: 0, end: 100 },
|
||||||
{ type: 'slider', height: 16, bottom: 8, borderColor: 'transparent', fillerColor: 'rgba(79,70,229,0.08)', handleStyle: { color: '#4f46e5' } },
|
{ type: 'slider', height: 16, bottom: 8, borderColor: 'transparent', fillerColor: 'rgba(79,70,229,0.08)', handleStyle: { color: '#4f46e5' } },
|
||||||
|
|||||||
Reference in New Issue
Block a user