YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 增加 YAML 配置与一键启动脚本

Browse Source
This commit is contained in:
caoxiaozhu
2026-07-21 16:15:01 +08:00
parent a5fbe5130a
commit 7c38e37b23
15 changed files with 954 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
backend/.env.example
View File

@@ -1,9 +1,18 @@
APP_NAME=YG Fine-Tune Platform API
APP_ENV=local
API_PREFIX=/api
LOG_LEVEL=INFO
LOG_DIR=./logs
LOG_FILE_PREFIX=backend
LOG_ERROR_FILE_PREFIX=error
LOG_MAX_BYTES=20971520
LOG_RETENTION_DAYS=10
# 默认自动读取 backend/config.yaml。设置此变量可切换到其他 YAML 文件。
BACKEND_CONFIG_FILE=./config.yaml
# 下列环境变量按需启用，并优先于 YAML 中的配置。
# DATABASE_BASE_URL=postgresql+psycopg://localhost:5432/yg_ft
# DATABASE_USERNAME=yg_ft
# DATABASE_PASSWORD=change_me
# DATABASE_URL=postgresql+psycopg://yg_ft:change_me@localhost:5432/yg_ft
# DATABASE_URL 是完整连接串覆盖项，优先级高于上面三个分离项。
# MODELTF_ROUTE_PREFIX=/modelTF
# FRONTEND_PORT=16801
# BACKEND_PORT=17861
# CORS_ALLOW_ORIGINS=http://localhost:16801,http://127.0.0.1:16801
# LOG_LEVEL=INFO
# LOG_DIR=./logs
# COMPUTE_MODE=real
# COMPUTE_STATUS_SYNC_MODE=polling
# COMPUTE_POLL_INTERVAL_SECONDS=3

54
backend/README.md
View File

@@ -32,17 +32,65 @@ backend/
services/ # 跨模块应用服务
workers/ # 后台任务入口
requirements.txt # 后端第三方依赖
config.example.yaml # 可提交的脱敏配置模板
config.yaml # 本地配置，已忽略，环境变量可覆盖
logs/ # 本地开发日志目录，生产环境建议挂载到独立日志盘
```
## 配置
先复制本地配置并填写数据库凭据：
```bash
cp config.example.yaml config.yaml
```
后端默认读取 `config.yaml`，配置优先级为：
```text
环境变量 > BACKEND_CONFIG_FILE 指定的 YAML > backend/config.yaml > 代码默认值
```
前后端端口、数据库地址、跨域来源、日志路径和计算状态轮询参数均可在 YAML 中
配置。敏感信息或不同环境的差异建议通过环境变量覆盖，不要写入仓库。YAML 中的
相对日志路径以该 YAML 文件所在目录为基准。使用其他配置文件时：
```bash
BACKEND_CONFIG_FILE=./config.prod.yaml uvicorn app.main:app --reload --port 17861
```
本机直连 PostgreSQL 时，数据库配置拆分为地址、用户名和密码：
```yaml
server:
frontend_port: 16801
backend_port: 17861
database:
url: postgresql+psycopg://localhost:5432/yg_ft
username: yg_ft
password: "change_me"
```
服务启动时会安全拼装完整连接串；用户名或密码中的特殊字符会自动编码。Docker
和生产环境仍可用完整的 `DATABASE_URL` 环境变量覆盖以上三项。
这里的 `5432` 是宿主机 PostgreSQL 默认端口；如果数据库容器映射到 `15432`，
将 YAML 中的端口改为 `15432` 即可。
`server.frontend_port` 和 `server.backend_port` 由根目录的
`scripts/start-dev.sh` 读取，并分别传给 Vite 和 Uvicorn。未配置
`app.cors_allow_origins` 时，本机 CORS 来源会跟随前端端口。直接手工执行
`uvicorn` 时仍需通过 `--port` 指定监听端口。
## 本地启动
```bash
cd backend
python -m venv .venv
.venv\Scripts\activate
/opt/miniconda3/bin/python3 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
uvicorn app.main:app --reload
uvicorn app.main:app --reload --port 17861
```
健康检查：

349
backend/app/core/config.py
View File

@@ -1,57 +1,326 @@
from dataclasses import dataclass
from __future__ import annotations
from dataclasses import dataclass, field
from functools import lru_cache
import os
from pathlib import Path
from typing import Any
from urllib.parse import quote, urlsplit, urlunsplit
import yaml
def _int_env(name: str, default: int) -> int:
raw = os.getenv(name)
if raw is None or raw == "":
BACKEND_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_CONFIG_PATH = BACKEND_ROOT / "config.yaml"
DEFAULT_DATABASE_BASE_URL = "postgresql+psycopg://localhost:5432/yg_ft"
DEFAULT_DATABASE_USERNAME = "yg_ft"
DEFAULT_DATABASE_PASSWORD = "change_me"
class ConfigurationError(ValueError):
"""Raised when the backend configuration cannot be parsed or validated."""
def _resolve_config_path(config_path: str | Path | None = None) -> tuple[Path, bool]:
configured_path = config_path or os.getenv("BACKEND_CONFIG_FILE")
is_explicit = configured_path is not None
path = Path(configured_path).expanduser() if configured_path else DEFAULT_CONFIG_PATH
if not path.is_absolute():
path = BACKEND_ROOT / path
return path.resolve(), is_explicit
def _load_yaml(config_path: str | Path | None = None) -> tuple[dict[str, Any], Path]:
path, is_explicit = _resolve_config_path(config_path)
if not path.exists():
if is_explicit:
raise ConfigurationError(f"Backend config file does not exist: {path}")
return {}, path
try:
loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
except yaml.YAMLError as exc:
raise ConfigurationError(f"Invalid YAML in backend config file {path}: {exc}") from exc
if loaded is None:
return {}, path
if not isinstance(loaded, dict):
raise ConfigurationError(f"Backend config root must be a mapping: {path}")
return loaded, path
def _yaml_value(config: dict[str, Any], section: str, key: str, default: Any) -> Any:
section_value = config.get(section, {})
if section_value is None:
return default
return int(raw)
if not isinstance(section_value, dict):
raise ConfigurationError(f"Config section '{section}' must be a mapping")
return section_value.get(key, default)
def _list_env(name: str, default: list[str]) -> list[str]:
raw = os.getenv(name)
if raw is None or raw.strip() == "":
return default
return [item.strip() for item in raw.split(",") if item.strip()]
def _string_setting(
env_name: str,
config: dict[str, Any],
section: str,
key: str,
default: str,
) -> str:
env_value = os.getenv(env_name)
if env_value is not None:
return env_value
value = _yaml_value(config, section, key, default)
return default if value is None else str(value)
def _int_setting(
env_name: str,
config: dict[str, Any],
section: str,
key: str,
default: int,
) -> int:
env_value = os.getenv(env_name)
value = (
env_value
if env_value not in (None, "")
else _yaml_value(config, section, key, default)
)
if isinstance(value, (bool, float)):
raise ConfigurationError(f"Config value {section}.{key} must be an integer")
try:
return int(value)
except (TypeError, ValueError) as exc:
raise ConfigurationError(f"Config value {section}.{key} must be an integer") from exc
def _port_setting(
env_name: str,
config: dict[str, Any],
key: str,
default: int,
) -> int:
port = _int_setting(env_name, config, "server", key, default)
if not 1 <= port <= 65535:
raise ConfigurationError(
f"Config value server.{key} must be between 1 and 65535"
)
return port
def _list_setting(
env_name: str,
config: dict[str, Any],
section: str,
key: str,
default: list[str],
) -> list[str]:
env_value = os.getenv(env_name)
value = (
env_value
if env_value and env_value.strip()
else _yaml_value(config, section, key, default)
)
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
if isinstance(value, list):
return [str(item).strip() for item in value if str(item).strip()]
raise ConfigurationError(
f"Config value {section}.{key} must be a list or comma-separated string"
)
def _path_setting(
env_name: str,
config: dict[str, Any],
section: str,
key: str,
default: str,
config_dir: Path,
) -> str:
env_value = os.getenv(env_name)
if env_value is not None:
return env_value
value = Path(str(_yaml_value(config, section, key, default))).expanduser()
if not value.is_absolute():
value = config_dir / value
return str(value.resolve())
def _build_database_url(base_url: str, username: str, password: str) -> str:
try:
parsed = urlsplit(base_url)
host = parsed.hostname
port = parsed.port
except ValueError as exc:
raise ConfigurationError("Config value database.url is not a valid URL") from exc
if not parsed.scheme or not host:
raise ConfigurationError(
"Config value database.url must include a scheme and host"
)
if parsed.scheme not in {"postgresql", "postgresql+psycopg"}:
raise ConfigurationError(
"Config value database.url must use postgresql or postgresql+psycopg"
)
if parsed.path in {"", "/"}:
raise ConfigurationError("Config value database.url must include a database name")
if not username:
raise ConfigurationError("Config value database.username cannot be empty")
if not password:
raise ConfigurationError("Config value database.password cannot be empty")
formatted_host = f"[{host}]" if ":" in host and not host.startswith("[") else host
host_and_port = f"{formatted_host}:{port}" if port is not None else formatted_host
credentials = f"{quote(username, safe='')}:{quote(password, safe='')}"
parts = (
parsed.scheme,
f"{credentials}@{host_and_port}",
parsed.path,
parsed.query,
parsed.fragment,
)
return urlunsplit(parts)
def _database_url(config: dict[str, Any]) -> str:
# 保留原有完整连接串环境变量，便于 Docker 和生产环境注入密钥。
complete_url_override = os.getenv("DATABASE_URL")
if complete_url_override is not None:
return complete_url_override
base_url = _string_setting(
"DATABASE_BASE_URL",
config,
"database",
"url",
DEFAULT_DATABASE_BASE_URL,
)
username = _string_setting(
"DATABASE_USERNAME",
config,
"database",
"username",
DEFAULT_DATABASE_USERNAME,
)
password = _string_setting(
"DATABASE_PASSWORD",
config,
"database",
"password",
DEFAULT_DATABASE_PASSWORD,
)
return _build_database_url(base_url, username, password)
@dataclass(frozen=True)
class Settings:
app_name: str = os.getenv("APP_NAME", "YG Fine-Tune Platform API")
app_env: str = os.getenv("APP_ENV", "local")
route_prefix: str = os.getenv("MODELTF_ROUTE_PREFIX", "/modelTF")
app_mode: str = os.getenv("APP_MODE", "local")
database_url: str = os.getenv("DATABASE_URL", "postgresql+psycopg://yg_ft:change_me@localhost:15432/yg_ft")
cors_allow_origins: list[str] = None # type: ignore[assignment]
compute_mode: str = os.getenv("COMPUTE_MODE", "real")
compute_status_sync_mode: str = os.getenv("COMPUTE_STATUS_SYNC_MODE", "polling")
compute_poll_interval_seconds: int = _int_env("COMPUTE_POLL_INTERVAL_SECONDS", 3)
log_level: str = os.getenv("LOG_LEVEL", "INFO")
log_dir: str = os.getenv("LOG_DIR", "./logs")
log_file_prefix: str = os.getenv("LOG_FILE_PREFIX", "backend")
log_error_file_prefix: str = os.getenv("LOG_ERROR_FILE_PREFIX", "error")
log_max_bytes: int = _int_env("LOG_MAX_BYTES", 20 * 1024 * 1024)
log_retention_days: int = _int_env("LOG_RETENTION_DAYS", 10)
app_name: str = "YG Fine-Tune Platform API"
app_env: str = "local"
route_prefix: str = "/modelTF"
app_mode: str = "local"
frontend_port: int = 16801
backend_port: int = 17861
database_url: str = field(
default="postgresql+psycopg://yg_ft:change_me@localhost:5432/yg_ft",
repr=False,
)
cors_allow_origins: list[str] = field(
default_factory=lambda: [
"http://localhost:16801",
"http://127.0.0.1:16801",
]
)
compute_mode: str = "real"
compute_status_sync_mode: str = "polling"
compute_poll_interval_seconds: int = 3
log_level: str = "INFO"
log_dir: str = "./logs"
log_file_prefix: str = "backend"
log_error_file_prefix: str = "error"
log_max_bytes: int = 20 * 1024 * 1024
log_retention_days: int = 10
def __post_init__(self) -> None:
object.__setattr__(
self,
"cors_allow_origins",
_list_env(
"CORS_ALLOW_ORIGINS",
[
"http://localhost:16801",
"http://127.0.0.1:16801",
"http://localhost:17861",
"http://127.0.0.1:17861",
],
),
def load_settings(config_path: str | Path | None = None) -> Settings:
config, resolved_config_path = _load_yaml(config_path)
frontend_port = _port_setting(
"FRONTEND_PORT", config, "frontend_port", Settings.frontend_port
)
backend_port = _port_setting(
"BACKEND_PORT", config, "backend_port", Settings.backend_port
)
if frontend_port == backend_port:
raise ConfigurationError(
"Config values server.frontend_port and server.backend_port must be different"
)
default_cors = [
f"http://localhost:{frontend_port}",
f"http://127.0.0.1:{frontend_port}",
]
return Settings(
app_name=_string_setting("APP_NAME", config, "app", "name", Settings.app_name),
app_env=_string_setting("APP_ENV", config, "app", "env", Settings.app_env),
route_prefix=_string_setting(
"MODELTF_ROUTE_PREFIX", config, "app", "route_prefix", Settings.route_prefix
),
app_mode=_string_setting("APP_MODE", config, "app", "mode", Settings.app_mode),
frontend_port=frontend_port,
backend_port=backend_port,
database_url=_database_url(config),
cors_allow_origins=_list_setting(
"CORS_ALLOW_ORIGINS", config, "app", "cors_allow_origins", default_cors
),
compute_mode=_string_setting(
"COMPUTE_MODE", config, "compute", "mode", Settings.compute_mode
),
compute_status_sync_mode=_string_setting(
"COMPUTE_STATUS_SYNC_MODE",
config,
"compute",
"status_sync_mode",
Settings.compute_status_sync_mode,
),
compute_poll_interval_seconds=_int_setting(
"COMPUTE_POLL_INTERVAL_SECONDS",
config,
"compute",
"poll_interval_seconds",
Settings.compute_poll_interval_seconds,
),
log_level=_string_setting(
"LOG_LEVEL", config, "logging", "level", Settings.log_level
),
log_dir=_path_setting(
"LOG_DIR",
config,
"logging",
"directory",
Settings.log_dir,
resolved_config_path.parent,
),
log_file_prefix=_string_setting(
"LOG_FILE_PREFIX", config, "logging", "file_prefix", Settings.log_file_prefix
),
log_error_file_prefix=_string_setting(
"LOG_ERROR_FILE_PREFIX",
config,
"logging",
"error_file_prefix",
Settings.log_error_file_prefix,
),
log_max_bytes=_int_setting(
"LOG_MAX_BYTES", config, "logging", "max_bytes", Settings.log_max_bytes
),
log_retention_days=_int_setting(
"LOG_RETENTION_DAYS",
config,
"logging",
"retention_days",
Settings.log_retention_days,
),
)
@lru_cache
def get_settings() -> Settings:
return Settings()
return load_settings()

6
backend/app/db/platform_store.py
View File

@@ -78,7 +78,10 @@ def verify_password(password: str, stored: str) -> tuple[bool, bool]:
def _psycopg_url(database_url: str) -> str:
return database_url.replace("postgresql+psycopg://", "postgresql://")
sqlalchemy_prefix = "postgresql+psycopg://"
if database_url.startswith(sqlalchemy_prefix):
return f"postgresql://{database_url[len(sqlalchemy_prefix):]}"
return database_url
def _pg_sql(sql: str) -> str:
@@ -1147,4 +1150,3 @@ def get_platform_store() -> PlatformStore:
if _store is None:
_store = PlatformStore()
return _store

6
backend/app/db/session.py
View File

@@ -1,14 +1,15 @@
from __future__ import annotations
import os
from collections.abc import Generator
from contextlib import contextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from app.core.config import get_settings
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+psycopg://yg_ft:change_me@localhost:15432/yg_ft")
DATABASE_URL = get_settings().database_url
engine = create_engine(
DATABASE_URL,
@@ -37,4 +38,3 @@ def session_scope() -> Generator[Session, None, None]:
raise
finally:
db.close()

31
backend/config.example.yaml Normal file
View File

@@ -0,0 +1,31 @@
# 后端配置模板。复制为 config.yaml 后填写本地数据库凭据。
# config.yaml 已加入 Git 忽略，不会提交真实密码。
app:
name: YG Fine-Tune Platform API
env: local
mode: local
route_prefix: /modelTF
server:
# 一键启动脚本统一读取这里的端口；端口被占用时只需修改这两项。
frontend_port: 16801
backend_port: 17861
database:
url: postgresql+psycopg://localhost:5432/yg_ft
username: yg_ft
password: "change_me"
compute:
mode: real
status_sync_mode: polling
poll_interval_seconds: 3
logging:
level: INFO
directory: ./logs
file_prefix: backend
error_file_prefix: error
max_bytes: 20971520
retention_days: 10

1
backend/pyproject.toml
View File

@@ -16,6 +16,7 @@ dependencies = [
"PyJWT>=2.8.0",
"passlib[bcrypt]>=1.7.4",
"python-dotenv>=1.0.1",
"PyYAML>=6.0.2",
]
[project.optional-dependencies]

1
backend/requirements.txt
View File

@@ -10,3 +10,4 @@ httpx>=0.27.0
PyJWT>=2.8.0
passlib[bcrypt]>=1.7.4
python-dotenv>=1.0.1
PyYAML>=6.0.2

196
backend/tests/test_config.py Normal file
View File

@@ -0,0 +1,196 @@
from __future__ import annotations
from pathlib import Path
import pytest
from app.core.config import ConfigurationError, load_settings
CONFIG_ENV_NAMES = (
"APP_NAME",
"APP_ENV",
"APP_MODE",
"MODELTF_ROUTE_PREFIX",
"FRONTEND_PORT",
"BACKEND_PORT",
"DATABASE_URL",
"DATABASE_BASE_URL",
"DATABASE_USERNAME",
"DATABASE_PASSWORD",
"CORS_ALLOW_ORIGINS",
"COMPUTE_MODE",
"COMPUTE_STATUS_SYNC_MODE",
"COMPUTE_POLL_INTERVAL_SECONDS",
"LOG_LEVEL",
"LOG_DIR",
"LOG_FILE_PREFIX",
"LOG_ERROR_FILE_PREFIX",
"LOG_MAX_BYTES",
"LOG_RETENTION_DAYS",
)
def clear_config_env(monkeypatch: pytest.MonkeyPatch) -> None:
for env_name in CONFIG_ENV_NAMES:
monkeypatch.delenv(env_name, raising=False)
def write_config(path: Path) -> None:
path.write_text(
"""
app:
name: YAML API
route_prefix: /yaml-api
cors_allow_origins:
- http://yaml.example
server:
frontend_port: 18001
backend_port: 18002
database:
url: postgresql+psycopg://db:5432/yaml
username: yaml-user
password: yaml-secret
compute:
mode: simulated
poll_interval_seconds: 9
logging:
directory: ./yaml-logs
max_bytes: 1024
retention_days: 2
""".strip(),
encoding="utf-8",
)
def test_load_settings_from_yaml(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
write_config(config_path)
settings = load_settings(config_path)
assert settings.app_name == "YAML API"
assert settings.route_prefix == "/yaml-api"
assert settings.frontend_port == 18001
assert settings.backend_port == 18002
assert settings.database_url == "postgresql+psycopg://yaml-user:yaml-secret@db:5432/yaml"
assert settings.cors_allow_origins == ["http://yaml.example"]
assert settings.compute_mode == "simulated"
assert settings.compute_poll_interval_seconds == 9
assert settings.log_dir == str(tmp_path / "yaml-logs")
assert settings.log_max_bytes == 1024
assert settings.log_retention_days == 2
def test_environment_overrides_yaml(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
write_config(config_path)
monkeypatch.setenv("DATABASE_URL", "postgresql+psycopg://env:secret@db:5432/env")
monkeypatch.setenv("CORS_ALLOW_ORIGINS", "http://one.example,http://two.example")
monkeypatch.setenv("COMPUTE_POLL_INTERVAL_SECONDS", "15")
monkeypatch.setenv("FRONTEND_PORT", "19001")
monkeypatch.setenv("BACKEND_PORT", "19002")
settings = load_settings(config_path)
assert settings.database_url == "postgresql+psycopg://env:secret@db:5432/env"
assert settings.cors_allow_origins == ["http://one.example", "http://two.example"]
assert settings.compute_poll_interval_seconds == 15
assert settings.frontend_port == 19001
assert settings.backend_port == 19002
def test_separate_database_credentials_are_encoded(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
write_config(config_path)
monkeypatch.setenv("DATABASE_USERNAME", "user@example.com")
monkeypatch.setenv("DATABASE_PASSWORD", "secret:/?#[]@")
settings = load_settings(config_path)
assert settings.database_url == (
"postgresql+psycopg://user%40example.com:secret%3A%2F%3F%23%5B%5D%40@db:5432/yaml"
)
def test_explicit_missing_config_is_rejected(tmp_path: Path) -> None:
with pytest.raises(ConfigurationError, match="does not exist"):
load_settings(tmp_path / "missing.yaml")
def test_default_cors_follows_frontend_port(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
config_path.write_text(
"server:\n frontend_port: 28001\n backend_port: 28002\n",
encoding="utf-8",
)
settings = load_settings(config_path)
assert settings.cors_allow_origins == [
"http://localhost:28001",
"http://127.0.0.1:28001",
]
@pytest.mark.parametrize("frontend_port", [0, 65536, "invalid", True])
def test_invalid_server_port_is_rejected(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
frontend_port: object,
) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
config_path.write_text(
"server:\n"
f" frontend_port: {str(frontend_port).lower()}\n"
" backend_port: 28002\n",
encoding="utf-8",
)
with pytest.raises(ConfigurationError, match="server.frontend_port"):
load_settings(config_path)
def test_frontend_and_backend_ports_must_differ(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
config_path.write_text(
"server:\n frontend_port: 28001\n backend_port: 28001\n",
encoding="utf-8",
)
with pytest.raises(ConfigurationError, match="must be different"):
load_settings(config_path)
@pytest.mark.parametrize(
("database_base_url", "message"),
[
("mysql://localhost:3306/yg_ft", "must use postgresql"),
("postgresql+psycopg://localhost:5432", "must include a database name"),
],
)
def test_invalid_database_url_is_rejected(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
database_base_url: str,
message: str,
) -> None:
clear_config_env(monkeypatch)
config_path = tmp_path / "config.yaml"
write_config(config_path)
monkeypatch.setenv("DATABASE_BASE_URL", database_base_url)
with pytest.raises(ConfigurationError, match=message):
load_settings(config_path)