55 lines
1.9 KiB
Python
55 lines
1.9 KiB
Python
|
"""Storage placement rules shared by dataset and data-processing flows."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from app.core.config import get_settings
|
||
|
|
|
||
|
|
|
||
|
|
_INLINE_TEXT_FORMATS = {
|
||
|
|
"txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv",
|
||
|
|
"yaml", "yml", "xml", "html", "text/plain", "application/json",
|
||
|
|
"application/jsonl", "text/csv",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def should_store_in_minio(
|
||
|
|
size_bytes: int | None,
|
||
|
|
*,
|
||
|
|
content_type: str | None = None,
|
||
|
|
file_format: str | None = None,
|
||
|
|
) -> bool:
|
||
|
|
"""Return whether a file is large enough to use the shared object store.
|
||
|
|
|
||
|
|
Small files remain inline in PostgreSQL so page previews and metadata reads
|
||
|
|
do not pay an object-storage round trip. MinIO is still mandatory for
|
||
|
|
large files when it is enabled.
|
||
|
|
"""
|
||
|
|
|
||
|
|
if not get_settings().minio_enabled:
|
||
|
|
return False
|
||
|
|
try:
|
||
|
|
size = max(0, int(size_bytes or 0))
|
||
|
|
except (TypeError, ValueError):
|
||
|
|
size = 0
|
||
|
|
if size > get_settings().minio_inline_max_bytes:
|
||
|
|
return True
|
||
|
|
# Binary office/document files remain in MinIO even when small because
|
||
|
|
# their original bytes cannot be safely represented by a text DB column.
|
||
|
|
normalized_format = str(file_format or "").strip().lower().lstrip(".")
|
||
|
|
normalized_type = str(content_type or "").strip().lower().split(";", 1)[0]
|
||
|
|
if normalized_format or normalized_type:
|
||
|
|
return not (
|
||
|
|
normalized_format in _INLINE_TEXT_FORMATS
|
||
|
|
or normalized_type in _INLINE_TEXT_FORMATS
|
||
|
|
or normalized_type.startswith("text/")
|
||
|
|
)
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str:
|
||
|
|
"""Return ``minio`` or ``database`` for a managed file."""
|
||
|
|
|
||
|
|
if str(requested or "").strip().lower() == "local":
|
||
|
|
return "database"
|
||
|
|
return "minio" if should_store_in_minio(size_bytes) else "database"
|