"""Storage placement rules shared by dataset and data-processing flows.""" from __future__ import annotations from app.core.config import get_settings _INLINE_TEXT_FORMATS = { "txt", "text", "md", "markdown", "json", "jsonl", "csv", "tsv", "yaml", "yml", "xml", "html", "text/plain", "application/json", "application/jsonl", "text/csv", } def should_store_in_minio( size_bytes: int | None, *, content_type: str | None = None, file_format: str | None = None, ) -> bool: """Return whether a file is large enough to use the shared object store. Small files remain inline in PostgreSQL so page previews and metadata reads do not pay an object-storage round trip. MinIO is still mandatory for large files when it is enabled. """ if not get_settings().minio_enabled: return False try: size = max(0, int(size_bytes or 0)) except (TypeError, ValueError): size = 0 if size > get_settings().minio_inline_max_bytes: return True # Binary office/document files remain in MinIO even when small because # their original bytes cannot be safely represented by a text DB column. normalized_format = str(file_format or "").strip().lower().lstrip(".") normalized_type = str(content_type or "").strip().lower().split(";", 1)[0] if normalized_format or normalized_type: return not ( normalized_format in _INLINE_TEXT_FORMATS or normalized_type in _INLINE_TEXT_FORMATS or normalized_type.startswith("text/") ) return False def storage_backend_for_size(size_bytes: int | None, *, requested: str | None = None) -> str: """Return ``minio`` or ``database`` for a managed file.""" if str(requested or "").strip().lower() == "local": return "database" return "minio" if should_store_in_minio(size_bytes) else "database"