From 2f48934e66164496d6e6fcb87d778e65abb8bfc6 Mon Sep 17 00:00:00 2001
From: caoxiaozhu <leocaoxiaozhu@gmail.com>
Date: Tue, 18 Aug 2026 15:48:20 +0800
Subject: [PATCH] =?UTF-8?q?fix(data=5Fprocess):=20=E4=BF=AE=E5=A4=8D=20tik?=
 =?UTF-8?q?token=20=E5=8A=A0=E8=BD=BD=E5=99=A8=E7=9A=84=E5=8F=98=E9=87=8F?=
 =?UTF-8?q?=E4=BD=9C=E7=94=A8=E5=9F=9F=E9=94=99=E8=AF=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

函数内部的 import tiktoken.core 会把 tiktoken 变成局部变量，导致
tiktoken.get_encoding 在联网下载前就抛 UnboundLocalError，被 except
吞掉后联网与本地缓存两条路径全部失效。移除该内部 import（模块顶部
已引入 tiktoken），切分测试全部通过，编码器成功下载并缓存到
~/.cache/tiktoken，此后离线环境也可正常加载。
---
 backend/app/modules/data_process/document_chunking.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/backend/app/modules/data_process/document_chunking.py b/backend/app/modules/data_process/document_chunking.py
index b418e79..2ad97ed 100644
--- a/backend/app/modules/data_process/document_chunking.py
+++ b/backend/app/modules/data_process/document_chunking.py
@@ -90,8 +90,10 @@ def _tokenizer() -> tiktoken.Encoding:
                         token, rank = line.split()
                         mergeable_ranks[base64.b64decode(token)] = int(rank)
 
-                # 构造 Encoding 对象
-                import tiktoken.core
+                # 构造 Encoding 对象（模块顶部已 import tiktoken，
+                # 此处不能再 import tiktoken.core，否则会把 tiktoken
+                # 变成局部变量，使函数开头的 tiktoken.get_encoding 抛
+                # UnboundLocalError）
                 return tiktoken.core.Encoding(
                     name="cl100k_base",
                     pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",