diff --git a/backend/app/modules/data_process/document_chunking.py b/backend/app/modules/data_process/document_chunking.py index b418e79..2ad97ed 100644 --- a/backend/app/modules/data_process/document_chunking.py +++ b/backend/app/modules/data_process/document_chunking.py @@ -90,8 +90,10 @@ def _tokenizer() -> tiktoken.Encoding: token, rank = line.split() mergeable_ranks[base64.b64decode(token)] = int(rank) - # 构造 Encoding 对象 - import tiktoken.core + # 构造 Encoding 对象(模块顶部已 import tiktoken, + # 此处不能再 import tiktoken.core,否则会把 tiktoken + # 变成局部变量,使函数开头的 tiktoken.get_encoding 抛 + # UnboundLocalError) return tiktoken.core.Encoding( name="cl100k_base", pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",