fix(data_process): 修复 tiktoken 加载器的变量作用域错误
函数内部的 import tiktoken.core 会把 tiktoken 变成局部变量,导致 tiktoken.get_encoding 在联网下载前就抛 UnboundLocalError,被 except 吞掉后联网与本地缓存两条路径全部失效。移除该内部 import(模块顶部 已引入 tiktoken),切分测试全部通过,编码器成功下载并缓存到 ~/.cache/tiktoken,此后离线环境也可正常加载。
This commit is contained in:
@@ -90,8 +90,10 @@ def _tokenizer() -> tiktoken.Encoding:
|
|||||||
token, rank = line.split()
|
token, rank = line.split()
|
||||||
mergeable_ranks[base64.b64decode(token)] = int(rank)
|
mergeable_ranks[base64.b64decode(token)] = int(rank)
|
||||||
|
|
||||||
# 构造 Encoding 对象
|
# 构造 Encoding 对象(模块顶部已 import tiktoken,
|
||||||
import tiktoken.core
|
# 此处不能再 import tiktoken.core,否则会把 tiktoken
|
||||||
|
# 变成局部变量,使函数开头的 tiktoken.get_encoding 抛
|
||||||
|
# UnboundLocalError)
|
||||||
return tiktoken.core.Encoding(
|
return tiktoken.core.Encoding(
|
||||||
name="cl100k_base",
|
name="cl100k_base",
|
||||||
pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
|
pat_str=r"""'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]++[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+""",
|
||||||
|
|||||||
Reference in New Issue
Block a user