Update tokenization_qwen.py

tokenization_qwen.py

@@ -27,12 +27,6 @@ logger = logging.getLogger(__name__)
 
 
 VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken", "ttf": "SimSun.ttf"}
-FONT_PATH = try_to_load_from_cache("Qwen/Qwen-VL-Chat", "SimSun.ttf")
-if FONT_PATH is None:
-    if not os.path.exists("SimSun.ttf"):
-        ttf = requests.get("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/SimSun.ttf")
-        open("SimSun.ttf", "wb").write(ttf.content)
-    FONT_PATH = "SimSun.ttf"
 
 PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
 ENDOFTEXT = "<|endoftext|>"
@@ -175,6 +169,24 @@ class QWenTokenizer(PreTrainedTokenizer):
         self.im_start_id = self.special_tokens[IMSTART]
         self.im_end_id = self.special_tokens[IMEND]
 
+    def __getstate__(self):
+        # for pickle lovers
+        state = self.__dict__.copy()
+        del state['tokenizer']
+        return state
+
+    def __setstate__(self, state):
+        # tokenizer is not python native; don't pass it; rebuild it
+        self.__dict__.update(state)
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        self.tokenizer = enc
+
+
     def __len__(self) -> int:
         return self.tokenizer.n_vocab
 
@@ -503,7 +515,7 @@ class VisImage:
 class Visualizer:
     def __init__(self, img_rgb, metadata=None, scale=1.0):
         self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8)
-        self.font_path = FONT_PATH
+        self.font_path = try_to_load_from_cache("Qwen/Qwen-VL-Chat", "SimSun.ttf")
         self.output = VisImage(self.img, scale=scale)
         self.cpu_device = torch.device("cpu")