Philip p208p2002

Python3 文字與unicode互轉

文字轉unicode較為簡單，用ord(x)即可

import re
def word2unicode(x):
    uni = hex(ord(x))
    uni = re.sub("^0x", "", uni).upper()
    return uni
word2unicode("字") # 5B57

pytorch模型部署時若遇到多執行緒(或是多個併發請求)會自動請求新的vram，使用完畢後也不會自動釋放，因此當API用一段時間後常常會出現 cuda out of memor 導致server崩潰。除此之外多執行緒爭奪資源也有機會讓程式變得不穩定。

有幾條思路可以解決

	# GPT2 BPE-Tokenizer token 轉 utf-8 處理
	# 轉換僅針對不在詞表內，以bytes形式表達的token(如中文字)

	from transformers import AutoTokenizer
	tokenizer = AutoTokenizer.from_pretrained("gpt2")
	word = "台"
	tokens = tokenizer.convert_ids_to_tokens(tokenizer(word,add_special_tokens=False)["input_ids"])
	print("tokens:",tokens)

	# 轉 utf-8

	# https://huggingface.co/docs/transformers/perplexity
	from typing import Any
	import torch
	from transformers import AutoModelForCausalLM, AutoTokenizer

	class PPL():
	def __init__(self, model_id="gpt2") -> None:
	self.model = AutoModelForCausalLM.from_pretrained(model_id)
	self.tokenizer = AutoTokenizer.from_pretrained(model_id)
	self.device = 'cpu'

	# $ pip install deepspeed>=0.9.3
	# $ deepspeed deepspeed_inference.py

	import os
	import deepspeed
	import torch
	from transformers import AutoTokenizer, AutoModelForCausalLM


	local_rank = int(os.getenv("LOCAL_RANK", "0"))

	<s>[INST] <<SYS>>你是一位中文母語使用者，你只能用中文對話<</SYS>>hello [/INST] 你好 (nǐ hǎo) </s>
	<s>[INST] 你是誰 [/INST] 我是 líng (wǒ shì líng) - I am Chinese. </s>
	<s>[INST] 說個笑話來聽聽 [/INST] 笑 (xì) - Sure, here's a Chinese joke for you </s>