from transformers import AutoTokenizer
#从huggingface加载模型，输入模型名称，即可加载对应的分词器
tokenizer=AutoTokenizer.from_pretrained("uer/roberta-xlarge-wwm-chinese-cluecorpussmall")


# 默认会存在 系统用户的【.cache->huggingface->hub->model-{模型名称}->snapshots】目录里面
# 指定保存模型到本地目录
tokenizer.save_pretrained("./roberta_tokenizer")

# 从本地加载tokenizer
tokenizer=AutoTokenizer.from_pretrained("./roberta_tokenizer")
tokenizer

# 分词
# 当前模型会拆分成一个个字，每个模型不一样
sen="弱小的我也有大梦想"
tokens=tokenizer.tokenize(sen)
tokens
# ['弱', '小', '的', '我', '也', '有', '大', '梦', '想']

# 中文 '##榭': 16588,  ##无意义。但是英文会把一个单词拆成词根
tokenizer.vocab

tokenizer.vocab_size

# 将词序转换成id序列，进入神经网络
ids=tokenizer.convert_tokens_to_ids(tokens)
ids
# [2483, 2207, 4638, 2769, 738, 3300, 1920, 3457, 2682]

# 将词序转成tokens
tokens=tokenizer.convert_ids_to_tokens(ids)
tokens
# ['弱', '小', '的', '我', '也', '有', '大', '梦', '想']

# 将token转回成字符串
str_sen = tokenizer.convert_tokens_to_string(tokens)
str_sen
# '弱 小 的 我 也 有 大 梦 想'

# 将词序转换成id序列，编码
ids=tokenizer.encode(sen)
ids
# [101, 2483, 2207, 4638, 2769, 738, 3300, 1920, 3457, 2682, 102]

# 序列多了 101和102 模型的词开始和结束，如果不需要
ids=tokenizer.encode(sen,add_special_tokens=False)


# 将id转换成字符串，解码
str_sen=tokenizer.decode(ids,skip_special_tokens=True)
str_sen

# 填充
ids=tokenizer.encode(sen,padding="max_length",max_length=15)
ids # [101, 2483, 2207, 4638, 2769, 738, 3300, 1920, 3457, 2682, 102, 0, 0, 0, 0]

# 截断
ids=tokenizer.encode(sen,max_length=5,truncation=True)
ids # [101, 2483, 2207, 4638, 102]

# 填充真实有效部分
inputs=tokenizer.encode_plus(sen,padding="max_length",max_length=15)
inputs

# 简洁写法
inputs=tokenizer(sen,padding="max_length",max_length=15)
inputs

sens=[
    "弱小的我也有大梦想",
    "有梦想谁都了不起",
    "迎着风追逐梦想"
]
inputs=tokenizer(sens)
inputs # {'input_ids': [[101, 2483, 2207, 4638, 2769, 738, 3300, 1920, 3457, 2682, 102], [101, 3300, 3457, 2682, 6443, 6963, 749, 679, 6629, 102], [101, 6816, 4708, 7599, 6841, 6852, 3457, 2682, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1]]}

sen="我有一个梦想Dreaming!"

# Fast基于Rust实现，slow基于Python实现

fast_tokenizer = AutoTokenizer.from_pretrained("uer/roberta-xlarge-wwm-chinese-cluecorpussmall")
fast_tokenizer
# slow_tokenizer = AutoTokenizer.from_pretrained("uer/roberta-xlarge-wwm-chinese-cluecorpussmall", use_fast=False)
# slow_tokenizer

# 远程仓库里面有自有实现的分词器代码 trust_remote_code调用
tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b",trust_remote_code=True)
tokenizer

from transformers import AutoModel

# 强制下载
model=AutoModel.from_pretrained("hfl/rbt3",force_download=True)  

# git下载：git clone "模型地址"  ，如果只需要模型的部分指定文件，可使用 git lfs clone "模型地址" --include="*.bin" 

# 离线下载
model=AutoModel.from_pretrained("/User/Download/hfl/rbt3")  

config=AutoConfig.from_pretrained("hfl/rbt3")

sen="弱小的我也有大梦想"
tokenizer=AutoTokenizer.from_pretrained("hfl/rbt3")
inputs=tokenizer(sen,return_tensors="pt")
inputs

model=AutoModel.from_pretrained("hfl/rbt3")
ouput=model(**tokenizer(sen,return_tensors="pt"))
ouput

form transformers import AutoModelForSequenceClassification
clz_model=AutoModelForSequenceClassification.from_pretrained("hfl/rbt3")
ouput=model(**tokenizer(sen,return_tensors="pt"))
ouput

from datasets import *
datasets=load_dataset("madao33/new-title-chinese")
datasets

datasets=load_dataset("super_glue","boolq")
datasets

datasets=load_dataset("madao33/new-title-chinese",split="train")
datasets