CLIP

数据集目录格式
└─Flickr8k
   ├─Flickr8k_Dataset
   |      └─Flickr8k_Dataset
   └─Flickr8k_text
          ├─Flickr8k.devImages.txt
          ├─Flickr8k.testImages.txt
          ├─Flickr8k.trainImages.txt
          └─Flickr8k.token.txt

数据集目录格式
└─cifar-100-python
   ├─meta
   ├─test  
   └─train  

# pretrain
python run_mindformer.py --config ./configs/clip/run_clip_vit_b_32_pretrain_flickr8k.yaml --run_mode train --train_dataset_dir [DATASET_PATH]

# evaluate
python run_mindformer.py --config ./configs/clip/run_clip_vit_b_32_zero_shot_image_classification_cifar100.yaml --run_mode eval --eval_dataset_dir [DATASET_PATH]

# predict
python run_mindformer.py --config ./configs/clip/run_clip_vit_b_32_zero_shot_image_classification_cifar100.yaml --run_mode predict --predict_data [PATH_TO_IMAGE]

import mindspore; mindspore.set_context(mode=0, device_id=0)
from mindformers import CLIPModel, CLIPConfig

CLIPModel.show_support_list()
# 输出：
# - support list of CLIPModel is:
# -    ['clip_vit_b_32', 'clip_vit_B_16', 'clip_vit_l_14', 'clip_vit_l_14@336']
# - -------------------------------------

# 模型标志加载模型
model = CLIPModel.from_pretrained("clip_vit_b_32")

#模型配置加载模型
config = CLIPConfig.from_pretrained("clip_vit_b_32")
# {'text_config': {'hidden_size': 512, 'vocab_size': 49408, 'max_position_embeddings': 77,
# 'num_hidden_layers': 12}, 'vision_config': {'hidden_size': 768, 'image_size': 224, 'patch_size': 32,
# 'num_hidden_layers': 12}, 'projection_dim': 512, 'ratio': 64, 'checkpoint_name_or_path': 'clip_vit_b_32',
# 'dtype': 'float16'}
model = CLIPModel(config)

import mindspore; mindspore.set_context(mode=0, device_id=0)
from mindformers.trainer import Trainer
from mindformers.tools.image_tools import load_image
# 初始化预训练任务
trainer = Trainer(task='contrastive_language_image_pretrain',
    model='clip_vit_b_32',
    train_dataset='./Flickr8k')
trainer.train() # 开启预训练

#初始化零样本图像分类下游任务
trainer = Trainer(task='zero_shot_image_classification',
    model='clip_vit_b_32',
    eval_dataset='./cifar-100-python')  
img = load_image("https://ascend-repo-modelzoo.obs.cn-east-2.myhuaweicloud.com/XFormer_for_mindspore/clip/sunflower.png")

# 方式1: 使用训练好的权重进行评估和推理
trainer.evaluate(eval_checkpoint=True)
predict_result = trainer.predict(predict_checkpoint=True, input_data=img, top_k=3)
print(predict_result)

# 方式2: 从obs下载训练好的权重并进行评估和推理
trainer.evaluate()  #下载权重进行评估
predict_result = trainer.predict(input_data=img, top_k=3)  #下载权重进行推理
print(predict_result)

import mindspore; mindspore.set_context(mode=0, device_id=0)
from mindformers import pipeline
from mindformers.tools.image_tools import load_image

classifier = pipeline("zero_shot_image_classification",
                      model="clip_vit_b_32",
                      candidate_labels=["sunflower", "tree", "dog", "cat", "toy"])
img = load_image("https://ascend-repo-modelzoo.obs.cn-east-2."
          "myhuaweicloud.com/XFormer_for_mindspore/clip/sunflower.png")
classifier(img)
# 输出
# [[{'score': 0.99995565, 'label': 'sunflower'}, {'score': 2.5318595e-05, 'label': 'toy'},
# {'score': 9.903885e-06, 'label': 'dog'}, {'score': 6.75336e-06, 'label': 'tree'},
# {'score': 2.396818e-06, 'label': 'cat'}]]

python mindformers/models/clip/convert_weight.py --torch_path "PATH OF ViT-B/32.pt" --mindspore_path "SAVE PATH OF clip_vit_b_32.ckpt"