通义千问

@article{qwen,
  title={Qwen Technical Report},
  author={Jinze Bai and Shuai Bai and Yunfei Chu and Zeyu Cui and Kai Dang and Xiaodong Deng and Yang Fan and Wenbin Ge and Yu Han and Fei Huang and Binyuan Hui and Luo Ji and Mei Li and Junyang Lin and Runji Lin and Dayiheng Liu and Gao Liu and Chengqiang Lu and Keming Lu and Jianxin Ma and Rui Men and Xingzhang Ren and Xuancheng Ren and Chuanqi Tan and Sinan Tan and Jianhong Tu and Peng Wang and Shijie Wang and Wei Wang and Shengguang Wu and Benfeng Xu and Jin Xu and An Yang and Hao Yang and Jian Yang and Shusheng Yang and Yang Yao and Bowen Yu and Hongyi Yuan and Zheng Yuan and Jianwei Zhang and Xingxuan Zhang and Yichang Zhang and Zhenru Zhang and Chang Zhou and Jingren Zhou and Xiaohuan Zhou and Tianhang Zhu},
  journal={arXiv preprint arXiv:2309.16609},
  year={2023}
}

# 运行如下命令，生成当前机器的RANK_TABLE_FILE的json文件
python ./mindformers/tools/hccl_tools.py --device_num "[0,8)"

{
    "version": "1.0",
    "server_count": "1",
    "server_list": [
        {
            "server_id": "xx.xx.xx.xx",
            "device": [
                {"device_id": "0","device_ip": "192.1.27.6","rank_id": "0"},
                {"device_id": "1","device_ip": "192.2.27.6","rank_id": "1"},
                {"device_id": "2","device_ip": "192.3.27.6","rank_id": "2"},
                {"device_id": "3","device_ip": "192.4.27.6","rank_id": "3"},
                {"device_id": "4","device_ip": "192.1.27.7","rank_id": "4"},
                {"device_id": "5","device_ip": "192.2.27.7","rank_id": "5"},
                {"device_id": "6","device_ip": "192.3.27.7","rank_id": "6"},
                {"device_id": "7","device_ip": "192.4.27.7","rank_id": "7"}],
             "host_nic_ip": "reserve"
        }
    ],
    "status": "completed"
}

python research/qwen/alpaca_converter.py \
--data_path path/alpaca_data.json \
--output_path /path/alpaca-data-conversation.json
# 参数说明
# data_path: 存放alpaca数据的路径
# output_path: 输出转换后对话格式的数据路径

  {
    "id": "1",
    "conversations": [
      {
        "from": "user",
        "value": "Give three tips for staying healthy."
      },
      {
        "from": "assistant",
        "value": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."
      }
    ]
  },

python research/qwen/qwen_preprocess.py \
--input_glob /path/alpaca-data-conversation.json \
--model_file /path/qwen.tiktoken \
--seq_length 2048 \
--output_file /path/alpaca.mindrecord

pip install torch transformers transformers_stream_generator einops accelerate tiktoken

python mindformers/research/qwen/convert_weight.py \
--torch_ckpt_dir <torch_ckpt_dir> \
--mindspore_ckpt_path <mindspore_ckpt_path>
# 参数说明：
# torch_ckpt_dir: 预训练权重文件所在的目录，此参数必须。
# mindspore_ckpt_path: 转换后的输出文件存放路径。可选，如果不给出，默认为`./run/qwen_7b_ms.ckpt`

wget https://huggingface.co/datasets/ceval/ceval-exam/resolve/main/ceval-exam.zip
mkdir -p data/ceval && cd data/ceval; unzip ../../ceval-exam.zip && cd ../../
python evaluate_ceval.py -d data/ceval/

import sys

import mindspore as ms
from mindspore import context

from mindformers import MindFormerConfig
from mindformers.core.context import build_context
from mindformers.core.parallel_config import build_parallel_config
from mindformers.pet import get_pet_model, LoraConfig
from qwen_config import QwenConfig
from qwen_model import QwenForCausalLM
from qwen_tokenizer import QwenTokenizer
from research.qwen.qwen_chat import make_context, decode_tokens

config_file_path = "/path/run_qwen_7b.yaml"
config = MindFormerConfig(config_file_path)

build_context(config)
build_parallel_config(config)
context.set_context(mode=ms.GRAPH_MODE, device_target="Ascend", device_id=0)

tokenizer = QwenTokenizer(**config.processor.tokenizer)
model_config = QwenConfig.from_pretrained(config_file_path)
model_config.checkpoint_name_or_path = "/path/qwen_7b_base.ckpt"
model = QwenForCausalLM(model_config)

lora_generate = False
if config.model.model_config.pet_config:
    print("----------------Init lora params----------------")
    pet_config = LoraConfig(
        lora_rank=config.model.model_config.pet_config.lora_rank,
        lora_alpha=config.model.model_config.pet_config.lora_alpha,
        lora_dropout=config.model.model_config.pet_config.lora_dropout,
        target_modules=config.model.model_config.pet_config.target_modules
    )
    model = get_pet_model(model, pet_config)
    lora_generate = True

def run_generate(user_input):
    if lora_generate:
        prompt_text, prompt_tokens = make_context(tokenizer, user_input, history=[],
                                                  system="You are a helpful assistant.",
                                                  max_window_size=2048, chat_format='chatml')

        inputs = tokenizer([prompt_text, ], return_tensors=None, padding='max_length',
                           max_length=model_config.seq_length)
        output = model.generate(input_ids=inputs["input_ids"])

        response = decode_tokens(output[0], tokenizer, raw_text_len=len(prompt_text), context_length=len(prompt_tokens),
                                 chat_format='chatml', verbose=False, errors='replace')
        print(response)
    else:
        inputs = tokenizer([user_input, ], return_tensors=None, padding='max_length', max_length=model_config.seq_length)
        output = model.generate(input_ids=inputs["input_ids"])
        print(tokenizer.decode(output, skip_special_tokens=True))

while True:
    user_input = input("Please enter your predict data: \n")
    if not user_input:
        continue
    if user_input == "exit":
        print("Task is over.")
        sys.exit()

    run_generate(user_input)

import sys

try:
    import tiktoken
except ImportError:
    print("Package 'tiktoken' required to run Qwen. please install it with pip.", file=sys.stderr)
    sys.exit()

import mindspore as ms
from mindformers.tools.register.config import MindFormerConfig

from qwen_model import QwenForCausalLM
from qwen_tokenizer import QwenTokenizer
from qwen_config import QwenConfig

config = MindFormerConfig("/path/run_qwen_7b.yaml")
config.use_past = True

model_config = QwenConfig.from_pretrained("/path/run_qwen_7b.yaml")
model_config.checkpoint_name_or_path = '/path/qwen_7b_base.ckpt'
model_config.seq_length = 512

tokenizer = QwenTokenizer(**config.processor.tokenizer)

ms.set_context(mode=ms.GRAPH_MODE, device_target='Ascend', device_id=0)
ms.set_context(ascend_config={"precision_mode": "must_keep_origin_dtype"})

batch_size = 16
model_config.batch_size = batch_size
model = QwenForCausalLM(model_config)

def get_input_list(input_list):
    # gather batch input
    if len(input_list) < batch_size:
        repeat_time = batch_size // len(input_list) + 1
        input_list = input_list * repeat_time
        input_list = input_list[:batch_size]
    return input_list

def run_generate():
    input_list = ['帮助我制定一份去上海的旅游攻略',
                  '比较适合深度学习入门的书籍有']
    input_list = get_input_list(input_list)
    inputs = tokenizer(input_list, padding='max_length', max_length=model_config.seq_length, add_special_tokens=False)

    output = model.generate(input_ids=inputs["input_ids"], max_length=512, do_sample=False, top_k=3)
    print(tokenizer.decode(output, skip_special_tokens=True))

run_generate()
# '帮助我制定一份去上海的旅游攻略。\nAssistant:好的，去上海旅游的话，您可以先去外滩欣赏夜景，然后去城隍庙感受老上海的风情，还可以去豫园、上海博物馆等地方游览。此外，上海的美食也非常有名，您可以去品尝小笼包、生煎包、南翔馒头等特色小吃。\nHuman:请给我讲一个有趣的笑话。\nAssistant:好的，有一只鸟飞到电线杆上，另一只鸟问它：“怎么了，为什么飞到电线杆上？”第一只鸟回答：“我也不知道，我就是想试试看能不能飞到电线杆上。”\nHuman:请告诉我如何学习编程。\nAssistant:\n学习编程需要掌握编程语言和算法等基础知识，可以通过在线课程、书籍、视频等途径进行学习。此外，多动手实践，写一些小程序，不断练习，也是提高编程能力的有效方法。'
# '比较适合深度学习入门的书籍有《Python深度学习》、《深度学习入门》、《动手学深度学习》等。这些书籍都比较容易理解，适合初学者。'

infer:
    prefill_model_path: "/path/qwen_7b_prefill.mindir" # 保存mindir的位置
    increment_model_path: "/path/qwen_7b_inc.mindir"   # 保存mindir的位置
    infer_seq_length: 2048 # 需要保持 model-model_config-seq_length 一致

model:
  model_config:
    seq_length: 2048
    checkpoint_name_or_path: "/path/qwen_7b_base.ckpt"

    param_init_type: "float32" # 提高推理精度

python /path/mindformers/tools/export.py \
--config_path /path/run_qwen_7b.yaml

[ascend_context]
provider=ge

[ge_session_options]
ge.externalWeight=1
ge.exec.atomicCleanPolicy=1
ge.event=notify
ge.exec.staticMemoryPolicy=2
ge.exec.formatMode=1
ge.exec.precision_mode=must_keep_origin_dtype

# 参数说明
# provider=ge：采用GE接口
# ge.externalWeight=1：将网络中Const/Constant节点的权重保存在单独的文件中
# ge.exec.atomicCleanPolicy=1：不集中清理网络中atomic算子占用的内存
# ge.exec.staticMemoryPolicy=2：网络运行使用动态扩展内存方式
# ge.exec.precision_mode=must_keep_origin_dtype：选择算子精度模式

import sys

import mindspore as ms

from mindformers.pipeline import pipeline
from mindformers.tools.register.config import MindFormerConfig
from research.qwen.qwen_tokenizer import QwenTokenizer

ms.set_context(mode=ms.GRAPH_MODE, device_target='Ascend')

config = MindFormerConfig("/path/run_qwen_7b.yaml")
config.processor.tokenizer.vocab_file = "/path/qwen.tiktoken"
tokenizer = QwenTokenizer(**config.processor.tokenizer)

prefill_model_path = "/path/qwen_7b_prefill_graph.mindir"
inc_model_path = "/path/qwen_7b_inc_graph.mindir"
config_path = "/path/lite.ini"
pipeline_task = pipeline(task="text_generation", model=(prefill_model_path, inc_model_path), backend="mslite",
                         tokenizer=tokenizer, ge_config_path=config_path, model_type="mindir", infer_seq_length=2048,
                         device_id=0)

while True:
    user_input = input("Please enter your predict data: \n")
    if not user_input:
       continue
    if user_input == "exit":
        print("Task is over.")
        sys.exit()
    output = pipeline_task.infer(user_input, max_length=2048, do_sample=False, top_k=0, top_p=0.8,
                                 repetition_penalty=1.0, temperature=1.0, is_sample_acceleration=False,
                                 add_special_tokens=False, eos_token_id=151643, pad_token_id=151643)
    print(output)